airfoil 0.4.0 → 0.4.2

package/src/utils/docker.ts

@@ -12,32 +12,126 @@ export const checkDockerVersion = () =>
 export const createDockerVolume = (volumeName: string) =>
   Effect.gen(function* () {
     const docker = yield* ChildProcess.make("docker", ["volume", "create", volumeName]);
+    const exitCode = yield* docker.exitCode;
+    if (exitCode !== 0) {
+      return yield* Effect.fail(new Error(`Docker volume create failed with code ${exitCode}`));
+    }
+  });
+
+export const createDockerNetwork = (networkName: string) =>
+  Effect.gen(function* () {
+    const docker = yield* ChildProcess.make("docker", ["network", "create", networkName]);
+    const exitCode = yield* docker.exitCode;
+    if (exitCode !== 0) {
+      return yield* Effect.fail(new Error(`Docker network create failed with code ${exitCode}`));
+    }
+  });
+
+export const removeDockerContainer = (containerName: string) =>
+  Effect.gen(function* () {
+    const docker = yield* ChildProcess.make("docker", ["rm", "-f", containerName], {
+      stdout: "ignore",
+      stderr: "ignore",
+    });
     yield* docker.exitCode;
   });
 
+export const removeDockerNetwork = (networkName: string) =>
+  Effect.gen(function* () {
+    const docker = yield* ChildProcess.make("docker", ["network", "rm", networkName], {
+      stdout: "ignore",
+      stderr: "ignore",
+    });
+    yield* docker.exitCode;
+  });
+
+export const pullDockerImage = (image: string) =>
+  Effect.gen(function* () {
+    const docker = yield* ChildProcess.make("docker", ["pull", image], {
+      stdout: "inherit",
+      stderr: "inherit",
+    });
+    const exitCode = yield* docker.exitCode;
+    if (exitCode !== 0) {
+      return yield* Effect.fail(new Error(`Docker pull failed with code ${exitCode}`));
+    }
+  });
+
 export const runDockerContainer = (image: string, volumeName: string) =>
   Effect.gen(function* () {
+    const networkName = "airfoil-dev";
+    const seaweedContainerName = "airfoil-seaweedfs";
+
+    yield* removeDockerContainer(seaweedContainerName);
+    yield* removeDockerNetwork(networkName);
+    yield* createDockerNetwork(networkName);
+    yield* Effect.addFinalizer(() =>
+      removeDockerNetwork(networkName).pipe(Effect.catchCause(() => Effect.void)),
+    );
+
+    const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+    const seaweed = ChildProcess.make("docker", [
+      "run",
+      "-d",
+      "--rm",
+      "--name",
+      seaweedContainerName,
+      "--network",
+      networkName,
+      "--network-alias",
+      "seaweedfs",
+      "-e",
+      "AWS_ACCESS_KEY_ID=wingsdevaccesskey",
+      "-e",
+      "AWS_SECRET_ACCESS_KEY=wingsdevsecretkey",
+      "-e",
+      "S3_BUCKET=default-bucket",
+      "-v",
+      `${volumeName}:/data`,
+      "-p",
+      "8333:8333",
+      "chrislusf/seaweedfs:4.29",
+      "weed",
+      "mini",
+      "-s3",
+      "-dir=/data",
+    ]);
+
+    yield* spawner.string(seaweed).pipe(Effect.map(String.trim));
+    yield* Effect.addFinalizer(() =>
+      removeDockerContainer(seaweedContainerName).pipe(Effect.catchCause(() => Effect.void)),
+    );
+
     const docker = yield* ChildProcess.make("docker", [
       "run",
       "-it",
       "--rm",
       "--name",
       "wings-dev",
-      "-v",
-      `${volumeName}:/tmp`,
+      "--network",
+      networkName,
+      "-e",
+      "AWS_ACCESS_KEY_ID=wingsdevaccesskey",
+      "-e",
+      "AWS_SECRET_ACCESS_KEY=wingsdevsecretkey",
+      "-e",
+      "AWS_ENDPOINT=http://seaweedfs:8333",
+      "-e",
+      "AWS_BUCKET_NAME=default-bucket",
+      "-e",
+      "AWS_DEFAULT_REGION=us-east-1",
+      "-e",
+      "AWS_ALLOW_HTTP=true",
       "-p",
       "7777:7777",
-      "-p",
-      "7780:7780",
       image,
       "dev",
-      "--http.address=0.0.0.0:7780",
-      "--metadata.address=0.0.0.0:7777",
+      "--grpc.address=0.0.0.0:7777",
     ]);
 
     const exitCode = yield* docker.exitCode;
 
     if (exitCode !== 0) {
-      yield* Effect.fail(new Error(`Docker process exited with code ${exitCode}`));
+      return yield* Effect.fail(new Error(`Docker process exited with code ${exitCode}`));
     }
   });

package/src/utils/options.ts

@@ -5,7 +5,7 @@ import { Flag } from "effect/unstable/cli";
  */
 export const hostOption = Flag.string("host").pipe(
   Flag.withDescription("Server host"),
-  Flag.withDefault("localhost"),
+  Flag.withDefault("127.0.0.1"),
 );
 
 export const portOption = Flag.integer("port").pipe(

package/src/utils/wings.ts

@@ -56,25 +56,29 @@ export const downloadWings = (version: string, targetPath: string, isStress = fa
     const client = yield* HttpClient.HttpClient;
 
     const filename = getBinaryFilename(isStress);
+    const tempPath = `${targetPath}.download`;
 
     const downloadPath = version === "latest" ? "latest/download" : `download/${version}`;
     const url = `${GITHUB_RELEASES_URL}/${downloadPath}/${filename}`;
 
     yield* fs.makeDirectory(WINGS_DIR, { recursive: true });
-
-    const response = yield* client.get(url);
-    const buffer = yield* HttpClientResponse.matchStatus({
-      200: (response) => response.arrayBuffer,
-      orElse: (response) => {
-        throw new Error(`Failed to download Wings from ${url}: ${response.status}`);
-      },
-    })(response);
-
-    yield* fs.writeFile(targetPath, new Uint8Array(buffer));
-
-    yield* verifyChecksum(version, targetPath, isStress);
-
-    yield* fs.chmod(targetPath, 0o755);
+    yield* fs.remove(tempPath).pipe(Effect.ignore);
+
+    const download = Effect.gen(function* () {
+      const response = yield* client.get(url);
+      const buffer = yield* HttpClientResponse.matchStatus({
+        200: (response) => response.arrayBuffer,
+        orElse: (response) =>
+          Effect.fail(new Error(`Failed to download Wings from ${url}: ${response.status}`)),
+      })(response);
+
+      yield* fs.writeFile(tempPath, new Uint8Array(buffer));
+      yield* verifyChecksum(version, tempPath, isStress);
+      yield* fs.chmod(tempPath, 0o755);
+      yield* fs.rename(tempPath, targetPath);
+    });
+
+    yield* download.pipe(Effect.onError(() => fs.remove(tempPath).pipe(Effect.ignore)));
   });
 
 /**

package/src/commands/cluster/data-lake/create-iceberg.ts

@@ -1,59 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { printTable } from "console-table-printer";
-import { Effect } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { hostOption, portOption } from "../../../utils/options.js";
-
-const parentOption = Flag.string("parent").pipe(
-  Flag.withDescription("Parent tenant in format: tenants/{tenant}"),
-);
-
-const dataLakeIdOption = Flag.string("data-lake-id").pipe(
-  Flag.withDescription("Unique identifier for the data lake"),
-);
-
-export const createDataLakeIcebergCommand = Command.make(
-  "iceberg",
-  {
-    parent: parentOption,
-    dataLakeId: dataLakeIdOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({ parent, dataLakeId, host, port }) =>
-    Effect.gen(function* () {
-      p.intro("🏞️  Create Iceberg Data Lake");
-
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Creating Iceberg data lake...");
-
-      const result = yield* WingsClusterMetadata.createDataLake({
-        parent,
-        dataLakeId,
-        dataLakeConfig: {
-          _tag: "iceberg",
-          iceberg: {},
-        },
-      }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to create data lake"))),
-      );
-
-      s.stop("Iceberg data lake created successfully");
-
-      yield* Effect.sync(() => {
-        printTable([
-          {
-            name: result.name,
-            type: "Iceberg",
-          },
-        ]);
-        p.outro("✓ Done");
-      });
-    }),
-).pipe(Command.withDescription("Create a new Iceberg data lake"));

package/src/commands/cluster/data-lake/create-parquet.ts

@@ -1,59 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { printTable } from "console-table-printer";
-import { Effect } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { hostOption, portOption } from "../../../utils/options.js";
-
-const parentOption = Flag.string("parent").pipe(
-  Flag.withDescription("Parent tenant in format: tenants/{tenant}"),
-);
-
-const dataLakeIdOption = Flag.string("data-lake-id").pipe(
-  Flag.withDescription("Unique identifier for the data lake"),
-);
-
-export const createDataLakeParquetCommand = Command.make(
-  "parquet",
-  {
-    parent: parentOption,
-    dataLakeId: dataLakeIdOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({ parent, dataLakeId, host, port }) =>
-    Effect.gen(function* () {
-      p.intro("🏞️  Create Parquet Data Lake");
-
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Creating Parquet data lake...");
-
-      const result = yield* WingsClusterMetadata.createDataLake({
-        parent,
-        dataLakeId,
-        dataLakeConfig: {
-          _tag: "parquet",
-          parquet: {},
-        },
-      }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to create data lake"))),
-      );
-
-      s.stop("Parquet data lake created successfully");
-
-      yield* Effect.sync(() => {
-        printTable([
-          {
-            name: result.name,
-            type: "Parquet",
-          },
-        ]);
-        p.outro("✓ Done");
-      });
-    }),
-).pipe(Command.withDescription("Create a new Parquet data lake"));

package/src/commands/cluster/data-lake/create.ts

@@ -1,10 +0,0 @@
-import { Effect } from "effect";
-import { Command } from "effect/unstable/cli";
-
-import { createDataLakeIcebergCommand } from "./create-iceberg.js";
-import { createDataLakeParquetCommand } from "./create-parquet.js";
-
-export const createDataLakeCommand = Command.make("create-data-lake", {}, () => Effect.void).pipe(
-  Command.withDescription("Create a new data lake"),
-  Command.withSubcommands([createDataLakeIcebergCommand, createDataLakeParquetCommand]),
-);

package/src/commands/cluster/data-lake/delete.ts

@@ -1,54 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { Effect } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { forceOption, hostOption, portOption } from "../../../utils/options.js";
-
-const nameOption = Flag.string("name").pipe(
-  Flag.withDescription("Data lake name in format: tenants/{tenant}/data-lakes/{data-lake}"),
-);
-
-export const deleteDataLakeCommand = Command.make(
-  "delete-data-lake",
-  {
-    name: nameOption,
-    force: forceOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({ name, force, host, port }) =>
-    Effect.gen(function* () {
-      p.intro("🗑️  Delete Data Lake");
-
-      if (!force) {
-        const confirm = yield* Effect.tryPromise({
-          try: () =>
-            p.confirm({
-              message: `Are you sure you want to delete data lake ${name}?`,
-              initialValue: false,
-            }),
-          catch: () => new Error("Failed to confirm deletion"),
-        });
-
-        if (p.isCancel(confirm) || !confirm) {
-          p.cancel("Deletion cancelled");
-          process.exit(0);
-        }
-      }
-
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Deleting data lake...");
-
-      yield* WingsClusterMetadata.deleteDataLake({ name }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to delete data lake"))),
-      );
-
-      s.stop("Data lake deleted successfully");
-      p.outro("✓ Done");
-    }),
-).pipe(Command.withDescription("Delete a data lake from the cluster"));

package/src/commands/cluster/data-lake/get.ts

@@ -1,40 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { printTable } from "console-table-printer";
-import { Effect } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { hostOption, portOption } from "../../../utils/options.js";
-
-const nameOption = Flag.string("name").pipe(
-  Flag.withDescription("Data lake name in format: tenants/{tenant}/data-lakes/{data-lake}"),
-);
-
-export const getDataLakeCommand = Command.make(
-  "get-data-lake",
-  {
-    name: nameOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({ name, host, port }) =>
-    Effect.gen(function* () {
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Fetching data lake...");
-
-      const dataLake = yield* WingsClusterMetadata.getDataLake({ name }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to get data lake"))),
-      );
-
-      s.stop("Data lake retrieved");
-
-      yield* Effect.sync(() => {
-        printTable([{ name: dataLake.name, type: dataLake.dataLakeConfig._tag || "-" }]);
-        p.outro("✓ Done");
-      });
-    }),
-).pipe(Command.withDescription("Get details of a specific data lake"));

package/src/commands/cluster/data-lake/list.ts

@@ -1,62 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { printTable } from "console-table-printer";
-import { Effect, Option } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { hostOption, pageSizeOption, pageTokenOption, portOption } from "../../../utils/options.js";
-
-const parentOption = Flag.string("parent").pipe(
-  Flag.withDescription("Parent tenant in format: tenants/{tenant}"),
-);
-
-export const listDataLakesCommand = Command.make(
-  "list-data-lakes",
-  {
-    parent: parentOption,
-    pageSize: pageSizeOption,
-    pageToken: pageTokenOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({ parent, pageSize, pageToken, host, port }) =>
-    Effect.gen(function* () {
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Fetching data lakes...");
-
-      const response = yield* WingsClusterMetadata.listDataLakes({
-        parent,
-        pageSize,
-        pageToken: Option.getOrUndefined(pageToken),
-      }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to list data lakes"))),
-      );
-
-      s.stop(`Found ${response.dataLakes.length} data lake(s)`);
-
-      yield* Effect.sync(() => {
-        if (response.dataLakes.length === 0) {
-          p.log.warn("No data lakes found");
-        } else {
-          printTable(
-            response.dataLakes.map(
-              (dataLake: { name: string; dataLakeConfig: { _tag?: string | null } }) => ({
-                name: dataLake.name,
-                type: dataLake.dataLakeConfig._tag || "-",
-              }),
-            ),
-          );
-        }
-
-        if (response.nextPageToken) {
-          p.log.info(`Next page token: ${response.nextPageToken}`);
-        }
-
-        p.outro("✓ Done");
-      });
-    }),
-).pipe(Command.withDescription("List all data lakes belonging to a tenant"));

package/src/commands/cluster/object-store/create-aws.ts

@@ -1,104 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { printTable } from "console-table-printer";
-import { Effect, Option } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { hostOption, portOption } from "../../../utils/options.js";
-
-const parentOption = Flag.string("parent").pipe(
-  Flag.withDescription("Parent tenant in format: tenants/{tenant}"),
-);
-
-const objectStoreIdOption = Flag.string("object-store-id").pipe(
-  Flag.withDescription("Unique identifier for the object store"),
-);
-
-const bucketNameOption = Flag.string("bucket-name").pipe(
-  Flag.withDescription("AWS S3 bucket name"),
-);
-
-const prefixOption = Flag.string("prefix").pipe(
-  Flag.withDescription("Bucket prefix (optional)"),
-  Flag.optional,
-);
-
-const accessKeyIdOption = Flag.string("access-key-id").pipe(
-  Flag.withDescription("AWS_ACCESS_KEY_ID"),
-);
-
-const secretAccessKeyOption = Flag.string("secret-access-key").pipe(
-  Flag.withDescription("AWS_SECRET_ACCESS_KEY"),
-);
-
-const regionOption = Flag.string("region").pipe(
-  Flag.withDescription("AWS_DEFAULT_REGION (e.g., us-east-1)"),
-  Flag.optional,
-);
-
-export const createObjectStoreAwsCommand = Command.make(
-  "aws",
-  {
-    parent: parentOption,
-    objectStoreId: objectStoreIdOption,
-    bucketName: bucketNameOption,
-    prefix: prefixOption,
-    accessKeyId: accessKeyIdOption,
-    secretAccessKey: secretAccessKeyOption,
-    region: regionOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({
-    parent,
-    objectStoreId,
-    bucketName,
-    prefix,
-    accessKeyId,
-    secretAccessKey,
-    region,
-    host,
-    port,
-  }) =>
-    Effect.gen(function* () {
-      p.intro("🗄️  Create AWS S3 Object Store");
-
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Creating AWS S3 object store...");
-
-      const result = yield* WingsClusterMetadata.createObjectStore({
-        parent,
-        objectStoreId,
-        objectStoreConfig: {
-          _tag: "aws",
-          aws: {
-            bucketName,
-            prefix: Option.getOrUndefined(prefix),
-            accessKeyId,
-            secretAccessKey,
-            region: Option.getOrUndefined(region),
-          },
-        },
-      }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to create AWS object store"))),
-      );
-
-      s.stop("AWS S3 object store created successfully");
-
-      yield* Effect.sync(() => {
-        printTable([
-          {
-            name: result.name,
-            type: "AWS S3",
-            bucket: bucketName,
-            region: Option.getOrUndefined(region) || "-",
-          },
-        ]);
-        p.outro("✓ Done");
-      });
-    }),
-).pipe(Command.withDescription("Create a new AWS S3 object store"));

package/src/commands/cluster/object-store/create-azure.ts

@@ -1,96 +0,0 @@
-import * as p from "@clack/prompts";
-import { WingsClusterMetadata } from "@useairfoil/wings";
-import { printTable } from "console-table-printer";
-import { Effect, Option } from "effect";
-import { Command, Flag } from "effect/unstable/cli";
-
-import { makeClusterMetadataLayer } from "../../../utils/client.js";
-import { hostOption, portOption } from "../../../utils/options.js";
-
-const parentOption = Flag.string("parent").pipe(
-  Flag.withDescription("Parent tenant in format: tenants/{tenant}"),
-);
-
-const objectStoreIdOption = Flag.string("object-store-id").pipe(
-  Flag.withDescription("Unique identifier for the object store"),
-);
-
-const containerNameOption = Flag.string("container-name").pipe(
-  Flag.withDescription("Azure container name"),
-);
-
-const prefixOption = Flag.string("prefix").pipe(
-  Flag.withDescription("Container prefix (optional)"),
-  Flag.optional,
-);
-
-const storageAccountNameOption = Flag.string("storage-account-name").pipe(
-  Flag.withDescription("AZURE_STORAGE_ACCOUNT_NAME"),
-);
-
-const storageAccountKeyOption = Flag.string("storage-account-key").pipe(
-  Flag.withDescription("AZURE_STORAGE_ACCOUNT_KEY"),
-);
-
-export const createObjectStoreAzureCommand = Command.make(
-  "azure",
-  {
-    parent: parentOption,
-    objectStoreId: objectStoreIdOption,
-    containerName: containerNameOption,
-    prefix: prefixOption,
-    storageAccountName: storageAccountNameOption,
-    storageAccountKey: storageAccountKeyOption,
-    host: hostOption,
-    port: portOption,
-  },
-  ({
-    parent,
-    objectStoreId,
-    containerName,
-    prefix,
-    storageAccountName,
-    storageAccountKey,
-    host,
-    port,
-  }) =>
-    Effect.gen(function* () {
-      p.intro("🗄️  Create Azure Blob Storage Object Store");
-
-      const layer = makeClusterMetadataLayer(host, port);
-
-      const s = p.spinner();
-      s.start("Creating Azure object store...");
-
-      const result = yield* WingsClusterMetadata.createObjectStore({
-        parent,
-        objectStoreId,
-        objectStoreConfig: {
-          _tag: "azure",
-          azure: {
-            containerName,
-            prefix: Option.getOrUndefined(prefix),
-            storageAccountName,
-            storageAccountKey,
-          },
-        },
-      }).pipe(
-        Effect.provide(layer),
-        Effect.tapError(() => Effect.sync(() => s.stop("Failed to create Azure object store"))),
-      );
-
-      s.stop("Azure object store created successfully");
-
-      yield* Effect.sync(() => {
-        printTable([
-          {
-            name: result.name,
-            type: "Azure Blob",
-            container: containerName,
-            account: storageAccountName,
-          },
-        ]);
-        p.outro("✓ Done");
-      });
-    }),
-).pipe(Command.withDescription("Create a new Azure Blob Storage object store"));