airfoil 0.1.0

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "airfoil",
+  "version": "0.1.0",
+  "module": "index.ts",
+  "type": "module",
+  "bin": {
+    "airfoil": "./dist/index.js"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check .",
+    "lint:fix": "pnpm lint --write",
+    "format": "biome format --write .",
+    "airfoil:dev": "bun run src/index.ts",
+    "build": "bun build src/index.ts --outdir dist --target node",
+    "compile": "bun build src/index.ts --compile --outfile airfoil",
+    "compile:all": "bun run src/scripts/compile-all.ts"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.5"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "dependencies": {
+    "@useairfoil/flight": "0.0.0",
+    "@useairfoil/wings": "0.0.0",
+    "@clack/prompts": "^0.11.0",
+    "@effect/cli": "^0.73.1",
+    "@effect/platform-node": "^0.104.1",
+    "console-table-printer": "^2.15.0",
+    "effect": "^3.19.15",
+    "nice-grpc": "^2.1.13",
+    "nice-grpc-common": "^2.0.2"
+  }
+}

package/src/commands/cluster/data-lake/create-iceberg.ts

@@ -0,0 +1,61 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { printTable } from "console-table-printer";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { hostOption, portOption } from "../../../utils/options.js";
+
+const parentOption = Options.text("parent").pipe(
+  Options.withDescription("Parent tenant in format: tenants/{tenant}"),
+);
+
+const dataLakeIdOption = Options.text("data-lake-id").pipe(
+  Options.withDescription("Unique identifier for the data lake"),
+);
+
+export const createDataLakeIcebergCommand = Command.make(
+  "iceberg",
+  {
+    parent: parentOption,
+    dataLakeId: dataLakeIdOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ parent, dataLakeId, host, port }) =>
+    Effect.gen(function* () {
+      p.intro("🏞️  Create Iceberg Data Lake");
+
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Creating Iceberg data lake...");
+
+      const result = yield* WingsClusterMetadata.createDataLake({
+        parent,
+        dataLakeId,
+        dataLakeConfig: {
+          _tag: "iceberg",
+          iceberg: {},
+        },
+      }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to create data lake")),
+        ),
+      );
+
+      s.stop("Iceberg data lake created successfully");
+
+      yield* Effect.sync(() => {
+        printTable([
+          {
+            name: result.name,
+            type: "Iceberg",
+          },
+        ]);
+        p.outro("✓ Done");
+      });
+    }).pipe(Effect.catchAll(handleCliError("Failed to create data lake"))),
+).pipe(Command.withDescription("Create a new Iceberg data lake"));

package/src/commands/cluster/data-lake/create-parquet.ts

@@ -0,0 +1,61 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { printTable } from "console-table-printer";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { hostOption, portOption } from "../../../utils/options.js";
+
+const parentOption = Options.text("parent").pipe(
+  Options.withDescription("Parent tenant in format: tenants/{tenant}"),
+);
+
+const dataLakeIdOption = Options.text("data-lake-id").pipe(
+  Options.withDescription("Unique identifier for the data lake"),
+);
+
+export const createDataLakeParquetCommand = Command.make(
+  "parquet",
+  {
+    parent: parentOption,
+    dataLakeId: dataLakeIdOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ parent, dataLakeId, host, port }) =>
+    Effect.gen(function* () {
+      p.intro("🏞️  Create Parquet Data Lake");
+
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Creating Parquet data lake...");
+
+      const result = yield* WingsClusterMetadata.createDataLake({
+        parent,
+        dataLakeId,
+        dataLakeConfig: {
+          _tag: "parquet",
+          parquet: {},
+        },
+      }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to create data lake")),
+        ),
+      );
+
+      s.stop("Parquet data lake created successfully");
+
+      yield* Effect.sync(() => {
+        printTable([
+          {
+            name: result.name,
+            type: "Parquet",
+          },
+        ]);
+        p.outro("✓ Done");
+      });
+    }).pipe(Effect.catchAll(handleCliError("Failed to create data lake"))),
+).pipe(Command.withDescription("Create a new Parquet data lake"));

package/src/commands/cluster/data-lake/create.ts

@@ -0,0 +1,16 @@
+import { Command } from "@effect/cli";
+import { Effect } from "effect";
+import { createDataLakeIcebergCommand } from "./create-iceberg.js";
+import { createDataLakeParquetCommand } from "./create-parquet.js";
+
+export const createDataLakeCommand = Command.make(
+  "create-data-lake",
+  {},
+  () => Effect.void,
+).pipe(
+  Command.withDescription("Create a new data lake"),
+  Command.withSubcommands([
+    createDataLakeIcebergCommand,
+    createDataLakeParquetCommand,
+  ]),
+);

package/src/commands/cluster/data-lake/delete.ts

@@ -0,0 +1,58 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { forceOption, hostOption, portOption } from "../../../utils/options.js";
+
+const nameOption = Options.text("name").pipe(
+  Options.withDescription(
+    "Data lake name in format: tenants/{tenant}/data-lakes/{data-lake}",
+  ),
+);
+
+export const deleteDataLakeCommand = Command.make(
+  "delete-data-lake",
+  {
+    name: nameOption,
+    force: forceOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ name, force, host, port }) =>
+    Effect.gen(function* () {
+      p.intro("🗑️  Delete Data Lake");
+
+      if (!force) {
+        const confirm = yield* Effect.tryPromise({
+          try: () =>
+            p.confirm({
+              message: `Are you sure you want to delete data lake ${name}?`,
+              initialValue: false,
+            }),
+          catch: () => new Error("Failed to confirm deletion"),
+        });
+
+        if (p.isCancel(confirm) || !confirm) {
+          p.cancel("Deletion cancelled");
+          process.exit(0);
+        }
+      }
+
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Deleting data lake...");
+
+      yield* WingsClusterMetadata.deleteDataLake({ name }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to delete data lake")),
+        ),
+      );
+
+      s.stop("Data lake deleted successfully");
+      p.outro("✓ Done");
+    }).pipe(Effect.catchAll(handleCliError("Failed to delete data lake"))),
+).pipe(Command.withDescription("Delete a data lake from the cluster"));

package/src/commands/cluster/data-lake/get.ts

@@ -0,0 +1,46 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { printTable } from "console-table-printer";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { hostOption, portOption } from "../../../utils/options.js";
+
+const nameOption = Options.text("name").pipe(
+  Options.withDescription(
+    "Data lake name in format: tenants/{tenant}/data-lakes/{data-lake}",
+  ),
+);
+
+export const getDataLakeCommand = Command.make(
+  "get-data-lake",
+  {
+    name: nameOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ name, host, port }) =>
+    Effect.gen(function* () {
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Fetching data lake...");
+
+      const dataLake = yield* WingsClusterMetadata.getDataLake({ name }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to get data lake")),
+        ),
+      );
+
+      s.stop("Data lake retrieved");
+
+      yield* Effect.sync(() => {
+        printTable([
+          { name: dataLake.name, type: dataLake.dataLakeConfig._tag || "-" },
+        ]);
+        p.outro("✓ Done");
+      });
+    }).pipe(Effect.catchAll(handleCliError("Failed to get data lake"))),
+).pipe(Command.withDescription("Get details of a specific data lake"));

package/src/commands/cluster/data-lake/list.ts

@@ -0,0 +1,72 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { printTable } from "console-table-printer";
+import { Effect, Option } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import {
+  hostOption,
+  pageSizeOption,
+  pageTokenOption,
+  portOption,
+} from "../../../utils/options.js";
+
+const parentOption = Options.text("parent").pipe(
+  Options.withDescription("Parent tenant in format: tenants/{tenant}"),
+);
+
+export const listDataLakesCommand = Command.make(
+  "list-data-lakes",
+  {
+    parent: parentOption,
+    pageSize: pageSizeOption,
+    pageToken: pageTokenOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ parent, pageSize, pageToken, host, port }) =>
+    Effect.gen(function* () {
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Fetching data lakes...");
+
+      const response = yield* WingsClusterMetadata.listDataLakes({
+        parent,
+        pageSize,
+        pageToken: Option.getOrUndefined(pageToken),
+      }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to list data lakes")),
+        ),
+      );
+
+      s.stop(`Found ${response.dataLakes.length} data lake(s)`);
+
+      yield* Effect.sync(() => {
+        if (response.dataLakes.length === 0) {
+          p.log.warn("No data lakes found");
+        } else {
+          printTable(
+            response.dataLakes.map(
+              (dataLake: {
+                name: string;
+                dataLakeConfig: { _tag?: string | null };
+              }) => ({
+                name: dataLake.name,
+                type: dataLake.dataLakeConfig._tag || "-",
+              }),
+            ),
+          );
+        }
+
+        if (response.nextPageToken) {
+          p.log.info(`Next page token: ${response.nextPageToken}`);
+        }
+
+        p.outro("✓ Done");
+      });
+    }).pipe(Effect.catchAll(handleCliError("Failed to list data lakes"))),
+).pipe(Command.withDescription("List all data lakes belonging to a tenant"));

package/src/commands/cluster/index.ts

@@ -0,0 +1,52 @@
+import { Command } from "@effect/cli";
+import { Effect } from "effect";
+import { createDataLakeCommand } from "./data-lake/create.js";
+import { deleteDataLakeCommand } from "./data-lake/delete.js";
+import { getDataLakeCommand } from "./data-lake/get.js";
+import { listDataLakesCommand } from "./data-lake/list.js";
+import { createNamespaceCommand } from "./namespace/create.js";
+import { deleteNamespaceCommand } from "./namespace/delete.js";
+import { getNamespaceCommand } from "./namespace/get.js";
+import { listNamespacesCommand } from "./namespace/list.js";
+import { createObjectStoreCommand } from "./object-store/create.js";
+import { deleteObjectStoreCommand } from "./object-store/delete.js";
+import { getObjectStoreCommand } from "./object-store/get.js";
+import { listObjectStoresCommand } from "./object-store/list.js";
+import { createTenantCommand } from "./tenant/create.js";
+import { deleteTenantCommand } from "./tenant/delete.js";
+import { getTenantCommand } from "./tenant/get.js";
+import { listTenantsCommand } from "./tenant/list.js";
+import { createTopicCommand } from "./topic/create.js";
+import { deleteTopicCommand } from "./topic/delete.js";
+import { getTopicCommand } from "./topic/get.js";
+import { listTopicsCommand } from "./topic/list.js";
+
+export const clusterCommand = Command.make(
+  "cluster",
+  {},
+  () => Effect.void,
+).pipe(
+  Command.withDescription("Interact with the cluster metadata server"),
+  Command.withSubcommands([
+    createTenantCommand,
+    listTenantsCommand,
+    getTenantCommand,
+    deleteTenantCommand,
+    createNamespaceCommand,
+    listNamespacesCommand,
+    getNamespaceCommand,
+    deleteNamespaceCommand,
+    createTopicCommand,
+    listTopicsCommand,
+    getTopicCommand,
+    deleteTopicCommand,
+    createObjectStoreCommand,
+    listObjectStoresCommand,
+    getObjectStoreCommand,
+    deleteObjectStoreCommand,
+    createDataLakeCommand,
+    listDataLakesCommand,
+    getDataLakeCommand,
+    deleteDataLakeCommand,
+  ]),
+);

package/src/commands/cluster/namespace/create.ts

@@ -0,0 +1,107 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { printTable } from "console-table-printer";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { hostOption, portOption } from "../../../utils/options.js";
+
+const parentOption = Options.text("parent").pipe(
+  Options.withDescription(
+    "Parent tenant in format: tenants/{tenant} (e.g., 'tenants/default')",
+  ),
+);
+
+const namespaceIdOption = Options.text("namespace-id").pipe(
+  Options.withDescription(
+    "Unique identifier for the namespace (e.g., 'production')",
+  ),
+);
+
+const flushSizeBytesOption = Options.integer("flush-size-bytes").pipe(
+  Options.withDescription(
+    "Size at which the current segment is flushed to object storage",
+  ),
+  Options.withDefault(0),
+);
+
+const flushIntervalMillisOption = Options.integer("flush-interval-millis").pipe(
+  Options.withDescription(
+    "Maximum interval at which the current segment is flushed (milliseconds)",
+  ),
+  Options.withDefault(0),
+);
+
+const objectStoreOption = Options.text("object-store").pipe(
+  Options.withDescription(
+    "Object store used by this namespace (format: tenants/{tenant}/object-stores/{object-store})",
+  ),
+);
+
+const dataLakeOption = Options.text("data-lake").pipe(
+  Options.withDescription(
+    "Data lake used by this namespace (format: tenants/{tenant}/data-lakes/{data-lake})",
+  ),
+);
+
+export const createNamespaceCommand = Command.make(
+  "create-namespace",
+  {
+    parent: parentOption,
+    namespaceId: namespaceIdOption,
+    flushSizeBytes: flushSizeBytesOption,
+    flushIntervalMillis: flushIntervalMillisOption,
+    objectStore: objectStoreOption,
+    dataLake: dataLakeOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({
+    parent,
+    namespaceId,
+    flushSizeBytes,
+    flushIntervalMillis,
+    objectStore,
+    dataLake,
+    host,
+    port,
+  }) =>
+    Effect.gen(function* () {
+      p.intro("📁 Create Namespace");
+
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Creating namespace...");
+
+      const namespace = yield* WingsClusterMetadata.createNamespace({
+        parent,
+        namespaceId,
+        flushSizeBytes: BigInt(flushSizeBytes),
+        flushIntervalMillis: BigInt(flushIntervalMillis),
+        objectStore,
+        dataLake,
+      }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to create namespace")),
+        ),
+      );
+
+      s.stop("Namespace created successfully");
+
+      yield* Effect.sync(() => {
+        printTable([
+          {
+            name: namespace.name,
+            flush_size_bytes: namespace.flushSizeBytes.toString(),
+            flush_interval_millis: namespace.flushIntervalMillis.toString(),
+            object_store: namespace.objectStore || "-",
+            data_lake: namespace.dataLake || "-",
+          },
+        ]);
+        p.outro("✓ Done");
+      });
+    }).pipe(Effect.catchAll(handleCliError("Failed to create namespace"))),
+).pipe(Command.withDescription("Create a new namespace belonging to a tenant"));

package/src/commands/cluster/namespace/delete.ts

@@ -0,0 +1,62 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { forceOption, hostOption, portOption } from "../../../utils/options.js";
+
+const nameOption = Options.text("name").pipe(
+  Options.withDescription(
+    "Namespace name in format: tenants/{tenant}/namespaces/{namespace}",
+  ),
+);
+
+export const deleteNamespaceCommand = Command.make(
+  "delete-namespace",
+  {
+    name: nameOption,
+    force: forceOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ name, force, host, port }) =>
+    Effect.gen(function* () {
+      p.intro("🗑️  Delete Namespace");
+
+      if (!force) {
+        const confirm = yield* Effect.tryPromise({
+          try: () =>
+            p.confirm({
+              message: `Are you sure you want to delete namespace ${name}?`,
+              initialValue: false,
+            }),
+          catch: () => new Error("Failed to confirm deletion"),
+        });
+
+        if (p.isCancel(confirm) || !confirm) {
+          p.cancel("Deletion cancelled");
+          process.exit(0);
+        }
+      }
+
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Deleting namespace...");
+
+      yield* WingsClusterMetadata.deleteNamespace({ name }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to delete namespace")),
+        ),
+      );
+
+      s.stop("Namespace deleted successfully");
+      p.outro("✓ Done");
+    }).pipe(Effect.catchAll(handleCliError("Failed to delete namespace"))),
+).pipe(
+  Command.withDescription(
+    "Delete a namespace from the cluster (fails if namespace has any topics)",
+  ),
+);

package/src/commands/cluster/namespace/get.ts

@@ -0,0 +1,52 @@
+import { WingsClusterMetadata } from "@airfoil/wings";
+import * as p from "@clack/prompts";
+import { Command, Options } from "@effect/cli";
+import { printTable } from "console-table-printer";
+import { Effect } from "effect";
+import { makeClusterMetadataLayer } from "../../../utils/client.js";
+import { handleCliError } from "../../../utils/effect.js";
+import { hostOption, portOption } from "../../../utils/options.js";
+
+const nameOption = Options.text("name").pipe(
+  Options.withDescription(
+    "Namespace name in format: tenants/{tenant}/namespaces/{namespace}",
+  ),
+);
+
+export const getNamespaceCommand = Command.make(
+  "get-namespace",
+  {
+    name: nameOption,
+    host: hostOption,
+    port: portOption,
+  },
+  ({ name, host, port }) =>
+    Effect.gen(function* () {
+      const layer = makeClusterMetadataLayer(host, port);
+
+      const s = p.spinner();
+      s.start("Fetching namespace...");
+
+      const namespace = yield* WingsClusterMetadata.getNamespace({ name }).pipe(
+        Effect.provide(layer),
+        Effect.tapError(() =>
+          Effect.sync(() => s.stop("Failed to get namespace")),
+        ),
+      );
+
+      s.stop("Namespace retrieved");
+
+      yield* Effect.sync(() => {
+        printTable([
+          {
+            name: namespace.name,
+            flush_size_bytes: namespace.flushSizeBytes.toString(),
+            flush_interval_millis: namespace.flushIntervalMillis.toString(),
+            object_store: namespace.objectStore || "-",
+            data_lake: namespace.dataLake || "-",
+          },
+        ]);
+        p.outro("✓ Done");
+      });
+    }).pipe(Effect.catchAll(handleCliError("Failed to get namespace"))),
+).pipe(Command.withDescription("Get details of a specific namespace"));