airfoil 0.4.5 → 0.4.7

package/src/commands/namespace.ts

@@ -0,0 +1,75 @@
+import { Effect, Option } from "effect";
+import { Argument, Command, Flag } from "effect/cli";
+
+import { getIcebergCatalog, parseNamespaceIdentifier } from "../utils/iceberg";
+import { catalogFlag } from "../utils/options";
+
+const namespaceArgument = Argument.String("namespace").pipe(
+  Argument.withDescription("Namespace name. Use dot-separated notation for nested namespaces"),
+);
+
+const parentFlag = Flag.String("parent").pipe(
+  Flag.withDescription("Parent namespace. Use dot-separated notation for nested namespaces"),
+  Flag.optional,
+);
+
+const createCommand = Command.make(
+  "create",
+  { namespace: namespaceArgument, catalog: catalogFlag },
+  ({ namespace, catalog }) =>
+    Effect.gen(function* () {
+      const id = yield* parseNamespaceIdentifier(namespace);
+      const iceberg = yield* getIcebergCatalog(catalog);
+      const created = yield* iceberg.createNamespace(id);
+
+      yield* Effect.log(created);
+    }),
+).pipe(Command.withDescription("Create a new namespace"));
+
+const getCommand = Command.make(
+  "get",
+  { namespace: namespaceArgument, catalog: catalogFlag },
+  ({ namespace, catalog }) =>
+    Effect.gen(function* () {
+      const id = yield* parseNamespaceIdentifier(namespace);
+      const iceberg = yield* getIcebergCatalog(catalog);
+      const metadata = yield* iceberg.loadNamespaceMetadata(id);
+
+      yield* Effect.log({ ...id, ...metadata });
+    }),
+).pipe(Command.withDescription("Get a namespace"));
+
+const deleteCommand = Command.make(
+  "delete",
+  { namespace: namespaceArgument, catalog: catalogFlag },
+  ({ namespace, catalog }) =>
+    Effect.gen(function* () {
+      const id = yield* parseNamespaceIdentifier(namespace);
+      const iceberg = yield* getIcebergCatalog(catalog);
+
+      yield* iceberg.dropNamespace(id);
+      yield* Effect.log(id);
+    }),
+).pipe(Command.withDescription("Delete a namespace"));
+
+const listCommand = Command.make(
+  "list",
+  { parent: parentFlag, catalog: catalogFlag },
+  ({ parent, catalog }) =>
+    Effect.gen(function* () {
+      const iceberg = yield* getIcebergCatalog(catalog);
+      const parentName = Option.getOrUndefined(parent);
+      const parentId =
+        parentName === undefined ? undefined : yield* parseNamespaceIdentifier(parentName);
+      const result = yield* iceberg.listNamespaces(
+        parentId === undefined ? undefined : { parent: parentId },
+      );
+
+      yield* Effect.log(result);
+    }),
+).pipe(Command.withDescription("List namespaces"));
+
+export const namespaceCommand = Command.make("namespace").pipe(
+  Command.withDescription("Manage namespaces"),
+  Command.withSubcommands([createCommand, getCommand, deleteCommand, listCommand]),
+);

package/src/commands/table.ts

@@ -0,0 +1,111 @@
+import { Effect, Option } from "effect";
+import { Argument, CliError, Command, Flag } from "effect/cli";
+
+import {
+  getIcebergCatalog,
+  listAllNamespaces,
+  listAllTables,
+  parseNamespaceIdentifier,
+  parseTableIdentifier,
+} from "../utils/iceberg";
+import { parseTableFields } from "../utils/json";
+import { catalogFlag } from "../utils/options";
+
+const tableArgument = Argument.String("table").pipe(
+  Argument.withDescription("Fully qualified table name"),
+);
+
+const namespaceArgument = Argument.String("namespace").pipe(
+  Argument.withDescription("Namespace name. Use dot-separated notation for nested namespaces"),
+  Argument.optional,
+);
+
+const allFlag = Flag.Boolean("all").pipe(
+  Flag.withDescription("List tables from all namespaces"),
+  Flag.withDefault(false),
+);
+
+const schemaFlag = Flag.String("schema").pipe(
+  Flag.withDescription("Iceberg table fields as a JSON array"),
+);
+
+const createCommand = Command.make(
+  "create",
+  { table: tableArgument, schema: schemaFlag, catalog: catalogFlag },
+  ({ table, schema, catalog }) =>
+    Effect.gen(function* () {
+      const id = yield* parseTableIdentifier(table);
+      const fields = yield* parseTableFields(schema);
+      const iceberg = yield* getIcebergCatalog(catalog);
+      const metadata = yield* iceberg.createTable(
+        { namespace: id.namespace },
+        { name: id.name, schema: { type: "struct", fields } },
+      );
+
+      yield* Effect.log({ ...id, metadata });
+    }),
+).pipe(Command.withDescription("Create a new table"));
+
+const getCommand = Command.make(
+  "get",
+  { table: tableArgument, catalog: catalogFlag },
+  ({ table, catalog }) =>
+    Effect.gen(function* () {
+      const id = yield* parseTableIdentifier(table);
+      const iceberg = yield* getIcebergCatalog(catalog);
+      const metadata = yield* iceberg.loadTable(id);
+
+      yield* Effect.log({ ...id, metadata });
+    }),
+).pipe(Command.withDescription("Get a table"));
+
+const deleteCommand = Command.make(
+  "delete",
+  { table: tableArgument, catalog: catalogFlag },
+  ({ table, catalog }) =>
+    Effect.gen(function* () {
+      const id = yield* parseTableIdentifier(table);
+      const iceberg = yield* getIcebergCatalog(catalog);
+
+      yield* iceberg.dropTable(id);
+      yield* Effect.log(id);
+    }),
+).pipe(Command.withDescription("Delete a table"));
+
+const listCommand = Command.make(
+  "list",
+  { namespace: namespaceArgument, all: allFlag, catalog: catalogFlag },
+  ({ namespace, all, catalog }) =>
+    Effect.gen(function* () {
+      const namespaceName = Option.getOrUndefined(namespace);
+
+      if (namespaceName === undefined && !all) {
+        return yield* new CliError.UserError({
+          cause: "Missing table list selector",
+          userMessage: "Provide a namespace or use --all",
+        });
+      }
+
+      if (namespaceName !== undefined && all) {
+        return yield* new CliError.UserError({
+          cause: "Conflicting table list selectors",
+          userMessage: "Provide either a namespace or --all, not both",
+        });
+      }
+
+      const iceberg = yield* getIcebergCatalog(catalog);
+      const tables =
+        namespaceName === undefined
+          ? (yield* Effect.forEach(yield* listAllNamespaces(iceberg), (namespace) =>
+              listAllTables(iceberg, namespace),
+            )).flat()
+          : yield* listAllTables(iceberg, yield* parseNamespaceIdentifier(namespaceName));
+
+      yield* Effect.log({ namespace: namespaceName, tables });
+    }),
+).pipe(Command.withDescription("List tables"));
+
+export const tableCommand = Command.make("table").pipe(
+  Command.withDescription("Manage tables"),
+  Command.withSubcommands([createCommand, getCommand, deleteCommand, listCommand]),
+);

package/src/index.ts

@@ -1,18 +1,22 @@
 #!/usr/bin/env node
 import { NodeRuntime, NodeServices } from "@effect/platform-node";
 import { Effect, Layer } from "effect";
-import { Command } from "effect/unstable/cli";
-import { FetchHttpClient } from "effect/unstable/http";
+import { Command } from "effect/cli";
+import { FetchHttpClient } from "effect/http";
 
 import packageJson from "../package.json";
-import { clusterCommand } from "./commands/cluster/index";
-import { devCommand } from "./commands/dev";
-import { sqlCommand } from "./commands/sql";
+import { catalogCommand } from "./commands/catalog";
+import { namespaceCommand } from "./commands/namespace";
+import { tableCommand } from "./commands/table";
+import { OutputLogger, reportError } from "./utils/logger";
+import { EngineUri, Output } from "./utils/options";
 
 const version = packageJson.version;
 
 const program = Command.make("airfoil", {}, () => Effect.void).pipe(
-  Command.withSubcommands([devCommand, sqlCommand, clusterCommand]),
+  Command.withSubcommands([catalogCommand, namespaceCommand, tableCommand]),
+  Command.provide(OutputLogger),
+  Command.withGlobalFlags([EngineUri, Output]),
 );
 
 const cli = Command.run(program, {
@@ -23,5 +27,7 @@ NodeRuntime.runMain(
   cli.pipe(
     Effect.provide(Layer.mergeAll(FetchHttpClient.layer, NodeServices.layer)),
     Effect.scoped,
+    Effect.tapCause(reportError),
   ),
+  { disableErrorReporting: true },
 );

package/src/utils/iceberg.ts

@@ -0,0 +1,58 @@
+import type { IcebergCatalog } from "@useairfoil/effect-iceberg";
+
+import { CatalogManager } from "@useairfoil/api";
+import { Effect, Schema } from "effect";
+
+import { EngineUri } from "./options";
+
+const NamespaceSegments = Schema.NonEmptyArray(Schema.NonEmptyString);
+const TableSegments = Schema.Array(Schema.NonEmptyString).check(Schema.isMinLength(2));
+
+export const parseNamespaceIdentifier = (name: string) =>
+  Schema.decodeUnknownEffect(NamespaceSegments)(name.split(".")).pipe(
+    Effect.map((namespace) => ({ namespace: Array.from(namespace) })),
+  );
+
+export const parseTableIdentifier = (qualifiedName: string) =>
+  Schema.decodeUnknownEffect(TableSegments)(qualifiedName.split(".")).pipe(
+    Effect.map((segments) => ({
+      namespace: segments.slice(0, -1),
+      name: segments[segments.length - 1]!,
+    })),
+  );
+
+export const getIcebergCatalog = Effect.fn("getIcebergCatalog")(function* (catalogId: string) {
+  const engineUri = yield* EngineUri;
+  const manager = yield* CatalogManager.make({ engineUri });
+
+  return yield* manager.getIcebergCatalog(catalogId);
+});
+
+export const listAllNamespaces = Effect.fn("listAllNamespaces")(function* (
+  iceberg: IcebergCatalog.Service,
+) {
+  let page = yield* iceberg.listNamespaces();
+  const namespaces = [...page.namespaces];
+
+  while (page.nextPageToken !== undefined) {
+    page = yield* iceberg.listNamespaces({ pageToken: page.nextPageToken });
+    namespaces.push(...page.namespaces);
+  }
+
+  return namespaces;
+});
+
+export const listAllTables = Effect.fn("listAllTables")(function* (
+  iceberg: IcebergCatalog.Service,
+  namespace: Parameters<IcebergCatalog.Service["listTables"]>[0],
+) {
+  let page = yield* iceberg.listTables(namespace);
+  const identifiers = [...page.identifiers];
+
+  while (page.nextPageToken !== undefined) {
+    page = yield* iceberg.listTables(namespace, { pageToken: page.nextPageToken });
+    identifiers.push(...page.identifiers);
+  }
+
+  return identifiers;
+});

package/src/utils/json.ts

@@ -0,0 +1,48 @@
+import type { IcebergType, TableField } from "iceberg-js";
+
+import { CatalogManager } from "@useairfoil/api";
+import { Schema } from "effect";
+
+const RestCatalogConfigJson = Schema.fromJsonString(CatalogManager.RestCatalogConfig);
+
+const DefaultValue = Schema.Union([Schema.Boolean, Schema.Number, Schema.String]);
+
+const IcebergTypeSchema: Schema.Codec<IcebergType> = Schema.suspend(() =>
+  Schema.Union([
+    Schema.String,
+    Schema.Struct({
+      type: Schema.Literal("struct"),
+      fields: Schema.mutable(Schema.Array(TableFieldSchema)),
+    }),
+    Schema.Struct({
+      type: Schema.Literal("list"),
+      "element-id": Schema.Int,
+      element: IcebergTypeSchema,
+      "element-required": Schema.Boolean,
+    }),
+    Schema.Struct({
+      type: Schema.Literal("map"),
+      "key-id": Schema.Int,
+      key: IcebergTypeSchema,
+      "value-id": Schema.Int,
+      value: IcebergTypeSchema,
+      "value-required": Schema.Boolean,
+    }),
+  ]),
+);
+
+const TableFieldSchema: Schema.Codec<TableField> = Schema.Struct({
+  id: Schema.Int,
+  name: Schema.NonEmptyString,
+  type: IcebergTypeSchema,
+  required: Schema.Boolean,
+  doc: Schema.optional(Schema.String),
+  "initial-default": Schema.optional(DefaultValue),
+  "write-default": Schema.optional(DefaultValue),
+});
+
+const TableFieldsJson = Schema.fromJsonString(Schema.mutable(Schema.Array(TableFieldSchema)));
+
+export const parseRestCatalogConfig = Schema.decodeUnknownEffect(RestCatalogConfigJson);
+
+export const parseTableFields = Schema.decodeUnknownEffect(TableFieldsJson);

package/src/utils/logger.ts

@@ -0,0 +1,67 @@
+import { encode } from "@toon-format/toon";
+import { Cause, Console, Effect, Formatter, Layer, Logger, Result, Runtime, Schema } from "effect";
+
+import { Output } from "./options";
+
+const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
+
+export type OutputFormat = "default" | "json";
+
+export const formatStructured = (value: unknown, output: OutputFormat) => {
+  const json = Formatter.formatJson(value);
+
+  return output === "json" ? json : encode(decodeJson(json));
+};
+
+const messageValue = (message: unknown) =>
+  Array.isArray(message) && message.length === 1 ? message[0] : message;
+
+const structuredLogger = (output: OutputFormat) =>
+  Logger.make<unknown, string>(({ message }) =>
+    formatStructured(messageValue(message), output),
+  ).pipe(Logger.withConsoleLog);
+
+// Errors are reported after the command ends, when its flags are gone. Keep the format here.
+let errorFormat: OutputFormat = "default";
+
+export const OutputLogger = Layer.unwrap(
+  Effect.map(Output, (output) => {
+    errorFormat = output;
+
+    return Logger.layer([structuredLogger(output)]);
+  }),
+);
+
+// Turns an error and its causes into plain JSON, so we do not walk the chain ourselves.
+const encodeError = Schema.encodeSync(Schema.Defect());
+const encodeErrorWithStack = Schema.encodeSync(Schema.Defect({ includeStack: true }));
+
+/**
+ * Prints a failed command as `error: message: ... cause: ...` instead of the stack dump
+ * `runMain` prints by default.
+ *
+ * Says nothing about Ctrl+C, or about errors the CLI already printed itself.
+ */
+export const reportError = (cause: Cause.Cause<unknown>) => {
+  if (Cause.hasInterruptsOnly(cause)) return Effect.void;
+
+  const error = Cause.squash(cause);
+
+  if (!Runtime.getErrorReported(error)) return Effect.void;
+
+  // Only an unexpected defect is worth a stack. A failure is explained by its message chain.
+  const defect = Cause.findDefect(cause);
+  const stack =
+    Result.isSuccess(defect) && defect.success instanceof Error ? defect.success.stack : undefined;
+
+  if (errorFormat === "json") {
+    const encode = stack === undefined ? encodeError : encodeErrorWithStack;
+
+    return Console.error(formatStructured({ error: encode(error) }, "json"));
+  }
+
+  // Toon has no multi-line string, so the stack only reads well under the block.
+  const block = formatStructured({ error: encodeError(error) }, "default");
+
+  return Console.error(stack === undefined ? block : `${block}\n\n${stack}`);
+};

package/src/utils/options.ts

@@ -1,35 +1,21 @@
-import { Flag } from "effect/unstable/cli";
+import { Config } from "effect";
+import { Flag, GlobalFlag } from "effect/cli";
 
-/**
- * Common server connection options
- */
-export const hostOption = Flag.string("host").pipe(
-  Flag.withDescription("Server host"),
-  Flag.withDefault("127.0.0.1"),
-);
-
-export const portOption = Flag.integer("port").pipe(
-  Flag.withDescription("Server port"),
-  Flag.withAlias("p"),
-  Flag.withDefault(7777),
-);
+export const EngineUri = GlobalFlag.Setting("uri")({
+  flag: Flag.String("uri").pipe(
+    Flag.withDefault("http://localhost:7777"),
+    Flag.withDescription("Engine URI"),
+  ),
+});
 
-/**
- * Common pagination options
- */
-export const pageSizeOption = Flag.integer("page-size").pipe(
-  Flag.withDescription("Number of items to return (max: 1000)"),
-  Flag.withDefault(100),
-);
-
-export const pageTokenOption = Flag.optional(
-  Flag.string("page-token").pipe(Flag.withDescription("Continuation token for pagination")),
-);
+export const Output = GlobalFlag.Setting("output")({
+  flag: Flag.Literals("output", ["default", "json"] as const).pipe(
+    Flag.withDefault("default"),
+    Flag.withDescription("Log output format"),
+  ),
+});
 
-/**
- * Force flag for destructive operations
- */
-export const forceOption = Flag.boolean("force").pipe(
-  Flag.withDescription("Skip confirmation prompt"),
-  Flag.withDefault(false),
+export const catalogFlag = Flag.String("catalog").pipe(
+  Flag.withDescription("Catalog ID. Alternatively, use the AIRFOIL_CATALOG environment variable"),
+  Flag.withFallbackConfig(Config.NonEmptyString("AIRFOIL_CATALOG")),
 );