@useairfoil/connector-kit 0.6.0 → 0.7.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@useairfoil/connector-kit",
-  "version": "0.6.0",
-  "repository": "useairfoil/typescript-sdk",
+  "version": "0.7.0",
+  "repository": "useairfoil/connector-kit",
   "files": [
     "dist",
     "src",
@@ -9,96 +9,110 @@
     "README.md"
   ],
   "type": "module",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "default": "./dist/index.js"
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
     },
     "./auth": {
-      "types": "./dist/auth/index.d.ts",
-      "import": "./dist/auth/index.js",
-      "default": "./dist/auth/index.js"
+      "types": "./dist/auth/index.d.mts",
+      "import": "./dist/auth/index.mjs",
+      "default": "./dist/auth/index.mjs"
     },
     "./connector-app": {
-      "types": "./dist/connector-app/index.d.ts",
-      "import": "./dist/connector-app/index.js",
-      "default": "./dist/connector-app/index.js"
+      "types": "./dist/connector-app/index.d.mts",
+      "import": "./dist/connector-app/index.mjs",
+      "default": "./dist/connector-app/index.mjs"
+    },
+    "./demo": {
+      "types": "./dist/demo/index.d.mts",
+      "import": "./dist/demo/index.mjs",
+      "default": "./dist/demo/index.mjs"
+    },
+    "./iceberg": {
+      "types": "./dist/iceberg/index.d.mts",
+      "import": "./dist/iceberg/index.mjs",
+      "default": "./dist/iceberg/index.mjs"
     },
     "./ingestion": {
-      "types": "./dist/ingestion/index.d.ts",
-      "import": "./dist/ingestion/index.js",
-      "default": "./dist/ingestion/index.js"
+      "types": "./dist/ingestion/index.d.mts",
+      "import": "./dist/ingestion/index.mjs",
+      "default": "./dist/ingestion/index.mjs"
+    },
+    "./ingestor": {
+      "types": "./dist/ingestor/index.d.mts",
+      "import": "./dist/ingestor/index.mjs",
+      "default": "./dist/ingestor/index.mjs"
     },
     "./manifest": {
-      "types": "./dist/manifest/index.d.ts",
-      "import": "./dist/manifest/index.js",
-      "default": "./dist/manifest/index.js"
+      "types": "./dist/manifest/index.d.mts",
+      "import": "./dist/manifest/index.mjs",
+      "default": "./dist/manifest/index.mjs"
     },
     "./metrics": {
-      "types": "./dist/metrics.d.ts",
-      "import": "./dist/metrics.js",
-      "default": "./dist/metrics.js"
-    },
-    "./publisher": {
-      "types": "./dist/publisher/index.d.ts",
-      "import": "./dist/publisher/index.js",
-      "default": "./dist/publisher/index.js"
+      "types": "./dist/metrics.d.mts",
+      "import": "./dist/metrics.mjs",
+      "default": "./dist/metrics.mjs"
     },
     "./runtime-config": {
-      "types": "./dist/runtime-config/index.d.ts",
-      "import": "./dist/runtime-config/index.js",
-      "default": "./dist/runtime-config/index.js"
+      "types": "./dist/runtime-config/index.d.mts",
+      "import": "./dist/runtime-config/index.mjs",
+      "default": "./dist/runtime-config/index.mjs"
     },
     "./state-store": {
-      "types": "./dist/state-store/index.d.ts",
-      "import": "./dist/state-store/index.js",
-      "default": "./dist/state-store/index.js"
+      "types": "./dist/state-store/index.d.mts",
+      "import": "./dist/state-store/index.mjs",
+      "default": "./dist/state-store/index.mjs"
     },
     "./status": {
-      "types": "./dist/status/index.d.ts",
-      "import": "./dist/status/index.js",
-      "default": "./dist/status/index.js"
+      "types": "./dist/status/index.d.mts",
+      "import": "./dist/status/index.mjs",
+      "default": "./dist/status/index.mjs"
     },
     "./webhook": {
-      "types": "./dist/webhook/index.d.ts",
-      "import": "./dist/webhook/index.js",
-      "default": "./dist/webhook/index.js"
+      "types": "./dist/webhook/index.d.mts",
+      "import": "./dist/webhook/index.mjs",
+      "default": "./dist/webhook/index.mjs"
     },
     "./errors": {
-      "types": "./dist/errors/index.d.ts",
-      "import": "./dist/errors/index.js",
-      "default": "./dist/errors/index.js"
+      "types": "./dist/errors/index.d.mts",
+      "import": "./dist/errors/index.mjs",
+      "default": "./dist/errors/index.mjs"
     }
   },
-  "scripts": {
-    "build": "tsdown",
-    "test": "vitest",
-    "test:ci": "vitest run",
-    "typecheck": "tsc --noEmit"
-  },
   "dependencies": {
-    "@effect/platform-node": "^4.0.0-rc.108",
-    "@useairfoil/flight": "0.4.6",
-    "@useairfoil/wings": "0.4.5"
+    "@effect/platform-node": "^4.1.0",
+    "@faker-js/faker": "10.6.0",
+    "apache-arrow": "^21.2.0",
+    "human-format": "^1.4.0",
+    "iceberg-js": "^1.0.0",
+    "@useairfoil/api": "0.5.0"
   },
   "devDependencies": {
-    "@effect/sql-pg": "^4.0.0-rc.108",
-    "@effect/vitest": "^4.0.0-rc.108",
+    "@effect/sql-pg": "^4.1.0",
+    "@effect/vitest": "^4.1.0",
     "@types/node": "^24.2.0",
-    "effect": "^4.0.0-rc.108",
+    "effect": "^4.1.0",
     "testcontainers": "^11.14.0",
-    "tsdown": "^0.15.6",
-    "vitest": "^4.1.0"
+    "tsdown": "^0.23.0",
+    "vitest": "^5.0.3",
+    "@useairfoil/testing": "0.2.0"
   },
   "peerDependencies": {
-    "effect": "^4.0.0-rc.108"
+    "effect": "^4.1.0"
   },
   "peerDependenciesMeta": {
     "effect": {
       "optional": false
     }
+  },
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest",
+    "test:ci": "vitest run",
+    "typecheck": "tsc --noEmit"
   }
-}
+}

package/src/catalog/retry.ts

@@ -0,0 +1,25 @@
+import { Effect, Schedule, type Duration } from "effect";
+
+export type TableRetryOptions = {
+  readonly interval: Duration.Input;
+  /** Retries after the initial attempt. */
+  readonly times: number;
+};
+
+/** Retries transient catalog responses at the request boundary. */
+export const retryCatalogRequest = <A, E extends { readonly status?: number }, R>(
+  request: Effect.Effect<A, E, R>,
+  options?: TableRetryOptions,
+) =>
+  options === undefined
+    ? request
+    : request.pipe(
+        Effect.retry({
+          schedule: Schedule.spaced(options.interval),
+          times: options.times,
+          while: (error) =>
+            error.status === 408 ||
+            error.status === 429 ||
+            (error.status !== undefined && error.status >= 500 && error.status <= 599),
+        }),
+      );

package/src/catalog/table.ts

@@ -0,0 +1,237 @@
+import type {
+  IcebergType,
+  StructField,
+  TableIdentifier,
+  TableMetadata,
+  TableSchema,
+} from "iceberg-js";
+
+import { CatalogManager } from "@useairfoil/api";
+import { Effect, type Schema } from "effect";
+import { HttpClient } from "effect/http";
+import { getCurrentSchema } from "iceberg-js";
+
+import { ConnectorError } from "../errors";
+import { IcebergSchemaError } from "../iceberg/error";
+import { compileTable, makeCreateRequest } from "../iceberg/table";
+import { retryCatalogRequest, type TableRetryOptions } from "./retry";
+
+type Catalog = Effect.Success<
+  ReturnType<CatalogManager.CatalogManagerService["getIcebergCatalog"]>
+>;
+
+type TableOptions = {
+  readonly retry?: TableRetryOptions;
+  readonly catalog: Pick<Catalog, "loadTable" | "createTable" | "createNamespaceIfNotExists">;
+  readonly identifier: TableIdentifier;
+  readonly location?: string | undefined;
+  readonly properties?: Readonly<Record<string, string>> | undefined;
+};
+
+type EngineTableOptions = {
+  readonly engineUri: string;
+  readonly catalogId: string;
+  readonly identifier: TableIdentifier;
+  readonly location?: string | undefined;
+  readonly properties?: Readonly<Record<string, string>> | undefined;
+};
+
+const tableError = (cause: unknown): ConnectorError => {
+  if (cause instanceof ConnectorError) return cause;
+
+  const message = cause instanceof Error ? cause.message : "Failed to prepare Iceberg table";
+  return new ConnectorError({
+    message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,
+    cause,
+  });
+};
+
+const compareTypes = (
+  expected: IcebergType,
+  actual: IcebergType,
+  path: string,
+): string | undefined => {
+  if (typeof expected === "string" || typeof actual === "string") {
+    return expected === actual ? undefined : path;
+  }
+
+  if (expected.type !== actual.type) return path;
+
+  switch (expected.type) {
+    case "struct":
+      if (actual.type !== "struct") return path;
+      return compareFields(expected.fields, actual.fields, path);
+    case "list":
+      if (actual.type !== "list" || expected["element-required"] !== actual["element-required"]) {
+        return path;
+      }
+      return compareTypes(expected.element, actual.element, `${path}.element`);
+    case "map":
+      if (actual.type !== "map" || expected["value-required"] !== actual["value-required"]) {
+        return path;
+      }
+      return (
+        compareTypes(expected.key, actual.key, `${path}.key`) ??
+        compareTypes(expected.value, actual.value, `${path}.value`)
+      );
+  }
+};
+
+const compareFields = (
+  expected: ReadonlyArray<StructField>,
+  actual: ReadonlyArray<StructField>,
+  path: string,
+): string | undefined => {
+  if (expected.length !== actual.length) return path;
+
+  for (const field of expected) {
+    const fieldPath = `${path}.${field.name}`;
+    const other = actual.find((candidate) => candidate.name === field.name);
+
+    if (!other || field.required !== other.required) return fieldPath;
+
+    const mismatch = compareTypes(field.type, other.type, fieldPath);
+    if (mismatch) return mismatch;
+  }
+};
+
+const verifyTable = (
+  metadata: TableMetadata,
+  expected: TableSchema,
+): Effect.Effect<TableSchema, ConnectorError> =>
+  Effect.try({
+    try: () => {
+      if (!Array.isArray(metadata.schemas)) throw new Error("Invalid table metadata");
+      if (metadata["format-version"] !== 3) {
+        throw new Error(
+          "Storage-engine tables must use Iceberg format v3. Migrate the table before ingesting.",
+        );
+      }
+      const partitionSpec = metadata["partition-specs"].find(
+        (spec) => spec["spec-id"] === metadata["default-spec-id"],
+      );
+      if (!partitionSpec) throw new Error("Table has no default partition spec");
+      if (partitionSpec.fields.length > 0) {
+        throw new Error("Storage-engine tables must be unpartitioned.");
+      }
+      if (metadata.properties?.["encryption.key-id"] !== undefined) {
+        throw new Error("Storage-engine tables must be unencrypted.");
+      }
+      const actual = getCurrentSchema(metadata);
+      if (!actual) throw new Error("Table has no current schema");
+
+      const path = compareFields(expected.fields, actual.fields, "$");
+      if (path) {
+        throw new Error(
+          `Table schema does not match at ${path}. Migrate the table or revert the schema change.`,
+        );
+      }
+
+      const identifierNames = (schema: TableSchema) =>
+        new Set(
+          schema.fields
+            .filter((field) => schema["identifier-field-ids"]?.includes(field.id))
+            .map((field) => field.name),
+        );
+      const expectedNames = identifierNames(expected);
+      const actualNames = identifierNames(actual);
+      if (
+        expectedNames.size !== actualNames.size ||
+        [...expectedNames].some((name) => !actualNames.has(name))
+      ) {
+        throw new Error(
+          "Table identifier fields do not match. Recreate the table or revert the schema change.",
+        );
+      }
+
+      return actual;
+    },
+    catch: (cause) =>
+      new ConnectorError({
+        message: cause instanceof Error ? cause.message : "Invalid table metadata",
+        cause,
+      }),
+  });
+
+/** Creates a table through the REST catalog, then checks the loaded schema. */
+export const createTable = (
+  schema: Schema.Top,
+  options: TableOptions,
+): Effect.Effect<TableSchema, ConnectorError> =>
+  Effect.gen(function* () {
+    const table = yield* compileTable(schema);
+    yield* options.catalog.createTable(
+      { namespace: options.identifier.namespace },
+      makeCreateRequest(table, {
+        name: options.identifier.name,
+        location: options.location,
+        properties: options.properties,
+      }),
+    );
+
+    const metadata = yield* options.catalog.loadTable(options.identifier);
+    return yield* verifyTable(metadata, table.schema);
+  }).pipe(Effect.mapError(tableError));
+
+/** Creates a missing table through an engine catalog and verifies its schema. */
+export const ensureTableFromEngine = (
+  schema: Schema.Top,
+  options: EngineTableOptions,
+): Effect.Effect<TableSchema, ConnectorError, HttpClient.HttpClient> =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const manager = yield* CatalogManager.make({ engineUri: options.engineUri });
+      const catalog = yield* manager.getIcebergCatalog(options.catalogId);
+      return yield* ensureTable(schema, {
+        catalog,
+        identifier: options.identifier,
+        location: options.location,
+        properties: options.properties,
+      });
+    }),
+  ).pipe(Effect.mapError(tableError));
+
+/** Loads a table, creating it when it is missing. */
+export const ensureTable = (
+  schema: Schema.Top,
+  options: TableOptions,
+): Effect.Effect<TableSchema, ConnectorError> =>
+  Effect.gen(function* () {
+    const table = yield* compileTable(schema);
+    const request = <A>(
+      effect: Effect.Effect<A, Effect.Error<ReturnType<Catalog["loadConfig"]>>>,
+    ) => retryCatalogRequest(effect, options.retry);
+    const metadata = yield* request(options.catalog.loadTable(options.identifier)).pipe(
+      Effect.catchIf(
+        (error) => error.status === 404,
+        () =>
+          request(
+            options.catalog.createNamespaceIfNotExists({ namespace: options.identifier.namespace }),
+          )
+            .pipe(
+              Effect.andThen(
+                request(
+                  options.catalog.createTable(
+                    { namespace: options.identifier.namespace },
+                    makeCreateRequest(table, {
+                      name: options.identifier.name,
+                      location: options.location,
+                      properties: options.properties,
+                    }),
+                  ),
+                ),
+              ),
+            )
+            .pipe(
+              // Another instance may have created the table. Reload and check it.
+              Effect.catchIf(
+                (error) => error.status === 409 || error.status === 412,
+                () => Effect.void,
+              ),
+              Effect.flatMap(() => request(options.catalog.loadTable(options.identifier))),
+            ),
+      ),
+    );
+
+    return yield* verifyTable(metadata, table.schema);
+  }).pipe(Effect.mapError(tableError));

package/src/connector-app/create-table.ts

@@ -0,0 +1,47 @@
+import { Effect, Option, type Schema } from "effect";
+import { Argument, CliError, Command, Flag } from "effect/cli";
+
+import { ensureTableFromEngine } from "../catalog/table";
+import { engineUri } from "../runtime-config/config";
+
+/** Creates the table command; existing tables must match the connector's row schemas. */
+export const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>
+  Command.make(
+    "create-table",
+    {
+      table: Argument.String("table"),
+      catalog: Flag.String("catalog"),
+      location: Flag.String("location").pipe(Flag.optional),
+      resource: Flag.String("resource").pipe(Flag.optional),
+    },
+    ({ table, catalog, location, resource }) =>
+      Effect.gen(function* () {
+        const namespace = table.split(".");
+        const name = namespace.pop();
+
+        if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) {
+          return yield* new CliError.UserError({
+            cause: "Invalid table name",
+            userMessage: "Use a table name like my.namespace.table",
+          });
+        }
+
+        const resourceName = Option.getOrElse(resource, () => name);
+        const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : undefined;
+        if (!schema) {
+          return yield* new CliError.UserError({
+            cause: "Unknown resource",
+            userMessage: `Unknown resource: ${resourceName}`,
+          });
+        }
+
+        const uri = yield* engineUri;
+        yield* ensureTableFromEngine(schema, {
+          engineUri: uri.href,
+          catalogId: catalog,
+          identifier: { namespace, name },
+          location: Option.getOrUndefined(location),
+        });
+        yield* Effect.log(`Prepared ${table}`);
+      }),
+  ).pipe(Command.withDescription("Create or verify an Iceberg table for a connector resource"));

package/src/connector-app/demo.ts

@@ -0,0 +1,47 @@
+import { DateTime, Effect, Option, Schema } from "effect";
+import { Command, Flag } from "effect/cli";
+import humanFormat from "human-format";
+
+import { run, type DemoDefinition } from "../demo";
+import { layerArrowFlightConfig } from "../ingestor/arrow-flight";
+
+/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */
+export const parseSize = (input: string): Option.Option<number> =>
+  Option.liftThrowable(() => humanFormat.parse.bytes(input))().pipe(
+    Option.filter((bytes) => Number.isSafeInteger(bytes) && bytes > 0),
+  );
+
+/**
+ * Creates the command that writes demo data over Arrow Flight, then exits.
+ * It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.
+ */
+export const makeDemoCommand = (demo: DemoDefinition) =>
+  Command.make(
+    "demo",
+    {
+      size: Flag.String("size").pipe(
+        Flag.filterMap(parseSize, () => "a size like 512kB, 128MB or 1.5GB"),
+      ),
+      seed: Flag.Int("seed").pipe(Flag.optional),
+      batchSize: Flag.Int("batch-size").pipe(
+        Flag.withSchema(Schema.Int.check(Schema.isGreaterThan(0))),
+        Flag.withDescription("Maximum rows per ingestion batch (default: 500)"),
+        Flag.optional,
+      ),
+    },
+    ({ size, seed, batchSize }) =>
+      Effect.gen(function* () {
+        const now = yield* DateTime.nowAsDate;
+
+        yield* run(demo, {
+          bytes: size,
+          seed: Option.getOrUndefined(seed),
+          batchSize: Option.getOrUndefined(batchSize),
+          now,
+        }).pipe(
+          Effect.provide(
+            layerArrowFlightConfig({ resources: [...demo.setup.tables, ...demo.group.tables] }),
+          ),
+        );
+      }),
+  ).pipe(Command.withDescription("Write demo data for the bound tables, then exit"));

package/src/connector-app/index.ts

@@ -1,2 +1,5 @@
 export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from "./check";
+export { makeCreateTableCommand } from "./create-table";
+export { makeDemoCommand, parseSize } from "./demo";
 export { start, type App, type StartOptions } from "./start";
+export { makeSetupCommand } from "./setup";

package/src/connector-app/setup.ts

@@ -0,0 +1,69 @@
+import { CatalogManager } from "@useairfoil/api";
+import { Duration, Effect, Option, Schema } from "effect";
+import { Command, Flag } from "effect/cli";
+
+import { retryCatalogRequest } from "../catalog/retry";
+import { ensureTable } from "../catalog/table";
+import * as RuntimeConfig from "../runtime-config";
+
+/** Creates or verifies all bound tables without loading provider credentials. */
+export const makeSetupCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>
+  Command.make(
+    "setup",
+    {
+      checkpointIntervalSec: Flag.Int("checkpoint-interval-sec").pipe(
+        Flag.withSchema(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 900 }))),
+        Flag.withDescription(
+          "Checkpoint interval for new tables in seconds (0 disables the interval delay)",
+        ),
+        Flag.optional,
+      ),
+      retryInterval: Flag.String("retry-interval").pipe(
+        Flag.withDefault("5 seconds"),
+        Flag.withSchema(Schema.DurationFromString),
+        Flag.filter(
+          (duration) => Duration.isPositive(duration) && Duration.isFinite(duration),
+          () => "a positive duration, such as 5 seconds",
+        ),
+        Flag.withDescription("Delay between transient catalog retries"),
+      ),
+      retries: Flag.Int("retries").pipe(
+        Flag.withDefault(10),
+        Flag.filter(
+          (value) => value >= 0,
+          () => "a non-negative number of retries",
+        ),
+        Flag.withDescription("Maximum retries per catalog request after the initial attempt"),
+      ),
+    },
+    ({ retryInterval, retries, checkpointIntervalSec }) =>
+      Effect.gen(function* () {
+        const source = {
+          resources: Object.entries(schemas).map(([name, rowSchema]) => ({ name, rowSchema })),
+        };
+        const tables = yield* RuntimeConfig.loadTableBindings(source);
+        const uri = yield* RuntimeConfig.engineUri;
+        const catalogId = yield* RuntimeConfig.catalog;
+        const manager = yield* CatalogManager.make({ engineUri: uri.href });
+        const retry = { interval: retryInterval, times: retries };
+        const properties = Option.match(checkpointIntervalSec, {
+          onNone: () => undefined,
+          onSome: (seconds) => ({ "airfoil.checkpoint.interval-sec": String(seconds) }),
+        });
+        const catalog = yield* retryCatalogRequest(manager.getIcebergCatalog(catalogId), retry);
+
+        // Sequential setup avoids bursting the catalog's table-creation quota.
+        for (const resource of source.resources) {
+          const binding = tables[resource.name];
+          if (!binding) continue;
+          yield* ensureTable(resource.rowSchema, {
+            catalog,
+            identifier: binding,
+            location: binding.location,
+            retry,
+            properties,
+          });
+          yield* Effect.logInfo(`Prepared ${[...binding.namespace, binding.name].join(".")}`);
+        }
+      }),
+  ).pipe(Command.withDescription("Create or verify the Iceberg tables in AIRFOIL_TABLE_BINDINGS"));

package/src/connector-app/start.ts

@@ -1,6 +1,6 @@
 import { NodeHttpServer } from "@effect/platform-node";
 import { DateTime, Effect } from "effect";
-import { HttpRouter } from "effect/unstable/http";
+import { HttpRouter } from "effect/http";
 import { createServer } from "node:http";
 
 import type { ConnectorDefinition } from "../core";