@useairfoil/connector-kit 0.6.1 → 0.7.0

package/src/iceberg/table.ts

@@ -1,5 +1,5 @@
 import type {
-  CommitTableRequest,
+  CreateTableRequest,
   IcebergType,
   MapType,
   StructField,
@@ -12,11 +12,10 @@ import { Effect, Schema, type SchemaRepresentation } from "effect";
 import { IcebergSchemaError } from "./error";
 
 /** Options for creating a table. */
-export type CreateTableCommitOptions = {
+export type CreateTableOptions = {
+  readonly name: string;
   /** Where the table's files go. The catalog may choose this when omitted. */
   readonly location?: string | undefined;
-  /** UUID for the table. Random by default. */
-  readonly uuid?: string | undefined;
   /** Extra table properties. These win over the ones from the schema. */
   readonly properties?: Readonly<Record<string, string>> | undefined;
 };
@@ -35,9 +34,6 @@ export type CompiledTable = {
   readonly description: string | undefined;
 };
 
-// Iceberg reserves its highest 200 field IDs.
-const maxFieldId = 2_147_483_447;
-
 const anonymousDeclaration =
   "Schemas without a name, such as Schema.Class, do not work here; use Schema.Struct";
 
@@ -121,7 +117,8 @@ class Compiler {
   readonly #activeObjects = new Set<SchemaRepresentation.Objects>();
   readonly #activeReferences = new Set<string>();
   readonly #activeSuspends = new Set<SchemaRepresentation.Suspend>();
-  readonly #fieldIds = new Map<number, string>();
+  // The REST request needs IDs. Only the catalog's returned IDs are authoritative.
+  #nextId = 0;
 
   constructor(private readonly references: SchemaRepresentation.References) {}
 
@@ -156,35 +153,6 @@ class Compiler {
     });
   }
 
-  #readId(annotations: Annotations | undefined, path: string, source: string): number {
-    const id = annotations?.fieldId;
-
-    if (typeof id !== "number") {
-      return fail(path, `Missing fieldId; add it with ${source}`);
-    }
-    if (!Number.isInteger(id)) return fail(path, "fieldId must be a whole number");
-    if (id < 1 || id > maxFieldId) {
-      return fail(path, `fieldId must be between 1 and ${maxFieldId}`);
-    }
-    const previousPath = this.#fieldIds.get(id);
-    if (previousPath !== undefined) {
-      return fail(path, `fieldId ${id} is already used at ${previousPath}`);
-    }
-
-    this.#fieldIds.set(id, path);
-    return id;
-  }
-
-  #readNodeId(representation: Representation, path: string): number {
-    return this.#withReference(representation, path, (resolved) =>
-      this.#readId(
-        resolveAnnotations(resolved),
-        path,
-        ".annotate({ fieldId }) on the schema itself",
-      ),
-    );
-  }
-
   #withReference<A>(
     representation: Representation,
     path: string,
@@ -231,7 +199,7 @@ class Compiler {
 
       const fieldPath = propertyPath(path, property.name);
       this.#checkRecursion(property.type, fieldPath);
-      const id = this.#readId(property.annotations, fieldPath, "Iceberg.field(id)");
+      const id = ++this.#nextId;
       const compiled = this.#compileType(property.type, fieldPath);
       const fieldDescription = description(property.annotations);
 
@@ -332,7 +300,7 @@ class Compiler {
 
     const elementPath = `${path}.element`;
     const elementRepresentation = representation.rest[0]!;
-    const elementId = this.#readNodeId(elementRepresentation, elementPath);
+    const elementId = ++this.#nextId;
     const compiled = this.#compileType(elementRepresentation, elementPath);
 
     return {
@@ -375,12 +343,12 @@ class Compiler {
     const valuePath = `${path}.value`;
     const keyRepresentation = representation.typeParameters[0]!;
     const valueRepresentation = representation.typeParameters[1]!;
-    const keyId = this.#readNodeId(keyRepresentation, keyPath);
+    const keyId = ++this.#nextId;
     const compiledKey = this.#compileType(keyRepresentation, keyPath);
 
     if (compiledKey.nullable) return fail(keyPath, "Map keys cannot be null");
 
-    const valueId = this.#readNodeId(valueRepresentation, valuePath);
+    const valueId = ++this.#nextId;
     const compiledValue = this.#compileType(valueRepresentation, valuePath);
 
     return {
@@ -448,42 +416,23 @@ export const compileTable = (
           }),
   });
 
-export const makeCommitRequest = (
+export const makeCreateRequest = (
   table: CompiledTable,
-  options: CreateTableCommitOptions,
-): CommitTableRequest => {
-  const properties = {
+  options: CreateTableOptions,
+): CreateTableRequest => ({
+  name: options.name,
+  schema: table.schema,
+  ...(options.location === undefined ? {} : { location: options.location }),
+  properties: {
     ...(table.description === undefined ? {} : { comment: table.description }),
     ...options.properties,
-  };
-
-  return {
-    requirements: [{ type: "assert-create" }],
-    updates: [
-      { action: "assign-uuid", uuid: options.uuid ?? crypto.randomUUID() },
-      { action: "upgrade-format-version", "format-version": 2 },
-      { action: "add-schema", schema: table.schema },
-      { action: "set-current-schema", "schema-id": -1 },
-      { action: "add-spec", spec: { "spec-id": 0, fields: [] } },
-      { action: "set-default-spec", "spec-id": -1 },
-      { action: "add-sort-order", "sort-order": { "order-id": 0, fields: [] } },
-      { action: "set-default-sort-order", "sort-order-id": -1 },
-      ...(options.location === undefined
-        ? []
-        : [{ action: "set-location" as const, location: options.location }]),
-      ...(Object.keys(properties).length === 0
-        ? []
-        : [{ action: "set-properties" as const, updates: properties }]),
-    ],
-  };
-};
+    "format-version": "3",
+  },
+});
 
 /**
- * Builds the commit that creates an Iceberg table from an Effect schema.
- *
- * The field IDs come from the schema, so the table has to keep them. Send the
- * commit with `commitTable`. Then load the table and check the IDs before you
- * write any rows. Don't use `createTable` here, because it can change the IDs.
+ * Builds a REST create-table request from an Effect schema.
+ * The catalog assigns the stored field IDs.
  *
  * A root `description` becomes the table `comment`, and a field `description`
  * becomes its `doc`. `title` and `examples` are dropped.
@@ -494,18 +443,19 @@ export const makeCommitRequest = (
  * @example
  * ```ts
  * const Row = Schema.Struct({
- *   id: Schema.String.pipe(Iceberg.field(1, { description: "Stable row identifier" })),
+ *   id: Schema.String.pipe(Schema.annotateKey({ description: "Row identifier" })),
  * })
  *   .annotate({ description: "Connector rows" })
  *   .pipe(Iceberg.identifier("id"))
  *
- * const request = yield* Iceberg.makeCreateTableCommitRequest(Row, {
+ * const request = yield* Iceberg.makeCreateTableRequest(Row, {
+ *   name: "rows",
  *   location: "s3://warehouse/rows"
  * })
  * ```
  */
-export const makeCreateTableCommitRequest = (
+export const makeCreateTableRequest = (
   schema: Schema.Top,
-  options: CreateTableCommitOptions,
-): Effect.Effect<CommitTableRequest, IcebergSchemaError> =>
-  Effect.map(compileTable(schema), (table) => makeCommitRequest(table, options));
+  options: CreateTableOptions,
+): Effect.Effect<CreateTableRequest, IcebergSchemaError> =>
+  Effect.map(compileTable(schema), (table) => makeCreateRequest(table, options));

package/src/ingestion/engine.ts

@@ -480,7 +480,7 @@ const runChanges = Effect.fnUntraced(function* (
     }).pipe(recordResourceError(connector, resource, "changes", "ingest"));
 
     const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
-    // Advance the durable cursor only after Wings acknowledges the page.
+    // Advance the durable cursor only after the engine acknowledges the page.
     yield* Effect.withSpan(
       store
         .setChangesState(resource.name, {

package/src/ingestor/{wings.ts → arrow-flight.ts}

@@ -1,20 +1,20 @@
 import type { RecordBatch, TypeMap } from "apache-arrow";
 import type { TableIdentifier } from "iceberg-js";
 
-import * as Wings from "@useairfoil/wings";
+import { CatalogManager } from "@useairfoil/api";
 import { Config, Effect, Layer } from "effect";
 import { FetchHttpClient } from "effect/http";
 
 import type { TableSource } from "../core/types";
 
-import { ensureTable } from "../catalog/table";
 import { ConnectorError } from "../errors";
+import { compileTable } from "../iceberg/table";
 import * as RuntimeConfig from "../runtime-config";
 import { makeRowEncoder, type RowEncoder } from "./arrow";
 import { Ingestor, type IngestorService } from "./service";
 
 /** One catalog and an Iceberg table for each resource that runs. */
-export type WingsIngestorConfig<Source extends TableSource = TableSource> = {
+export type ArrowFlightIngestorConfig<Source extends TableSource = TableSource> = {
   readonly connector: Source;
   readonly catalog: string;
   readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;
@@ -33,17 +33,13 @@ type PreparedTable = {
 
 const connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });
 
-/** Loads each table schema and opens one Wings stream per bound resource. */
-export const layerWings = <const Source extends TableSource>(
-  config: WingsIngestorConfig<Source>,
-): Layer.Layer<Ingestor, ConnectorError, Wings.CatalogManager.CatalogManager> =>
+/** Compiles local schemas and opens one Arrow Flight stream per bound resource. */
+export const layerArrowFlight = <const Source extends TableSource>(
+  config: ArrowFlightIngestorConfig<Source>,
+): Layer.Layer<Ingestor, ConnectorError, CatalogManager.CatalogManager> =>
   Layer.effect(Ingestor)(
     Effect.gen(function* () {
-      const manager = yield* Wings.CatalogManager.CatalogManager;
-      const catalog = yield* manager
-        .getIcebergCatalog(config.catalog)
-        .pipe(Effect.mapError((cause) => connectorError("Failed to load Wings catalog", cause)));
-
+      const manager = yield* CatalogManager.CatalogManager;
       const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;
 
       const prepared: Array<PreparedTable> = [];
@@ -51,15 +47,11 @@ export const layerWings = <const Source extends TableSource>(
         const binding = bindings[resource.name];
         if (!binding) continue;
 
-        const { namespace, name, location } = binding;
+        const { namespace, name } = binding;
         const identifier = { namespace, name };
-        const schema = yield* ensureTable(resource.rowSchema, {
-          catalog,
-          identifier,
-          location,
-        }).pipe(
+        const { schema } = yield* compileTable(resource.rowSchema).pipe(
           Effect.mapError((cause) =>
-            connectorError(`Failed to prepare table for ${resource.name}: ${cause.message}`, cause),
+            connectorError(`Invalid schema for ${resource.name}: ${cause.message}`, cause),
           ),
         );
 
@@ -118,26 +110,32 @@ export const layerWings = <const Source extends TableSource>(
     }),
   );
 
-/** Reads the Wings URL, catalog and table bindings from hosted config. */
-export const layerWingsConfig = (connector: TableSource) =>
+/** Configures Arrow Flight ingestion from the hosted URL, catalog and table bindings. */
+export const layerArrowFlightConfig = (connector: TableSource) =>
   Layer.unwrap(
     Effect.gen(function* () {
-      const { catalog, uri } = yield* Config.all({
+      const { catalog, uri, flightUri } = yield* Config.all({
         catalog: RuntimeConfig.catalog,
-        uri: RuntimeConfig.wingsUri,
-      }).pipe(Effect.mapError((cause) => connectorError("Invalid Wings configuration", cause)));
+        uri: RuntimeConfig.engineUri,
+        flightUri: RuntimeConfig.flightUri,
+      }).pipe(
+        Effect.mapError((cause) => connectorError("Invalid Arrow Flight configuration", cause)),
+      );
 
       if (uri.protocol !== "http:" && uri.protocol !== "https:") {
-        return yield* Effect.fail(connectorError("Invalid Wings URL"));
+        return yield* Effect.fail(connectorError("Invalid Arrow Flight URL"));
+      }
+      if (flightUri.protocol !== "http:" && flightUri.protocol !== "https:") {
+        return yield* Effect.fail(connectorError("Invalid Arrow Flight URL"));
       }
 
       const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(
         Effect.mapError((cause) => connectorError(cause.message, cause)),
       );
-      const manager = Wings.CatalogManager.layer({ baseUrl: uri.href }).pipe(
+      const manager = CatalogManager.layer({ engineUri: uri.href, flightUri: flightUri.href }).pipe(
         Layer.provide(FetchHttpClient.layer),
       );
 
-      return layerWings({ connector, catalog, tables }).pipe(Layer.provide(manager));
+      return layerArrowFlight({ connector, catalog, tables }).pipe(Layer.provide(manager));
     }),
   );

package/src/ingestor/console.ts

@@ -2,7 +2,7 @@ import { Effect, Layer } from "effect";
 
 import { Ingestor } from "./service";
 
-/** Logs batches without sending them to Wings. */
+/** Logs batches without sending them over Arrow Flight. */
 export const layerConsole = Layer.succeed(Ingestor)({
   ingest: ({ resource, source, batch }) =>
     Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(

package/src/ingestor/index.ts

@@ -6,4 +6,8 @@ export {
   type IngestResult,
   type IngestSource,
 } from "./service";
-export { layerWings, layerWingsConfig, type WingsIngestorConfig } from "./wings";
+export {
+  layerArrowFlight,
+  layerArrowFlightConfig,
+  type ArrowFlightIngestorConfig,
+} from "./arrow-flight";

package/src/runtime-config/config.ts

@@ -7,8 +7,13 @@ export const httpPort = Config.Port(PlatformRuntimeKey.httpPort).pipe(
   Config.withDefault(PlatformRuntimeDefault.httpPort),
 );
 
-/** Wings URL, including the protocol. */
-export const wingsUri = Config.URL(PlatformRuntimeKey.wingsUri);
+/** Engine URL, including the protocol. */
+export const engineUri = Config.URL(PlatformRuntimeKey.engineUri);
 
-/** Wings catalog ID. */
+/** Arrow Flight URL. Uses the engine URL when unset. */
+export const flightUri = engineUri.pipe(
+  Config.flatMap((uri) => Config.URL(PlatformRuntimeKey.flightUri).pipe(Config.withDefault(uri))),
+);
+
+/** Catalog ID. */
 export const catalog = Config.NonEmptyString(PlatformRuntimeKey.catalog);

package/src/runtime-config/constants.ts

@@ -8,7 +8,8 @@ export const PlatformRuntimeKey = {
   connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
   stateTable: "AIRFOIL_STATE_TABLE",
   postgresConnectionString: "POSTGRES_CONNECTION_STRING",
-  wingsUri: "WINGS_URI",
+  engineUri: "ENGINE_URI",
+  flightUri: "FLIGHT_URI",
   catalog: "AIRFOIL_CATALOG",
   otelEnabled: "OTEL_ENABLED",
   otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",

package/src/runtime-config/index.ts

@@ -1,5 +1,5 @@
 export { RuntimeConfigError, type RuntimeConfigErrorCode } from "./error";
-export { catalog, httpPort, wingsUri } from "./config";
+export { catalog, engineUri, flightUri, httpPort } from "./config";
 export { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
 export { layerHosted } from "./layer";
 export { type ResolvedTableBindings, type TableBinding, loadTableBindings } from "./table-bindings";

package/src/runtime-config/table-bindings.ts

@@ -2,15 +2,17 @@ import type { TableIdentifier } from "iceberg-js";
 
 import { Config, Effect, Schema } from "effect";
 
-import type { TableSource } from "../core/types";
-
 import { PlatformRuntimeKey } from "./constants";
 import { RuntimeConfigError } from "./error";
 
+type BindingSource = {
+  readonly resources: ReadonlyArray<{ readonly name: string }>;
+};
+
 /** An Iceberg table and its storage location for each connector resource. */
 export type TableBinding = TableIdentifier & { readonly location?: string | undefined };
 
-export type ResolvedTableBindings<Source extends TableSource> = Readonly<
+export type ResolvedTableBindings<Source extends BindingSource> = Readonly<
   Partial<Record<Source["resources"][number]["name"], TableBinding>>
 >;
 
@@ -31,7 +33,7 @@ const fail = (code: "TABLE_BINDINGS_INVALID" | "TABLE_BINDINGS_MISMATCH", messag
   new RuntimeConfigError({ code, message, cause: new Error(message) });
 
 /** Reads hosted table bindings and checks they name resources of the connector. */
-export const loadTableBindings = <const Source extends TableSource>(
+export const loadTableBindings = <const Source extends BindingSource>(
   connector: Source,
 ): Effect.Effect<ResolvedTableBindings<Source>, RuntimeConfigError> =>
   Effect.gen(function* () {

package/src/telemetry.ts

@@ -1,4 +1,4 @@
-import { Cause, Config, Effect, Layer, Metric, Option, type Duration } from "effect";
+import { Cause, Config, Effect, Layer, Metric, Option, Schema, type Duration } from "effect";
 import { Headers } from "effect/http";
 import * as Observability from "effect/observability";
 
@@ -60,7 +60,14 @@ const maxErrorMessageLength = 500;
 const OtlpEnvConfig = Config.all({
   enabled: Config.Boolean(PlatformRuntimeKey.otelEnabled).pipe(Config.withDefault(false)),
   baseUrl: Config.option(Config.String(PlatformRuntimeKey.otelExporterOtlpEndpoint)),
-  rawHeaders: Config.option(Config.String(PlatformRuntimeKey.otelExporterOtlpHeaders)),
+  headers: Config.option(
+    Config.Record(
+      Schema.String,
+      Schema.StringFromUriComponent,
+      PlatformRuntimeKey.otelExporterOtlpHeaders,
+      { trim: true },
+    ),
+  ),
 });
 
 const defaultRedactedHeaders: ReadonlyArray<string | RegExp> = [
@@ -74,19 +81,6 @@ const defaultRedactedHeaders: ReadonlyArray<string | RegExp> = [
   /token/i,
 ];
 
-const parseOtelHeaders = (value: string): Record<string, string> =>
-  Object.fromEntries(
-    value
-      .split(",")
-      .map((entry) => entry.trim())
-      .filter(Boolean)
-      .flatMap((entry) => {
-        const separator = entry.indexOf("=");
-        if (separator < 1) return [];
-        return [[entry.slice(0, separator).trim(), entry.slice(separator + 1).trim()]];
-      }),
-  );
-
 const appendPath = (baseUrl: string, path: string) =>
   `${baseUrl.replace(/\/$/, "")}/${path.replace(/^\//, "")}`;
 
@@ -254,9 +248,7 @@ export const layerOtlpTracing = (options: OtlpTracingOptions = {}) =>
         {
           enabled: raw.enabled,
           endpoint: Option.getOrUndefined(raw.baseUrl),
-          headers: Option.isSome(raw.rawHeaders)
-            ? parseOtelHeaders(raw.rawHeaders.value)
-            : undefined,
+          headers: Option.getOrUndefined(raw.headers),
         },
         options,
       );
@@ -276,7 +268,7 @@ export const layerOtlpMetrics = (options: OtlpMetricsOptions = {}) =>
         );
       return Observability.OtlpMetrics.layer({
         url: appendPath(endpoint, "/v1/metrics"),
-        headers: Option.isSome(raw.rawHeaders) ? parseOtelHeaders(raw.rawHeaders.value) : undefined,
+        headers: Option.getOrUndefined(raw.headers),
         exportInterval: options.exportInterval,
         temporality: "cumulative",
       }).pipe(Layer.provide(Observability.OtlpSerialization.layerJson));

package/dist/connector-app-C-L4QqG6.mjs

@@ -1,123 +0,0 @@
-import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
-import { n as layerWingsConfig, r as createTableFromWings } from "./wings-I56hGtNv.mjs";
-import { s as wingsUri } from "./runtime-config-Oa55gAew.mjs";
-import { n as run } from "./demo-B30Vlkk2.mjs";
-import { n as run$1 } from "./ingestion-q0WwqUSt.mjs";
-import { ByteSize, DateTime, Effect, Option, Tuple } from "effect";
-import { Argument, CliError, Command, Flag } from "effect/cli";
-import "effect/http";
-import { NodeHttpServer } from "@effect/platform-node";
-import { createServer } from "node:http";
-//#region src/connector-app/check.ts
-const ok = { _tag: "ok" };
-const error = (message) => ({
-	_tag: "error",
-	message
-});
-const fromEntries = (entries) => Object.fromEntries(entries);
-const resultsFor = (names, result) => fromEntries(names.map((name) => Tuple.make(name, result)));
-/**
-* Builds the connector service from its layer and validates only the selected
-* resources. The layer remains open until every check finishes.
-*/
-function check(service, layer, options) {
-	if (options.resources.length === 0) return Effect.succeed(fromEntries([]));
-	return service.pipe(Effect.flatMap((connector) => Effect.forEach(options.resources, (name) => {
-		const resource = connector.resources.find((candidate) => candidate.name === name);
-		return (resource ? resource.check.pipe(Effect.match({
-			onFailure: (failure) => error(failure.message),
-			onSuccess: () => ok
-		})) : Effect.succeed(error(`Unknown connector resource: ${name}`))).pipe(Effect.map((result) => Tuple.make(name, result)));
-	}).pipe(Effect.map(fromEntries))), Effect.provide(layer), Effect.match({
-		onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
-		onSuccess: (results) => results
-	}));
-}
-//#endregion
-//#region src/connector-app/create-table.ts
-/** Creates the table command for a connector's row schemas. */
-const makeCreateTableCommand = (schemas) => Command.make("create-table", {
-	table: Argument.String("table"),
-	catalog: Flag.String("catalog"),
-	location: Flag.String("location").pipe(Flag.optional),
-	resource: Flag.String("resource").pipe(Flag.optional)
-}, ({ table, catalog, location, resource }) => Effect.gen(function* () {
-	const namespace = table.split(".");
-	const name = namespace.pop();
-	if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) return yield* new CliError.UserError({
-		cause: "Invalid table name",
-		userMessage: "Use a table name like my.namespace.table"
-	});
-	const resourceName = Option.getOrElse(resource, () => name);
-	const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : void 0;
-	if (!schema) return yield* new CliError.UserError({
-		cause: "Unknown resource",
-		userMessage: `Unknown resource: ${resourceName}`
-	});
-	const uri = yield* wingsUri;
-	yield* createTableFromWings(schema, {
-		wingsUri: uri.href,
-		catalogId: catalog,
-		identifier: {
-			namespace,
-			name
-		},
-		location: Option.getOrUndefined(location)
-	});
-	yield* Effect.log(`Created ${table}`);
-})).pipe(Command.withDescription("Create an Iceberg table for a connector resource"));
-//#endregion
-//#region src/connector-app/demo.ts
-/** Reads sizes such as `512KiB`, `128MiB` or `1GiB`. */
-const parseSize = (input) => {
-	if (!/^\d+(KiB|MiB|GiB)$/.test(input)) return Option.none();
-	return ByteSize.fromString(input).pipe(Option.flatMap(ByteSize.toNumber), Option.filter((bytes) => bytes > 0));
-};
-/**
-* Creates the command that writes the demo data to Wings, then exits.
-* It reads Wings and table bindings from Config, so provide `RuntimeConfig.layerHosted()`.
-*/
-const makeDemoCommand = (demo) => Command.make("demo", {
-	size: Flag.String("size").pipe(Flag.filterMap(parseSize, () => "a size like 512KiB, 128MiB or 1GiB")),
-	seed: Flag.Int("seed").pipe(Flag.withDefault(1))
-}, ({ size, seed }) => Effect.gen(function* () {
-	const now = yield* DateTime.nowAsDate;
-	yield* run(demo, {
-		bytes: size,
-		seed,
-		now
-	}).pipe(Effect.provide(layerWingsConfig({ resources: [...demo.setup.tables, ...demo.group.tables] })));
-})).pipe(Command.withDescription("Write demo data for the bound tables, then exit"));
-//#endregion
-//#region src/connector-app/start.ts
-const start = (app, options) => Effect.gen(function* () {
-	const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
-	const routes = app.webhooks?.map((route) => route.path) ?? [];
-	yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
-		port: options.port,
-		routes
-	}));
-	return yield* run$1(app, {
-		initialCutoff,
-		webhook: {
-			routes: app.webhooks ?? [],
-			healthPath: options.healthPath ?? "/health",
-			metricsPath: options.metricsPath,
-			statusPath: options.statusPath,
-			disableHttpLogger: options.disableHttpLogger ?? true
-		}
-	}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
-});
-//#endregion
-//#region src/connector-app/index.ts
-var connector_app_exports = /* @__PURE__ */ __exportAll({
-	check: () => check,
-	makeCreateTableCommand: () => makeCreateTableCommand,
-	makeDemoCommand: () => makeDemoCommand,
-	parseSize: () => parseSize,
-	start: () => start
-});
-//#endregion
-export { makeCreateTableCommand as a, parseSize as i, start as n, check as o, makeDemoCommand as r, connector_app_exports as t };
-
-//# sourceMappingURL=connector-app-C-L4QqG6.mjs.map

package/dist/connector-app-C-L4QqG6.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"connector-app-C-L4QqG6.mjs","names":["runIngestion"],"sources":["../src/connector-app/check.ts","../src/connector-app/create-table.ts","../src/connector-app/demo.ts","../src/connector-app/start.ts","../src/connector-app/index.ts"],"sourcesContent":["import { Context, Effect, Layer, Tuple } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition, ResourceName } from \"../core/types\";\n\nexport type ResourceCheckResult =\n  | { readonly _tag: \"ok\" }\n  | { readonly _tag: \"error\"; readonly message: string };\n\nexport type CheckOptions<Name extends string> = {\n  readonly resources: ReadonlyArray<Name>;\n};\n\ntype ResourceCheckResults<Name extends PropertyKey> = {\n  readonly [Key in Name]: ResourceCheckResult;\n};\n\nexport type CheckResult<\n  Resources extends ReadonlyArray<ResourceDefinition>,\n  Name extends ResourceName<Resources> = ResourceName<Resources>,\n> = ResourceCheckResults<Name>;\n\ntype CheckFailure = { readonly message: string };\n\nconst ok: ResourceCheckResult = { _tag: \"ok\" };\n\nconst error = (message: string): ResourceCheckResult => ({ _tag: \"error\", message });\n\nconst fromEntries = <Name extends string>(\n  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,\n): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;\n\nconst resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>\n  fromEntries(names.map((name) => Tuple.make(name, result)));\n\n/**\n * Builds the connector service from its layer and validates only the selected\n * resources. The layer remains open until every check finishes.\n */\nexport function check<\n  Identifier,\n  const Resources extends ReadonlyArray<ResourceDefinition>,\n  E extends CheckFailure,\n  R,\n  const Name extends ResourceName<Resources>,\n>(\n  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,\n  layer: Layer.Layer<Identifier, E, R>,\n  options: CheckOptions<Name>,\n): Effect.Effect<CheckResult<Resources, Name>, never, R> {\n  if (options.resources.length === 0) {\n    return Effect.succeed(fromEntries<Name>([]));\n  }\n\n  return service.pipe(\n    Effect.flatMap((connector) =>\n      Effect.forEach(options.resources, (name) => {\n        const resource = connector.resources.find((candidate) => candidate.name === name);\n        const result = resource\n          ? resource.check.pipe(\n              Effect.match({\n                onFailure: (failure) => error(failure.message),\n                onSuccess: () => ok,\n              }),\n            )\n          : Effect.succeed(error(`Unknown connector resource: ${name}`));\n\n        return result.pipe(Effect.map((result) => Tuple.make(name, result)));\n      }).pipe(Effect.map(fromEntries)),\n    ),\n    Effect.provide(layer),\n    Effect.match({\n      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),\n      onSuccess: (results) => results,\n    }),\n  );\n}\n","import { Effect, Option, type Schema } from \"effect\";\nimport { Argument, CliError, Command, Flag } from \"effect/cli\";\n\nimport { createTableFromWings } from \"../catalog/table\";\nimport { wingsUri } from \"../runtime-config/config\";\n\n/** Creates the table command for a connector's row schemas. */\nexport const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>\n  Command.make(\n    \"create-table\",\n    {\n      table: Argument.String(\"table\"),\n      catalog: Flag.String(\"catalog\"),\n      location: Flag.String(\"location\").pipe(Flag.optional),\n      resource: Flag.String(\"resource\").pipe(Flag.optional),\n    },\n    ({ table, catalog, location, resource }) =>\n      Effect.gen(function* () {\n        const namespace = table.split(\".\");\n        const name = namespace.pop();\n\n        if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) {\n          return yield* new CliError.UserError({\n            cause: \"Invalid table name\",\n            userMessage: \"Use a table name like my.namespace.table\",\n          });\n        }\n\n        const resourceName = Option.getOrElse(resource, () => name);\n        const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : undefined;\n        if (!schema) {\n          return yield* new CliError.UserError({\n            cause: \"Unknown resource\",\n            userMessage: `Unknown resource: ${resourceName}`,\n          });\n        }\n\n        const uri = yield* wingsUri;\n        yield* createTableFromWings(schema, {\n          wingsUri: uri.href,\n          catalogId: catalog,\n          identifier: { namespace, name },\n          location: Option.getOrUndefined(location),\n        });\n        yield* Effect.log(`Created ${table}`);\n      }),\n  ).pipe(Command.withDescription(\"Create an Iceberg table for a connector resource\"));\n","import { ByteSize, DateTime, Effect, Option } from \"effect\";\nimport { Command, Flag } from \"effect/cli\";\n\nimport { run, type DemoDefinition } from \"../demo\";\nimport { layerWingsConfig } from \"../ingestor/wings\";\n\n/** Reads sizes such as `512KiB`, `128MiB` or `1GiB`. */\nexport const parseSize = (input: string): Option.Option<number> => {\n  if (!/^\\d+(KiB|MiB|GiB)$/.test(input)) return Option.none();\n\n  return ByteSize.fromString(input).pipe(\n    Option.flatMap(ByteSize.toNumber),\n    Option.filter((bytes) => bytes > 0),\n  );\n};\n\n/**\n * Creates the command that writes the demo data to Wings, then exits.\n * It reads Wings and table bindings from Config, so provide `RuntimeConfig.layerHosted()`.\n */\nexport const makeDemoCommand = (demo: DemoDefinition) =>\n  Command.make(\n    \"demo\",\n    {\n      size: Flag.String(\"size\").pipe(\n        Flag.filterMap(parseSize, () => \"a size like 512KiB, 128MiB or 1GiB\"),\n      ),\n      seed: Flag.Int(\"seed\").pipe(Flag.withDefault(1)),\n    },\n    ({ size, seed }) =>\n      Effect.gen(function* () {\n        const now = yield* DateTime.nowAsDate;\n\n        yield* run(demo, { bytes: size, seed, now }).pipe(\n          Effect.provide(\n            layerWingsConfig({\n              resources: [...demo.setup.tables, ...demo.group.tables],\n            }),\n          ),\n        );\n      }),\n  ).pipe(Command.withDescription(\"Write demo data for the bound tables, then exit\"));\n","import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n  readonly port: number;\n  readonly initialCutoff?: Cursor.Value;\n  readonly healthPath?: HttpRouter.PathInput;\n  readonly metricsPath?: HttpRouter.PathInput | false;\n  readonly statusPath?: HttpRouter.PathInput | false;\n  readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n  Effect.gen(function* () {\n    const initialCutoff =\n      options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n    yield* Effect.logInfo(\"webhook server ready\").pipe(\n      Effect.annotateLogs({ port: options.port, routes }),\n    );\n\n    return yield* runIngestion(app, {\n      initialCutoff,\n      webhook: {\n        routes: app.webhooks ?? [],\n        healthPath: options.healthPath ?? \"/health\",\n        metricsPath: options.metricsPath,\n        statusPath: options.statusPath,\n        disableHttpLogger: options.disableHttpLogger ?? true,\n      },\n    }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n  });\n","export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from \"./check\";\nexport { makeCreateTableCommand } from \"./create-table\";\nexport { makeDemoCommand, parseSize } from \"./demo\";\nexport { start, type App, type StartOptions } from \"./start\";\n"],"mappings":";;;;;;;;;;;AAuBA,MAAM,KAA0B,EAAE,MAAM,KAAK;AAE7C,MAAM,SAAS,aAA0C;CAAE,MAAM;CAAS;AAAQ;AAElF,MAAM,eACJ,YAC+B,OAAO,YAAY,OAAO;AAE3D,MAAM,cAAmC,OAA4B,WACnE,YAAY,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;;;;;AAM3D,SAAgB,MAOd,SACA,OACA,SACuD;CACvD,IAAI,QAAQ,UAAU,WAAW,GAC/B,OAAO,OAAO,QAAQ,YAAkB,CAAC,CAAC,CAAC;CAG7C,OAAO,QAAQ,KACb,OAAO,SAAS,cACd,OAAO,QAAQ,QAAQ,YAAY,SAAS;EAC1C,MAAM,WAAW,UAAU,UAAU,MAAM,cAAc,UAAU,SAAS,IAAI;EAUhF,QATe,WACX,SAAS,MAAM,KACb,OAAO,MAAM;GACX,YAAY,YAAY,MAAM,QAAQ,OAAO;GAC7C,iBAAiB;EACnB,CAAC,CACH,IACA,OAAO,QAAQ,MAAM,+BAA+B,MAAM,CAAC,EAAA,CAEjD,KAAK,OAAO,KAAK,WAAW,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;CACrE,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,WAAW,CAAC,CACjC,GACA,OAAO,QAAQ,KAAK,GACpB,OAAO,MAAM;EACX,YAAY,YAAY,WAAW,QAAQ,WAAW,MAAM,QAAQ,OAAO,CAAC;EAC5E,YAAY,YAAY;CAC1B,CAAC,CACH;AACF;;;;ACpEA,MAAa,0BAA0B,YACrC,QAAQ,KACN,gBACA;CACE,OAAO,SAAS,OAAO,OAAO;CAC9B,SAAS,KAAK,OAAO,SAAS;CAC9B,UAAU,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,KAAK,QAAQ;CACpD,UAAU,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,KAAK,QAAQ;AACtD,IACC,EAAE,OAAO,SAAS,UAAU,eAC3B,OAAO,IAAI,aAAa;CACtB,MAAM,YAAY,MAAM,MAAM,GAAG;CACjC,MAAM,OAAO,UAAU,IAAI;CAE3B,IAAI,CAAC,QAAQ,UAAU,WAAW,KAAK,UAAU,MAAM,SAAS,KAAK,WAAW,CAAC,GAC/E,OAAO,OAAO,IAAI,SAAS,UAAU;EACnC,OAAO;EACP,aAAa;CACf,CAAC;CAGH,MAAM,eAAe,OAAO,UAAU,gBAAgB,IAAI;CAC1D,MAAM,SAAS,OAAO,OAAO,SAAS,YAAY,IAAI,QAAQ,gBAAgB,KAAA;CAC9E,IAAI,CAAC,QACH,OAAO,OAAO,IAAI,SAAS,UAAU;EACnC,OAAO;EACP,aAAa,qBAAqB;CACpC,CAAC;CAGH,MAAM,MAAM,OAAO;CACnB,OAAO,qBAAqB,QAAQ;EAClC,UAAU,IAAI;EACd,WAAW;EACX,YAAY;GAAE;GAAW;EAAK;EAC9B,UAAU,OAAO,eAAe,QAAQ;CAC1C,CAAC;CACD,OAAO,OAAO,IAAI,WAAW,OAAO;AACtC,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,kDAAkD,CAAC;;;;ACvCpF,MAAa,aAAa,UAAyC;CACjE,IAAI,CAAC,qBAAqB,KAAK,KAAK,GAAG,OAAO,OAAO,KAAK;CAE1D,OAAO,SAAS,WAAW,KAAK,CAAC,CAAC,KAChC,OAAO,QAAQ,SAAS,QAAQ,GAChC,OAAO,QAAQ,UAAU,QAAQ,CAAC,CACpC;AACF;;;;;AAMA,MAAa,mBAAmB,SAC9B,QAAQ,KACN,QACA;CACE,MAAM,KAAK,OAAO,MAAM,CAAC,CAAC,KACxB,KAAK,UAAU,iBAAiB,oCAAoC,CACtE;CACA,MAAM,KAAK,IAAI,MAAM,CAAC,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC;AACjD,IACC,EAAE,MAAM,WACP,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,SAAS;CAE5B,OAAO,IAAI,MAAM;EAAE,OAAO;EAAM;EAAM;CAAI,CAAC,CAAC,CAAC,KAC3C,OAAO,QACL,iBAAiB,EACf,WAAW,CAAC,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,MAAM,MAAM,EACxD,CAAC,CACH,CACF;AACF,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,iDAAiD,CAAC;;;ACpBnF,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;CACnF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,IAAI,KAAK,CAAC;CAE5D,OAAO,OAAO,QAAQ,sBAAsB,CAAC,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;CAAO,CAAC,CACpD;CAEA,OAAO,OAAOA,MAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,CAAC;GACzB,YAAY,QAAQ,cAAc;GAClC,aAAa,QAAQ;GACrB,YAAY,QAAQ;GACpB,mBAAmB,QAAQ,qBAAqB;EAClD;CACF,CAAC,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,KAAK,CAAC,CAAC,CAAC;AACpF,CAAC"}

package/dist/constants-D-GQzpHZ.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants-D-GQzpHZ.mjs","names":[],"sources":["../src/runtime-config/constants.ts"],"sourcesContent":["import { HashSet } from \"effect\";\n\n/** Shared runtime keys owned by Airfoil rather than an individual connector. */\nexport const PlatformRuntimeKey = {\n  configPath: \"AIRFOIL_CONFIG_PATH\",\n  httpPort: \"AIRFOIL_HTTP_PORT\",\n  tableBindings: \"AIRFOIL_TABLE_BINDINGS\",\n  connectorInstanceId: \"AIRFOIL_CONNECTOR_INSTANCE_ID\",\n  stateTable: \"AIRFOIL_STATE_TABLE\",\n  postgresConnectionString: \"POSTGRES_CONNECTION_STRING\",\n  wingsUri: \"WINGS_URI\",\n  catalog: \"AIRFOIL_CATALOG\",\n  otelEnabled: \"OTEL_ENABLED\",\n  otelExporterOtlpEndpoint: \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n  otelExporterOtlpHeaders: \"OTEL_EXPORTER_OTLP_HEADERS\",\n  otelServiceName: \"OTEL_SERVICE_NAME\",\n  otelServiceVersion: \"OTEL_SERVICE_VERSION\",\n  otelResourceAttributes: \"OTEL_RESOURCE_ATTRIBUTES\",\n} as const;\n\n/** Defaults for optional platform-owned runtime configuration. */\nexport const PlatformRuntimeDefault = {\n  httpPort: 8080,\n  stateTable: \"_airfoil_connectors_state\",\n} as const;\n\nconst platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));\n\n/** @internal */\nexport const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);\n"],"mappings":";;;AAGA,MAAa,qBAAqB;CAChC,YAAY;CACZ,UAAU;CACV,eAAe;CACf,qBAAqB;CACrB,YAAY;CACZ,0BAA0B;CAC1B,UAAU;CACV,SAAS;CACT,aAAa;CACb,0BAA0B;CAC1B,yBAAyB;CACzB,iBAAiB;CACjB,oBAAoB;CACpB,wBAAwB;AAC1B;;AAGA,MAAa,yBAAyB;CACpC,UAAU;CACV,YAAY;AACd;AAEA,MAAM,sBAAsB,QAAQ,aAAa,OAAO,OAAO,kBAAkB,CAAC;;AAGlF,MAAa,wBAAwB,QAAyB,QAAQ,IAAI,qBAAqB,GAAG"}