@useairfoil/connector-kit 0.6.0 → 0.7.0

package/dist/arrow-flight-DDQ0ktx9.mjs

@@ -0,0 +1,251 @@
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { n as compileTable } from "./table-CxM09xf7.mjs";
+import { a as catalog, n as loadTableBindings, o as engineUri, s as flightUri } from "./runtime-config-Bn8Fivt1.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { Config, Effect, Layer, Schema } from "effect";
+import { CatalogManager } from "@useairfoil/api";
+import { FetchHttpClient } from "effect/http";
+import { parseDecimalType, parseFixedType } from "iceberg-js";
+import { Bool, DateDay, Decimal, Field, FixedSizeBinary, Float32, Float64, Int32, Int64, LargeBinary, List, Map_, RecordBatch, Schema as Schema$1, Struct as Struct$1, TimeMicrosecond, TimestampMicrosecond, Utf8, makeBuilder } from "apache-arrow";
+//#region src/ingestor/arrow.ts
+const bytes = (length) => Schema.Uint8Array.check(Schema.makeFilter((value) => value.byteLength === length || `expected ${length} bytes`));
+const int32 = Schema.Number.check(Schema.isInt(), Schema.isBetween({
+	minimum: -2147483648,
+	maximum: 2147483647
+}));
+const int64 = Schema.BigInt.check(Schema.isBetweenBigInt({
+	minimum: -(2n ** 63n),
+	maximum: 2n ** 63n - 1n
+}));
+const float32 = Schema.Finite.check(Schema.makeFilter((value) => Number.isFinite(Math.fround(value)) || "expected a 32-bit number"));
+const microsecondsInDay = Schema.BigInt.check(Schema.isBetweenBigInt({
+	minimum: 0n,
+	maximum: 86399999999n
+}));
+const primitives = {
+	boolean: {
+		type: new Bool(),
+		row: Schema.Boolean
+	},
+	int: {
+		type: new Int32(),
+		row: int32
+	},
+	long: {
+		type: new Int64(),
+		row: int64
+	},
+	float: {
+		type: new Float32(),
+		row: float32
+	},
+	double: {
+		type: new Float64(),
+		row: Schema.Finite
+	},
+	string: {
+		type: new Utf8(),
+		row: Schema.String
+	},
+	uuid: {
+		type: new FixedSizeBinary(16),
+		row: bytes(16)
+	},
+	binary: {
+		type: new LargeBinary(),
+		row: Schema.Uint8Array
+	},
+	date: {
+		type: new DateDay(),
+		row: Schema.Date
+	},
+	time: {
+		type: new TimeMicrosecond(),
+		row: microsecondsInDay
+	},
+	timestamp: {
+		type: new TimestampMicrosecond(),
+		row: Schema.Date
+	},
+	timestamptz: {
+		type: new TimestampMicrosecond("UTC"),
+		row: Schema.Date
+	}
+};
+const primitive = (type) => {
+	const known = primitives[type];
+	if (known) return known;
+	const decimal = parseDecimalType(type);
+	if (decimal) {
+		if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) throw new Error(`Invalid Iceberg decimal type ${type}`);
+		return {
+			type: new Decimal(decimal.scale, decimal.precision),
+			row: Schema.instanceOf(Uint32Array).check(Schema.makeFilter((value) => value.length === 4 || "expected four words"))
+		};
+	}
+	const fixed = parseFixedType(type);
+	if (fixed) {
+		if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);
+		return {
+			type: new FixedSizeBinary(fixed.length),
+			row: bytes(fixed.length)
+		};
+	}
+	throw new Error(`Unsupported Iceberg type ${type}`);
+};
+/** Absent and null both become null in Arrow. */
+const optional = (row) => Schema.optional(Schema.NullOr(row));
+const fieldMetadata = (id) => /* @__PURE__ */ new Map([["PARQUET:field_id", String(id)]]);
+/** Same as `primitive`, plus struct, list and map. */
+const compileType = (type) => {
+	if (typeof type === "string") return primitive(type);
+	if (type.type === "struct") {
+		const fields = type.fields.map((field) => ({
+			...field,
+			compiled: compileType(field.type)
+		}));
+		return {
+			type: new Struct$1(fields.map((field) => new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)))),
+			row: Schema.Struct(Object.fromEntries(fields.map((field) => [field.name, field.required ? field.compiled.row : optional(field.compiled.row)])))
+		};
+	}
+	if (type.type === "list") {
+		const element = compileType(type.element);
+		const required = type["element-required"];
+		return {
+			type: new List(new Field("element", element.type, !required, fieldMetadata(type["element-id"]))),
+			row: Schema.Array(required ? element.row : Schema.NullishOr(element.row))
+		};
+	}
+	const key = compileType(type.key);
+	const value = compileType(type.value);
+	const required = type["value-required"];
+	const entries = new Struct$1([new Field("key", key.type, false, fieldMetadata(type["key-id"])), new Field("value", value.type, !required, fieldMetadata(type["value-id"]))]);
+	return {
+		type: new Map_(new Field("entries", entries, false)),
+		row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row))
+	};
+};
+const makeRowEncoder = (tableSchema, version) => Effect.try({
+	try: () => {
+		const fields = tableSchema.fields.map((field) => ({
+			id: field.id,
+			name: field.name,
+			...compileType(field.type)
+		}));
+		const fieldNames = new Set(fields.map((field) => field.name));
+		const identifierIds = new Set(tableSchema["identifier-field-ids"]);
+		const keys = tableSchema.fields.filter((field) => identifierIds.has(field.id)).map((field) => field.name);
+		const required = /* @__PURE__ */ new Set([...keys, version]);
+		if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);
+		const struct = new Struct$1(fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))));
+		const schema = new Schema$1(struct.children);
+		const decodeRow = Schema.decodeUnknownSync(Schema.Struct(Object.fromEntries(fields.map((field) => [field.name, required.has(field.name) ? field.row : optional(field.row)]))), { onExcessProperty: "error" });
+		const decodeAt = (row, index) => {
+			try {
+				return decodeRow(row);
+			} catch (cause) {
+				const detail = cause instanceof Error ? cause.message : String(cause);
+				throw new Error(`row ${index}: ${detail}`, { cause });
+			}
+		};
+		return (rows) => Effect.try({
+			try: () => {
+				const builder = makeBuilder({
+					type: struct,
+					nullValues: [null, void 0]
+				});
+				for (const [index, row] of rows.entries()) builder.append(decodeAt(row, index));
+				return new RecordBatch(schema, builder.finish().flush());
+			},
+			catch: (cause) => new ConnectorError({
+				message: "Failed to encode rows",
+				cause
+			})
+		});
+	},
+	catch: (cause) => new ConnectorError({
+		message: "Failed to convert Iceberg schema",
+		cause
+	})
+});
+//#endregion
+//#region src/ingestor/arrow-flight.ts
+const connectorError = (message, cause) => new ConnectorError({
+	message,
+	cause
+});
+/** Compiles local schemas and opens one Arrow Flight stream per bound resource. */
+const layerArrowFlight = (config) => Layer.effect(Ingestor)(Effect.gen(function* () {
+	const manager = yield* CatalogManager.CatalogManager;
+	const bindings = config.tables;
+	const prepared = [];
+	for (const resource of config.connector.resources) {
+		const binding = bindings[resource.name];
+		if (!binding) continue;
+		const { namespace, name } = binding;
+		const identifier = {
+			namespace,
+			name
+		};
+		const { schema } = yield* compileTable(resource.rowSchema).pipe(Effect.mapError((cause) => connectorError(`Invalid schema for ${resource.name}: ${cause.message}`, cause)));
+		const encode = yield* makeRowEncoder(schema, resource.version);
+		prepared.push({
+			resource: resource.name,
+			identifier,
+			encode
+		});
+	}
+	const entries = /* @__PURE__ */ new Map();
+	for (const { resource, identifier, encode } of prepared) {
+		const ingestor = yield* manager.ingestor({
+			catalog: config.catalog,
+			namespace: identifier.namespace,
+			table: identifier.name
+		}).pipe(Effect.mapError((cause) => connectorError(`Failed to open ingestor for ${resource}`, cause)));
+		entries.set(resource, {
+			encode,
+			push: (batch) => ingestor.push(batch).pipe(Effect.mapError((cause) => connectorError(`Failed to ingest rows for ${resource}`, cause)))
+		});
+	}
+	const service = {
+		resourceNames: prepared.map(({ resource }) => resource),
+		ingest: ({ resource, batch }) => {
+			const entry = entries.get(resource);
+			if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));
+			if (batch.rows.length === 0) return Effect.succeed({
+				rows: 0,
+				bytes: 0
+			});
+			return entry.encode(batch.rows).pipe(Effect.flatMap((encoded) => entry.push(encoded).pipe(Effect.as({
+				rows: batch.rows.length,
+				bytes: encoded.data.byteLength
+			}))));
+		}
+	};
+	return Ingestor.of(service);
+}));
+/** Configures Arrow Flight ingestion from the hosted URL, catalog and table bindings. */
+const layerArrowFlightConfig = (connector) => Layer.unwrap(Effect.gen(function* () {
+	const { catalog: catalog$1, uri, flightUri: flightUri$1 } = yield* Config.all({
+		catalog,
+		uri: engineUri,
+		flightUri
+	}).pipe(Effect.mapError((cause) => connectorError("Invalid Arrow Flight configuration", cause)));
+	if (uri.protocol !== "http:" && uri.protocol !== "https:") return yield* Effect.fail(connectorError("Invalid Arrow Flight URL"));
+	if (flightUri$1.protocol !== "http:" && flightUri$1.protocol !== "https:") return yield* Effect.fail(connectorError("Invalid Arrow Flight URL"));
+	const tables = yield* loadTableBindings(connector).pipe(Effect.mapError((cause) => connectorError(cause.message, cause)));
+	const manager = CatalogManager.layer({
+		engineUri: uri.href,
+		flightUri: flightUri$1.href
+	}).pipe(Layer.provide(FetchHttpClient.layer));
+	return layerArrowFlight({
+		connector,
+		catalog: catalog$1,
+		tables
+	}).pipe(Layer.provide(manager));
+}));
+//#endregion
+export { layerArrowFlightConfig as n, layerArrowFlight as t };
+
+//# sourceMappingURL=arrow-flight-DDQ0ktx9.mjs.map

package/dist/arrow-flight-DDQ0ktx9.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-flight-DDQ0ktx9.mjs","names":["Struct","ArrowSchema","RuntimeConfig.catalog","RuntimeConfig.engineUri","RuntimeConfig.flightUri","flightUri","RuntimeConfig.loadTableBindings"],"sources":["../src/ingestor/arrow.ts","../src/ingestor/arrow-flight.ts"],"sourcesContent":["import {\n  Bool,\n  DateDay,\n  Decimal,\n  Field,\n  FixedSizeBinary,\n  Float32,\n  Float64,\n  Int32,\n  Int64,\n  LargeBinary,\n  List,\n  Map_,\n  RecordBatch,\n  Schema as ArrowSchema,\n  Struct,\n  TimeMicrosecond,\n  TimestampMicrosecond,\n  Utf8,\n  makeBuilder,\n  type DataType,\n  type TypeMap,\n} from \"apache-arrow\";\nimport { Effect, Schema } from \"effect\";\nimport {\n  parseDecimalType,\n  parseFixedType,\n  type IcebergType,\n  type PrimitiveType,\n  type TableSchema,\n} from \"iceberg-js\";\n\nimport { ConnectorError } from \"../errors\";\n\n// Nothing converts an Iceberg schema to Arrow for us, so we do it here.\n// Each type also gets a schema for its values, because Arrow does not reject a bad\n// value, it converts it: \"1\" becomes 1, and 1.5 becomes 1.\n\ntype RowSchema = Schema.Codec<unknown, unknown>;\n\n/** One Iceberg type: the Arrow type, and the values it accepts. */\ntype Compiled = {\n  readonly type: DataType;\n  readonly row: RowSchema;\n};\n\n// Arrow pads or cuts binary values to fit.\nconst bytes = (length: number) =>\n  Schema.Uint8Array.check(\n    Schema.makeFilter(\n      (value: Uint8Array) => value.byteLength === length || `expected ${length} bytes`,\n    ),\n  );\n\n// Arrow wraps big integers instead of failing, so check the range.\nconst int32 = Schema.Number.check(\n  Schema.isInt(),\n  Schema.isBetween({ minimum: -2_147_483_648, maximum: 2_147_483_647 }),\n);\n\nconst int64 = Schema.BigInt.check(\n  Schema.isBetweenBigInt({ minimum: -(2n ** 63n), maximum: 2n ** 63n - 1n }),\n);\n\n// Arrow turns a number outside the 32-bit range into Infinity.\nconst float32 = Schema.Finite.check(\n  Schema.makeFilter(\n    (value: number) => Number.isFinite(Math.fround(value)) || \"expected a 32-bit number\",\n  ),\n);\n\nconst microsecondsInDay = Schema.BigInt.check(\n  Schema.isBetweenBigInt({ minimum: 0n, maximum: 86_399_999_999n }),\n);\n\nconst primitives: Record<string, Compiled> = {\n  boolean: { type: new Bool(), row: Schema.Boolean },\n  int: { type: new Int32(), row: int32 },\n  long: { type: new Int64(), row: int64 },\n  float: { type: new Float32(), row: float32 },\n  double: { type: new Float64(), row: Schema.Finite },\n  string: { type: new Utf8(), row: Schema.String },\n  uuid: { type: new FixedSizeBinary(16), row: bytes(16) },\n  binary: { type: new LargeBinary(), row: Schema.Uint8Array },\n  date: { type: new DateDay(), row: Schema.Date },\n  time: { type: new TimeMicrosecond(), row: microsecondsInDay },\n  timestamp: { type: new TimestampMicrosecond(), row: Schema.Date },\n  timestamptz: { type: new TimestampMicrosecond(\"UTC\"), row: Schema.Date },\n};\n\nconst primitive = (type: PrimitiveType): Compiled => {\n  const known = primitives[type];\n\n  if (known) return known;\n\n  // Nothing checks the catalog metadata, and Arrow takes any precision.\n  // Better to fail here than at write time.\n  const decimal = parseDecimalType(type);\n\n  if (decimal) {\n    if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) {\n      throw new Error(`Invalid Iceberg decimal type ${type}`);\n    }\n\n    return {\n      type: new Decimal(decimal.scale, decimal.precision),\n      row: Schema.instanceOf(Uint32Array).check(\n        Schema.makeFilter((value: Uint32Array) => value.length === 4 || \"expected four words\"),\n      ),\n    };\n  }\n\n  const fixed = parseFixedType(type);\n\n  if (fixed) {\n    if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);\n\n    return { type: new FixedSizeBinary(fixed.length), row: bytes(fixed.length) };\n  }\n\n  throw new Error(`Unsupported Iceberg type ${type}`);\n};\n\n/** Absent and null both become null in Arrow. */\nconst optional = (row: RowSchema) => Schema.optional(Schema.NullOr(row));\n\nconst fieldMetadata = (id: number): Map<string, string> =>\n  new Map([[\"PARQUET:field_id\", String(id)]]);\n\n/** Same as `primitive`, plus struct, list and map. */\nconst compileType = (type: IcebergType): Compiled => {\n  if (typeof type === \"string\") return primitive(type);\n\n  if (type.type === \"struct\") {\n    const fields = type.fields.map((field) => ({ ...field, compiled: compileType(field.type) }));\n\n    return {\n      type: new Struct(\n        fields.map(\n          (field) =>\n            new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)),\n        ),\n      ),\n      row: Schema.Struct(\n        Object.fromEntries(\n          fields.map((field) => [\n            field.name,\n            field.required ? field.compiled.row : optional(field.compiled.row),\n          ]),\n        ),\n      ),\n    };\n  }\n\n  if (type.type === \"list\") {\n    const element = compileType(type.element);\n    const required = type[\"element-required\"];\n\n    return {\n      type: new List(\n        new Field(\"element\", element.type, !required, fieldMetadata(type[\"element-id\"])),\n      ),\n      row: Schema.Array(required ? element.row : Schema.NullishOr(element.row)),\n    };\n  }\n\n  const key = compileType(type.key);\n  const value = compileType(type.value);\n  const required = type[\"value-required\"];\n  const entries = new Struct([\n    new Field(\"key\", key.type, false, fieldMetadata(type[\"key-id\"])),\n    new Field(\"value\", value.type, !required, fieldMetadata(type[\"value-id\"])),\n  ]);\n\n  return {\n    type: new Map_(new Field(\"entries\", entries, false)),\n    row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row)),\n  };\n};\n\nexport type RowEncoder = (\n  rows: ReadonlyArray<object>,\n) => Effect.Effect<RecordBatch<TypeMap>, ConnectorError>;\n\nexport const makeRowEncoder = (\n  tableSchema: TableSchema,\n  version: string,\n): Effect.Effect<RowEncoder, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      const fields = tableSchema.fields.map((field) => ({\n        id: field.id,\n        name: field.name,\n        ...compileType(field.type),\n      }));\n      const fieldNames = new Set(fields.map((field) => field.name));\n      const identifierIds = new Set(tableSchema[\"identifier-field-ids\"]);\n      const keys = tableSchema.fields\n        .filter((field) => identifierIds.has(field.id))\n        .map((field) => field.name);\n      // Other columns may be missing: a delete only sends the key and version.\n      const required = new Set([...keys, version]);\n\n      if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);\n\n      const struct = new Struct(\n        fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))),\n      );\n      const schema = new ArrowSchema(struct.children);\n\n      const decodeRow = Schema.decodeUnknownSync(\n        Schema.Struct(\n          Object.fromEntries(\n            fields.map((field) => [\n              field.name,\n              required.has(field.name) ? field.row : optional(field.row),\n            ]),\n          ),\n        ),\n        { onExcessProperty: \"error\" },\n      );\n\n      // Schema says which field is wrong. This adds the row number.\n      const decodeAt = (row: object, index: number) => {\n        try {\n          return decodeRow(row) as Record<string, unknown>;\n        } catch (cause) {\n          const detail = cause instanceof Error ? cause.message : String(cause);\n          throw new Error(`row ${index}: ${detail}`, { cause });\n        }\n      };\n\n      return (rows: ReadonlyArray<object>) =>\n        Effect.try({\n          try: () => {\n            const builder = makeBuilder({ type: struct, nullValues: [null, undefined] });\n\n            for (const [index, row] of rows.entries()) {\n              builder.append(decodeAt(row, index) as Struct<TypeMap>[\"TValue\"]);\n            }\n\n            return new RecordBatch(schema, builder.finish().flush());\n          },\n          catch: (cause) => new ConnectorError({ message: \"Failed to encode rows\", cause }),\n        });\n    },\n    catch: (cause) => new ConnectorError({ message: \"Failed to convert Iceberg schema\", cause }),\n  });\n","import type { RecordBatch, TypeMap } from \"apache-arrow\";\nimport type { TableIdentifier } from \"iceberg-js\";\n\nimport { CatalogManager } from \"@useairfoil/api\";\nimport { Config, Effect, Layer } from \"effect\";\nimport { FetchHttpClient } from \"effect/http\";\n\nimport type { TableSource } from \"../core/types\";\n\nimport { ConnectorError } from \"../errors\";\nimport { compileTable } from \"../iceberg/table\";\nimport * as RuntimeConfig from \"../runtime-config\";\nimport { makeRowEncoder, type RowEncoder } from \"./arrow\";\nimport { Ingestor, type IngestorService } from \"./service\";\n\n/** One catalog and an Iceberg table for each resource that runs. */\nexport type ArrowFlightIngestorConfig<Source extends TableSource = TableSource> = {\n  readonly connector: Source;\n  readonly catalog: string;\n  readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;\n};\n\ntype IngestorEntry = {\n  readonly encode: RowEncoder;\n  readonly push: (batch: RecordBatch<TypeMap>) => Effect.Effect<void, ConnectorError>;\n};\n\ntype PreparedTable = {\n  readonly resource: string;\n  readonly identifier: TableIdentifier;\n  readonly encode: RowEncoder;\n};\n\nconst connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });\n\n/** Compiles local schemas and opens one Arrow Flight stream per bound resource. */\nexport const layerArrowFlight = <const Source extends TableSource>(\n  config: ArrowFlightIngestorConfig<Source>,\n): Layer.Layer<Ingestor, ConnectorError, CatalogManager.CatalogManager> =>\n  Layer.effect(Ingestor)(\n    Effect.gen(function* () {\n      const manager = yield* CatalogManager.CatalogManager;\n      const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;\n\n      const prepared: Array<PreparedTable> = [];\n      for (const resource of config.connector.resources) {\n        const binding = bindings[resource.name];\n        if (!binding) continue;\n\n        const { namespace, name } = binding;\n        const identifier = { namespace, name };\n        const { schema } = yield* compileTable(resource.rowSchema).pipe(\n          Effect.mapError((cause) =>\n            connectorError(`Invalid schema for ${resource.name}: ${cause.message}`, cause),\n          ),\n        );\n\n        const encode = yield* makeRowEncoder(schema, resource.version);\n        prepared.push({ resource: resource.name, identifier, encode });\n      }\n\n      const entries = new Map<string, IngestorEntry>();\n      for (const { resource, identifier, encode } of prepared) {\n        const ingestor = yield* manager\n          .ingestor({\n            catalog: config.catalog,\n            namespace: identifier.namespace,\n            table: identifier.name,\n          })\n          .pipe(\n            Effect.mapError((cause) =>\n              connectorError(`Failed to open ingestor for ${resource}`, cause),\n            ),\n          );\n\n        entries.set(resource, {\n          encode,\n          push: (batch) =>\n            ingestor\n              .push(batch)\n              .pipe(\n                Effect.mapError((cause) =>\n                  connectorError(`Failed to ingest rows for ${resource}`, cause),\n                ),\n              ),\n        });\n      }\n\n      const service: IngestorService = {\n        resourceNames: prepared.map(({ resource }) => resource),\n        ingest: ({ resource, batch }) => {\n          const entry = entries.get(resource);\n\n          if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));\n          if (batch.rows.length === 0) return Effect.succeed({ rows: 0, bytes: 0 });\n\n          return entry\n            .encode(batch.rows)\n            .pipe(\n              Effect.flatMap((encoded) =>\n                entry\n                  .push(encoded)\n                  .pipe(Effect.as({ rows: batch.rows.length, bytes: encoded.data.byteLength })),\n              ),\n            );\n        },\n      };\n\n      return Ingestor.of(service);\n    }),\n  );\n\n/** Configures Arrow Flight ingestion from the hosted URL, catalog and table bindings. */\nexport const layerArrowFlightConfig = (connector: TableSource) =>\n  Layer.unwrap(\n    Effect.gen(function* () {\n      const { catalog, uri, flightUri } = yield* Config.all({\n        catalog: RuntimeConfig.catalog,\n        uri: RuntimeConfig.engineUri,\n        flightUri: RuntimeConfig.flightUri,\n      }).pipe(\n        Effect.mapError((cause) => connectorError(\"Invalid Arrow Flight configuration\", cause)),\n      );\n\n      if (uri.protocol !== \"http:\" && uri.protocol !== \"https:\") {\n        return yield* Effect.fail(connectorError(\"Invalid Arrow Flight URL\"));\n      }\n      if (flightUri.protocol !== \"http:\" && flightUri.protocol !== \"https:\") {\n        return yield* Effect.fail(connectorError(\"Invalid Arrow Flight URL\"));\n      }\n\n      const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(\n        Effect.mapError((cause) => connectorError(cause.message, cause)),\n      );\n      const manager = CatalogManager.layer({ engineUri: uri.href, flightUri: flightUri.href }).pipe(\n        Layer.provide(FetchHttpClient.layer),\n      );\n\n      return layerArrowFlight({ connector, catalog, tables }).pipe(Layer.provide(manager));\n    }),\n  );\n"],"mappings":";;;;;;;;;;AA+CA,MAAM,SAAS,WACb,OAAO,WAAW,MAChB,OAAO,YACJ,UAAsB,MAAM,eAAe,UAAU,YAAY,OAAO,OAC3E,CACF;AAGF,MAAM,QAAQ,OAAO,OAAO,MAC1B,OAAO,MAAM,GACb,OAAO,UAAU;CAAE,SAAS;CAAgB,SAAS;AAAc,CAAC,CACtE;AAEA,MAAM,QAAQ,OAAO,OAAO,MAC1B,OAAO,gBAAgB;CAAE,SAAS,EAAE,MAAM;CAAM,SAAS,MAAM,MAAM;AAAG,CAAC,CAC3E;AAGA,MAAM,UAAU,OAAO,OAAO,MAC5B,OAAO,YACJ,UAAkB,OAAO,SAAS,KAAK,OAAO,KAAK,CAAC,KAAK,0BAC5D,CACF;AAEA,MAAM,oBAAoB,OAAO,OAAO,MACtC,OAAO,gBAAgB;CAAE,SAAS;CAAI,SAAS;AAAgB,CAAC,CAClE;AAEA,MAAM,aAAuC;CAC3C,SAAS;EAAE,MAAM,IAAI,KAAK;EAAG,KAAK,OAAO;CAAQ;CACjD,KAAK;EAAE,MAAM,IAAI,MAAM;EAAG,KAAK;CAAM;CACrC,MAAM;EAAE,MAAM,IAAI,MAAM;EAAG,KAAK;CAAM;CACtC,OAAO;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK;CAAQ;CAC3C,QAAQ;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK,OAAO;CAAO;CAClD,QAAQ;EAAE,MAAM,IAAI,KAAK;EAAG,KAAK,OAAO;CAAO;CAC/C,MAAM;EAAE,MAAM,IAAI,gBAAgB,EAAE;EAAG,KAAK,MAAM,EAAE;CAAE;CACtD,QAAQ;EAAE,MAAM,IAAI,YAAY;EAAG,KAAK,OAAO;CAAW;CAC1D,MAAM;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK,OAAO;CAAK;CAC9C,MAAM;EAAE,MAAM,IAAI,gBAAgB;EAAG,KAAK;CAAkB;CAC5D,WAAW;EAAE,MAAM,IAAI,qBAAqB;EAAG,KAAK,OAAO;CAAK;CAChE,aAAa;EAAE,MAAM,IAAI,qBAAqB,KAAK;EAAG,KAAK,OAAO;CAAK;AACzE;AAEA,MAAM,aAAa,SAAkC;CACnD,MAAM,QAAQ,WAAW;CAEzB,IAAI,OAAO,OAAO;CAIlB,MAAM,UAAU,iBAAiB,IAAI;CAErC,IAAI,SAAS;EACX,IAAI,QAAQ,YAAY,KAAK,QAAQ,YAAY,MAAM,QAAQ,QAAQ,QAAQ,WAC7E,MAAM,IAAI,MAAM,gCAAgC,MAAM;EAGxD,OAAO;GACL,MAAM,IAAI,QAAQ,QAAQ,OAAO,QAAQ,SAAS;GAClD,KAAK,OAAO,WAAW,WAAW,CAAC,CAAC,MAClC,OAAO,YAAY,UAAuB,MAAM,WAAW,KAAK,qBAAqB,CACvF;EACF;CACF;CAEA,MAAM,QAAQ,eAAe,IAAI;CAEjC,IAAI,OAAO;EACT,IAAI,MAAM,SAAS,GAAG,MAAM,IAAI,MAAM,8BAA8B,MAAM;EAE1E,OAAO;GAAE,MAAM,IAAI,gBAAgB,MAAM,MAAM;GAAG,KAAK,MAAM,MAAM,MAAM;EAAE;CAC7E;CAEA,MAAM,IAAI,MAAM,4BAA4B,MAAM;AACpD;;AAGA,MAAM,YAAY,QAAmB,OAAO,SAAS,OAAO,OAAO,GAAG,CAAC;AAEvE,MAAM,iBAAiB,uBACrB,IAAI,IAAI,CAAC,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC,CAAC;;AAG5C,MAAM,eAAe,SAAgC;CACnD,IAAI,OAAO,SAAS,UAAU,OAAO,UAAU,IAAI;CAEnD,IAAI,KAAK,SAAS,UAAU;EAC1B,MAAM,SAAS,KAAK,OAAO,KAAK,WAAW;GAAE,GAAG;GAAO,UAAU,YAAY,MAAM,IAAI;EAAE,EAAE;EAE3F,OAAO;GACL,MAAM,IAAIA,SACR,OAAO,KACJ,UACC,IAAI,MAAM,MAAM,MAAM,MAAM,SAAS,MAAM,CAAC,MAAM,UAAU,cAAc,MAAM,EAAE,CAAC,CACvF,CACF;GACA,KAAK,OAAO,OACV,OAAO,YACL,OAAO,KAAK,UAAU,CACpB,MAAM,MACN,MAAM,WAAW,MAAM,SAAS,MAAM,SAAS,MAAM,SAAS,GAAG,CACnE,CAAC,CACH,CACF;EACF;CACF;CAEA,IAAI,KAAK,SAAS,QAAQ;EACxB,MAAM,UAAU,YAAY,KAAK,OAAO;EACxC,MAAM,WAAW,KAAK;EAEtB,OAAO;GACL,MAAM,IAAI,KACR,IAAI,MAAM,WAAW,QAAQ,MAAM,CAAC,UAAU,cAAc,KAAK,aAAa,CAAC,CACjF;GACA,KAAK,OAAO,MAAM,WAAW,QAAQ,MAAM,OAAO,UAAU,QAAQ,GAAG,CAAC;EAC1E;CACF;CAEA,MAAM,MAAM,YAAY,KAAK,GAAG;CAChC,MAAM,QAAQ,YAAY,KAAK,KAAK;CACpC,MAAM,WAAW,KAAK;CACtB,MAAM,UAAU,IAAIA,SAAO,CACzB,IAAI,MAAM,OAAO,IAAI,MAAM,OAAO,cAAc,KAAK,SAAS,CAAC,GAC/D,IAAI,MAAM,SAAS,MAAM,MAAM,CAAC,UAAU,cAAc,KAAK,WAAW,CAAC,CAC3E,CAAC;CAED,OAAO;EACL,MAAM,IAAI,KAAK,IAAI,MAAM,WAAW,SAAS,KAAK,CAAC;EACnD,KAAK,OAAO,YAAY,IAAI,KAAK,WAAW,MAAM,MAAM,OAAO,UAAU,MAAM,GAAG,CAAC;CACrF;AACF;AAMA,MAAa,kBACX,aACA,YAEA,OAAO,IAAI;CACT,WAAW;EACT,MAAM,SAAS,YAAY,OAAO,KAAK,WAAW;GAChD,IAAI,MAAM;GACV,MAAM,MAAM;GACZ,GAAG,YAAY,MAAM,IAAI;EAC3B,EAAE;EACF,MAAM,aAAa,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,IAAI,CAAC;EAC5D,MAAM,gBAAgB,IAAI,IAAI,YAAY,uBAAuB;EACjE,MAAM,OAAO,YAAY,OACtB,QAAQ,UAAU,cAAc,IAAI,MAAM,EAAE,CAAC,CAAC,CAC9C,KAAK,UAAU,MAAM,IAAI;EAE5B,MAAM,2BAAW,IAAI,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC;EAE3C,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,MAAM,IAAI,MAAM,0BAA0B,SAAS;EAEjF,MAAM,SAAS,IAAIA,SACjB,OAAO,KAAK,UAAU,IAAI,MAAM,MAAM,MAAM,MAAM,MAAM,MAAM,cAAc,MAAM,EAAE,CAAC,CAAC,CACxF;EACA,MAAM,SAAS,IAAIC,SAAY,OAAO,QAAQ;EAE9C,MAAM,YAAY,OAAO,kBACvB,OAAO,OACL,OAAO,YACL,OAAO,KAAK,UAAU,CACpB,MAAM,MACN,SAAS,IAAI,MAAM,IAAI,IAAI,MAAM,MAAM,SAAS,MAAM,GAAG,CAC3D,CAAC,CACH,CACF,GACA,EAAE,kBAAkB,QAAQ,CAC9B;EAGA,MAAM,YAAY,KAAa,UAAkB;GAC/C,IAAI;IACF,OAAO,UAAU,GAAG;GACtB,SAAS,OAAO;IACd,MAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;IACpE,MAAM,IAAI,MAAM,OAAO,MAAM,IAAI,UAAU,EAAE,MAAM,CAAC;GACtD;EACF;EAEA,QAAQ,SACN,OAAO,IAAI;GACT,WAAW;IACT,MAAM,UAAU,YAAY;KAAE,MAAM;KAAQ,YAAY,CAAC,MAAM,KAAA,CAAS;IAAE,CAAC;IAE3E,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GACtC,QAAQ,OAAO,SAAS,KAAK,KAAK,CAA8B;IAGlE,OAAO,IAAI,YAAY,QAAQ,QAAQ,OAAO,CAAC,CAAC,MAAM,CAAC;GACzD;GACA,QAAQ,UAAU,IAAI,eAAe;IAAE,SAAS;IAAyB;GAAM,CAAC;EAClF,CAAC;CACL;CACA,QAAQ,UAAU,IAAI,eAAe;EAAE,SAAS;EAAoC;CAAM,CAAC;AAC7F,CAAC;;;ACtNH,MAAM,kBAAkB,SAAiB,UAAoB,IAAI,eAAe;CAAE;CAAS;AAAM,CAAC;;AAGlG,MAAa,oBACX,WAEA,MAAM,OAAO,QAAQ,CAAC,CACpB,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,eAAe;CACtC,MAAM,WAAgE,OAAO;CAE7E,MAAM,WAAiC,CAAC;CACxC,KAAK,MAAM,YAAY,OAAO,UAAU,WAAW;EACjD,MAAM,UAAU,SAAS,SAAS;EAClC,IAAI,CAAC,SAAS;EAEd,MAAM,EAAE,WAAW,SAAS;EAC5B,MAAM,aAAa;GAAE;GAAW;EAAK;EACrC,MAAM,EAAE,WAAW,OAAO,aAAa,SAAS,SAAS,CAAC,CAAC,KACzD,OAAO,UAAU,UACf,eAAe,sBAAsB,SAAS,KAAK,IAAI,MAAM,WAAW,KAAK,CAC/E,CACF;EAEA,MAAM,SAAS,OAAO,eAAe,QAAQ,SAAS,OAAO;EAC7D,SAAS,KAAK;GAAE,UAAU,SAAS;GAAM;GAAY;EAAO,CAAC;CAC/D;CAEA,MAAM,0BAAU,IAAI,IAA2B;CAC/C,KAAK,MAAM,EAAE,UAAU,YAAY,YAAY,UAAU;EACvD,MAAM,WAAW,OAAO,QACrB,SAAS;GACR,SAAS,OAAO;GAChB,WAAW,WAAW;GACtB,OAAO,WAAW;EACpB,CAAC,CAAC,CACD,KACC,OAAO,UAAU,UACf,eAAe,+BAA+B,YAAY,KAAK,CACjE,CACF;EAEF,QAAQ,IAAI,UAAU;GACpB;GACA,OAAO,UACL,SACG,KAAK,KAAK,CAAC,CACX,KACC,OAAO,UAAU,UACf,eAAe,6BAA6B,YAAY,KAAK,CAC/D,CACF;EACN,CAAC;CACH;CAEA,MAAM,UAA2B;EAC/B,eAAe,SAAS,KAAK,EAAE,eAAe,QAAQ;EACtD,SAAS,EAAE,UAAU,YAAY;GAC/B,MAAM,QAAQ,QAAQ,IAAI,QAAQ;GAElC,IAAI,CAAC,OAAO,OAAO,OAAO,KAAK,eAAe,oBAAoB,UAAU,CAAC;GAC7E,IAAI,MAAM,KAAK,WAAW,GAAG,OAAO,OAAO,QAAQ;IAAE,MAAM;IAAG,OAAO;GAAE,CAAC;GAExE,OAAO,MACJ,OAAO,MAAM,IAAI,CAAC,CAClB,KACC,OAAO,SAAS,YACd,MACG,KAAK,OAAO,CAAC,CACb,KAAK,OAAO,GAAG;IAAE,MAAM,MAAM,KAAK;IAAQ,OAAO,QAAQ,KAAK;GAAW,CAAC,CAAC,CAChF,CACF;EACJ;CACF;CAEA,OAAO,SAAS,GAAG,OAAO;AAC5B,CAAC,CACH;;AAGF,MAAa,0BAA0B,cACrC,MAAM,OACJ,OAAO,IAAI,aAAa;CACtB,MAAM,EAAE,SAAA,WAAS,KAAK,WAAA,gBAAc,OAAO,OAAO,IAAI;EAC3CC;EACT,KAAKC;EACMC;CACb,CAAC,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,eAAe,sCAAsC,KAAK,CAAC,CACxF;CAEA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,OAAO,OAAO,OAAO,KAAK,eAAe,0BAA0B,CAAC;CAEtE,IAAIC,YAAU,aAAa,WAAWA,YAAU,aAAa,UAC3D,OAAO,OAAO,OAAO,KAAK,eAAe,0BAA0B,CAAC;CAGtE,MAAM,SAAS,OAAOC,kBAAgC,SAAS,CAAC,CAAC,KAC/D,OAAO,UAAU,UAAU,eAAe,MAAM,SAAS,KAAK,CAAC,CACjE;CACA,MAAM,UAAU,eAAe,MAAM;EAAE,WAAW,IAAI;EAAM,WAAWD,YAAU;CAAK,CAAC,CAAC,CAAC,KACvF,MAAM,QAAQ,gBAAgB,KAAK,CACrC;CAEA,OAAO,iBAAiB;EAAE;EAAW,SAAA;EAAS;CAAO,CAAC,CAAC,CAAC,KAAK,MAAM,QAAQ,OAAO,CAAC;AACrF,CAAC,CACH"}

package/dist/auth/index.d.mts

@@ -0,0 +1,2 @@
+import { i as makeTokenCache, n as ExpiringToken, r as TokenCache } from "../index-BX8G7d41.mjs";
+export { ExpiringToken, TokenCache, makeTokenCache };

package/dist/{auth-DvU_CZ7C.js → auth/index.mjs}

@@ -1,6 +1,5 @@
-import { t as __export } from "./chunk-Bp6m_JJh.js";
-import { Cache, Duration, Effect, Exit, Number, Redacted } from "effect";
-
+import { t as __exportAll } from "../rolldown-runtime-D7D4PA-g.mjs";
+import { Cache, Duration, Effect, Exit, Number } from "effect";
 //#region src/auth/token-cache.ts
 /**
 * Creates a lazy, in-memory token cache around an acquisition effect.
@@ -32,11 +31,10 @@ const makeTokenCache = (options) => Effect.gen(function* () {
 		invalidate: Cache.invalidate(cache, void 0)
 	};
 });
-
 //#endregion
 //#region src/auth/index.ts
-var auth_exports = /* @__PURE__ */ __export({ makeTokenCache: () => makeTokenCache });
-
+var auth_exports = /* @__PURE__ */ __exportAll({ makeTokenCache: () => makeTokenCache });
 //#endregion
-export { makeTokenCache as n, auth_exports as t };
-//# sourceMappingURL=auth-DvU_CZ7C.js.map
+export { makeTokenCache, auth_exports as t };
+
+//# sourceMappingURL=index.mjs.map

package/dist/auth/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../src/auth/token-cache.ts","../../src/auth/index.ts"],"sourcesContent":["import { Cache, Duration, Effect, Exit, Number, Redacted } from \"effect\";\n\nexport interface ExpiringToken {\n  /** The credential kept in memory by the cache. */\n  readonly value: Redacted.Redacted<string>;\n  /** The lifetime reported by the provider. */\n  readonly expiresIn: Duration.Duration;\n}\n\nexport interface TokenCache<E> {\n  /** Returns the cached token or performs one shared acquisition. */\n  readonly get: Effect.Effect<Redacted.Redacted<string>, E>;\n  /** Removes the cached token so the next `get` reacquires it. */\n  readonly invalidate: Effect.Effect<void>;\n}\n\n/**\n * Creates a lazy, in-memory token cache around an acquisition effect.\n *\n * `refreshFactor` controls how much of the provider lifetime is cached. It is\n * clamped between `0` and `1`.\n *\n * @example\n * ```ts\n * const cache = yield* makeTokenCache({\n *   acquire: exchangeCredentials,\n *   refreshFactor: 0.5,\n * })\n *\n * const token = yield* cache.get\n * ```\n */\nexport const makeTokenCache = <E, R>(options: {\n  readonly acquire: Effect.Effect<ExpiringToken, E, R>;\n  readonly refreshFactor?: number;\n}): Effect.Effect<TokenCache<E>, never, R> =>\n  Effect.gen(function* () {\n    const refreshFactor = Number.clamp(options.refreshFactor ?? 0.5, {\n      minimum: 0,\n      maximum: 1,\n    });\n    const cache = yield* Cache.makeWith((_: void) => options.acquire, {\n      capacity: 1,\n      timeToLive: (exit) =>\n        Exit.isSuccess(exit)\n          ? Duration.millis(Duration.toMillis(exit.value.expiresIn) * refreshFactor)\n          : Duration.zero,\n    });\n\n    return {\n      get: Cache.get(cache, undefined).pipe(Effect.map((token) => token.value)),\n      invalidate: Cache.invalidate(cache, undefined),\n    };\n  });\n","export * from \"./token-cache\";\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAgCA,MAAa,kBAAwB,YAInC,OAAO,IAAI,aAAa;CACtB,MAAM,gBAAgB,OAAO,MAAM,QAAQ,iBAAiB,IAAK;EAC/D,SAAS;EACT,SAAS;CACX,CAAC;CACD,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAY,QAAQ,SAAS;EAChE,UAAU;EACV,aAAa,SACX,KAAK,UAAU,IAAI,IACf,SAAS,OAAO,SAAS,SAAS,KAAK,MAAM,SAAS,IAAI,aAAa,IACvE,SAAS;CACjB,CAAC;CAED,OAAO;EACL,KAAK,MAAM,IAAI,OAAO,KAAA,CAAS,CAAC,CAAC,KAAK,OAAO,KAAK,UAAU,MAAM,KAAK,CAAC;EACxE,YAAY,MAAM,WAAW,OAAO,KAAA,CAAS;CAC/C;AACF,CAAC"}

package/dist/connector-app/index.d.mts

@@ -0,0 +1,2 @@
+import { _ as check, a as start, c as makeCreateTableCommand, g as ResourceCheckResult, h as CheckResult, i as StartOptions, m as CheckOptions, n as makeSetupCommand, o as makeDemoCommand, r as App, s as parseSize } from "../index-CZKx2Yce.mjs";
+export { type App, type CheckOptions, type CheckResult, type ResourceCheckResult, type StartOptions, check, makeCreateTableCommand, makeDemoCommand, makeSetupCommand, parseSize, start };

package/dist/connector-app/index.mjs

@@ -0,0 +1,2 @@
+import { a as parseSize, i as makeDemoCommand, n as makeSetupCommand, o as makeCreateTableCommand, r as start, s as check } from "../connector-app-C6_dZnhE.mjs";
+export { check, makeCreateTableCommand, makeDemoCommand, makeSetupCommand, parseSize, start };

package/dist/connector-app-C6_dZnhE.mjs

@@ -0,0 +1,256 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { a as IcebergSchemaError, n as compileTable, r as makeCreateRequest } from "./table-CxM09xf7.mjs";
+import { a as catalog, n as loadTableBindings, o as engineUri } from "./runtime-config-Bn8Fivt1.mjs";
+import { n as run } from "./demo-DqTJ7TWI.mjs";
+import { n as layerArrowFlightConfig } from "./arrow-flight-DDQ0ktx9.mjs";
+import { n as run$1 } from "./ingestion-DoCUwm5N.mjs";
+import { DateTime, Duration, Effect, Option, Schedule, Schema, Tuple } from "effect";
+import { Argument, CliError, Command, Flag } from "effect/cli";
+import { CatalogManager } from "@useairfoil/api";
+import "effect/http";
+import { getCurrentSchema } from "iceberg-js";
+import humanFormat from "human-format";
+import { NodeHttpServer } from "@effect/platform-node";
+import { createServer } from "node:http";
+//#region src/connector-app/check.ts
+const ok = { _tag: "ok" };
+const error = (message) => ({
+	_tag: "error",
+	message
+});
+const fromEntries = (entries) => Object.fromEntries(entries);
+const resultsFor = (names, result) => fromEntries(names.map((name) => Tuple.make(name, result)));
+/**
+* Builds the connector service from its layer and validates only the selected
+* resources. The layer remains open until every check finishes.
+*/
+function check(service, layer, options) {
+	if (options.resources.length === 0) return Effect.succeed(fromEntries([]));
+	return service.pipe(Effect.flatMap((connector) => Effect.forEach(options.resources, (name) => {
+		const resource = connector.resources.find((candidate) => candidate.name === name);
+		return (resource ? resource.check.pipe(Effect.match({
+			onFailure: (failure) => error(failure.message),
+			onSuccess: () => ok
+		})) : Effect.succeed(error(`Unknown connector resource: ${name}`))).pipe(Effect.map((result) => Tuple.make(name, result)));
+	}).pipe(Effect.map(fromEntries))), Effect.provide(layer), Effect.match({
+		onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
+		onSuccess: (results) => results
+	}));
+}
+//#endregion
+//#region src/catalog/retry.ts
+/** Retries transient catalog responses at the request boundary. */
+const retryCatalogRequest = (request, options) => options === void 0 ? request : request.pipe(Effect.retry({
+	schedule: Schedule.spaced(options.interval),
+	times: options.times,
+	while: (error) => error.status === 408 || error.status === 429 || error.status !== void 0 && error.status >= 500 && error.status <= 599
+}));
+//#endregion
+//#region src/catalog/table.ts
+const tableError = (cause) => {
+	if (cause instanceof ConnectorError) return cause;
+	const message = cause instanceof Error ? cause.message : "Failed to prepare Iceberg table";
+	return new ConnectorError({
+		message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,
+		cause
+	});
+};
+const compareTypes = (expected, actual, path) => {
+	if (typeof expected === "string" || typeof actual === "string") return expected === actual ? void 0 : path;
+	if (expected.type !== actual.type) return path;
+	switch (expected.type) {
+		case "struct":
+			if (actual.type !== "struct") return path;
+			return compareFields(expected.fields, actual.fields, path);
+		case "list":
+			if (actual.type !== "list" || expected["element-required"] !== actual["element-required"]) return path;
+			return compareTypes(expected.element, actual.element, `${path}.element`);
+		case "map":
+			if (actual.type !== "map" || expected["value-required"] !== actual["value-required"]) return path;
+			return compareTypes(expected.key, actual.key, `${path}.key`) ?? compareTypes(expected.value, actual.value, `${path}.value`);
+	}
+};
+const compareFields = (expected, actual, path) => {
+	if (expected.length !== actual.length) return path;
+	for (const field of expected) {
+		const fieldPath = `${path}.${field.name}`;
+		const other = actual.find((candidate) => candidate.name === field.name);
+		if (!other || field.required !== other.required) return fieldPath;
+		const mismatch = compareTypes(field.type, other.type, fieldPath);
+		if (mismatch) return mismatch;
+	}
+};
+const verifyTable = (metadata, expected) => Effect.try({
+	try: () => {
+		if (!Array.isArray(metadata.schemas)) throw new Error("Invalid table metadata");
+		if (metadata["format-version"] !== 3) throw new Error("Storage-engine tables must use Iceberg format v3. Migrate the table before ingesting.");
+		const partitionSpec = metadata["partition-specs"].find((spec) => spec["spec-id"] === metadata["default-spec-id"]);
+		if (!partitionSpec) throw new Error("Table has no default partition spec");
+		if (partitionSpec.fields.length > 0) throw new Error("Storage-engine tables must be unpartitioned.");
+		if (metadata.properties?.["encryption.key-id"] !== void 0) throw new Error("Storage-engine tables must be unencrypted.");
+		const actual = getCurrentSchema(metadata);
+		if (!actual) throw new Error("Table has no current schema");
+		const path = compareFields(expected.fields, actual.fields, "$");
+		if (path) throw new Error(`Table schema does not match at ${path}. Migrate the table or revert the schema change.`);
+		const identifierNames = (schema) => new Set(schema.fields.filter((field) => schema["identifier-field-ids"]?.includes(field.id)).map((field) => field.name));
+		const expectedNames = identifierNames(expected);
+		const actualNames = identifierNames(actual);
+		if (expectedNames.size !== actualNames.size || [...expectedNames].some((name) => !actualNames.has(name))) throw new Error("Table identifier fields do not match. Recreate the table or revert the schema change.");
+		return actual;
+	},
+	catch: (cause) => new ConnectorError({
+		message: cause instanceof Error ? cause.message : "Invalid table metadata",
+		cause
+	})
+});
+/** Creates a missing table through an engine catalog and verifies its schema. */
+const ensureTableFromEngine = (schema, options) => Effect.scoped(Effect.gen(function* () {
+	const catalog = yield* (yield* CatalogManager.make({ engineUri: options.engineUri })).getIcebergCatalog(options.catalogId);
+	return yield* ensureTable(schema, {
+		catalog,
+		identifier: options.identifier,
+		location: options.location,
+		properties: options.properties
+	});
+})).pipe(Effect.mapError(tableError));
+/** Loads a table, creating it when it is missing. */
+const ensureTable = (schema, options) => Effect.gen(function* () {
+	const table = yield* compileTable(schema);
+	const request = (effect) => retryCatalogRequest(effect, options.retry);
+	const metadata = yield* request(options.catalog.loadTable(options.identifier)).pipe(Effect.catchIf((error) => error.status === 404, () => request(options.catalog.createNamespaceIfNotExists({ namespace: options.identifier.namespace })).pipe(Effect.andThen(request(options.catalog.createTable({ namespace: options.identifier.namespace }, makeCreateRequest(table, {
+		name: options.identifier.name,
+		location: options.location,
+		properties: options.properties
+	}))))).pipe(Effect.catchIf((error) => error.status === 409 || error.status === 412, () => Effect.void), Effect.flatMap(() => request(options.catalog.loadTable(options.identifier))))));
+	return yield* verifyTable(metadata, table.schema);
+}).pipe(Effect.mapError(tableError));
+//#endregion
+//#region src/connector-app/create-table.ts
+/** Creates the table command; existing tables must match the connector's row schemas. */
+const makeCreateTableCommand = (schemas) => Command.make("create-table", {
+	table: Argument.String("table"),
+	catalog: Flag.String("catalog"),
+	location: Flag.String("location").pipe(Flag.optional),
+	resource: Flag.String("resource").pipe(Flag.optional)
+}, ({ table, catalog, location, resource }) => Effect.gen(function* () {
+	const namespace = table.split(".");
+	const name = namespace.pop();
+	if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) return yield* new CliError.UserError({
+		cause: "Invalid table name",
+		userMessage: "Use a table name like my.namespace.table"
+	});
+	const resourceName = Option.getOrElse(resource, () => name);
+	const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : void 0;
+	if (!schema) return yield* new CliError.UserError({
+		cause: "Unknown resource",
+		userMessage: `Unknown resource: ${resourceName}`
+	});
+	const uri = yield* engineUri;
+	yield* ensureTableFromEngine(schema, {
+		engineUri: uri.href,
+		catalogId: catalog,
+		identifier: {
+			namespace,
+			name
+		},
+		location: Option.getOrUndefined(location)
+	});
+	yield* Effect.log(`Prepared ${table}`);
+})).pipe(Command.withDescription("Create or verify an Iceberg table for a connector resource"));
+//#endregion
+//#region src/connector-app/demo.ts
+/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */
+const parseSize = (input) => Option.liftThrowable(() => humanFormat.parse.bytes(input))().pipe(Option.filter((bytes) => Number.isSafeInteger(bytes) && bytes > 0));
+/**
+* Creates the command that writes demo data over Arrow Flight, then exits.
+* It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.
+*/
+const makeDemoCommand = (demo) => Command.make("demo", {
+	size: Flag.String("size").pipe(Flag.filterMap(parseSize, () => "a size like 512kB, 128MB or 1.5GB")),
+	seed: Flag.Int("seed").pipe(Flag.optional),
+	batchSize: Flag.Int("batch-size").pipe(Flag.withSchema(Schema.Int.check(Schema.isGreaterThan(0))), Flag.withDescription("Maximum rows per ingestion batch (default: 500)"), Flag.optional)
+}, ({ size, seed, batchSize }) => Effect.gen(function* () {
+	const now = yield* DateTime.nowAsDate;
+	yield* run(demo, {
+		bytes: size,
+		seed: Option.getOrUndefined(seed),
+		batchSize: Option.getOrUndefined(batchSize),
+		now
+	}).pipe(Effect.provide(layerArrowFlightConfig({ resources: [...demo.setup.tables, ...demo.group.tables] })));
+})).pipe(Command.withDescription("Write demo data for the bound tables, then exit"));
+//#endregion
+//#region src/connector-app/start.ts
+const start = (app, options) => Effect.gen(function* () {
+	const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+	const routes = app.webhooks?.map((route) => route.path) ?? [];
+	yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
+		port: options.port,
+		routes
+	}));
+	return yield* run$1(app, {
+		initialCutoff,
+		webhook: {
+			routes: app.webhooks ?? [],
+			healthPath: options.healthPath ?? "/health",
+			metricsPath: options.metricsPath,
+			statusPath: options.statusPath,
+			disableHttpLogger: options.disableHttpLogger ?? true
+		}
+	}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
+});
+//#endregion
+//#region src/connector-app/setup.ts
+/** Creates or verifies all bound tables without loading provider credentials. */
+const makeSetupCommand = (schemas) => Command.make("setup", {
+	checkpointIntervalSec: Flag.Int("checkpoint-interval-sec").pipe(Flag.withSchema(Schema.Int.check(Schema.isBetween({
+		minimum: 0,
+		maximum: 900
+	}))), Flag.withDescription("Checkpoint interval for new tables in seconds (0 disables the interval delay)"), Flag.optional),
+	retryInterval: Flag.String("retry-interval").pipe(Flag.withDefault("5 seconds"), Flag.withSchema(Schema.DurationFromString), Flag.filter((duration) => Duration.isPositive(duration) && Duration.isFinite(duration), () => "a positive duration, such as 5 seconds"), Flag.withDescription("Delay between transient catalog retries")),
+	retries: Flag.Int("retries").pipe(Flag.withDefault(10), Flag.filter((value) => value >= 0, () => "a non-negative number of retries"), Flag.withDescription("Maximum retries per catalog request after the initial attempt"))
+}, ({ retryInterval, retries, checkpointIntervalSec }) => Effect.gen(function* () {
+	const source = { resources: Object.entries(schemas).map(([name, rowSchema]) => ({
+		name,
+		rowSchema
+	})) };
+	const tables = yield* loadTableBindings(source);
+	const uri = yield* engineUri;
+	const catalogId = yield* catalog;
+	const manager = yield* CatalogManager.make({ engineUri: uri.href });
+	const retry = {
+		interval: retryInterval,
+		times: retries
+	};
+	const properties = Option.match(checkpointIntervalSec, {
+		onNone: () => void 0,
+		onSome: (seconds) => ({ "airfoil.checkpoint.interval-sec": String(seconds) })
+	});
+	const catalog$1 = yield* retryCatalogRequest(manager.getIcebergCatalog(catalogId), retry);
+	for (const resource of source.resources) {
+		const binding = tables[resource.name];
+		if (!binding) continue;
+		yield* ensureTable(resource.rowSchema, {
+			catalog: catalog$1,
+			identifier: binding,
+			location: binding.location,
+			retry,
+			properties
+		});
+		yield* Effect.logInfo(`Prepared ${[...binding.namespace, binding.name].join(".")}`);
+	}
+})).pipe(Command.withDescription("Create or verify the Iceberg tables in AIRFOIL_TABLE_BINDINGS"));
+//#endregion
+//#region src/connector-app/index.ts
+var connector_app_exports = /* @__PURE__ */ __exportAll({
+	check: () => check,
+	makeCreateTableCommand: () => makeCreateTableCommand,
+	makeDemoCommand: () => makeDemoCommand,
+	makeSetupCommand: () => makeSetupCommand,
+	parseSize: () => parseSize,
+	start: () => start
+});
+//#endregion
+export { parseSize as a, makeDemoCommand as i, makeSetupCommand as n, makeCreateTableCommand as o, start as r, check as s, connector_app_exports as t };
+
+//# sourceMappingURL=connector-app-C6_dZnhE.mjs.map