@useairfoil/connector-kit 0.6.0 → 0.6.1

package/dist/webhook/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../src/webhook/index.ts"],"sourcesContent":["import type { ResourceDefinition, WebhookRoute, WebhookRouteInput } from \"../core/types\";\n\nexport { router } from \"./server\";\nexport type { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput } from \"./types\";\n\nexport const route = <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(\n  definition: WebhookRouteInput<Resources, Payload>,\n): WebhookRoute<Resources, Payload> => definition;\n"],"mappings":";;;;;;;AAKA,MAAa,SACX,eACqC"}

package/dist/wings-I56hGtNv.mjs

@@ -0,0 +1,325 @@
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { a as IcebergSchemaError, n as compileTable, r as makeCommitRequest } from "./table-V0tgUC5i.mjs";
+import { a as catalog, n as loadTableBindings, s as wingsUri } from "./runtime-config-Oa55gAew.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { Config, Effect, Layer, Schema } from "effect";
+import * as Wings from "@useairfoil/wings";
+import { FetchHttpClient } from "effect/http";
+import { getCurrentSchema, parseDecimalType, parseFixedType } from "iceberg-js";
+import { Bool, DateDay, Decimal, Field, FixedSizeBinary, Float32, Float64, Int32, Int64, LargeBinary, List, Map_, RecordBatch, Schema as Schema$1, Struct as Struct$1, TimeMicrosecond, TimestampMicrosecond, Utf8, makeBuilder } from "apache-arrow";
+//#region src/catalog/table.ts
+const tableError = (cause) => {
+	if (cause instanceof ConnectorError) return cause;
+	const message = cause instanceof Error ? cause.message : "Failed to prepare Iceberg table";
+	return new ConnectorError({
+		message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,
+		cause
+	});
+};
+const compareTypes = (expected, actual, path) => {
+	if (typeof expected === "string" || typeof actual === "string") return expected === actual ? void 0 : path;
+	if (expected.type !== actual.type) return path;
+	switch (expected.type) {
+		case "struct":
+			if (actual.type !== "struct") return path;
+			return compareFields(expected.fields, actual.fields, path);
+		case "list":
+			if (actual.type !== "list" || expected["element-id"] !== actual["element-id"] || expected["element-required"] !== actual["element-required"]) return path;
+			return compareTypes(expected.element, actual.element, `${path}.element`);
+		case "map":
+			if (actual.type !== "map" || expected["key-id"] !== actual["key-id"] || expected["value-id"] !== actual["value-id"] || expected["value-required"] !== actual["value-required"]) return path;
+			return compareTypes(expected.key, actual.key, `${path}.key`) ?? compareTypes(expected.value, actual.value, `${path}.value`);
+	}
+};
+const compareFields = (expected, actual, path) => {
+	if (expected.length !== actual.length) return path;
+	for (const field of expected) {
+		const fieldPath = `${path}.${field.name}`;
+		const other = actual.find((candidate) => candidate.name === field.name);
+		if (!other || field.id !== other.id || field.required !== other.required) return fieldPath;
+		const mismatch = compareTypes(field.type, other.type, fieldPath);
+		if (mismatch) return mismatch;
+	}
+};
+const verifyTable = (metadata, expected) => Effect.try({
+	try: () => {
+		if (!Array.isArray(metadata.schemas)) throw new Error("Invalid table metadata");
+		const actual = getCurrentSchema(metadata);
+		if (!actual) throw new Error("Table has no current schema");
+		const path = compareFields(expected.fields, actual.fields, "$");
+		if (path) throw new Error(`Table schema does not match at ${path}. Migrate the table or revert the schema change.`);
+		const expectedIds = new Set(expected["identifier-field-ids"]);
+		const actualIds = new Set(actual["identifier-field-ids"]);
+		if (expectedIds.size !== actualIds.size || [...expectedIds].some((id) => !actualIds.has(id))) throw new Error("Table identifier fields do not match. Recreate the table or revert the schema change.");
+		return actual;
+	},
+	catch: (cause) => new ConnectorError({
+		message: cause instanceof Error ? cause.message : "Invalid table metadata",
+		cause
+	})
+});
+/** Creates a table with the IDs from its Effect schema, then checks the loaded schema. */
+const createTable = (schema, options) => Effect.gen(function* () {
+	const table = yield* compileTable(schema);
+	yield* options.catalog.commitTable(options.identifier, makeCommitRequest(table, { location: options.location }));
+	const metadata = yield* options.catalog.loadTable(options.identifier);
+	return yield* verifyTable(metadata, table.schema);
+}).pipe(Effect.mapError(tableError));
+/** Creates a table through a Wings catalog. */
+const createTableFromWings = (schema, options) => Effect.scoped(Effect.gen(function* () {
+	const catalog = yield* (yield* Wings.CatalogManager.make({ baseUrl: options.wingsUri })).getIcebergCatalog(options.catalogId);
+	return yield* createTable(schema, {
+		catalog,
+		identifier: options.identifier,
+		location: options.location
+	});
+})).pipe(Effect.mapError(tableError));
+/** Loads a table, creating it when it is missing. */
+const ensureTable = (schema, options) => Effect.gen(function* () {
+	const table = yield* compileTable(schema);
+	const metadata = yield* options.catalog.loadTable(options.identifier).pipe(Effect.catchIf((error) => error.status === 404, () => options.catalog.commitTable(options.identifier, makeCommitRequest(table, { location: options.location })).pipe(Effect.catchIf((error) => error.status === 409 || error.status === 412, () => Effect.void), Effect.flatMap(() => options.catalog.loadTable(options.identifier)))));
+	return yield* verifyTable(metadata, table.schema);
+}).pipe(Effect.mapError(tableError));
+//#endregion
+//#region src/ingestor/arrow.ts
+const bytes = (length) => Schema.Uint8Array.check(Schema.makeFilter((value) => value.byteLength === length || `expected ${length} bytes`));
+const int32 = Schema.Number.check(Schema.isInt(), Schema.isBetween({
+	minimum: -2147483648,
+	maximum: 2147483647
+}));
+const int64 = Schema.BigInt.check(Schema.isBetweenBigInt({
+	minimum: -(2n ** 63n),
+	maximum: 2n ** 63n - 1n
+}));
+const float32 = Schema.Finite.check(Schema.makeFilter((value) => Number.isFinite(Math.fround(value)) || "expected a 32-bit number"));
+const microsecondsInDay = Schema.BigInt.check(Schema.isBetweenBigInt({
+	minimum: 0n,
+	maximum: 86399999999n
+}));
+const primitives = {
+	boolean: {
+		type: new Bool(),
+		row: Schema.Boolean
+	},
+	int: {
+		type: new Int32(),
+		row: int32
+	},
+	long: {
+		type: new Int64(),
+		row: int64
+	},
+	float: {
+		type: new Float32(),
+		row: float32
+	},
+	double: {
+		type: new Float64(),
+		row: Schema.Finite
+	},
+	string: {
+		type: new Utf8(),
+		row: Schema.String
+	},
+	uuid: {
+		type: new FixedSizeBinary(16),
+		row: bytes(16)
+	},
+	binary: {
+		type: new LargeBinary(),
+		row: Schema.Uint8Array
+	},
+	date: {
+		type: new DateDay(),
+		row: Schema.Date
+	},
+	time: {
+		type: new TimeMicrosecond(),
+		row: microsecondsInDay
+	},
+	timestamp: {
+		type: new TimestampMicrosecond(),
+		row: Schema.Date
+	},
+	timestamptz: {
+		type: new TimestampMicrosecond("UTC"),
+		row: Schema.Date
+	}
+};
+const primitive = (type) => {
+	const known = primitives[type];
+	if (known) return known;
+	const decimal = parseDecimalType(type);
+	if (decimal) {
+		if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) throw new Error(`Invalid Iceberg decimal type ${type}`);
+		return {
+			type: new Decimal(decimal.scale, decimal.precision),
+			row: Schema.instanceOf(Uint32Array).check(Schema.makeFilter((value) => value.length === 4 || "expected four words"))
+		};
+	}
+	const fixed = parseFixedType(type);
+	if (fixed) {
+		if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);
+		return {
+			type: new FixedSizeBinary(fixed.length),
+			row: bytes(fixed.length)
+		};
+	}
+	throw new Error(`Unsupported Iceberg type ${type}`);
+};
+/** Absent and null both become null in Arrow. */
+const optional = (row) => Schema.optional(Schema.NullOr(row));
+const fieldMetadata = (id) => /* @__PURE__ */ new Map([["PARQUET:field_id", String(id)]]);
+/** Same as `primitive`, plus struct, list and map. */
+const compileType = (type) => {
+	if (typeof type === "string") return primitive(type);
+	if (type.type === "struct") {
+		const fields = type.fields.map((field) => ({
+			...field,
+			compiled: compileType(field.type)
+		}));
+		return {
+			type: new Struct$1(fields.map((field) => new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)))),
+			row: Schema.Struct(Object.fromEntries(fields.map((field) => [field.name, field.required ? field.compiled.row : optional(field.compiled.row)])))
+		};
+	}
+	if (type.type === "list") {
+		const element = compileType(type.element);
+		const required = type["element-required"];
+		return {
+			type: new List(new Field("element", element.type, !required, fieldMetadata(type["element-id"]))),
+			row: Schema.Array(required ? element.row : Schema.NullishOr(element.row))
+		};
+	}
+	const key = compileType(type.key);
+	const value = compileType(type.value);
+	const required = type["value-required"];
+	const entries = new Struct$1([new Field("key", key.type, false, fieldMetadata(type["key-id"])), new Field("value", value.type, !required, fieldMetadata(type["value-id"]))]);
+	return {
+		type: new Map_(new Field("entries", entries, false)),
+		row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row))
+	};
+};
+const makeRowEncoder = (tableSchema, version) => Effect.try({
+	try: () => {
+		const fields = tableSchema.fields.map((field) => ({
+			id: field.id,
+			name: field.name,
+			...compileType(field.type)
+		}));
+		const fieldNames = new Set(fields.map((field) => field.name));
+		const identifierIds = new Set(tableSchema["identifier-field-ids"]);
+		const keys = tableSchema.fields.filter((field) => identifierIds.has(field.id)).map((field) => field.name);
+		const required = /* @__PURE__ */ new Set([...keys, version]);
+		if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);
+		const struct = new Struct$1(fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))));
+		const schema = new Schema$1(struct.children);
+		const decodeRow = Schema.decodeUnknownSync(Schema.Struct(Object.fromEntries(fields.map((field) => [field.name, required.has(field.name) ? field.row : optional(field.row)]))), { onExcessProperty: "error" });
+		const decodeAt = (row, index) => {
+			try {
+				return decodeRow(row);
+			} catch (cause) {
+				const detail = cause instanceof Error ? cause.message : String(cause);
+				throw new Error(`row ${index}: ${detail}`, { cause });
+			}
+		};
+		return (rows) => Effect.try({
+			try: () => {
+				const builder = makeBuilder({
+					type: struct,
+					nullValues: [null, void 0]
+				});
+				for (const [index, row] of rows.entries()) builder.append(decodeAt(row, index));
+				return new RecordBatch(schema, builder.finish().flush());
+			},
+			catch: (cause) => new ConnectorError({
+				message: "Failed to encode rows",
+				cause
+			})
+		});
+	},
+	catch: (cause) => new ConnectorError({
+		message: "Failed to convert Iceberg schema",
+		cause
+	})
+});
+//#endregion
+//#region src/ingestor/wings.ts
+const connectorError = (message, cause) => new ConnectorError({
+	message,
+	cause
+});
+/** Loads each table schema and opens one Wings stream per bound resource. */
+const layerWings = (config) => Layer.effect(Ingestor)(Effect.gen(function* () {
+	const manager = yield* Wings.CatalogManager.CatalogManager;
+	const catalog = yield* manager.getIcebergCatalog(config.catalog).pipe(Effect.mapError((cause) => connectorError("Failed to load Wings catalog", cause)));
+	const bindings = config.tables;
+	const prepared = [];
+	for (const resource of config.connector.resources) {
+		const binding = bindings[resource.name];
+		if (!binding) continue;
+		const { namespace, name, location } = binding;
+		const identifier = {
+			namespace,
+			name
+		};
+		const schema = yield* ensureTable(resource.rowSchema, {
+			catalog,
+			identifier,
+			location
+		}).pipe(Effect.mapError((cause) => connectorError(`Failed to prepare table for ${resource.name}: ${cause.message}`, cause)));
+		const encode = yield* makeRowEncoder(schema, resource.version);
+		prepared.push({
+			resource: resource.name,
+			identifier,
+			encode
+		});
+	}
+	const entries = /* @__PURE__ */ new Map();
+	for (const { resource, identifier, encode } of prepared) {
+		const ingestor = yield* manager.ingestor({
+			catalog: config.catalog,
+			namespace: identifier.namespace,
+			table: identifier.name
+		}).pipe(Effect.mapError((cause) => connectorError(`Failed to open ingestor for ${resource}`, cause)));
+		entries.set(resource, {
+			encode,
+			push: (batch) => ingestor.push(batch).pipe(Effect.mapError((cause) => connectorError(`Failed to ingest rows for ${resource}`, cause)))
+		});
+	}
+	const service = {
+		resourceNames: prepared.map(({ resource }) => resource),
+		ingest: ({ resource, batch }) => {
+			const entry = entries.get(resource);
+			if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));
+			if (batch.rows.length === 0) return Effect.succeed({
+				rows: 0,
+				bytes: 0
+			});
+			return entry.encode(batch.rows).pipe(Effect.flatMap((encoded) => entry.push(encoded).pipe(Effect.as({
+				rows: batch.rows.length,
+				bytes: encoded.data.byteLength
+			}))));
+		}
+	};
+	return Ingestor.of(service);
+}));
+/** Reads the Wings URL, catalog and table bindings from hosted config. */
+const layerWingsConfig = (connector) => Layer.unwrap(Effect.gen(function* () {
+	const { catalog: catalog$1, uri } = yield* Config.all({
+		catalog,
+		uri: wingsUri
+	}).pipe(Effect.mapError((cause) => connectorError("Invalid Wings configuration", cause)));
+	if (uri.protocol !== "http:" && uri.protocol !== "https:") return yield* Effect.fail(connectorError("Invalid Wings URL"));
+	const tables = yield* loadTableBindings(connector).pipe(Effect.mapError((cause) => connectorError(cause.message, cause)));
+	const manager = Wings.CatalogManager.layer({ baseUrl: uri.href }).pipe(Layer.provide(FetchHttpClient.layer));
+	return layerWings({
+		connector,
+		catalog: catalog$1,
+		tables
+	}).pipe(Layer.provide(manager));
+}));
+//#endregion
+export { layerWingsConfig as n, createTableFromWings as r, layerWings as t };
+
+//# sourceMappingURL=wings-I56hGtNv.mjs.map

package/dist/wings-I56hGtNv.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"wings-I56hGtNv.mjs","names":["Struct","ArrowSchema","RuntimeConfig.catalog","RuntimeConfig.wingsUri","RuntimeConfig.loadTableBindings"],"sources":["../src/catalog/table.ts","../src/ingestor/arrow.ts","../src/ingestor/wings.ts"],"sourcesContent":["import type {\n  IcebergType,\n  StructField,\n  TableIdentifier,\n  TableMetadata,\n  TableSchema,\n} from \"iceberg-js\";\n\nimport * as Wings from \"@useairfoil/wings\";\nimport { Effect, type Schema } from \"effect\";\nimport { HttpClient } from \"effect/http\";\nimport { getCurrentSchema } from \"iceberg-js\";\n\nimport { ConnectorError } from \"../errors\";\nimport { IcebergSchemaError } from \"../iceberg/error\";\nimport { compileTable, makeCommitRequest } from \"../iceberg/table\";\n\ntype Catalog = Effect.Success<\n  ReturnType<Wings.CatalogManager.CatalogManagerService[\"getIcebergCatalog\"]>\n>;\n\ntype TableOptions = {\n  readonly catalog: Catalog;\n  readonly identifier: TableIdentifier;\n  readonly location?: string | undefined;\n};\n\ntype WingsTableOptions = {\n  readonly wingsUri: string;\n  readonly catalogId: string;\n  readonly identifier: TableIdentifier;\n  readonly location?: string | undefined;\n};\n\nconst tableError = (cause: unknown): ConnectorError => {\n  if (cause instanceof ConnectorError) return cause;\n\n  const message = cause instanceof Error ? cause.message : \"Failed to prepare Iceberg table\";\n  return new ConnectorError({\n    message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,\n    cause,\n  });\n};\n\nconst compareTypes = (\n  expected: IcebergType,\n  actual: IcebergType,\n  path: string,\n): string | undefined => {\n  if (typeof expected === \"string\" || typeof actual === \"string\") {\n    return expected === actual ? undefined : path;\n  }\n\n  if (expected.type !== actual.type) return path;\n\n  switch (expected.type) {\n    case \"struct\":\n      if (actual.type !== \"struct\") return path;\n      return compareFields(expected.fields, actual.fields, path);\n    case \"list\":\n      if (\n        actual.type !== \"list\" ||\n        expected[\"element-id\"] !== actual[\"element-id\"] ||\n        expected[\"element-required\"] !== actual[\"element-required\"]\n      ) {\n        return path;\n      }\n      return compareTypes(expected.element, actual.element, `${path}.element`);\n    case \"map\":\n      if (\n        actual.type !== \"map\" ||\n        expected[\"key-id\"] !== actual[\"key-id\"] ||\n        expected[\"value-id\"] !== actual[\"value-id\"] ||\n        expected[\"value-required\"] !== actual[\"value-required\"]\n      ) {\n        return path;\n      }\n      return (\n        compareTypes(expected.key, actual.key, `${path}.key`) ??\n        compareTypes(expected.value, actual.value, `${path}.value`)\n      );\n  }\n};\n\nconst compareFields = (\n  expected: ReadonlyArray<StructField>,\n  actual: ReadonlyArray<StructField>,\n  path: string,\n): string | undefined => {\n  if (expected.length !== actual.length) return path;\n\n  for (const field of expected) {\n    const fieldPath = `${path}.${field.name}`;\n    const other = actual.find((candidate) => candidate.name === field.name);\n\n    if (!other || field.id !== other.id || field.required !== other.required) return fieldPath;\n\n    const mismatch = compareTypes(field.type, other.type, fieldPath);\n    if (mismatch) return mismatch;\n  }\n};\n\nconst verifyTable = (\n  metadata: TableMetadata,\n  expected: TableSchema,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      if (!Array.isArray(metadata.schemas)) throw new Error(\"Invalid table metadata\");\n      const actual = getCurrentSchema(metadata);\n      if (!actual) throw new Error(\"Table has no current schema\");\n\n      const path = compareFields(expected.fields, actual.fields, \"$\");\n      if (path) {\n        throw new Error(\n          `Table schema does not match at ${path}. Migrate the table or revert the schema change.`,\n        );\n      }\n\n      const expectedIds = new Set(expected[\"identifier-field-ids\"]);\n      const actualIds = new Set(actual[\"identifier-field-ids\"]);\n      if (\n        expectedIds.size !== actualIds.size ||\n        [...expectedIds].some((id) => !actualIds.has(id))\n      ) {\n        throw new Error(\n          \"Table identifier fields do not match. Recreate the table or revert the schema change.\",\n        );\n      }\n\n      return actual;\n    },\n    catch: (cause) =>\n      new ConnectorError({\n        message: cause instanceof Error ? cause.message : \"Invalid table metadata\",\n        cause,\n      }),\n  });\n\n/** Creates a table with the IDs from its Effect schema, then checks the loaded schema. */\nexport const createTable = (\n  schema: Schema.Top,\n  options: TableOptions,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.gen(function* () {\n    const table = yield* compileTable(schema);\n    yield* options.catalog.commitTable(\n      options.identifier,\n      makeCommitRequest(table, { location: options.location }),\n    );\n\n    const metadata = yield* options.catalog.loadTable(options.identifier);\n    return yield* verifyTable(metadata, table.schema);\n  }).pipe(Effect.mapError(tableError));\n\n/** Creates a table through a Wings catalog. */\nexport const createTableFromWings = (\n  schema: Schema.Top,\n  options: WingsTableOptions,\n): Effect.Effect<TableSchema, ConnectorError, HttpClient.HttpClient> =>\n  Effect.scoped(\n    Effect.gen(function* () {\n      const manager = yield* Wings.CatalogManager.make({ baseUrl: options.wingsUri });\n      const catalog = yield* manager.getIcebergCatalog(options.catalogId);\n      return yield* createTable(schema, {\n        catalog,\n        identifier: options.identifier,\n        location: options.location,\n      });\n    }),\n  ).pipe(Effect.mapError(tableError));\n\n/** Loads a table, creating it when it is missing. */\nexport const ensureTable = (\n  schema: Schema.Top,\n  options: TableOptions,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.gen(function* () {\n    const table = yield* compileTable(schema);\n    const metadata = yield* options.catalog.loadTable(options.identifier).pipe(\n      Effect.catchIf(\n        (error) => error.status === 404,\n        () =>\n          options.catalog\n            .commitTable(\n              options.identifier,\n              makeCommitRequest(table, { location: options.location }),\n            )\n            .pipe(\n              // Another instance may have created the table. Reload and check it.\n              Effect.catchIf(\n                (error) => error.status === 409 || error.status === 412,\n                () => Effect.void,\n              ),\n              Effect.flatMap(() => options.catalog.loadTable(options.identifier)),\n            ),\n      ),\n    );\n\n    return yield* verifyTable(metadata, table.schema);\n  }).pipe(Effect.mapError(tableError));\n","import {\n  Bool,\n  DateDay,\n  Decimal,\n  Field,\n  FixedSizeBinary,\n  Float32,\n  Float64,\n  Int32,\n  Int64,\n  LargeBinary,\n  List,\n  Map_,\n  RecordBatch,\n  Schema as ArrowSchema,\n  Struct,\n  TimeMicrosecond,\n  TimestampMicrosecond,\n  Utf8,\n  makeBuilder,\n  type DataType,\n  type TypeMap,\n} from \"apache-arrow\";\nimport { Effect, Schema } from \"effect\";\nimport {\n  parseDecimalType,\n  parseFixedType,\n  type IcebergType,\n  type PrimitiveType,\n  type TableSchema,\n} from \"iceberg-js\";\n\nimport { ConnectorError } from \"../errors\";\n\n// Nothing converts an Iceberg schema to Arrow for us, so we do it here.\n// Each type also gets a schema for its values, because Arrow does not reject a bad\n// value, it converts it: \"1\" becomes 1, and 1.5 becomes 1.\n\ntype RowSchema = Schema.Codec<unknown, unknown>;\n\n/** One Iceberg type: the Arrow type, and the values it accepts. */\ntype Compiled = {\n  readonly type: DataType;\n  readonly row: RowSchema;\n};\n\n// Arrow pads or cuts binary values to fit.\nconst bytes = (length: number) =>\n  Schema.Uint8Array.check(\n    Schema.makeFilter(\n      (value: Uint8Array) => value.byteLength === length || `expected ${length} bytes`,\n    ),\n  );\n\n// Arrow wraps big integers instead of failing, so check the range.\nconst int32 = Schema.Number.check(\n  Schema.isInt(),\n  Schema.isBetween({ minimum: -2_147_483_648, maximum: 2_147_483_647 }),\n);\n\nconst int64 = Schema.BigInt.check(\n  Schema.isBetweenBigInt({ minimum: -(2n ** 63n), maximum: 2n ** 63n - 1n }),\n);\n\n// Arrow turns a number outside the 32-bit range into Infinity.\nconst float32 = Schema.Finite.check(\n  Schema.makeFilter(\n    (value: number) => Number.isFinite(Math.fround(value)) || \"expected a 32-bit number\",\n  ),\n);\n\nconst microsecondsInDay = Schema.BigInt.check(\n  Schema.isBetweenBigInt({ minimum: 0n, maximum: 86_399_999_999n }),\n);\n\nconst primitives: Record<string, Compiled> = {\n  boolean: { type: new Bool(), row: Schema.Boolean },\n  int: { type: new Int32(), row: int32 },\n  long: { type: new Int64(), row: int64 },\n  float: { type: new Float32(), row: float32 },\n  double: { type: new Float64(), row: Schema.Finite },\n  string: { type: new Utf8(), row: Schema.String },\n  uuid: { type: new FixedSizeBinary(16), row: bytes(16) },\n  binary: { type: new LargeBinary(), row: Schema.Uint8Array },\n  date: { type: new DateDay(), row: Schema.Date },\n  time: { type: new TimeMicrosecond(), row: microsecondsInDay },\n  timestamp: { type: new TimestampMicrosecond(), row: Schema.Date },\n  timestamptz: { type: new TimestampMicrosecond(\"UTC\"), row: Schema.Date },\n};\n\nconst primitive = (type: PrimitiveType): Compiled => {\n  const known = primitives[type];\n\n  if (known) return known;\n\n  // Nothing checks the catalog metadata, and Arrow takes any precision.\n  // Better to fail here than at write time.\n  const decimal = parseDecimalType(type);\n\n  if (decimal) {\n    if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) {\n      throw new Error(`Invalid Iceberg decimal type ${type}`);\n    }\n\n    return {\n      type: new Decimal(decimal.scale, decimal.precision),\n      row: Schema.instanceOf(Uint32Array).check(\n        Schema.makeFilter((value: Uint32Array) => value.length === 4 || \"expected four words\"),\n      ),\n    };\n  }\n\n  const fixed = parseFixedType(type);\n\n  if (fixed) {\n    if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);\n\n    return { type: new FixedSizeBinary(fixed.length), row: bytes(fixed.length) };\n  }\n\n  throw new Error(`Unsupported Iceberg type ${type}`);\n};\n\n/** Absent and null both become null in Arrow. */\nconst optional = (row: RowSchema) => Schema.optional(Schema.NullOr(row));\n\nconst fieldMetadata = (id: number): Map<string, string> =>\n  new Map([[\"PARQUET:field_id\", String(id)]]);\n\n/** Same as `primitive`, plus struct, list and map. */\nconst compileType = (type: IcebergType): Compiled => {\n  if (typeof type === \"string\") return primitive(type);\n\n  if (type.type === \"struct\") {\n    const fields = type.fields.map((field) => ({ ...field, compiled: compileType(field.type) }));\n\n    return {\n      type: new Struct(\n        fields.map(\n          (field) =>\n            new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)),\n        ),\n      ),\n      row: Schema.Struct(\n        Object.fromEntries(\n          fields.map((field) => [\n            field.name,\n            field.required ? field.compiled.row : optional(field.compiled.row),\n          ]),\n        ),\n      ),\n    };\n  }\n\n  if (type.type === \"list\") {\n    const element = compileType(type.element);\n    const required = type[\"element-required\"];\n\n    return {\n      type: new List(\n        new Field(\"element\", element.type, !required, fieldMetadata(type[\"element-id\"])),\n      ),\n      row: Schema.Array(required ? element.row : Schema.NullishOr(element.row)),\n    };\n  }\n\n  const key = compileType(type.key);\n  const value = compileType(type.value);\n  const required = type[\"value-required\"];\n  const entries = new Struct([\n    new Field(\"key\", key.type, false, fieldMetadata(type[\"key-id\"])),\n    new Field(\"value\", value.type, !required, fieldMetadata(type[\"value-id\"])),\n  ]);\n\n  return {\n    type: new Map_(new Field(\"entries\", entries, false)),\n    row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row)),\n  };\n};\n\nexport type RowEncoder = (\n  rows: ReadonlyArray<object>,\n) => Effect.Effect<RecordBatch<TypeMap>, ConnectorError>;\n\nexport const makeRowEncoder = (\n  tableSchema: TableSchema,\n  version: string,\n): Effect.Effect<RowEncoder, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      const fields = tableSchema.fields.map((field) => ({\n        id: field.id,\n        name: field.name,\n        ...compileType(field.type),\n      }));\n      const fieldNames = new Set(fields.map((field) => field.name));\n      const identifierIds = new Set(tableSchema[\"identifier-field-ids\"]);\n      const keys = tableSchema.fields\n        .filter((field) => identifierIds.has(field.id))\n        .map((field) => field.name);\n      // Other columns may be missing: a delete only sends the key and version.\n      const required = new Set([...keys, version]);\n\n      if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);\n\n      const struct = new Struct(\n        fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))),\n      );\n      const schema = new ArrowSchema(struct.children);\n\n      const decodeRow = Schema.decodeUnknownSync(\n        Schema.Struct(\n          Object.fromEntries(\n            fields.map((field) => [\n              field.name,\n              required.has(field.name) ? field.row : optional(field.row),\n            ]),\n          ),\n        ),\n        { onExcessProperty: \"error\" },\n      );\n\n      // Schema says which field is wrong. This adds the row number.\n      const decodeAt = (row: object, index: number) => {\n        try {\n          return decodeRow(row) as Record<string, unknown>;\n        } catch (cause) {\n          const detail = cause instanceof Error ? cause.message : String(cause);\n          throw new Error(`row ${index}: ${detail}`, { cause });\n        }\n      };\n\n      return (rows: ReadonlyArray<object>) =>\n        Effect.try({\n          try: () => {\n            const builder = makeBuilder({ type: struct, nullValues: [null, undefined] });\n\n            for (const [index, row] of rows.entries()) {\n              builder.append(decodeAt(row, index) as Struct<TypeMap>[\"TValue\"]);\n            }\n\n            return new RecordBatch(schema, builder.finish().flush());\n          },\n          catch: (cause) => new ConnectorError({ message: \"Failed to encode rows\", cause }),\n        });\n    },\n    catch: (cause) => new ConnectorError({ message: \"Failed to convert Iceberg schema\", cause }),\n  });\n","import type { RecordBatch, TypeMap } from \"apache-arrow\";\nimport type { TableIdentifier } from \"iceberg-js\";\n\nimport * as Wings from \"@useairfoil/wings\";\nimport { Config, Effect, Layer } from \"effect\";\nimport { FetchHttpClient } from \"effect/http\";\n\nimport type { TableSource } from \"../core/types\";\n\nimport { ensureTable } from \"../catalog/table\";\nimport { ConnectorError } from \"../errors\";\nimport * as RuntimeConfig from \"../runtime-config\";\nimport { makeRowEncoder, type RowEncoder } from \"./arrow\";\nimport { Ingestor, type IngestorService } from \"./service\";\n\n/** One catalog and an Iceberg table for each resource that runs. */\nexport type WingsIngestorConfig<Source extends TableSource = TableSource> = {\n  readonly connector: Source;\n  readonly catalog: string;\n  readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;\n};\n\ntype IngestorEntry = {\n  readonly encode: RowEncoder;\n  readonly push: (batch: RecordBatch<TypeMap>) => Effect.Effect<void, ConnectorError>;\n};\n\ntype PreparedTable = {\n  readonly resource: string;\n  readonly identifier: TableIdentifier;\n  readonly encode: RowEncoder;\n};\n\nconst connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });\n\n/** Loads each table schema and opens one Wings stream per bound resource. */\nexport const layerWings = <const Source extends TableSource>(\n  config: WingsIngestorConfig<Source>,\n): Layer.Layer<Ingestor, ConnectorError, Wings.CatalogManager.CatalogManager> =>\n  Layer.effect(Ingestor)(\n    Effect.gen(function* () {\n      const manager = yield* Wings.CatalogManager.CatalogManager;\n      const catalog = yield* manager\n        .getIcebergCatalog(config.catalog)\n        .pipe(Effect.mapError((cause) => connectorError(\"Failed to load Wings catalog\", cause)));\n\n      const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;\n\n      const prepared: Array<PreparedTable> = [];\n      for (const resource of config.connector.resources) {\n        const binding = bindings[resource.name];\n        if (!binding) continue;\n\n        const { namespace, name, location } = binding;\n        const identifier = { namespace, name };\n        const schema = yield* ensureTable(resource.rowSchema, {\n          catalog,\n          identifier,\n          location,\n        }).pipe(\n          Effect.mapError((cause) =>\n            connectorError(`Failed to prepare table for ${resource.name}: ${cause.message}`, cause),\n          ),\n        );\n\n        const encode = yield* makeRowEncoder(schema, resource.version);\n        prepared.push({ resource: resource.name, identifier, encode });\n      }\n\n      const entries = new Map<string, IngestorEntry>();\n      for (const { resource, identifier, encode } of prepared) {\n        const ingestor = yield* manager\n          .ingestor({\n            catalog: config.catalog,\n            namespace: identifier.namespace,\n            table: identifier.name,\n          })\n          .pipe(\n            Effect.mapError((cause) =>\n              connectorError(`Failed to open ingestor for ${resource}`, cause),\n            ),\n          );\n\n        entries.set(resource, {\n          encode,\n          push: (batch) =>\n            ingestor\n              .push(batch)\n              .pipe(\n                Effect.mapError((cause) =>\n                  connectorError(`Failed to ingest rows for ${resource}`, cause),\n                ),\n              ),\n        });\n      }\n\n      const service: IngestorService = {\n        resourceNames: prepared.map(({ resource }) => resource),\n        ingest: ({ resource, batch }) => {\n          const entry = entries.get(resource);\n\n          if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));\n          if (batch.rows.length === 0) return Effect.succeed({ rows: 0, bytes: 0 });\n\n          return entry\n            .encode(batch.rows)\n            .pipe(\n              Effect.flatMap((encoded) =>\n                entry\n                  .push(encoded)\n                  .pipe(Effect.as({ rows: batch.rows.length, bytes: encoded.data.byteLength })),\n              ),\n            );\n        },\n      };\n\n      return Ingestor.of(service);\n    }),\n  );\n\n/** Reads the Wings URL, catalog and table bindings from hosted config. */\nexport const layerWingsConfig = (connector: TableSource) =>\n  Layer.unwrap(\n    Effect.gen(function* () {\n      const { catalog, uri } = yield* Config.all({\n        catalog: RuntimeConfig.catalog,\n        uri: RuntimeConfig.wingsUri,\n      }).pipe(Effect.mapError((cause) => connectorError(\"Invalid Wings configuration\", cause)));\n\n      if (uri.protocol !== \"http:\" && uri.protocol !== \"https:\") {\n        return yield* Effect.fail(connectorError(\"Invalid Wings URL\"));\n      }\n\n      const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(\n        Effect.mapError((cause) => connectorError(cause.message, cause)),\n      );\n      const manager = Wings.CatalogManager.layer({ baseUrl: uri.href }).pipe(\n        Layer.provide(FetchHttpClient.layer),\n      );\n\n      return layerWings({ connector, catalog, tables }).pipe(Layer.provide(manager));\n    }),\n  );\n"],"mappings":";;;;;;;;;;AAkCA,MAAM,cAAc,UAAmC;CACrD,IAAI,iBAAiB,gBAAgB,OAAO;CAE5C,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;CACzD,OAAO,IAAI,eAAe;EACxB,SAAS,iBAAiB,qBAAqB,GAAG,MAAM,KAAK,IAAI,YAAY;EAC7E;CACF,CAAC;AACH;AAEA,MAAM,gBACJ,UACA,QACA,SACuB;CACvB,IAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UACpD,OAAO,aAAa,SAAS,KAAA,IAAY;CAG3C,IAAI,SAAS,SAAS,OAAO,MAAM,OAAO;CAE1C,QAAQ,SAAS,MAAjB;EACE,KAAK;GACH,IAAI,OAAO,SAAS,UAAU,OAAO;GACrC,OAAO,cAAc,SAAS,QAAQ,OAAO,QAAQ,IAAI;EAC3D,KAAK;GACH,IACE,OAAO,SAAS,UAChB,SAAS,kBAAkB,OAAO,iBAClC,SAAS,wBAAwB,OAAO,qBAExC,OAAO;GAET,OAAO,aAAa,SAAS,SAAS,OAAO,SAAS,GAAG,KAAK,SAAS;EACzE,KAAK;GACH,IACE,OAAO,SAAS,SAChB,SAAS,cAAc,OAAO,aAC9B,SAAS,gBAAgB,OAAO,eAChC,SAAS,sBAAsB,OAAO,mBAEtC,OAAO;GAET,OACE,aAAa,SAAS,KAAK,OAAO,KAAK,GAAG,KAAK,KAAK,KACpD,aAAa,SAAS,OAAO,OAAO,OAAO,GAAG,KAAK,OAAO;CAEhE;AACF;AAEA,MAAM,iBACJ,UACA,QACA,SACuB;CACvB,IAAI,SAAS,WAAW,OAAO,QAAQ,OAAO;CAE9C,KAAK,MAAM,SAAS,UAAU;EAC5B,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM;EACnC,MAAM,QAAQ,OAAO,MAAM,cAAc,UAAU,SAAS,MAAM,IAAI;EAEtE,IAAI,CAAC,SAAS,MAAM,OAAO,MAAM,MAAM,MAAM,aAAa,MAAM,UAAU,OAAO;EAEjF,MAAM,WAAW,aAAa,MAAM,MAAM,MAAM,MAAM,SAAS;EAC/D,IAAI,UAAU,OAAO;CACvB;AACF;AAEA,MAAM,eACJ,UACA,aAEA,OAAO,IAAI;CACT,WAAW;EACT,IAAI,CAAC,MAAM,QAAQ,SAAS,OAAO,GAAG,MAAM,IAAI,MAAM,wBAAwB;EAC9E,MAAM,SAAS,iBAAiB,QAAQ;EACxC,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,6BAA6B;EAE1D,MAAM,OAAO,cAAc,SAAS,QAAQ,OAAO,QAAQ,GAAG;EAC9D,IAAI,MACF,MAAM,IAAI,MACR,kCAAkC,KAAK,iDACzC;EAGF,MAAM,cAAc,IAAI,IAAI,SAAS,uBAAuB;EAC5D,MAAM,YAAY,IAAI,IAAI,OAAO,uBAAuB;EACxD,IACE,YAAY,SAAS,UAAU,QAC/B,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,GAEhD,MAAM,IAAI,MACR,uFACF;EAGF,OAAO;CACT;CACA,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,iBAAiB,QAAQ,MAAM,UAAU;EAClD;CACF,CAAC;AACL,CAAC;;AAGH,MAAa,eACX,QACA,YAEA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,aAAa,MAAM;CACxC,OAAO,QAAQ,QAAQ,YACrB,QAAQ,YACR,kBAAkB,OAAO,EAAE,UAAU,QAAQ,SAAS,CAAC,CACzD;CAEA,MAAM,WAAW,OAAO,QAAQ,QAAQ,UAAU,QAAQ,UAAU;CACpE,OAAO,OAAO,YAAY,UAAU,MAAM,MAAM;AAClD,CAAC,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;AAGrC,MAAa,wBACX,QACA,YAEA,OAAO,OACL,OAAO,IAAI,aAAa;CAEtB,MAAM,UAAU,QAAO,OADA,MAAM,eAAe,KAAK,EAAE,SAAS,QAAQ,SAAS,CAAC,EAAA,CAC/C,kBAAkB,QAAQ,SAAS;CAClE,OAAO,OAAO,YAAY,QAAQ;EAChC;EACA,YAAY,QAAQ;EACpB,UAAU,QAAQ;CACpB,CAAC;AACH,CAAC,CACH,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;AAGpC,MAAa,eACX,QACA,YAEA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,aAAa,MAAM;CACxC,MAAM,WAAW,OAAO,QAAQ,QAAQ,UAAU,QAAQ,UAAU,CAAC,CAAC,KACpE,OAAO,SACJ,UAAU,MAAM,WAAW,WAE1B,QAAQ,QACL,YACC,QAAQ,YACR,kBAAkB,OAAO,EAAE,UAAU,QAAQ,SAAS,CAAC,CACzD,CAAC,CACA,KAEC,OAAO,SACJ,UAAU,MAAM,WAAW,OAAO,MAAM,WAAW,WAC9C,OAAO,IACf,GACA,OAAO,cAAc,QAAQ,QAAQ,UAAU,QAAQ,UAAU,CAAC,CACpE,CACN,CACF;CAEA,OAAO,OAAO,YAAY,UAAU,MAAM,MAAM;AAClD,CAAC,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;;ACzJrC,MAAM,SAAS,WACb,OAAO,WAAW,MAChB,OAAO,YACJ,UAAsB,MAAM,eAAe,UAAU,YAAY,OAAO,OAC3E,CACF;AAGF,MAAM,QAAQ,OAAO,OAAO,MAC1B,OAAO,MAAM,GACb,OAAO,UAAU;CAAE,SAAS;CAAgB,SAAS;AAAc,CAAC,CACtE;AAEA,MAAM,QAAQ,OAAO,OAAO,MAC1B,OAAO,gBAAgB;CAAE,SAAS,EAAE,MAAM;CAAM,SAAS,MAAM,MAAM;AAAG,CAAC,CAC3E;AAGA,MAAM,UAAU,OAAO,OAAO,MAC5B,OAAO,YACJ,UAAkB,OAAO,SAAS,KAAK,OAAO,KAAK,CAAC,KAAK,0BAC5D,CACF;AAEA,MAAM,oBAAoB,OAAO,OAAO,MACtC,OAAO,gBAAgB;CAAE,SAAS;CAAI,SAAS;AAAgB,CAAC,CAClE;AAEA,MAAM,aAAuC;CAC3C,SAAS;EAAE,MAAM,IAAI,KAAK;EAAG,KAAK,OAAO;CAAQ;CACjD,KAAK;EAAE,MAAM,IAAI,MAAM;EAAG,KAAK;CAAM;CACrC,MAAM;EAAE,MAAM,IAAI,MAAM;EAAG,KAAK;CAAM;CACtC,OAAO;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK;CAAQ;CAC3C,QAAQ;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK,OAAO;CAAO;CAClD,QAAQ;EAAE,MAAM,IAAI,KAAK;EAAG,KAAK,OAAO;CAAO;CAC/C,MAAM;EAAE,MAAM,IAAI,gBAAgB,EAAE;EAAG,KAAK,MAAM,EAAE;CAAE;CACtD,QAAQ;EAAE,MAAM,IAAI,YAAY;EAAG,KAAK,OAAO;CAAW;CAC1D,MAAM;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK,OAAO;CAAK;CAC9C,MAAM;EAAE,MAAM,IAAI,gBAAgB;EAAG,KAAK;CAAkB;CAC5D,WAAW;EAAE,MAAM,IAAI,qBAAqB;EAAG,KAAK,OAAO;CAAK;CAChE,aAAa;EAAE,MAAM,IAAI,qBAAqB,KAAK;EAAG,KAAK,OAAO;CAAK;AACzE;AAEA,MAAM,aAAa,SAAkC;CACnD,MAAM,QAAQ,WAAW;CAEzB,IAAI,OAAO,OAAO;CAIlB,MAAM,UAAU,iBAAiB,IAAI;CAErC,IAAI,SAAS;EACX,IAAI,QAAQ,YAAY,KAAK,QAAQ,YAAY,MAAM,QAAQ,QAAQ,QAAQ,WAC7E,MAAM,IAAI,MAAM,gCAAgC,MAAM;EAGxD,OAAO;GACL,MAAM,IAAI,QAAQ,QAAQ,OAAO,QAAQ,SAAS;GAClD,KAAK,OAAO,WAAW,WAAW,CAAC,CAAC,MAClC,OAAO,YAAY,UAAuB,MAAM,WAAW,KAAK,qBAAqB,CACvF;EACF;CACF;CAEA,MAAM,QAAQ,eAAe,IAAI;CAEjC,IAAI,OAAO;EACT,IAAI,MAAM,SAAS,GAAG,MAAM,IAAI,MAAM,8BAA8B,MAAM;EAE1E,OAAO;GAAE,MAAM,IAAI,gBAAgB,MAAM,MAAM;GAAG,KAAK,MAAM,MAAM,MAAM;EAAE;CAC7E;CAEA,MAAM,IAAI,MAAM,4BAA4B,MAAM;AACpD;;AAGA,MAAM,YAAY,QAAmB,OAAO,SAAS,OAAO,OAAO,GAAG,CAAC;AAEvE,MAAM,iBAAiB,uBACrB,IAAI,IAAI,CAAC,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC,CAAC;;AAG5C,MAAM,eAAe,SAAgC;CACnD,IAAI,OAAO,SAAS,UAAU,OAAO,UAAU,IAAI;CAEnD,IAAI,KAAK,SAAS,UAAU;EAC1B,MAAM,SAAS,KAAK,OAAO,KAAK,WAAW;GAAE,GAAG;GAAO,UAAU,YAAY,MAAM,IAAI;EAAE,EAAE;EAE3F,OAAO;GACL,MAAM,IAAIA,SACR,OAAO,KACJ,UACC,IAAI,MAAM,MAAM,MAAM,MAAM,SAAS,MAAM,CAAC,MAAM,UAAU,cAAc,MAAM,EAAE,CAAC,CACvF,CACF;GACA,KAAK,OAAO,OACV,OAAO,YACL,OAAO,KAAK,UAAU,CACpB,MAAM,MACN,MAAM,WAAW,MAAM,SAAS,MAAM,SAAS,MAAM,SAAS,GAAG,CACnE,CAAC,CACH,CACF;EACF;CACF;CAEA,IAAI,KAAK,SAAS,QAAQ;EACxB,MAAM,UAAU,YAAY,KAAK,OAAO;EACxC,MAAM,WAAW,KAAK;EAEtB,OAAO;GACL,MAAM,IAAI,KACR,IAAI,MAAM,WAAW,QAAQ,MAAM,CAAC,UAAU,cAAc,KAAK,aAAa,CAAC,CACjF;GACA,KAAK,OAAO,MAAM,WAAW,QAAQ,MAAM,OAAO,UAAU,QAAQ,GAAG,CAAC;EAC1E;CACF;CAEA,MAAM,MAAM,YAAY,KAAK,GAAG;CAChC,MAAM,QAAQ,YAAY,KAAK,KAAK;CACpC,MAAM,WAAW,KAAK;CACtB,MAAM,UAAU,IAAIA,SAAO,CACzB,IAAI,MAAM,OAAO,IAAI,MAAM,OAAO,cAAc,KAAK,SAAS,CAAC,GAC/D,IAAI,MAAM,SAAS,MAAM,MAAM,CAAC,UAAU,cAAc,KAAK,WAAW,CAAC,CAC3E,CAAC;CAED,OAAO;EACL,MAAM,IAAI,KAAK,IAAI,MAAM,WAAW,SAAS,KAAK,CAAC;EACnD,KAAK,OAAO,YAAY,IAAI,KAAK,WAAW,MAAM,MAAM,OAAO,UAAU,MAAM,GAAG,CAAC;CACrF;AACF;AAMA,MAAa,kBACX,aACA,YAEA,OAAO,IAAI;CACT,WAAW;EACT,MAAM,SAAS,YAAY,OAAO,KAAK,WAAW;GAChD,IAAI,MAAM;GACV,MAAM,MAAM;GACZ,GAAG,YAAY,MAAM,IAAI;EAC3B,EAAE;EACF,MAAM,aAAa,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,IAAI,CAAC;EAC5D,MAAM,gBAAgB,IAAI,IAAI,YAAY,uBAAuB;EACjE,MAAM,OAAO,YAAY,OACtB,QAAQ,UAAU,cAAc,IAAI,MAAM,EAAE,CAAC,CAAC,CAC9C,KAAK,UAAU,MAAM,IAAI;EAE5B,MAAM,2BAAW,IAAI,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC;EAE3C,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,MAAM,IAAI,MAAM,0BAA0B,SAAS;EAEjF,MAAM,SAAS,IAAIA,SACjB,OAAO,KAAK,UAAU,IAAI,MAAM,MAAM,MAAM,MAAM,MAAM,MAAM,cAAc,MAAM,EAAE,CAAC,CAAC,CACxF;EACA,MAAM,SAAS,IAAIC,SAAY,OAAO,QAAQ;EAE9C,MAAM,YAAY,OAAO,kBACvB,OAAO,OACL,OAAO,YACL,OAAO,KAAK,UAAU,CACpB,MAAM,MACN,SAAS,IAAI,MAAM,IAAI,IAAI,MAAM,MAAM,SAAS,MAAM,GAAG,CAC3D,CAAC,CACH,CACF,GACA,EAAE,kBAAkB,QAAQ,CAC9B;EAGA,MAAM,YAAY,KAAa,UAAkB;GAC/C,IAAI;IACF,OAAO,UAAU,GAAG;GACtB,SAAS,OAAO;IACd,MAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;IACpE,MAAM,IAAI,MAAM,OAAO,MAAM,IAAI,UAAU,EAAE,MAAM,CAAC;GACtD;EACF;EAEA,QAAQ,SACN,OAAO,IAAI;GACT,WAAW;IACT,MAAM,UAAU,YAAY;KAAE,MAAM;KAAQ,YAAY,CAAC,MAAM,KAAA,CAAS;IAAE,CAAC;IAE3E,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GACtC,QAAQ,OAAO,SAAS,KAAK,KAAK,CAA8B;IAGlE,OAAO,IAAI,YAAY,QAAQ,QAAQ,OAAO,CAAC,CAAC,MAAM,CAAC;GACzD;GACA,QAAQ,UAAU,IAAI,eAAe;IAAE,SAAS;IAAyB;GAAM,CAAC;EAClF,CAAC;CACL;CACA,QAAQ,UAAU,IAAI,eAAe;EAAE,SAAS;EAAoC;CAAM,CAAC;AAC7F,CAAC;;;ACtNH,MAAM,kBAAkB,SAAiB,UAAoB,IAAI,eAAe;CAAE;CAAS;AAAM,CAAC;;AAGlG,MAAa,cACX,WAEA,MAAM,OAAO,QAAQ,CAAC,CACpB,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,MAAM,eAAe;CAC5C,MAAM,UAAU,OAAO,QACpB,kBAAkB,OAAO,OAAO,CAAC,CACjC,KAAK,OAAO,UAAU,UAAU,eAAe,gCAAgC,KAAK,CAAC,CAAC;CAEzF,MAAM,WAAgE,OAAO;CAE7E,MAAM,WAAiC,CAAC;CACxC,KAAK,MAAM,YAAY,OAAO,UAAU,WAAW;EACjD,MAAM,UAAU,SAAS,SAAS;EAClC,IAAI,CAAC,SAAS;EAEd,MAAM,EAAE,WAAW,MAAM,aAAa;EACtC,MAAM,aAAa;GAAE;GAAW;EAAK;EACrC,MAAM,SAAS,OAAO,YAAY,SAAS,WAAW;GACpD;GACA;GACA;EACF,CAAC,CAAC,CAAC,KACD,OAAO,UAAU,UACf,eAAe,+BAA+B,SAAS,KAAK,IAAI,MAAM,WAAW,KAAK,CACxF,CACF;EAEA,MAAM,SAAS,OAAO,eAAe,QAAQ,SAAS,OAAO;EAC7D,SAAS,KAAK;GAAE,UAAU,SAAS;GAAM;GAAY;EAAO,CAAC;CAC/D;CAEA,MAAM,0BAAU,IAAI,IAA2B;CAC/C,KAAK,MAAM,EAAE,UAAU,YAAY,YAAY,UAAU;EACvD,MAAM,WAAW,OAAO,QACrB,SAAS;GACR,SAAS,OAAO;GAChB,WAAW,WAAW;GACtB,OAAO,WAAW;EACpB,CAAC,CAAC,CACD,KACC,OAAO,UAAU,UACf,eAAe,+BAA+B,YAAY,KAAK,CACjE,CACF;EAEF,QAAQ,IAAI,UAAU;GACpB;GACA,OAAO,UACL,SACG,KAAK,KAAK,CAAC,CACX,KACC,OAAO,UAAU,UACf,eAAe,6BAA6B,YAAY,KAAK,CAC/D,CACF;EACN,CAAC;CACH;CAEA,MAAM,UAA2B;EAC/B,eAAe,SAAS,KAAK,EAAE,eAAe,QAAQ;EACtD,SAAS,EAAE,UAAU,YAAY;GAC/B,MAAM,QAAQ,QAAQ,IAAI,QAAQ;GAElC,IAAI,CAAC,OAAO,OAAO,OAAO,KAAK,eAAe,oBAAoB,UAAU,CAAC;GAC7E,IAAI,MAAM,KAAK,WAAW,GAAG,OAAO,OAAO,QAAQ;IAAE,MAAM;IAAG,OAAO;GAAE,CAAC;GAExE,OAAO,MACJ,OAAO,MAAM,IAAI,CAAC,CAClB,KACC,OAAO,SAAS,YACd,MACG,KAAK,OAAO,CAAC,CACb,KAAK,OAAO,GAAG;IAAE,MAAM,MAAM,KAAK;IAAQ,OAAO,QAAQ,KAAK;GAAW,CAAC,CAAC,CAChF,CACF;EACJ;CACF;CAEA,OAAO,SAAS,GAAG,OAAO;AAC5B,CAAC,CACH;;AAGF,MAAa,oBAAoB,cAC/B,MAAM,OACJ,OAAO,IAAI,aAAa;CACtB,MAAM,EAAE,SAAA,WAAS,QAAQ,OAAO,OAAO,IAAI;EAChCC;EACT,KAAKC;CACP,CAAC,CAAC,CAAC,KAAK,OAAO,UAAU,UAAU,eAAe,+BAA+B,KAAK,CAAC,CAAC;CAExF,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,OAAO,OAAO,OAAO,KAAK,eAAe,mBAAmB,CAAC;CAG/D,MAAM,SAAS,OAAOC,kBAAgC,SAAS,CAAC,CAAC,KAC/D,OAAO,UAAU,UAAU,eAAe,MAAM,SAAS,KAAK,CAAC,CACjE;CACA,MAAM,UAAU,MAAM,eAAe,MAAM,EAAE,SAAS,IAAI,KAAK,CAAC,CAAC,CAAC,KAChE,MAAM,QAAQ,gBAAgB,KAAK,CACrC;CAEA,OAAO,WAAW;EAAE;EAAW,SAAA;EAAS;CAAO,CAAC,CAAC,CAAC,KAAK,MAAM,QAAQ,OAAO,CAAC;AAC/E,CAAC,CACH"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@useairfoil/connector-kit",
-  "version": "0.6.0",
-  "repository": "useairfoil/typescript-sdk",
+  "version": "0.6.1",
+  "repository": "useairfoil/connector-kit",
   "files": [
     "dist",
     "src",
@@ -9,68 +9,78 @@
     "README.md"
   ],
   "type": "module",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "default": "./dist/index.js"
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
     },
     "./auth": {
-      "types": "./dist/auth/index.d.ts",
-      "import": "./dist/auth/index.js",
-      "default": "./dist/auth/index.js"
+      "types": "./dist/auth/index.d.mts",
+      "import": "./dist/auth/index.mjs",
+      "default": "./dist/auth/index.mjs"
     },
     "./connector-app": {
-      "types": "./dist/connector-app/index.d.ts",
-      "import": "./dist/connector-app/index.js",
-      "default": "./dist/connector-app/index.js"
+      "types": "./dist/connector-app/index.d.mts",
+      "import": "./dist/connector-app/index.mjs",
+      "default": "./dist/connector-app/index.mjs"
+    },
+    "./demo": {
+      "types": "./dist/demo/index.d.mts",
+      "import": "./dist/demo/index.mjs",
+      "default": "./dist/demo/index.mjs"
+    },
+    "./iceberg": {
+      "types": "./dist/iceberg/index.d.mts",
+      "import": "./dist/iceberg/index.mjs",
+      "default": "./dist/iceberg/index.mjs"
     },
     "./ingestion": {
-      "types": "./dist/ingestion/index.d.ts",
-      "import": "./dist/ingestion/index.js",
-      "default": "./dist/ingestion/index.js"
+      "types": "./dist/ingestion/index.d.mts",
+      "import": "./dist/ingestion/index.mjs",
+      "default": "./dist/ingestion/index.mjs"
+    },
+    "./ingestor": {
+      "types": "./dist/ingestor/index.d.mts",
+      "import": "./dist/ingestor/index.mjs",
+      "default": "./dist/ingestor/index.mjs"
     },
     "./manifest": {
-      "types": "./dist/manifest/index.d.ts",
-      "import": "./dist/manifest/index.js",
-      "default": "./dist/manifest/index.js"
+      "types": "./dist/manifest/index.d.mts",
+      "import": "./dist/manifest/index.mjs",
+      "default": "./dist/manifest/index.mjs"
     },
     "./metrics": {
-      "types": "./dist/metrics.d.ts",
-      "import": "./dist/metrics.js",
-      "default": "./dist/metrics.js"
-    },
-    "./publisher": {
-      "types": "./dist/publisher/index.d.ts",
-      "import": "./dist/publisher/index.js",
-      "default": "./dist/publisher/index.js"
+      "types": "./dist/metrics.d.mts",
+      "import": "./dist/metrics.mjs",
+      "default": "./dist/metrics.mjs"
     },
     "./runtime-config": {
-      "types": "./dist/runtime-config/index.d.ts",
-      "import": "./dist/runtime-config/index.js",
-      "default": "./dist/runtime-config/index.js"
+      "types": "./dist/runtime-config/index.d.mts",
+      "import": "./dist/runtime-config/index.mjs",
+      "default": "./dist/runtime-config/index.mjs"
     },
     "./state-store": {
-      "types": "./dist/state-store/index.d.ts",
-      "import": "./dist/state-store/index.js",
-      "default": "./dist/state-store/index.js"
+      "types": "./dist/state-store/index.d.mts",
+      "import": "./dist/state-store/index.mjs",
+      "default": "./dist/state-store/index.mjs"
     },
     "./status": {
-      "types": "./dist/status/index.d.ts",
-      "import": "./dist/status/index.js",
-      "default": "./dist/status/index.js"
+      "types": "./dist/status/index.d.mts",
+      "import": "./dist/status/index.mjs",
+      "default": "./dist/status/index.mjs"
     },
     "./webhook": {
-      "types": "./dist/webhook/index.d.ts",
-      "import": "./dist/webhook/index.js",
-      "default": "./dist/webhook/index.js"
+      "types": "./dist/webhook/index.d.mts",
+      "import": "./dist/webhook/index.mjs",
+      "default": "./dist/webhook/index.mjs"
     },
     "./errors": {
-      "types": "./dist/errors/index.d.ts",
-      "import": "./dist/errors/index.js",
-      "default": "./dist/errors/index.js"
+      "types": "./dist/errors/index.d.mts",
+      "import": "./dist/errors/index.mjs",
+      "default": "./dist/errors/index.mjs"
     }
   },
   "scripts": {
@@ -80,21 +90,24 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@effect/platform-node": "^4.0.0-rc.108",
-    "@useairfoil/flight": "0.4.6",
-    "@useairfoil/wings": "0.4.5"
+    "@effect/platform-node": "^4.0.7",
+    "@faker-js/faker": "10.6.0",
+    "@useairfoil/wings": "0.4.7",
+    "apache-arrow": "^21.1.0",
+    "iceberg-js": "^1.0.0"
   },
   "devDependencies": {
-    "@effect/sql-pg": "^4.0.0-rc.108",
-    "@effect/vitest": "^4.0.0-rc.108",
+    "@effect/sql-pg": "^4.0.7",
+    "@effect/vitest": "^4.0.7",
     "@types/node": "^24.2.0",
-    "effect": "^4.0.0-rc.108",
+    "@useairfoil/wings-testing": "0.1.6",
+    "effect": "^4.0.7",
     "testcontainers": "^11.14.0",
-    "tsdown": "^0.15.6",
-    "vitest": "^4.1.0"
+    "tsdown": "^0.23.0",
+    "vitest": "^5.0.3"
   },
   "peerDependencies": {
-    "effect": "^4.0.0-rc.108"
+    "effect": "^4.0.7"
   },
   "peerDependenciesMeta": {
     "effect": {