@useairfoil/connector-kit 0.6.0 → 0.7.0

package/dist/index-CV6V6fyx.d.mts

@@ -0,0 +1,79 @@
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
+import { _ as ResourceRequiredField, b as ResourceSchema } from "./types-NtSv6gwl.mjs";
+import { i as Ingestor, n as IngestResult } from "./service-BRoy_EId.mjs";
+import { Effect, Schema } from "effect";
+import { Faker } from "@faker-js/faker";
+//#region src/demo/generate.d.ts
+/** What one run shares: the Faker and the time the run started. */
+type DemoContext = {
+  readonly faker: Faker;
+  readonly now: Date;
+};
+//#endregion
+//#region src/demo/definition.d.ts
+/** A table a demo writes: the same name, row schema and version as its resource. */
+type DemoTable<Name extends string = string, S extends ResourceSchema = ResourceSchema, Version extends string = string> = {
+  readonly name: Name;
+  readonly rowSchema: S;
+  readonly version: Version;
+};
+declare const table: <const Name extends string, S extends ResourceSchema, const Version extends ResourceRequiredField<S>>(definition: DemoTable<Name, S, Version>) => DemoTable<Name, S, Version>;
+/** Rows by table. Required fields a row leaves out get placeholder values. */
+type DemoRows<Tables extends ReadonlyArray<DemoTable>> = { readonly [Table in Tables[number] as Table["name"]]?: ReadonlyArray<Partial<Schema.Schema.Type<Table["rowSchema"]>>>; };
+type DemoDefinition<Setup extends ReadonlyArray<DemoTable> = ReadonlyArray<DemoTable>, Group extends ReadonlyArray<DemoTable> = ReadonlyArray<DemoTable>> = {
+  readonly name: string;
+  /** Tables written once, before any group, such as a product catalog. */
+  readonly setup: {
+    readonly tables: Setup;
+    readonly rows: (context: DemoContext) => DemoRows<Setup>;
+  };
+  /**
+   * Tables for one related unit, such as a customer and their orders. The run
+   * adds groups until it reaches the size. Build keys from the index, so they
+   * are unique and other groups can point to them.
+   */
+  readonly group: {
+    readonly tables: Group;
+    readonly rows: (index: number, context: DemoContext) => DemoRows<Group>;
+    /**
+     * Groups written per chunk, 1000 by default. A run can go over the size by
+     * one chunk, so lower it when a group is big.
+     */
+    readonly groupsPerChunk?: number;
+    /**
+     * The number of groups there are, when the provider only keeps so much
+     * history. The run stops there, even below the size.
+     */
+    readonly count?: number;
+  };
+};
+declare const define: <const Setup extends ReadonlyArray<DemoTable>, const Group extends ReadonlyArray<DemoTable>>(definition: DemoDefinition<Setup, Group>) => DemoDefinition<Setup, Group>;
+//#endregion
+//#region src/demo/run.d.ts
+type RunOptions = {
+  /** Stop once this many Arrow bytes are written. */
+  readonly bytes: number;
+  /** Omit for random data on each run. */
+  readonly seed?: number;
+  /** The time data is made around. Same seed, time and tables give the same rows. */
+  readonly now: Date;
+  /** Maximum rows per ingestion batch. Defaults to 500. Must be a positive integer. */
+  readonly batchSize?: number;
+  /** Groups per chunk. Overrides the demo's. */
+  readonly groupsPerChunk?: number;
+};
+type DemoTotals = IngestResult & {
+  readonly tables: Readonly<Record<string, IngestResult>>;
+};
+/**
+ * Writes the setup rows, then groups, until it reaches the size or the demo's
+ * count. It can go over the size by one chunk. Only bound tables are
+ * written; keys still point to the others.
+ */
+declare const run: (demo: DemoDefinition, options: RunOptions) => Effect.Effect<DemoTotals, ConnectorError, Ingestor>;
+declare namespace index_d_exports {
+  export { DemoContext, DemoDefinition, DemoRows, DemoTable, DemoTotals, RunOptions, define, run, table };
+}
+//#endregion
+export { DemoDefinition as a, define as c, run as i, table as l, DemoTotals as n, DemoRows as o, RunOptions as r, DemoTable as s, index_d_exports as t, DemoContext as u };
+//# sourceMappingURL=index-CV6V6fyx.d.mts.map

package/dist/{index-C5Ma7V7t.d.ts → index-CZKx2Yce.d.mts}

@@ -1,12 +1,22 @@
-import { t as ConnectorError } from "./index-DKoRwyFn.js";
-import { b as WebhookHandler, d as ResourceMutation, f as ResourceName, g as ResourceSchema, i as DeleteValue, l as ResourceDefinition, n as ConnectorDefinition, r as Cursor, s as PageFetch, t as ChangesFetch } from "./types-jVD5Ys43.js";
-import { i as Publisher } from "./service-CUjGhe-7.js";
-import { i as StateStore } from "./index-DYpEMJf-.js";
-import { Config, Context, Duration, Effect, Layer } from "effect";
-import { HttpRouter } from "effect/unstable/http";
-import * as effect_unstable_http_HttpClient0 from "effect/unstable/http/HttpClient";
-import * as effect_unstable_http_HttpServerError0 from "effect/unstable/http/HttpServerError";
-
+import "./index-BX8G7d41.mjs";
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
+import "./annotations-vzSzXtr-.mjs";
+import { _ as ResourceRequiredField, b as ResourceSchema, d as ResourceDefinition, h as ResourceName, i as Cursor$1, l as PageFetch, n as ChangesFetch, r as ConnectorDefinition, t as ChangesFeed } from "./types-NtSv6gwl.mjs";
+import { a as DemoDefinition } from "./index-CV6V6fyx.mjs";
+import { i as Ingestor } from "./service-BRoy_EId.mjs";
+import "./index-BTAbexBp.mjs";
+import { i as StateStore } from "./index-CgvFomO6.mjs";
+import "./index-DeTwIUKl.mjs";
+import "./index-uFa2wwYo.mjs";
+import { f as RuntimeConfigError, t as index_d_exports$1 } from "./index-fHIJJoRe.mjs";
+import "./index-eFuoMZZ3.mjs";
+import "./metrics-DoJ0WeKu.mjs";
+import "./index-Byr8NTdG.mjs";
+import "./index-dHS59EZl.mjs";
+import { Config, Context, Duration, Effect, Layer, Option, Schema } from "effect";
+import { CliError, Command } from "effect/cli";
+import { CatalogManager } from "@useairfoil/api";
+import { HttpRouter } from "effect/http";
 //#region src/connector-app/check.d.ts
 type ResourceCheckResult = {
   readonly _tag: "ok";
@@ -17,7 +27,7 @@ type ResourceCheckResult = {
 type CheckOptions<Name extends string> = {
   readonly resources: ReadonlyArray<Name>;
 };
-type ResourceCheckResults<Name extends PropertyKey> = { readonly [Key in Name]: ResourceCheckResult };
+type ResourceCheckResults<Name extends PropertyKey> = { readonly [Key in Name]: ResourceCheckResult; };
 type CheckResult<Resources extends ReadonlyArray<ResourceDefinition>, Name extends ResourceName<Resources> = ResourceName<Resources>> = ResourceCheckResults<Name>;
 type CheckFailure = {
   readonly message: string;
@@ -30,25 +40,20 @@ declare function check<Identifier, const Resources extends ReadonlyArray<Resourc
 //#endregion
 //#region src/core/builder.d.ts
 declare const Connector: {
-  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(definition: ConnectorDefinition<Resources>) => ConnectorDefinition<Resources>;
+  define: <const Resources extends ReadonlyArray<ResourceDefinition>, const FeedResources extends ReadonlyArray<Resources[number]> = Resources>(definition: ConnectorDefinition<Resources, FeedResources>) => ConnectorDefinition<Resources, FeedResources>;
 };
 declare const Resource: {
-  entity: <S extends ResourceSchema, Payload = never, R = never, const Name extends string = string>(definition: ResourceDefinition<S, Payload, R, Name>) => ResourceDefinition<S, Payload, R, Name>;
-  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) => WebhookHandler<Row, Payload>;
-  upsert: <Row extends object>(row: Row) => ResourceMutation<Row>;
-  delete: (options: {
-    readonly key: DeleteValue;
-    readonly version: DeleteValue;
-  }) => ResourceMutation<never>;
+  entity: <S extends ResourceSchema, Payload = never, R = never, const Name extends string = string, const Version extends ResourceRequiredField<S> = ResourceRequiredField<S>>(definition: ResourceDefinition<S, Payload, R, Name, Version>) => ResourceDefinition<S, Payload, R, Name, Version>;
 };
 declare const Fetch: {
   page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => PageFetch<Row, R>;
   changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => ChangesFetch<Row, R>;
+  feed: <const Resources extends ReadonlyArray<ResourceDefinition>, R = never>(definition: ChangesFeed<Resources, R>) => ChangesFeed<Resources, R>;
 };
-declare const Cursor$1: {
-  string: () => Cursor.Definition<string>;
-  number: () => Cursor.Definition<number>;
-  isoDateTime: () => Cursor.Definition<string>;
+declare const Cursor: {
+  string: () => Cursor$1.Definition<string>;
+  number: () => Cursor$1.Definition<number>;
+  isoDateTime: () => Cursor$1.Definition<string>;
   nowIsoDateTime: Effect.Effect<string, never, never>;
 };
 declare namespace telemetry_d_exports {
@@ -59,7 +64,7 @@ declare const SpanName: {
   readonly batchProcess: "connector.batch.process";
   readonly webhookDecode: "connector.webhook.decode";
   readonly webhookHandle: "connector.webhook.handle";
-  readonly publish: "connector.publish";
+  readonly ingest: "connector.ingest";
   readonly stateSet: "connector.state.set";
   readonly apiFetch: "connector.api.fetch";
 };
@@ -69,13 +74,13 @@ declare const Attr: {
   readonly connectorResourcesCount: "airfoil.connector.resources.count";
   readonly resourceName: "airfoil.resource.name";
   readonly resourceSource: "airfoil.resource.source";
-  readonly batchMutations: "airfoil.batch.mutations";
+  readonly batchRows: "airfoil.batch.rows";
   readonly batchOutcome: "airfoil.batch.outcome";
   readonly webhookPath: "airfoil.webhook.path";
   readonly webhookOutcome: "airfoil.webhook.outcome";
   readonly syncState: "airfoil.connector.sync.state";
   readonly apiRetryReason: "airfoil.connector.api.retry.reason";
-  readonly publisherSuccess: "airfoil.publisher.success";
+  readonly ingestionSuccess: "airfoil.ingestion.success";
   readonly stateKey: "airfoil.state.key";
   readonly apiPath: "airfoil.api.path";
   readonly errorPhase: "airfoil.error.phase";
@@ -120,18 +125,18 @@ type OtlpMetricsOptions = {
  * Trace-only OTLP export with sensitive HTTP header redaction.
  * Takes direct config values — no `ConfigProvider` required.
  */
-declare const layer: (config?: OtlpTracingConfig, options?: OtlpTracingOptions) => Layer.Layer<never, Error, effect_unstable_http_HttpClient0.HttpClient>;
+declare const layer: (config?: OtlpTracingConfig, options?: OtlpTracingOptions) => Layer.Layer<never, Error, import("effect/http/HttpClient").HttpClient>;
 /**
  * Trace-only OTLP export with sensitive HTTP header redaction.
  * Takes Effect Config-wrapped values — reads from `ConfigProvider`.
  *
  * @example
  * Telemetry.layerConfig({
- *   enabled: Config.boolean("MY_OTEL_ENABLED"),
- *   endpoint: Config.string("MY_COLLECTOR_URL"),
+ *   enabled: Config.Boolean("MY_OTEL_ENABLED"),
+ *   endpoint: Config.String("MY_COLLECTOR_URL"),
  * })
  */
-declare const layerConfig: (config: Config.Wrap<OtlpTracingConfig>, options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
+declare const layerConfig: (config: Config.Wrap<OtlpTracingConfig>, options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, import("effect/http/HttpClient").HttpClient>;
 /**
  * Trace-only OTLP export with sensitive HTTP header redaction.
  * Zero-config shortcut that reads the standard `OTEL_*` env var names.
@@ -140,28 +145,58 @@ declare const layerConfig: (config: Config.Wrap<OtlpTracingConfig>, options?: Ot
  * `OTEL_EXPORTER_OTLP_HEADERS`. Effect's OTLP resource handling reads
  * `OTEL_SERVICE_NAME`, `OTEL_SERVICE_VERSION`, and `OTEL_RESOURCE_ATTRIBUTES`.
  */
-declare const layerOtlpTracing: (options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
+declare const layerOtlpTracing: (options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, import("effect/http/HttpClient").HttpClient>;
 /** Exports connector metrics over OTLP when telemetry is enabled. */
-declare const layerOtlpMetrics: (options?: OtlpMetricsOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
+declare const layerOtlpMetrics: (options?: OtlpMetricsOptions) => Layer.Layer<never, Error | Config.ConfigError, import("effect/http/HttpClient").HttpClient>;
 /** Enables OTLP traces and metrics with shared environment configuration. */
-declare const layerOtlp: (options?: OtlpTracingOptions & OtlpMetricsOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
+declare const layerOtlp: (options?: OtlpTracingOptions & OtlpMetricsOptions) => Layer.Layer<never, Error | Config.ConfigError, import("effect/http/HttpClient").HttpClient>;
 /** Periodically logs the in-memory metric snapshot for local debugging. */
 declare const layerMetricsConsoleDump: (interval?: Duration.Input) => Layer.Layer<never, never, never>;
 //#endregion
+//#region src/connector-app/create-table.d.ts
+/** Creates the table command; existing tables must match the connector's row schemas. */
+declare const makeCreateTableCommand: (schemas: Readonly<Record<string, Schema.Top>>) => Command.Command<"create-table", {
+  readonly table: string;
+  readonly catalog: string;
+  readonly location: Option.Option<string>;
+  readonly resource: Option.Option<string>;
+}, {}, ConnectorError | import("effect/Config").ConfigError | CliError.UserError, import("effect/http/HttpClient").HttpClient>;
+//#endregion
+//#region src/connector-app/demo.d.ts
+/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */
+declare const parseSize: (input: string) => Option.Option<number>;
+/**
+ * Creates the command that writes demo data over Arrow Flight, then exits.
+ * It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.
+ */
+declare const makeDemoCommand: (demo: DemoDefinition) => Command.Command<"demo", {
+  readonly size: number;
+  readonly seed: Option.Option<number>;
+  readonly batchSize: Option.Option<number>;
+}, {}, ConnectorError, never>;
+//#endregion
 //#region src/connector-app/start.d.ts
 type App = ConnectorDefinition;
 type StartOptions = {
   readonly port: number;
-  readonly initialCutoff?: Cursor.Value;
+  readonly initialCutoff?: Cursor$1.Value;
   readonly healthPath?: HttpRouter.PathInput;
   readonly metricsPath?: HttpRouter.PathInput | false;
   readonly statusPath?: HttpRouter.PathInput | false;
   readonly disableHttpLogger?: boolean;
 };
-declare const start: (app: App, options: StartOptions) => Effect.Effect<void, ConnectorError | effect_unstable_http_HttpServerError0.ServeError, Publisher | StateStore>;
+declare const start: (app: App, options: StartOptions) => Effect.Effect<void, ConnectorError | import("effect/http/HttpServerError").ServeError, StateStore | Ingestor>;
+//#endregion
+//#region src/connector-app/setup.d.ts
+/** Creates or verifies all bound tables without loading provider credentials. */
+declare const makeSetupCommand: (schemas: Readonly<Record<string, Schema.Top>>) => Command.Command<"setup", {
+  readonly checkpointIntervalSec: Option.Option<number>;
+  readonly retryInterval: Duration.Duration;
+  readonly retries: number;
+}, {}, ConnectorError | import("effect/Config").ConfigError | RuntimeConfigError | CatalogManager.CatalogManagerError, import("effect/http/HttpClient").HttpClient | import("effect/Scope").Scope>;
 declare namespace index_d_exports {
-  export { App, CheckOptions, CheckResult, ResourceCheckResult, StartOptions, check, start };
+  export { App, CheckOptions, CheckResult, ResourceCheckResult, StartOptions, check, makeCreateTableCommand, makeDemoCommand, makeSetupCommand, parseSize, start };
 }
 //#endregion
-export { telemetry_d_exports as a, Fetch as c, CheckResult as d, ResourceCheckResult as f, start as i, Resource as l, App as n, Connector as o, check as p, StartOptions as r, Cursor$1 as s, index_d_exports as t, CheckOptions as u };
-//# sourceMappingURL=index-C5Ma7V7t.d.ts.map
+export { check as _, start as a, makeCreateTableCommand as c, Cursor as d, Fetch as f, ResourceCheckResult as g, CheckResult as h, StartOptions as i, telemetry_d_exports as l, CheckOptions as m, makeSetupCommand as n, makeDemoCommand as o, Resource as p, App as r, parseSize as s, index_d_exports as t, Connector as u };
+//# sourceMappingURL=index-CZKx2Yce.d.mts.map

package/dist/{index-DYpEMJf-.d.ts → index-CgvFomO6.d.mts}

@@ -1,7 +1,6 @@
-import { t as ConnectorError } from "./index-DKoRwyFn.js";
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
 import { Config, Context, Effect, Layer, Schema } from "effect";
-import { SqlClient } from "effect/unstable/sql";
-
+import { SqlClient } from "effect/sql";
 //#region src/state-store/schema.d.ts
 /** Durable progress for a finite historical backfill. */
 declare const BackfillStateSchema: Schema.Struct<{
@@ -16,7 +15,7 @@ declare const ChangesStateSchema: Schema.Struct<{
   readonly lastSuccessAt: Schema.optional<Schema.String>;
 }>;
 declare const StateSourceSchema: Schema.Literals<readonly ["backfill", "changes"]>;
-declare const StateOperationSchema: Schema.Literals<readonly ["fetch", "publish", "checkpoint"]>;
+declare const StateOperationSchema: Schema.Literals<readonly ["fetch", "ingest", "checkpoint"]>;
 /** Complete persisted state assembled from independently stored components. */
 declare const ResourceStateSchema: Schema.Struct<{
   readonly backfill: Schema.optional<Schema.Struct<{
@@ -32,8 +31,8 @@ declare const ResourceStateSchema: Schema.Struct<{
   readonly lastError: Schema.optional<Schema.Struct<{
     readonly at: Schema.String;
     readonly source: Schema.Literals<readonly ["backfill", "changes"]>;
-    readonly operation: Schema.Literals<readonly ["fetch", "publish", "checkpoint"]>;
-    readonly code: Schema.Literals<readonly ["fetch_failed", "publish_failed", "checkpoint_failed"]>;
+    readonly operation: Schema.Literals<readonly ["fetch", "ingest", "checkpoint"]>;
+    readonly code: Schema.Literals<readonly ["fetch_failed", "ingest_failed", "checkpoint_failed"]>;
     readonly message: Schema.String;
   }>>;
 }>;
@@ -92,4 +91,4 @@ declare namespace index_d_exports {
 }
 //#endregion
 export { StateStoreService as a, StateStore as i, layerSql as n, layerMemory as r, index_d_exports as t };
-//# sourceMappingURL=index-DYpEMJf-.d.ts.map
+//# sourceMappingURL=index-CgvFomO6.d.mts.map

package/dist/{index-bfjH7Yb0.d.ts → index-DeTwIUKl.d.mts}

@@ -1,10 +1,9 @@
-import { t as ConnectorError } from "./index-DKoRwyFn.js";
-import { l as ResourceDefinition, n as ConnectorDefinition, r as Cursor, x as WebhookRoute } from "./types-jVD5Ys43.js";
-import { i as Publisher } from "./service-CUjGhe-7.js";
-import { i as StateStore } from "./index-DYpEMJf-.js";
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
+import { d as ResourceDefinition, i as Cursor, k as WebhookRoute, r as ConnectorDefinition } from "./types-NtSv6gwl.mjs";
+import { i as Ingestor } from "./service-BRoy_EId.mjs";
+import { i as StateStore } from "./index-CgvFomO6.mjs";
 import { Effect } from "effect";
-import { HttpRouter, HttpServer } from "effect/unstable/http";
-
+import { HttpRouter, HttpServer } from "effect/http";
 //#region src/ingestion/engine.d.ts
 type RunBaseOptions = {
   readonly initialCutoff?: Cursor.Value;
@@ -25,14 +24,14 @@ type RunWebhookOptions = RunOptions & {
   readonly webhook: NonNullable<RunOptions["webhook"]>;
 };
 /**
- * Runs all connector ingestion loops. Callers provide the Publisher and
+ * Runs all connector ingestion loops. Callers provide the Ingestor and
  * StateStore layers; webhook mode additionally requires an HttpServer layer.
  */
-declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options?: RunNoWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Publisher>;
-declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options: RunWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
+declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options?: RunNoWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Ingestor>;
+declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options: RunWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Ingestor | HttpServer.HttpServer>;
 declare namespace index_d_exports {
   export { RunOptions, run };
 }
 //#endregion
 export { RunOptions as n, run as r, index_d_exports as t };
-//# sourceMappingURL=index-bfjH7Yb0.d.ts.map
+//# sourceMappingURL=index-DeTwIUKl.d.mts.map

package/dist/{index-DKoRwyFn.d.ts → index-DkVNO6ks.d.mts}

@@ -1,8 +1,5 @@
-import * as effect_Types0 from "effect/Types";
-import * as effect_Cause0 from "effect/Cause";
-
 //#region src/errors/connector-error.d.ts
-declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
+declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
   readonly _tag: "ConnectorError";
 } & Readonly<A>;
 declare class ConnectorError extends ConnectorError_base<{
@@ -11,4 +8,4 @@ declare class ConnectorError extends ConnectorError_base<{
 }> {}
 //#endregion
 export { ConnectorError as t };
-//# sourceMappingURL=index-DKoRwyFn.d.ts.map
+//# sourceMappingURL=index-DkVNO6ks.d.mts.map

package/dist/{index-BKw7jA2a.d.ts → index-dHS59EZl.d.mts}

@@ -1,9 +1,8 @@
-import { C as WebhookRouteInput, S as WebhookRouteContext, c as ResourceBatch, l as ResourceDefinition, x as WebhookRoute, y as WebhookAckMode } from "./types-jVD5Ys43.js";
-import { i as Publisher } from "./service-CUjGhe-7.js";
+import { A as WebhookRouteContext, D as WebhookAckMode, d as ResourceDefinition, j as WebhookRouteInput, k as WebhookRoute, u as ResourceBatch } from "./types-NtSv6gwl.mjs";
+import { i as Ingestor } from "./service-BRoy_EId.mjs";
+import "./index-uFa2wwYo.mjs";
 import { Context, Queue } from "effect";
-import { HttpRouter } from "effect/unstable/http";
-import * as effect_Layer1 from "effect/Layer";
-
+import { HttpRouter } from "effect/http";
 //#region src/webhook/server.d.ts
 type QueuedWebhookBatch = {
   readonly resource: string;
@@ -13,13 +12,16 @@ declare const WebhookQueue_base: Context.ServiceClass<WebhookQueue, "@useairfoil
   readonly queue: Queue.Queue<QueuedWebhookBatch>;
 }>;
 declare class WebhookQueue extends WebhookQueue_base {}
-declare const router: (routes: ReadonlyArray<WebhookRoute>, options: {
+type RouterOptions = {
   readonly connectorName: string;
-}) => effect_Layer1.Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", Publisher> | HttpRouter.Request<"Requires", WebhookQueue>>;
+  /** Events for other resources are skipped. */
+  readonly resourceNames: ReadonlyArray<string>;
+};
+declare const router: (routes: ReadonlyArray<WebhookRoute>, options: RouterOptions) => import("effect/Layer").Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", Ingestor> | HttpRouter.Request<"Requires", WebhookQueue>>;
 declare namespace index_d_exports {
   export { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput, route, router };
 }
 declare const route: <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(definition: WebhookRouteInput<Resources, Payload>) => WebhookRoute<Resources, Payload>;
 //#endregion
 export { route as n, router as r, index_d_exports as t };
-//# sourceMappingURL=index-BKw7jA2a.d.ts.map
+//# sourceMappingURL=index-dHS59EZl.d.mts.map

package/dist/{index-CR9HJDdr.d.ts → index-eFuoMZZ3.d.mts}

@@ -1,6 +1,4 @@
 import { Config, Option, Redacted, Schema, Types } from "effect";
-import * as effect_Effect0 from "effect/Effect";
-
 //#region src/manifest/number-schema.d.ts
 type NumberConstraints = {
   readonly integer?: boolean;
@@ -46,28 +44,28 @@ type ConfigFieldSpec = ConfigFieldDefinition & {
   readonly name: string;
 };
 /** Pairs an Effect runtime config with the metadata used to generate its form field. */
-type ConfigField<A$1> = {
-  readonly config: Config.Config<A$1>;
+type ConfigField<A> = {
+  readonly config: Config.Config<A>;
   readonly spec: ConfigFieldDefinition;
 };
 type FieldBaseOptions = {
   readonly runtimeKey: string;
   readonly description?: string;
 };
-type FieldOptions<A$1 extends string | number | boolean = string | number | boolean> = FieldBaseOptions & ({
+type FieldOptions<A extends string | number | boolean = string | number | boolean> = FieldBaseOptions & ({
   readonly required?: true;
-  readonly default?: A$1;
+  readonly default?: A;
 } | {
   readonly required: false;
-  readonly default: A$1;
+  readonly default: A;
 });
 type NumberFieldOptions = FieldOptions<number> & NumberConstraints;
 type FieldsRecord = Record<string, ConfigField<unknown>>;
-type ConfigOf<Fields$1 extends FieldsRecord> = { readonly [Key in keyof Fields$1]: Fields$1[Key] extends ConfigField<infer A> ? A : never };
+type ConfigOf<Fields extends FieldsRecord> = { readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A> ? A : never; };
 /** Effect config and serializable manifest metadata produced from one field definition. */
-type ConnectorConfigDef<Fields$1 extends FieldsRecord> = {
-  readonly fields: { readonly [Key in keyof Fields$1]: Fields$1[Key] extends ConfigField<infer A> ? Config.Config<A> : never };
-  readonly config: Config.Config<ConfigOf<Fields$1>>;
+type ConnectorConfigDef<Fields extends FieldsRecord> = {
+  readonly fields: { readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A> ? Config.Config<A> : never; };
+  readonly config: Config.Config<ConfigOf<Fields>>;
   readonly spec: ReadonlyArray<ConfigFieldSpec>;
 };
 type ConfigValuesOf<Def extends ConnectorConfigDef<FieldsRecord>> = Def extends ConnectorConfigDef<infer Fields> ? ConfigOf<Fields> : never;
@@ -86,7 +84,7 @@ declare const secret: (options: FieldBaseOptions & {
   readonly required?: true;
 }) => ConfigField<Redacted.Redacted<string>>;
 /** Makes an existing field optional and represents a missing value as `Option.none()`. */
-declare const optional: <A$1>(field: ConfigField<A$1>) => ConfigField<Option.Option<A$1>>;
+declare const optional: <A>(field: ConfigField<A>) => ConfigField<Option.Option<A>>;
 /**
  * Defines user-facing connector config once for both Effect runtime loading and
  * frontend form generation. The object keys become logical form/API field names;
@@ -103,7 +101,7 @@ declare const optional: <A$1>(field: ConfigField<A$1>) => ConfigField<Option.Opt
  * const formFields = config.spec;
  * ```
  */
-declare const define: <const Fields$1 extends FieldsRecord>(fields: Fields$1) => ConnectorConfigDef<Fields$1>;
+declare const define$1: <const Fields extends FieldsRecord>(fields: Fields) => ConnectorConfigDef<Fields>;
 //#endregion
 //#region src/manifest/manifest.d.ts
 type ResourceCapability = "backfill" | "changes" | "webhook";
@@ -131,11 +129,11 @@ type ConnectorManifest = {
  * });
  * ```
  */
-declare const define$1: <const Manifest extends ConnectorManifest>(manifest: Manifest) => Manifest;
+declare const define: <const Manifest extends ConnectorManifest>(manifest: Manifest) => Manifest;
 //#endregion
 //#region src/manifest/schema.d.ts
-type FieldValueOf<Field$1 extends ConfigFieldSpec> = Field$1["type"] extends "number" ? number : Field$1["type"] extends "boolean" ? boolean : Field$1["values"] extends ReadonlyArray<string> ? Field$1["values"][number] : string;
-type ConnectorConfigValues<Manifest extends ConnectorManifest> = Types.Simplify<{ readonly [Field in Manifest["config"][number] as Field["required"] extends true ? Field["name"] : never]: FieldValueOf<Field> } & { readonly [Field in Manifest["config"][number] as Field["required"] extends true ? never : Field["name"]]?: FieldValueOf<Field> | undefined }> & object;
+type FieldValueOf<Field extends ConfigFieldSpec> = Field["type"] extends "number" ? number : Field["type"] extends "boolean" ? boolean : Field["values"] extends ReadonlyArray<string> ? Field["values"][number] : string;
+type ConnectorConfigValues<Manifest extends ConnectorManifest> = Types.Simplify<{ readonly [Field in Manifest["config"][number] as Field["required"] extends true ? Field["name"] : never]: FieldValueOf<Field>; } & { readonly [Field in Manifest["config"][number] as Field["required"] extends true ? never : Field["name"]]?: FieldValueOf<Field> | undefined; }> & object;
 type ConnectorConfigSchema<Manifest extends ConnectorManifest> = Schema.Codec<ConnectorConfigValues<Manifest>, unknown>;
 /** Flat primitive document keyed by the runtime keys consumed by Effect Config. */
 type RuntimeConfigDocument = Readonly<Record<string, string | number | boolean>>;
@@ -153,7 +151,7 @@ type RuntimeConfigDocument = Readonly<Record<string, string | number | boolean>>
  */
 declare const configSchema: <const Manifest extends ConnectorManifest>(manifest: Manifest) => ConnectorConfigSchema<Manifest>;
 /** Decodes browser/API config, applies manifest defaults, and rejects undeclared fields. */
-declare const decodeConfig: <const Manifest extends ConnectorManifest>(manifest: Manifest, input: unknown) => effect_Effect0.Effect<ConnectorConfigValues<Manifest>, Schema.SchemaError, never>;
+declare const decodeConfig: <const Manifest extends ConnectorManifest>(manifest: Manifest, input: unknown) => import("effect/Effect").Effect<ConnectorConfigValues<Manifest>, Schema.SchemaError, never>;
 /**
  * Maps validated logical values to the flat runtime-key document mounted in a pod.
  * Missing optional values are omitted.
@@ -166,8 +164,8 @@ declare const decodeConfig: <const Manifest extends ConnectorManifest>(manifest:
  */
 declare const toRuntimeDocument: <const Manifest extends ConnectorManifest>(manifest: Manifest, values: ConnectorConfigValues<Manifest>) => RuntimeConfigDocument;
 declare namespace index_d_exports {
-  export { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef, ConnectorConfigSchema, ConnectorConfigValues, ConnectorManifest, ResourceCapability, RuntimeConfigDocument, boolean, configSchema, decodeConfig, define$1 as define, define as defineConfig, number, optional, secret, select, string, toRuntimeDocument };
+  export { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef, ConnectorConfigSchema, ConnectorConfigValues, ConnectorManifest, ResourceCapability, RuntimeConfigDocument, boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };
 }
 //#endregion
-export { number as _, configSchema as a, select as b, ConnectorManifest as c, ConfigField as d, ConfigFieldSpec as f, define as g, boolean as h, RuntimeConfigDocument as i, ResourceCapability as l, ConnectorConfigDef as m, ConnectorConfigSchema as n, decodeConfig as o, ConfigValuesOf as p, ConnectorConfigValues as r, toRuntimeDocument as s, index_d_exports as t, define$1 as u, optional as v, string as x, secret as y };
-//# sourceMappingURL=index-CR9HJDdr.d.ts.map
+export { number as _, configSchema as a, select as b, ConnectorManifest as c, ConfigField as d, ConfigFieldSpec as f, define$1 as g, boolean as h, RuntimeConfigDocument as i, ResourceCapability as l, ConnectorConfigDef as m, ConnectorConfigSchema as n, decodeConfig as o, ConfigValuesOf as p, ConnectorConfigValues as r, toRuntimeDocument as s, index_d_exports as t, define as u, optional as v, string as x, secret as y };
+//# sourceMappingURL=index-eFuoMZZ3.d.mts.map

package/dist/{index-ZSuiZX5H.d.ts → index-fHIJJoRe.d.mts}

@@ -1,13 +1,8 @@
-import { f as ResourceName, n as ConnectorDefinition } from "./types-jVD5Ys43.js";
 import { Config, Effect, FileSystem } from "effect";
-import * as Wings from "@useairfoil/wings";
-import * as effect_Types0 from "effect/Types";
-import * as effect_Cause0 from "effect/Cause";
-import * as effect_Layer0 from "effect/Layer";
-
+import { TableIdentifier } from "iceberg-js";
 //#region src/runtime-config/error.d.ts
 type RuntimeConfigErrorCode = "CONFIG_PATH_MISSING" | "CONFIG_FILE_UNREADABLE" | "CONFIG_JSON_INVALID" | "TABLE_BINDINGS_MISSING" | "TABLE_BINDINGS_INVALID" | "TABLE_BINDINGS_MISMATCH";
-declare const RuntimeConfigError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
+declare const RuntimeConfigError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
   readonly _tag: "RuntimeConfigError";
 } & Readonly<A>;
 /** Expected hosted-config failure with a stable code and diagnostic cause. */
@@ -20,11 +15,12 @@ declare class RuntimeConfigError extends RuntimeConfigError_base<{
 //#region src/runtime-config/config.d.ts
 /** Shared HTTP port used by hosted and sandbox connector entrypoints. */
 declare const httpPort: Config.Config<number>;
-/** Shared Wings client configuration supplied by the hosting platform. */
-declare const wingsClient: {
-  host: Config.Config<string>;
-  namespace: Config.Config<string>;
-};
+/** Engine URL, including the protocol. */
+declare const engineUri: Config.Config<URL>;
+/** Arrow Flight URL. Uses the engine URL when unset. */
+declare const flightUri: Config.Config<URL>;
+/** Catalog ID. */
+declare const catalog: Config.Config<string>;
 //#endregion
 //#region src/runtime-config/constants.d.ts
 /** Shared runtime keys owned by Airfoil rather than an individual connector. */
@@ -35,8 +31,9 @@ declare const PlatformRuntimeKey: {
   readonly connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID";
   readonly stateTable: "AIRFOIL_STATE_TABLE";
   readonly postgresConnectionString: "POSTGRES_CONNECTION_STRING";
-  readonly wingsHost: "WINGS_HOST";
-  readonly wingsNamespace: "WINGS_NAMESPACE";
+  readonly engineUri: "ENGINE_URI";
+  readonly flightUri: "FLIGHT_URI";
+  readonly catalog: "AIRFOIL_CATALOG";
   readonly otelEnabled: "OTEL_ENABLED";
   readonly otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT";
   readonly otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS";
@@ -60,26 +57,24 @@ declare const PlatformRuntimeDefault: {
  * program.pipe(Effect.provide(layerHosted()));
  * ```
  */
-declare const layerHosted: () => effect_Layer0.Layer<never, RuntimeConfigError, FileSystem.FileSystem>;
+declare const layerHosted: () => import("effect/Layer").Layer<never, RuntimeConfigError, FileSystem.FileSystem>;
 //#endregion
 //#region src/runtime-config/table-bindings.d.ts
-/** Table binding shape accepted in `AIRFOIL_TABLE_BINDINGS`. */
-type HostedTableBinding = string | {
-  readonly name: string;
-  readonly partition?: Wings.PartitionValue.Json;
+type BindingSource = {
+  readonly resources: ReadonlyArray<{
+    readonly name: string;
+  }>;
 };
-/** Validated table binding ready for the Wings publisher. */
-type ResolvedTableBinding = {
-  readonly name: string;
-  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
+/** An Iceberg table and its storage location for each connector resource. */
+type TableBinding = TableIdentifier & {
+  readonly location?: string | undefined;
 };
-/** One resolved binding for each resource in a connector manifest. */
-type ResolvedTableBindings<Connector extends ConnectorDefinition> = Readonly<Record<ResourceName<Connector["resources"]>, ResolvedTableBinding>>;
-/** Loads and validates the complete hosted Wings table map for a connector. */
-declare const loadTableBindings: <const Connector extends ConnectorDefinition>(connector: Connector) => Effect.Effect<ResolvedTableBindings<Connector>, RuntimeConfigError>;
+type ResolvedTableBindings<Source extends BindingSource> = Readonly<Partial<Record<Source["resources"][number]["name"], TableBinding>>>;
+/** Reads hosted table bindings and checks they name resources of the connector. */
+declare const loadTableBindings: <const Source extends BindingSource>(connector: Source) => Effect.Effect<ResolvedTableBindings<Source>, RuntimeConfigError>;
 declare namespace index_d_exports {
-  export { HostedTableBinding, PlatformRuntimeDefault, PlatformRuntimeKey, ResolvedTableBinding, ResolvedTableBindings, RuntimeConfigError, RuntimeConfigErrorCode, httpPort, layerHosted, loadTableBindings, wingsClient };
+  export { PlatformRuntimeDefault, PlatformRuntimeKey, ResolvedTableBindings, RuntimeConfigError, RuntimeConfigErrorCode, TableBinding, catalog, engineUri, flightUri, httpPort, layerHosted, loadTableBindings };
 }
 //#endregion
-export { loadTableBindings as a, PlatformRuntimeKey as c, RuntimeConfigError as d, RuntimeConfigErrorCode as f, ResolvedTableBindings as i, httpPort as l, HostedTableBinding as n, layerHosted as o, ResolvedTableBinding as r, PlatformRuntimeDefault as s, index_d_exports as t, wingsClient as u };
-//# sourceMappingURL=index-ZSuiZX5H.d.ts.map
+export { layerHosted as a, catalog as c, httpPort as d, RuntimeConfigError as f, loadTableBindings as i, engineUri as l, ResolvedTableBindings as n, PlatformRuntimeDefault as o, RuntimeConfigErrorCode as p, TableBinding as r, PlatformRuntimeKey as s, index_d_exports as t, flightUri as u };
+//# sourceMappingURL=index-fHIJJoRe.d.mts.map

package/dist/index-uFa2wwYo.d.mts

@@ -0,0 +1,27 @@
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
+import { E as TableSource } from "./types-NtSv6gwl.mjs";
+import { a as IngestorService, i as Ingestor, n as IngestResult, r as IngestSource, t as IngestOptions } from "./service-BRoy_EId.mjs";
+import { n as ResolvedTableBindings, t as index_d_exports$1 } from "./index-fHIJJoRe.mjs";
+import { Layer } from "effect";
+import { CatalogManager } from "@useairfoil/api";
+//#region src/ingestor/console.d.ts
+/** Logs batches without sending them over Arrow Flight. */
+declare const layerConsole: Layer.Layer<Ingestor, never, never>;
+//#endregion
+//#region src/ingestor/arrow-flight.d.ts
+/** One catalog and an Iceberg table for each resource that runs. */
+type ArrowFlightIngestorConfig<Source extends TableSource = TableSource> = {
+  readonly connector: Source;
+  readonly catalog: string;
+  readonly tables: ResolvedTableBindings<Source>;
+};
+/** Compiles local schemas and opens one Arrow Flight stream per bound resource. */
+declare const layerArrowFlight: <const Source extends TableSource>(config: ArrowFlightIngestorConfig<Source>) => Layer.Layer<Ingestor, ConnectorError, CatalogManager.CatalogManager>;
+/** Configures Arrow Flight ingestion from the hosted URL, catalog and table bindings. */
+declare const layerArrowFlightConfig: (connector: TableSource) => Layer.Layer<Ingestor, ConnectorError, never>;
+declare namespace index_d_exports {
+  export { ArrowFlightIngestorConfig, IngestOptions, IngestResult, IngestSource, Ingestor, IngestorService, layerArrowFlight, layerArrowFlightConfig, layerConsole };
+}
+//#endregion
+export { layerConsole as a, layerArrowFlightConfig as i, ArrowFlightIngestorConfig as n, layerArrowFlight as r, index_d_exports as t };
+//# sourceMappingURL=index-uFa2wwYo.d.mts.map