@useairfoil/connector-kit 0.5.3 → 0.6.1

package/src/ingestor/index.ts

@@ -0,0 +1,9 @@
+export { layerConsole } from "./console";
+export {
+  Ingestor,
+  type IngestOptions,
+  type IngestorService,
+  type IngestResult,
+  type IngestSource,
+} from "./service";
+export { layerWings, layerWingsConfig, type WingsIngestorConfig } from "./wings";

package/src/ingestor/instrumented.ts

@@ -0,0 +1,61 @@
+import { Effect, Metric } from "effect";
+
+import type { ResourceBatch } from "../core/types";
+
+import * as Metrics from "../metrics";
+import { Attr, EventAttr, EventName, SpanName, addCurrentSpanEvent } from "../telemetry";
+import { Ingestor, type IngestSource } from "./service";
+
+export const ingestBatch = Effect.fnUntraced(function* (options: {
+  readonly connector?: string;
+  readonly resource: string;
+  readonly source: IngestSource;
+  readonly batch: ResourceBatch;
+}) {
+  const metricAttributes = Metrics.withResourceAttributes({
+    connector: options.connector ?? "unknown",
+    resource: options.resource,
+    source: options.source,
+  });
+  const batchOutcome = (outcome: "success" | "error") =>
+    Metric.withAttributes(Metrics.batches, {
+      ...metricAttributes,
+      [Attr.batchOutcome]: outcome,
+    });
+
+  const ingestor = yield* Ingestor;
+
+  const ingest = ingestor.ingest(options).pipe(
+    Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })),
+    Effect.tapError(() =>
+      Effect.annotateCurrentSpan({
+        [Attr.ingestionSuccess]: false,
+        [Attr.errorPhase]: "ingest",
+      }),
+    ),
+  );
+
+  // onError keeps the failure, so the engine still sees it and skips the checkpoint.
+  const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: "producer" }).pipe(
+    Effect.trackDuration(Metric.withAttributes(Metrics.ingestDuration, metricAttributes)),
+    Effect.onError(() => Metric.update(batchOutcome("error"), 1)),
+  );
+
+  yield* Metric.update(batchOutcome("success"), 1);
+  yield* Metric.update(
+    Metric.withAttributes(Metrics.rowsIngested, metricAttributes),
+    options.batch.rows.length,
+  );
+  yield* Metric.update(
+    Metric.withAttributes(Metrics.batchSize, metricAttributes),
+    options.batch.rows.length,
+  );
+
+  if (options.batch.cursor !== undefined) {
+    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {
+      [EventAttr.batchCursor]: options.batch.cursor,
+    });
+  }
+
+  return result;
+});

package/src/ingestor/service.ts

@@ -0,0 +1,30 @@
+import { Context, type Effect } from "effect";
+
+import type { ResourceBatch } from "../core/types";
+import type { ConnectorError } from "../errors";
+
+export type IngestSource = "backfill" | "changes" | "webhook" | "demo";
+
+/** A resource batch and the source that produced it. */
+export type IngestOptions = {
+  readonly resource: string;
+  readonly source: IngestSource;
+  readonly batch: ResourceBatch;
+};
+
+/** Rows written, and their size as uncompressed Arrow buffers. */
+export type IngestResult = {
+  readonly rows: number;
+  readonly bytes: number;
+};
+
+/** Handles batches from connector sources. */
+export interface IngestorService {
+  readonly ingest: (options: IngestOptions) => Effect.Effect<IngestResult, ConnectorError>;
+  /** Resources it has tables for. Only these run. Without it, every resource runs. */
+  readonly resourceNames?: ReadonlyArray<string>;
+}
+
+export class Ingestor extends Context.Service<Ingestor, IngestorService>()(
+  "@useairfoil/connector-kit/Ingestor",
+) {}

package/src/ingestor/wings.ts

@@ -0,0 +1,143 @@
+import type { RecordBatch, TypeMap } from "apache-arrow";
+import type { TableIdentifier } from "iceberg-js";
+
+import * as Wings from "@useairfoil/wings";
+import { Config, Effect, Layer } from "effect";
+import { FetchHttpClient } from "effect/http";
+
+import type { TableSource } from "../core/types";
+
+import { ensureTable } from "../catalog/table";
+import { ConnectorError } from "../errors";
+import * as RuntimeConfig from "../runtime-config";
+import { makeRowEncoder, type RowEncoder } from "./arrow";
+import { Ingestor, type IngestorService } from "./service";
+
+/** One catalog and an Iceberg table for each resource that runs. */
+export type WingsIngestorConfig<Source extends TableSource = TableSource> = {
+  readonly connector: Source;
+  readonly catalog: string;
+  readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;
+};
+
+type IngestorEntry = {
+  readonly encode: RowEncoder;
+  readonly push: (batch: RecordBatch<TypeMap>) => Effect.Effect<void, ConnectorError>;
+};
+
+type PreparedTable = {
+  readonly resource: string;
+  readonly identifier: TableIdentifier;
+  readonly encode: RowEncoder;
+};
+
+const connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });
+
+/** Loads each table schema and opens one Wings stream per bound resource. */
+export const layerWings = <const Source extends TableSource>(
+  config: WingsIngestorConfig<Source>,
+): Layer.Layer<Ingestor, ConnectorError, Wings.CatalogManager.CatalogManager> =>
+  Layer.effect(Ingestor)(
+    Effect.gen(function* () {
+      const manager = yield* Wings.CatalogManager.CatalogManager;
+      const catalog = yield* manager
+        .getIcebergCatalog(config.catalog)
+        .pipe(Effect.mapError((cause) => connectorError("Failed to load Wings catalog", cause)));
+
+      const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;
+
+      const prepared: Array<PreparedTable> = [];
+      for (const resource of config.connector.resources) {
+        const binding = bindings[resource.name];
+        if (!binding) continue;
+
+        const { namespace, name, location } = binding;
+        const identifier = { namespace, name };
+        const schema = yield* ensureTable(resource.rowSchema, {
+          catalog,
+          identifier,
+          location,
+        }).pipe(
+          Effect.mapError((cause) =>
+            connectorError(`Failed to prepare table for ${resource.name}: ${cause.message}`, cause),
+          ),
+        );
+
+        const encode = yield* makeRowEncoder(schema, resource.version);
+        prepared.push({ resource: resource.name, identifier, encode });
+      }
+
+      const entries = new Map<string, IngestorEntry>();
+      for (const { resource, identifier, encode } of prepared) {
+        const ingestor = yield* manager
+          .ingestor({
+            catalog: config.catalog,
+            namespace: identifier.namespace,
+            table: identifier.name,
+          })
+          .pipe(
+            Effect.mapError((cause) =>
+              connectorError(`Failed to open ingestor for ${resource}`, cause),
+            ),
+          );
+
+        entries.set(resource, {
+          encode,
+          push: (batch) =>
+            ingestor
+              .push(batch)
+              .pipe(
+                Effect.mapError((cause) =>
+                  connectorError(`Failed to ingest rows for ${resource}`, cause),
+                ),
+              ),
+        });
+      }
+
+      const service: IngestorService = {
+        resourceNames: prepared.map(({ resource }) => resource),
+        ingest: ({ resource, batch }) => {
+          const entry = entries.get(resource);
+
+          if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));
+          if (batch.rows.length === 0) return Effect.succeed({ rows: 0, bytes: 0 });
+
+          return entry
+            .encode(batch.rows)
+            .pipe(
+              Effect.flatMap((encoded) =>
+                entry
+                  .push(encoded)
+                  .pipe(Effect.as({ rows: batch.rows.length, bytes: encoded.data.byteLength })),
+              ),
+            );
+        },
+      };
+
+      return Ingestor.of(service);
+    }),
+  );
+
+/** Reads the Wings URL, catalog and table bindings from hosted config. */
+export const layerWingsConfig = (connector: TableSource) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const { catalog, uri } = yield* Config.all({
+        catalog: RuntimeConfig.catalog,
+        uri: RuntimeConfig.wingsUri,
+      }).pipe(Effect.mapError((cause) => connectorError("Invalid Wings configuration", cause)));
+
+      if (uri.protocol !== "http:" && uri.protocol !== "https:") {
+        return yield* Effect.fail(connectorError("Invalid Wings URL"));
+      }
+
+      const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(
+        Effect.mapError((cause) => connectorError(cause.message, cause)),
+      );
+      const manager = Wings.CatalogManager.layer({ baseUrl: uri.href }).pipe(
+        Layer.provide(FetchHttpClient.layer),
+      );
+
+      return layerWings({ connector, catalog, tables }).pipe(Layer.provide(manager));
+    }),
+  );

package/src/manifest/config.ts

@@ -111,7 +111,7 @@ const withDefault = <A extends string | number | boolean>(
 
 /** Defines a required non-empty string, optionally with a default. */
 export const string = (options: FieldOptions<string>): ConfigField<string> => ({
-  config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
+  config: withDefault(Config.NonEmptyString(options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "string",
@@ -145,7 +145,7 @@ export const number = (options: NumberFieldOptions): ConfigField<number> => {
 
 /** Defines a boolean, optionally with a default. */
 export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> => ({
-  config: withDefault(Config.boolean(options.runtimeKey), options.default),
+  config: withDefault(Config.Boolean(options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "boolean",
@@ -160,7 +160,7 @@ export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> =>
 export const select = <const Values extends readonly [string, ...Array<string>]>(
   options: FieldOptions<Values[number]> & { readonly values: Values },
 ): ConfigField<Values[number]> => ({
-  config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
+  config: withDefault(Config.Literals(options.values, options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "select",

package/src/metrics.ts

@@ -1,9 +1,21 @@
-import { DateTime, Effect, Metric, Option } from "effect";
+import {
+  Cause,
+  DateTime,
+  type Duration,
+  Effect,
+  Metric,
+  Option,
+  Predicate,
+  Schedule,
+} from "effect";
+import { HttpClientError, HttpClientResponse } from "effect/http";
 
 import type { SyncState } from "./core/types";
 
-export type BatchSource = "backfill" | "changes" | "webhook";
-export type BatchOutcome = "accepted" | "rejected" | "error";
+import { Attr } from "./telemetry";
+
+export type BatchSource = "backfill" | "changes" | "webhook" | "demo";
+export type BatchOutcome = "success" | "error";
 export type ApiRetryReason = "transport" | "timeout" | "rate_limit" | "server_error";
 export type WebhookOutcome =
   | "ok"
@@ -41,61 +53,83 @@ export type ApiRetryMetricAttrs = {
   readonly reason: ApiRetryReason;
 };
 
-export const entitiesUpsertedTotal = Metric.counter("airfoil_connector_entities_upserted_total", {
-  description: "Total entity upsert mutations published by connector sources",
+export const batches = Metric.counter("airfoil.connector.batches", {
+  description: "Total resource batches ingested by connector sources",
   incremental: true,
+  attributes: { unit: "{batch}" },
 });
 
-export const entitiesDeletedTotal = Metric.counter("airfoil_connector_entities_deleted_total", {
-  description: "Total entity delete mutations published by connector sources",
+export const rowsIngested = Metric.counter("airfoil.connector.rows.ingested", {
+  description: "Total resource rows successfully ingested",
   incremental: true,
+  attributes: { unit: "{row}" },
 });
 
-export const batchesTotal = Metric.counter("airfoil_connector_batches_total", {
-  description: "Total resource batches published by connector sources",
-  incremental: true,
-});
-
-export const batchSize = Metric.histogram("airfoil_connector_batch_size", {
-  description: "Distribution of resource mutation batch sizes",
+export const batchSize = Metric.histogram("airfoil.connector.batch.size", {
+  description: "Distribution of resource row batch sizes",
   boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],
+  attributes: { unit: "{row}" },
 });
 
-export const publishDuration = Metric.timer("airfoil_connector_publish_duration_ms", {
-  description: "Publisher call duration in milliseconds",
+export const ingestDuration = Metric.timer("airfoil.connector.ingest.duration", {
+  description: "Ingestor call duration in milliseconds",
   boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],
+  attributes: { unit: "ms" },
 });
 
-export const webhookRequestsTotal = Metric.counter("airfoil_connector_webhook_requests_total", {
+export const webhookRequests = Metric.counter("airfoil.connector.webhook.requests", {
   description: "Total webhook HTTP requests handled by connector routes",
   incremental: true,
+  attributes: { unit: "{request}" },
 });
 
-export const webhookQueueDepth = Metric.gauge("airfoil_connector_webhook_queue_depth", {
-  description: "Current queued webhook batches waiting for publish",
+export const webhookQueueDepth = Metric.gauge("airfoil.connector.webhook.queue.depth", {
+  description: "Current queued webhook batches waiting for ingestion",
+  attributes: { unit: "{batch}" },
 });
 
-export const syncState = Metric.gauge("airfoil_connector_sync_state", {
+export const syncState = Metric.gauge("airfoil.connector.sync.state", {
   description: "Connector resource sync state as a Prometheus state-set gauge",
+  attributes: { unit: "1" },
 });
 
-export const lastSuccessTimestamp = Metric.gauge(
-  "airfoil_connector_last_success_timestamp_seconds",
-  {
-    description: "Unix timestamp of the last successful durable source checkpoint",
-  },
-);
+export const lastSuccessTimestamp = Metric.gauge("airfoil.connector.last_success.timestamp", {
+  description: "Unix timestamp of the last successful durable source checkpoint",
+  attributes: { unit: "s" },
+});
 
-export const apiRetriesTotal = Metric.counter("airfoil_connector_api_retries_total", {
+export const apiRetries = Metric.counter("airfoil.connector.api.retries", {
   description: "Total provider API retries performed by the connector",
   incremental: true,
+  attributes: { unit: "{retry}" },
+});
+
+const resourceAttributes = (attrs: ResourceMetricAttrs) => ({
+  [Attr.connectorName]: attrs.connector,
+  [Attr.resourceName]: attrs.resource,
+  [Attr.resourceSource]: attrs.source,
+});
+
+const syncAttributes = (attrs: SyncStateAttrs) => ({
+  [Attr.connectorName]: attrs.connector,
+  [Attr.resourceName]: attrs.resource,
 });
 
 export const recordWebhookRequest = (attrs: WebhookMetricAttrs) =>
-  Metric.update(Metric.withAttributes(webhookRequestsTotal, attrs), 1);
+  Metric.update(
+    Metric.withAttributes(webhookRequests, {
+      [Attr.connectorName]: attrs.connector,
+      [Attr.webhookPath]: attrs.path,
+      [Attr.webhookOutcome]: attrs.outcome,
+    }),
+    1,
+  );
 
 export const setWebhookQueueDepth = (connector: string, depth: number) =>
-  Metric.update(Metric.withAttributes(webhookQueueDepth, { connector }), depth);
+  Metric.update(
+    Metric.withAttributes(webhookQueueDepth, { [Attr.connectorName]: connector }),
+    depth,
+  );
 
 export const setLastSuccessTimestamp = (attrs: LastSuccessMetricAttrs, lastSuccessAt?: string) => {
   const timestamp =
@@ -105,11 +139,66 @@ export const setLastSuccessTimestamp = (attrs: LastSuccessMetricAttrs, lastSucce
           Option.map((dateTime) => DateTime.toEpochMillis(dateTime) / 1000),
           Option.getOrElse(() => 0),
         );
-  return Metric.update(Metric.withAttributes(lastSuccessTimestamp, attrs), timestamp);
+  return Metric.update(
+    Metric.withAttributes(lastSuccessTimestamp, {
+      [Attr.connectorName]: attrs.connector,
+      [Attr.resourceName]: attrs.resource,
+      [Attr.resourceSource]: attrs.source,
+    }),
+    timestamp,
+  );
 };
 
 export const recordApiRetry = (attrs: ApiRetryMetricAttrs) =>
-  Metric.update(Metric.withAttributes(apiRetriesTotal, attrs), 1);
+  Metric.update(
+    Metric.withAttributes(apiRetries, {
+      [Attr.connectorName]: attrs.connector,
+      [Attr.apiRetryReason]: attrs.reason,
+    }),
+    1,
+  );
+
+// Same statuses as `HttpClient.retryTransient`.
+const transientStatuses = new Set([408, 429, 500, 502, 503, 504]);
+
+// A connector only retries a `403` when the provider uses it for rate limits.
+const reasonForStatus = (status: number): ApiRetryReason =>
+  status === 408 ? "timeout" : status === 429 || status === 403 ? "rate_limit" : "server_error";
+
+// Effect exports the response type ID, not a guard.
+const isResponse = (value: unknown): value is HttpClientResponse.HttpClientResponse =>
+  Predicate.hasProperty(value, HttpClientResponse.TypeId);
+
+/** Why `HttpClient.retryTransient` retries. Undefined for a response it keeps. */
+const retryReason = (value: unknown): ApiRetryReason | undefined => {
+  if (isResponse(value)) {
+    return transientStatuses.has(value.status) ? reasonForStatus(value.status) : undefined;
+  }
+  if (HttpClientError.isHttpClientError(value) && value.reason._tag === "StatusCodeError") {
+    return reasonForStatus(value.reason.response.status);
+  }
+  return Cause.isTimeoutError(value) ? "timeout" : "transport";
+};
+
+/**
+ * A jittered exponential schedule for `HttpClient.retryTransient` that records
+ * each retry with its reason.
+ */
+export const retrySchedule = (options: {
+  readonly connector: string;
+  readonly baseDelay: Duration.Input;
+  readonly times: number;
+}) =>
+  Schedule.exponential(options.baseDelay).pipe(
+    Schedule.jittered,
+    Schedule.upTo({ times: options.times }),
+    Schedule.tap(({ input }) => {
+      const reason = retryReason(input);
+      return reason === undefined
+        ? Effect.void
+        : recordApiRetry({ connector: options.connector, reason });
+    }),
+  );
 
 const syncStates = ["pending", "backfilling", "live", "error"] as const;
 
@@ -119,10 +208,13 @@ export const setSyncState = (attrs: SyncStateAttrs, current: SyncState) =>
     (state) =>
       Metric.update(
         Metric.withAttributes(syncState, {
-          ...attrs,
-          state,
+          ...syncAttributes(attrs),
+          [Attr.syncState]: state,
         }),
         state === current ? 1 : 0,
       ),
     { discard: true },
   );
+
+/** @internal Converts the ergonomic SDK input to canonical OTel attributes. */
+export const withResourceAttributes = resourceAttributes;

package/src/runtime-config/config.ts

@@ -0,0 +1,14 @@
+import { Config } from "effect";
+
+import { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
+
+/** Shared HTTP port used by hosted and sandbox connector entrypoints. */
+export const httpPort = Config.Port(PlatformRuntimeKey.httpPort).pipe(
+  Config.withDefault(PlatformRuntimeDefault.httpPort),
+);
+
+/** Wings URL, including the protocol. */
+export const wingsUri = Config.URL(PlatformRuntimeKey.wingsUri);
+
+/** Wings catalog ID. */
+export const catalog = Config.NonEmptyString(PlatformRuntimeKey.catalog);

package/src/runtime-config/constants.ts

@@ -3,11 +3,13 @@ import { HashSet } from "effect";
 /** Shared runtime keys owned by Airfoil rather than an individual connector. */
 export const PlatformRuntimeKey = {
   configPath: "AIRFOIL_CONFIG_PATH",
+  httpPort: "AIRFOIL_HTTP_PORT",
+  tableBindings: "AIRFOIL_TABLE_BINDINGS",
   connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
   stateTable: "AIRFOIL_STATE_TABLE",
   postgresConnectionString: "POSTGRES_CONNECTION_STRING",
-  wingsHost: "WINGS_HOST",
-  wingsNamespace: "WINGS_NAMESPACE",
+  wingsUri: "WINGS_URI",
+  catalog: "AIRFOIL_CATALOG",
   otelEnabled: "OTEL_ENABLED",
   otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
   otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS",
@@ -18,6 +20,7 @@ export const PlatformRuntimeKey = {
 
 /** Defaults for optional platform-owned runtime configuration. */
 export const PlatformRuntimeDefault = {
+  httpPort: 8080,
   stateTable: "_airfoil_connectors_state",
 } as const;
 

package/src/runtime-config/error.ts

@@ -3,7 +3,10 @@ import { Data } from "effect";
 export type RuntimeConfigErrorCode =
   | "CONFIG_PATH_MISSING"
   | "CONFIG_FILE_UNREADABLE"
-  | "CONFIG_JSON_INVALID";
+  | "CONFIG_JSON_INVALID"
+  | "TABLE_BINDINGS_MISSING"
+  | "TABLE_BINDINGS_INVALID"
+  | "TABLE_BINDINGS_MISMATCH";
 
 /** Expected hosted-config failure with a stable code and diagnostic cause. */
 export class RuntimeConfigError extends Data.TaggedError("RuntimeConfigError")<{

package/src/runtime-config/index.ts

@@ -1,3 +1,5 @@
 export { RuntimeConfigError, type RuntimeConfigErrorCode } from "./error";
+export { catalog, httpPort, wingsUri } from "./config";
 export { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
 export { layerHosted } from "./layer";
+export { type ResolvedTableBindings, type TableBinding, loadTableBindings } from "./table-bindings";

package/src/runtime-config/layer.ts

@@ -8,7 +8,7 @@ const providerFromFile: Effect.Effect<
   RuntimeConfigError,
   FileSystem.FileSystem
 > = Effect.gen(function* () {
-  const configPath = yield* Config.nonEmptyString(PlatformRuntimeKey.configPath).pipe(
+  const configPath = yield* Config.NonEmptyString(PlatformRuntimeKey.configPath).pipe(
     Effect.mapError(
       (error) =>
         new RuntimeConfigError({

package/src/runtime-config/table-bindings.ts

@@ -0,0 +1,92 @@
+import type { TableIdentifier } from "iceberg-js";
+
+import { Config, Effect, Schema } from "effect";
+
+import type { TableSource } from "../core/types";
+
+import { PlatformRuntimeKey } from "./constants";
+import { RuntimeConfigError } from "./error";
+
+/** An Iceberg table and its storage location for each connector resource. */
+export type TableBinding = TableIdentifier & { readonly location?: string | undefined };
+
+export type ResolvedTableBindings<Source extends TableSource> = Readonly<
+  Partial<Record<Source["resources"][number]["name"], TableBinding>>
+>;
+
+const TableIdentifierSchema = Schema.Struct({
+  namespace: Schema.NonEmptyArray(Schema.NonEmptyString),
+  name: Schema.NonEmptyString,
+  location: Schema.optional(Schema.NonEmptyString),
+});
+
+const RawBindingsSchema = Schema.fromJsonString(
+  Schema.Record(Schema.String, TableIdentifierSchema),
+);
+
+const asError = (cause: unknown): Error =>
+  cause instanceof Error ? cause : new Error(String(cause));
+
+const fail = (code: "TABLE_BINDINGS_INVALID" | "TABLE_BINDINGS_MISMATCH", message: string) =>
+  new RuntimeConfigError({ code, message, cause: new Error(message) });
+
+/** Reads hosted table bindings and checks they name resources of the connector. */
+export const loadTableBindings = <const Source extends TableSource>(
+  connector: Source,
+): Effect.Effect<ResolvedTableBindings<Source>, RuntimeConfigError> =>
+  Effect.gen(function* () {
+    const raw = yield* Config.NonEmptyString(PlatformRuntimeKey.tableBindings).pipe(
+      Effect.mapError(
+        (cause) =>
+          new RuntimeConfigError({
+            code: "TABLE_BINDINGS_MISSING",
+            message: `${PlatformRuntimeKey.tableBindings} is required in hosted mode`,
+            cause: asError(cause),
+          }),
+      ),
+    );
+
+    const decoded = yield* Schema.decodeUnknownEffect(RawBindingsSchema)(raw, {
+      onExcessProperty: "error",
+    }).pipe(
+      Effect.mapError(
+        (cause) =>
+          new RuntimeConfigError({
+            code: "TABLE_BINDINGS_INVALID",
+            message: "Hosted table bindings are invalid",
+            cause: asError(cause),
+          }),
+      ),
+    );
+
+    const actual = Object.keys(decoded);
+    if (actual.length === 0) {
+      return yield* Effect.fail(
+        fail("TABLE_BINDINGS_INVALID", "Hosted table bindings name no resources"),
+      );
+    }
+
+    // Bindings come from the platform, resources from the connector. An unknown name
+    // means one of them is out of date, and we could write to the wrong table.
+    const expected = new Set(connector.resources.map((resource) => resource.name));
+    const unknown = actual.filter((name) => !expected.has(name));
+    if (unknown.length > 0) {
+      return yield* Effect.fail(
+        fail(
+          "TABLE_BINDINGS_MISMATCH",
+          `Hosted table bindings name resources the connector doesn't have (${unknown.length})`,
+        ),
+      );
+    }
+
+    return Object.fromEntries(
+      Object.entries(decoded).map(([resource, identifier]) => [
+        resource,
+        {
+          namespace: Array.from(identifier.namespace),
+          name: identifier.name,
+          ...(identifier.location === undefined ? {} : { location: identifier.location }),
+        },
+      ]),
+    ) as ResolvedTableBindings<Source>;
+  });

package/src/state-store/layer-key-value-store.ts

@@ -1,5 +1,5 @@
 import { DateTime, Effect, Layer, Option, Schema } from "effect";
-import { KeyValueStore } from "effect/unstable/persistence";
+import { KeyValueStore } from "effect/persistence";
 
 import { ConnectorError } from "../errors";
 import { resourceComponentKey, resourceErrorKey } from "./keys";
@@ -33,8 +33,13 @@ const resourceErrorContext = (
   switch (operation) {
     case "fetch":
       return { source, operation, code: "fetch_failed", message: `${sourceName} fetch failed` };
-    case "publish":
-      return { source, operation, code: "publish_failed", message: `${sourceName} publish failed` };
+    case "ingest":
+      return {
+        source,
+        operation,
+        code: "ingest_failed",
+        message: `${sourceName} ingestion failed`,
+      };
     case "checkpoint":
       return {
         source,