@useairfoil/connector-kit 0.6.0 → 0.6.1

package/src/ingestor/index.ts

@@ -0,0 +1,9 @@
+export { layerConsole } from "./console";
+export {
+  Ingestor,
+  type IngestOptions,
+  type IngestorService,
+  type IngestResult,
+  type IngestSource,
+} from "./service";
+export { layerWings, layerWingsConfig, type WingsIngestorConfig } from "./wings";

package/src/ingestor/instrumented.ts

@@ -0,0 +1,61 @@
+import { Effect, Metric } from "effect";
+
+import type { ResourceBatch } from "../core/types";
+
+import * as Metrics from "../metrics";
+import { Attr, EventAttr, EventName, SpanName, addCurrentSpanEvent } from "../telemetry";
+import { Ingestor, type IngestSource } from "./service";
+
+export const ingestBatch = Effect.fnUntraced(function* (options: {
+  readonly connector?: string;
+  readonly resource: string;
+  readonly source: IngestSource;
+  readonly batch: ResourceBatch;
+}) {
+  const metricAttributes = Metrics.withResourceAttributes({
+    connector: options.connector ?? "unknown",
+    resource: options.resource,
+    source: options.source,
+  });
+  const batchOutcome = (outcome: "success" | "error") =>
+    Metric.withAttributes(Metrics.batches, {
+      ...metricAttributes,
+      [Attr.batchOutcome]: outcome,
+    });
+
+  const ingestor = yield* Ingestor;
+
+  const ingest = ingestor.ingest(options).pipe(
+    Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })),
+    Effect.tapError(() =>
+      Effect.annotateCurrentSpan({
+        [Attr.ingestionSuccess]: false,
+        [Attr.errorPhase]: "ingest",
+      }),
+    ),
+  );
+
+  // onError keeps the failure, so the engine still sees it and skips the checkpoint.
+  const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: "producer" }).pipe(
+    Effect.trackDuration(Metric.withAttributes(Metrics.ingestDuration, metricAttributes)),
+    Effect.onError(() => Metric.update(batchOutcome("error"), 1)),
+  );
+
+  yield* Metric.update(batchOutcome("success"), 1);
+  yield* Metric.update(
+    Metric.withAttributes(Metrics.rowsIngested, metricAttributes),
+    options.batch.rows.length,
+  );
+  yield* Metric.update(
+    Metric.withAttributes(Metrics.batchSize, metricAttributes),
+    options.batch.rows.length,
+  );
+
+  if (options.batch.cursor !== undefined) {
+    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {
+      [EventAttr.batchCursor]: options.batch.cursor,
+    });
+  }
+
+  return result;
+});

package/src/ingestor/service.ts

@@ -0,0 +1,30 @@
+import { Context, type Effect } from "effect";
+
+import type { ResourceBatch } from "../core/types";
+import type { ConnectorError } from "../errors";
+
+export type IngestSource = "backfill" | "changes" | "webhook" | "demo";
+
+/** A resource batch and the source that produced it. */
+export type IngestOptions = {
+  readonly resource: string;
+  readonly source: IngestSource;
+  readonly batch: ResourceBatch;
+};
+
+/** Rows written, and their size as uncompressed Arrow buffers. */
+export type IngestResult = {
+  readonly rows: number;
+  readonly bytes: number;
+};
+
+/** Handles batches from connector sources. */
+export interface IngestorService {
+  readonly ingest: (options: IngestOptions) => Effect.Effect<IngestResult, ConnectorError>;
+  /** Resources it has tables for. Only these run. Without it, every resource runs. */
+  readonly resourceNames?: ReadonlyArray<string>;
+}
+
+export class Ingestor extends Context.Service<Ingestor, IngestorService>()(
+  "@useairfoil/connector-kit/Ingestor",
+) {}

package/src/ingestor/wings.ts

@@ -0,0 +1,143 @@
+import type { RecordBatch, TypeMap } from "apache-arrow";
+import type { TableIdentifier } from "iceberg-js";
+
+import * as Wings from "@useairfoil/wings";
+import { Config, Effect, Layer } from "effect";
+import { FetchHttpClient } from "effect/http";
+
+import type { TableSource } from "../core/types";
+
+import { ensureTable } from "../catalog/table";
+import { ConnectorError } from "../errors";
+import * as RuntimeConfig from "../runtime-config";
+import { makeRowEncoder, type RowEncoder } from "./arrow";
+import { Ingestor, type IngestorService } from "./service";
+
+/** One catalog and an Iceberg table for each resource that runs. */
+export type WingsIngestorConfig<Source extends TableSource = TableSource> = {
+  readonly connector: Source;
+  readonly catalog: string;
+  readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;
+};
+
+type IngestorEntry = {
+  readonly encode: RowEncoder;
+  readonly push: (batch: RecordBatch<TypeMap>) => Effect.Effect<void, ConnectorError>;
+};
+
+type PreparedTable = {
+  readonly resource: string;
+  readonly identifier: TableIdentifier;
+  readonly encode: RowEncoder;
+};
+
+const connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });
+
+/** Loads each table schema and opens one Wings stream per bound resource. */
+export const layerWings = <const Source extends TableSource>(
+  config: WingsIngestorConfig<Source>,
+): Layer.Layer<Ingestor, ConnectorError, Wings.CatalogManager.CatalogManager> =>
+  Layer.effect(Ingestor)(
+    Effect.gen(function* () {
+      const manager = yield* Wings.CatalogManager.CatalogManager;
+      const catalog = yield* manager
+        .getIcebergCatalog(config.catalog)
+        .pipe(Effect.mapError((cause) => connectorError("Failed to load Wings catalog", cause)));
+
+      const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;
+
+      const prepared: Array<PreparedTable> = [];
+      for (const resource of config.connector.resources) {
+        const binding = bindings[resource.name];
+        if (!binding) continue;
+
+        const { namespace, name, location } = binding;
+        const identifier = { namespace, name };
+        const schema = yield* ensureTable(resource.rowSchema, {
+          catalog,
+          identifier,
+          location,
+        }).pipe(
+          Effect.mapError((cause) =>
+            connectorError(`Failed to prepare table for ${resource.name}: ${cause.message}`, cause),
+          ),
+        );
+
+        const encode = yield* makeRowEncoder(schema, resource.version);
+        prepared.push({ resource: resource.name, identifier, encode });
+      }
+
+      const entries = new Map<string, IngestorEntry>();
+      for (const { resource, identifier, encode } of prepared) {
+        const ingestor = yield* manager
+          .ingestor({
+            catalog: config.catalog,
+            namespace: identifier.namespace,
+            table: identifier.name,
+          })
+          .pipe(
+            Effect.mapError((cause) =>
+              connectorError(`Failed to open ingestor for ${resource}`, cause),
+            ),
+          );
+
+        entries.set(resource, {
+          encode,
+          push: (batch) =>
+            ingestor
+              .push(batch)
+              .pipe(
+                Effect.mapError((cause) =>
+                  connectorError(`Failed to ingest rows for ${resource}`, cause),
+                ),
+              ),
+        });
+      }
+
+      const service: IngestorService = {
+        resourceNames: prepared.map(({ resource }) => resource),
+        ingest: ({ resource, batch }) => {
+          const entry = entries.get(resource);
+
+          if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));
+          if (batch.rows.length === 0) return Effect.succeed({ rows: 0, bytes: 0 });
+
+          return entry
+            .encode(batch.rows)
+            .pipe(
+              Effect.flatMap((encoded) =>
+                entry
+                  .push(encoded)
+                  .pipe(Effect.as({ rows: batch.rows.length, bytes: encoded.data.byteLength })),
+              ),
+            );
+        },
+      };
+
+      return Ingestor.of(service);
+    }),
+  );
+
+/** Reads the Wings URL, catalog and table bindings from hosted config. */
+export const layerWingsConfig = (connector: TableSource) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const { catalog, uri } = yield* Config.all({
+        catalog: RuntimeConfig.catalog,
+        uri: RuntimeConfig.wingsUri,
+      }).pipe(Effect.mapError((cause) => connectorError("Invalid Wings configuration", cause)));
+
+      if (uri.protocol !== "http:" && uri.protocol !== "https:") {
+        return yield* Effect.fail(connectorError("Invalid Wings URL"));
+      }
+
+      const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(
+        Effect.mapError((cause) => connectorError(cause.message, cause)),
+      );
+      const manager = Wings.CatalogManager.layer({ baseUrl: uri.href }).pipe(
+        Layer.provide(FetchHttpClient.layer),
+      );
+
+      return layerWings({ connector, catalog, tables }).pipe(Layer.provide(manager));
+    }),
+  );

package/src/manifest/config.ts

@@ -111,7 +111,7 @@ const withDefault = <A extends string | number | boolean>(
 
 /** Defines a required non-empty string, optionally with a default. */
 export const string = (options: FieldOptions<string>): ConfigField<string> => ({
-  config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
+  config: withDefault(Config.NonEmptyString(options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "string",
@@ -145,7 +145,7 @@ export const number = (options: NumberFieldOptions): ConfigField<number> => {
 
 /** Defines a boolean, optionally with a default. */
 export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> => ({
-  config: withDefault(Config.boolean(options.runtimeKey), options.default),
+  config: withDefault(Config.Boolean(options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "boolean",
@@ -160,7 +160,7 @@ export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> =>
 export const select = <const Values extends readonly [string, ...Array<string>]>(
   options: FieldOptions<Values[number]> & { readonly values: Values },
 ): ConfigField<Values[number]> => ({
-  config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
+  config: withDefault(Config.Literals(options.values, options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "select",

package/src/metrics.ts

@@ -1,11 +1,21 @@
-import { DateTime, Effect, Metric, Option } from "effect";
+import {
+  Cause,
+  DateTime,
+  type Duration,
+  Effect,
+  Metric,
+  Option,
+  Predicate,
+  Schedule,
+} from "effect";
+import { HttpClientError, HttpClientResponse } from "effect/http";
 
 import type { SyncState } from "./core/types";
 
 import { Attr } from "./telemetry";
 
-export type BatchSource = "backfill" | "changes" | "webhook";
-export type BatchOutcome = "accepted" | "rejected" | "error";
+export type BatchSource = "backfill" | "changes" | "webhook" | "demo";
+export type BatchOutcome = "success" | "error";
 export type ApiRetryReason = "transport" | "timeout" | "rate_limit" | "server_error";
 export type WebhookOutcome =
   | "ok"
@@ -43,32 +53,26 @@ export type ApiRetryMetricAttrs = {
   readonly reason: ApiRetryReason;
 };
 
-export const entitiesUpserted = Metric.counter("airfoil.connector.entity.upserts", {
-  description: "Total entity upsert mutations published by connector sources",
-  incremental: true,
-  attributes: { unit: "{mutation}" },
-});
-
-export const entitiesDeleted = Metric.counter("airfoil.connector.entity.deletes", {
-  description: "Total entity delete mutations published by connector sources",
+export const batches = Metric.counter("airfoil.connector.batches", {
+  description: "Total resource batches ingested by connector sources",
   incremental: true,
-  attributes: { unit: "{mutation}" },
+  attributes: { unit: "{batch}" },
 });
 
-export const batches = Metric.counter("airfoil.connector.batches", {
-  description: "Total resource batches published by connector sources",
+export const rowsIngested = Metric.counter("airfoil.connector.rows.ingested", {
+  description: "Total resource rows successfully ingested",
   incremental: true,
-  attributes: { unit: "{batch}" },
+  attributes: { unit: "{row}" },
 });
 
 export const batchSize = Metric.histogram("airfoil.connector.batch.size", {
-  description: "Distribution of resource mutation batch sizes",
+  description: "Distribution of resource row batch sizes",
   boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],
-  attributes: { unit: "{mutation}" },
+  attributes: { unit: "{row}" },
 });
 
-export const publishDuration = Metric.timer("airfoil.connector.publish.duration", {
-  description: "Publisher call duration in milliseconds",
+export const ingestDuration = Metric.timer("airfoil.connector.ingest.duration", {
+  description: "Ingestor call duration in milliseconds",
   boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],
   attributes: { unit: "ms" },
 });
@@ -80,7 +84,7 @@ export const webhookRequests = Metric.counter("airfoil.connector.webhook.request
 });
 
 export const webhookQueueDepth = Metric.gauge("airfoil.connector.webhook.queue.depth", {
-  description: "Current queued webhook batches waiting for publish",
+  description: "Current queued webhook batches waiting for ingestion",
   attributes: { unit: "{batch}" },
 });
 
@@ -154,6 +158,48 @@ export const recordApiRetry = (attrs: ApiRetryMetricAttrs) =>
     1,
   );
 
+// Same statuses as `HttpClient.retryTransient`.
+const transientStatuses = new Set([408, 429, 500, 502, 503, 504]);
+
+// A connector only retries a `403` when the provider uses it for rate limits.
+const reasonForStatus = (status: number): ApiRetryReason =>
+  status === 408 ? "timeout" : status === 429 || status === 403 ? "rate_limit" : "server_error";
+
+// Effect exports the response type ID, not a guard.
+const isResponse = (value: unknown): value is HttpClientResponse.HttpClientResponse =>
+  Predicate.hasProperty(value, HttpClientResponse.TypeId);
+
+/** Why `HttpClient.retryTransient` retries. Undefined for a response it keeps. */
+const retryReason = (value: unknown): ApiRetryReason | undefined => {
+  if (isResponse(value)) {
+    return transientStatuses.has(value.status) ? reasonForStatus(value.status) : undefined;
+  }
+  if (HttpClientError.isHttpClientError(value) && value.reason._tag === "StatusCodeError") {
+    return reasonForStatus(value.reason.response.status);
+  }
+  return Cause.isTimeoutError(value) ? "timeout" : "transport";
+};
+
+/**
+ * A jittered exponential schedule for `HttpClient.retryTransient` that records
+ * each retry with its reason.
+ */
+export const retrySchedule = (options: {
+  readonly connector: string;
+  readonly baseDelay: Duration.Input;
+  readonly times: number;
+}) =>
+  Schedule.exponential(options.baseDelay).pipe(
+    Schedule.jittered,
+    Schedule.upTo({ times: options.times }),
+    Schedule.tap(({ input }) => {
+      const reason = retryReason(input);
+      return reason === undefined
+        ? Effect.void
+        : recordApiRetry({ connector: options.connector, reason });
+    }),
+  );
+
 const syncStates = ["pending", "backfilling", "live", "error"] as const;
 
 export const setSyncState = (attrs: SyncStateAttrs, current: SyncState) =>

package/src/runtime-config/config.ts

@@ -1,16 +1,14 @@
-import type { WingsClientOptions } from "@useairfoil/wings/wings-client";
-
 import { Config } from "effect";
 
 import { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
 
 /** Shared HTTP port used by hosted and sandbox connector entrypoints. */
-export const httpPort = Config.port(PlatformRuntimeKey.httpPort).pipe(
+export const httpPort = Config.Port(PlatformRuntimeKey.httpPort).pipe(
   Config.withDefault(PlatformRuntimeDefault.httpPort),
 );
 
-/** Shared Wings client configuration supplied by the hosting platform. */
-export const wingsClient = {
-  host: Config.nonEmptyString(PlatformRuntimeKey.wingsHost),
-  namespace: Config.nonEmptyString(PlatformRuntimeKey.wingsNamespace),
-} satisfies Config.Wrap<WingsClientOptions>;
+/** Wings URL, including the protocol. */
+export const wingsUri = Config.URL(PlatformRuntimeKey.wingsUri);
+
+/** Wings catalog ID. */
+export const catalog = Config.NonEmptyString(PlatformRuntimeKey.catalog);

package/src/runtime-config/constants.ts

@@ -8,8 +8,8 @@ export const PlatformRuntimeKey = {
   connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
   stateTable: "AIRFOIL_STATE_TABLE",
   postgresConnectionString: "POSTGRES_CONNECTION_STRING",
-  wingsHost: "WINGS_HOST",
-  wingsNamespace: "WINGS_NAMESPACE",
+  wingsUri: "WINGS_URI",
+  catalog: "AIRFOIL_CATALOG",
   otelEnabled: "OTEL_ENABLED",
   otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
   otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS",

package/src/runtime-config/index.ts

@@ -1,10 +1,5 @@
 export { RuntimeConfigError, type RuntimeConfigErrorCode } from "./error";
-export { httpPort, wingsClient } from "./config";
+export { catalog, httpPort, wingsUri } from "./config";
 export { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
 export { layerHosted } from "./layer";
-export {
-  type HostedTableBinding,
-  type ResolvedTableBinding,
-  type ResolvedTableBindings,
-  loadTableBindings,
-} from "./table-bindings";
+export { type ResolvedTableBindings, type TableBinding, loadTableBindings } from "./table-bindings";

package/src/runtime-config/layer.ts

@@ -8,7 +8,7 @@ const providerFromFile: Effect.Effect<
   RuntimeConfigError,
   FileSystem.FileSystem
 > = Effect.gen(function* () {
-  const configPath = yield* Config.nonEmptyString(PlatformRuntimeKey.configPath).pipe(
+  const configPath = yield* Config.NonEmptyString(PlatformRuntimeKey.configPath).pipe(
     Effect.mapError(
       (error) =>
         new RuntimeConfigError({