@useairfoil/connector-kit 0.6.0 → 0.7.0

package/src/index.ts

@@ -1,10 +1,12 @@
 export * from "./core";
 export * as Auth from "./auth";
 export * as ConnectorApp from "./connector-app";
+export * as Demo from "./demo";
+export * as Iceberg from "./iceberg";
 export * as Ingestion from "./ingestion";
+export * as Ingestor from "./ingestor";
 export * as Manifest from "./manifest";
 export * as Metrics from "./metrics";
-export * as Publisher from "./publisher";
 export * as RuntimeConfig from "./runtime-config";
 export * as StateStore from "./state-store";
 export * as Status from "./status";

package/src/ingestion/engine.ts

@@ -1,8 +1,9 @@
-import { DateTime, Effect, Layer, Queue } from "effect";
-import { HttpRouter, type HttpServer, HttpServerResponse } from "effect/unstable/http";
-import * as Observability from "effect/unstable/observability";
+import { Array as Arr, DateTime, Effect, Layer, Order, Queue } from "effect";
+import { HttpRouter, type HttpServer, HttpServerResponse } from "effect/http";
+import * as Observability from "effect/observability";
 
 import type {
+  ChangesFeed,
   ConnectorDefinition,
   Cursor,
   ResourceDefinition,
@@ -12,11 +13,11 @@ import type {
 import type { StateOperation, StateSource } from "../state-store/schema";
 
 import { ConnectorError } from "../errors";
+import { ingestBatch } from "../ingestor/instrumented";
+import { Ingestor } from "../ingestor/service";
 import * as Metrics from "../metrics";
-import { publishBatch } from "../publisher/instrumented";
-import { Publisher } from "../publisher/service";
 import { StateStore } from "../state-store";
-import { deriveSyncState, normalizeCursor } from "../state-store/state";
+import { deriveSyncState, hasChanges, normalizeCursor } from "../state-store/state";
 import { statusRoute } from "../status";
 import { Attr, SpanName, annotateError } from "../telemetry";
 import { router, WebhookQueue, type QueuedWebhookBatch } from "../webhook/server";
@@ -44,47 +45,70 @@ type RunWebhookOptions = RunOptions & {
 };
 
 /**
- * Runs all connector ingestion loops. Callers provide the Publisher and
+ * Runs all connector ingestion loops. Callers provide the Ingestor and
  * StateStore layers; webhook mode additionally requires an HttpServer layer.
  */
 export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
   connector: ConnectorDefinition<Resources>,
   options?: RunNoWebhookOptions,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher>;
+): Effect.Effect<void, ConnectorError, StateStore | Ingestor>;
 export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
   connector: ConnectorDefinition<Resources>,
   options: RunWebhookOptions,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
-export function run(connector: ConnectorDefinition, options?: RunOptions) {
-  const queueLayer = makeWebhookQueueLayer(connector.name);
-  const runtimeLayer = options?.webhook
-    ? Layer.mergeAll(
-        queueLayer,
-        makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),
-      )
-    : Layer.empty;
-
+): Effect.Effect<void, ConnectorError, StateStore | Ingestor | HttpServer.HttpServer>;
+export function run(definition: ConnectorDefinition, options?: RunOptions) {
   return Effect.gen(function* () {
-    const initialCutoff =
-      options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
-    yield* Effect.logInfo("Connector started").pipe(
-      Effect.annotateLogs({
-        [Attr.connectorName]: connector.name,
-        resources: connector.resources.length,
-      }),
-    );
+    // Check the whole feed, even the resources that don't run.
+    if (definition.changes) yield* validateFeed(definition, definition.changes);
+    const ingestor = yield* Ingestor;
+    const connector = selectResources(definition, ingestor.resourceNames);
+    const queueLayer = makeWebhookQueueLayer(connector.name);
+    const runtimeLayer = options?.webhook
+      ? Layer.mergeAll(
+          queueLayer,
+          makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),
+        )
+      : Layer.empty;
 
-    const sourceRun = runIngestion(connector, initialCutoff);
-    const webhookRun = options?.webhook
-      ? runWebhookRuntime(connector, options.webhook.routes)
-      : Effect.void;
+    return yield* Effect.gen(function* () {
+      const initialCutoff =
+        options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+      yield* Effect.logInfo("Connector started").pipe(
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resources: connector.resources.length,
+        }),
+      );
 
-    return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(
-      Effect.asVoid,
-    );
-  }).pipe(Effect.provide(runtimeLayer));
+      const sourceRun = runIngestion(connector, initialCutoff);
+      const webhookRun = options?.webhook
+        ? runWebhookRuntime(connector, options.webhook.routes)
+        : Effect.void;
+
+      return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(
+        Effect.asVoid,
+      );
+    }).pipe(Effect.provide(runtimeLayer));
+  });
 }
 
+const selectResources = (
+  connector: ConnectorDefinition,
+  names: ReadonlyArray<string> | undefined,
+): ConnectorDefinition => {
+  if (names === undefined) return connector;
+  const isSelected = (resource: ResourceDefinition) => names.includes(resource.name);
+  const { changes, ...rest } = connector;
+  const feedResources = changes?.resources.filter(isSelected) ?? [];
+  return {
+    ...rest,
+    resources: connector.resources.filter(isSelected),
+    ...(changes && feedResources.length > 0
+      ? { changes: { ...changes, resources: feedResources } }
+      : {}),
+  };
+};
+
 const makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>
   Layer.effect(WebhookQueue)(
     Effect.gen(function* () {
@@ -111,7 +135,7 @@ const runWebhookQueueConsumer = (connector: ConnectorDefinition) =>
       const batch = yield* Queue.take(queue.queue);
       const depth = yield* Queue.size(queue.queue);
       yield* Metrics.setWebhookQueueDepth(connector.name, depth);
-      yield* publishBatch({
+      yield* ingestBatch({
         connector: connector.name,
         resource: batch.resource,
         source: "webhook",
@@ -134,7 +158,10 @@ const makeWebhookServerLayer = (
   const metricsPath = options.metricsPath === undefined ? "/metrics" : options.metricsPath;
   const statusPath = options.statusPath === undefined ? "/status" : options.statusPath;
   const app = Layer.mergeAll(
-    router(options.routes, { connectorName: connector.name }),
+    router(options.routes, {
+      connectorName: connector.name,
+      resourceNames: connector.resources.map((resource) => resource.name),
+    }),
     HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))),
     ...(metricsPath === false
       ? []
@@ -150,14 +177,57 @@ const makeWebhookServerLayer = (
 const runIngestion = (
   connector: ConnectorDefinition,
   initialCutoff: Cursor.Value,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
-  Effect.forEach(
-    connector.resources,
-    (resource) => runResourceSources(connector, resource, initialCutoff),
-    {
-      concurrency: "unbounded",
-    },
-  ).pipe(Effect.asVoid);
+): Effect.Effect<void, ConnectorError, StateStore | Ingestor> =>
+  Effect.gen(function* () {
+    const feed = connector.changes;
+    yield* Effect.all(
+      [
+        Effect.forEach(
+          connector.resources,
+          (resource) => runResourceSources(connector, resource, initialCutoff),
+          { concurrency: "unbounded" },
+        ),
+        feed ? runChangesFeed(connector, feed, initialCutoff) : Effect.void,
+      ],
+      { concurrency: "unbounded" },
+    );
+  });
+
+const bySavedAt = Order.mapInput(
+  Order.String,
+  (state: NonNullable<ResourceState["changes"]>) => state.lastSuccessAt ?? "",
+);
+
+const validateFeed = (connector: ConnectorDefinition, feed: ChangesFeed) =>
+  Effect.forEach(feed.resources, (member, index) => {
+    if (feed.resources.findIndex((item) => item.name === member.name) !== index) {
+      return Effect.fail(
+        new ConnectorError({ message: `Resource ${member.name} is in the changes feed twice` }),
+      );
+    }
+    const resource = connector.resources.find((item) => item.name === member.name);
+    if (!resource) {
+      return Effect.fail(
+        new ConnectorError({
+          message: `Changes feed resource ${member.name} is not a resource of connector ${connector.name}`,
+        }),
+      );
+    }
+    if (resource.changes) {
+      return Effect.fail(
+        new ConnectorError({
+          message: `Resource ${member.name} has its own changes and is also in the changes feed`,
+        }),
+      );
+    }
+    return Effect.void;
+  }).pipe(
+    Effect.andThen(
+      feed.resources.length === 0
+        ? Effect.fail(new ConnectorError({ message: "The changes feed has no resources" }))
+        : Effect.void,
+    ),
+  );
 
 const initializeResourceState = (
   existing: ResourceState | undefined,
@@ -207,9 +277,9 @@ const runResourceSources = (
   return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(
     Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })),
     Effect.asVoid,
-    Effect.catch(() =>
+    Effect.catch((error) =>
       Metrics.setSyncState({ connector: connector.name, resource: resource.name }, "error").pipe(
-        Effect.andThen(Effect.logError("Connector resource initialization failed")),
+        Effect.andThen(Effect.logError("Connector resource initialization failed", error)),
         Effect.annotateLogs({
           [Attr.connectorName]: connector.name,
           resource: resource.name,
@@ -250,8 +320,8 @@ const isolateSourceFailure = <A, R>(
 ): Effect.Effect<A, never, R> =>
   // Only typed source errors are isolated. Defects and interruption still stop the runtime.
   effect.pipe(
-    Effect.catch(() =>
-      Effect.logError("Connector source parked").pipe(
+    Effect.catch((error) =>
+      Effect.logError("Connector source parked", error).pipe(
         Effect.annotateLogs({
           [Attr.connectorName]: connector.name,
           resource: resource.name,
@@ -270,8 +340,9 @@ const refreshRuntimeStatus = Effect.fnUntraced(function* (
   const state = yield* getInitializedState(resource.name, initialCutoff);
   yield* Metrics.setSyncState(
     { connector: connector.name, resource: resource.name },
-    deriveSyncState(resource, state),
+    deriveSyncState(connector, resource, state),
   );
+
   return state;
 });
 
@@ -283,15 +354,16 @@ const initializeRuntimeStatus = Effect.fnUntraced(function* (
   const state = yield* getInitializedState(resource.name, initialCutoff);
   yield* Metrics.setSyncState(
     { connector: connector.name, resource: resource.name },
-    deriveSyncState(resource, state),
+    deriveSyncState(connector, resource, state),
   );
+
   if (resource.backfill) {
     yield* Metrics.setLastSuccessTimestamp(
       { connector: connector.name, resource: resource.name, source: "backfill" },
       state.backfill?.lastSuccessAt,
     );
   }
-  if (resource.changes) {
+  if (hasChanges(connector, resource)) {
     yield* Metrics.setLastSuccessTimestamp(
       { connector: connector.name, resource: resource.name, source: "changes" },
       state.changes?.lastSuccessAt,
@@ -328,15 +400,15 @@ const runBackfill = Effect.fnUntraced(function* (
       })
       .pipe(recordResourceError(connector, resource, "backfill", "fetch"));
 
-    yield* publishBatch({
+    yield* ingestBatch({
       connector: connector.name,
       resource: resource.name,
       source: "backfill",
       batch: {
         cursor: page.nextPageCursor ?? backfill.cutoff,
-        mutations: page.mutations,
+        rows: page.rows,
       },
-    }).pipe(recordResourceError(connector, resource, "backfill", "publish"));
+    }).pipe(recordResourceError(connector, resource, "backfill", "ingest"));
 
     const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
     state = {
@@ -349,7 +421,7 @@ const runBackfill = Effect.fnUntraced(function* (
       },
     };
 
-    // Checkpoint only after publish succeeds so a crash cannot skip an
+    // Checkpoint only after ingestion succeeds so a crash cannot skip an
     // unacknowledged page; replay therefore remains at-least-once.
     yield* Effect.withSpan(
       store
@@ -397,18 +469,18 @@ const runChanges = Effect.fnUntraced(function* (
       .fetch({ cursor })
       .pipe(recordResourceError(connector, resource, "changes", "fetch"));
 
-    yield* publishBatch({
+    yield* ingestBatch({
       connector: connector.name,
       resource: resource.name,
       source: "changes",
       batch: {
         cursor: page.cursor,
-        mutations: page.mutations,
+        rows: page.rows,
       },
-    }).pipe(recordResourceError(connector, resource, "changes", "publish"));
+    }).pipe(recordResourceError(connector, resource, "changes", "ingest"));
 
     const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
-    // Advance the durable cursor only after Wings accepts the page.
+    // Advance the durable cursor only after the engine acknowledges the page.
     yield* Effect.withSpan(
       store
         .setChangesState(resource.name, {
@@ -433,6 +505,114 @@ const runChanges = Effect.fnUntraced(function* (
       yield* refreshRuntimeStatus(connector, resource, initialCutoff);
     }
 
-    yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+    if (!page.hasMore) {
+      yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+    }
   }
 });
+
+// Every member resource stores the feed cursor. All rows are ingested before any
+// cursor is written. If a crash stops the writes partway, the next run starts
+// from the oldest cursor.
+const runChangesFeed = (
+  connector: ConnectorDefinition,
+  feed: ChangesFeed,
+  initialCutoff: Cursor.Value,
+): Effect.Effect<void, never, StateStore | Ingestor> =>
+  Effect.gen(function* () {
+    const store = yield* StateStore;
+    const members = feed.resources;
+    const selected = new Set(members.map((resource) => resource.name));
+    // Any failure stops the whole feed, so it is an error for every resource in it.
+    const recordFeedError =
+      (operation: StateOperation) =>
+      <A, R>(effect: Effect.Effect<A, ConnectorError, R>) =>
+        effect.pipe(
+          Effect.tapError(() =>
+            Effect.forEach(
+              members,
+              (resource) =>
+                Metrics.setSyncState(
+                  { connector: connector.name, resource: resource.name },
+                  "error",
+                ).pipe(Effect.andThen(store.setResourceError(resource.name, "changes", operation))),
+              { discard: true },
+            ),
+          ),
+        );
+
+    while (true) {
+      const states = yield* Effect.forEach(members, (resource) =>
+        store.getResourceState(resource.name),
+      );
+      // A resource that was off has an older cursor. Resume from the oldest, so it
+      // gets the changes it missed. The others read some changes again.
+      const saved = states.flatMap((state) => (state?.changes ? [state.changes] : []));
+      const cursor = Arr.isReadonlyArrayNonEmpty(saved)
+        ? Arr.min(saved, bySavedAt).cursor
+        : initialCutoff;
+      const result = yield* feed.fetch({ cursor, selected }).pipe(recordFeedError("fetch"));
+
+      yield* Effect.forEach(
+        members,
+        (resource) =>
+          ingestBatch({
+            connector: connector.name,
+            resource: resource.name,
+            source: "changes",
+            batch: {
+              cursor: result.cursor,
+              rows: result.rows[resource.name] ?? [],
+            },
+          }),
+        { discard: true },
+      ).pipe(recordFeedError("ingest"));
+
+      const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+      yield* Effect.forEach(
+        members,
+        (resource) =>
+          Effect.withSpan(
+            store
+              .setChangesState(resource.name, {
+                cursor: normalizeCursor(result.cursor),
+                lastSuccessAt,
+              })
+              .pipe(
+                Effect.andThen(
+                  Metrics.setLastSuccessTimestamp(
+                    { connector: connector.name, resource: resource.name, source: "changes" },
+                    lastSuccessAt,
+                  ),
+                ),
+                Effect.andThen(store.clearResourceError(resource.name, "changes")),
+                Effect.tapError((error) => annotateError("state_set", error)),
+              ),
+            SpanName.stateSet,
+            { attributes: { [Attr.stateKey]: resource.name } },
+          ),
+        { discard: true },
+      ).pipe(recordFeedError("checkpoint"));
+
+      yield* Effect.forEach(members, (resource, index) =>
+        states[index]?.lastError?.source === "changes"
+          ? refreshRuntimeStatus(connector, resource, initialCutoff)
+          : Effect.void,
+      );
+
+      if (!result.hasMore) {
+        yield* Effect.sleep(feed.interval ?? "1 minute");
+      }
+    }
+  }).pipe(
+    // Only typed errors park the feed. Defects and interruption still stop the runtime.
+    Effect.catch((error) =>
+      Effect.logError("Connector changes feed parked", error).pipe(
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resources: feed.resources.map((resource) => resource.name).join(","),
+        }),
+        Effect.andThen(Effect.never),
+      ),
+    ),
+  );

package/src/ingestor/arrow-flight.ts

@@ -0,0 +1,141 @@
+import type { RecordBatch, TypeMap } from "apache-arrow";
+import type { TableIdentifier } from "iceberg-js";
+
+import { CatalogManager } from "@useairfoil/api";
+import { Config, Effect, Layer } from "effect";
+import { FetchHttpClient } from "effect/http";
+
+import type { TableSource } from "../core/types";
+
+import { ConnectorError } from "../errors";
+import { compileTable } from "../iceberg/table";
+import * as RuntimeConfig from "../runtime-config";
+import { makeRowEncoder, type RowEncoder } from "./arrow";
+import { Ingestor, type IngestorService } from "./service";
+
+/** One catalog and an Iceberg table for each resource that runs. */
+export type ArrowFlightIngestorConfig<Source extends TableSource = TableSource> = {
+  readonly connector: Source;
+  readonly catalog: string;
+  readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;
+};
+
+type IngestorEntry = {
+  readonly encode: RowEncoder;
+  readonly push: (batch: RecordBatch<TypeMap>) => Effect.Effect<void, ConnectorError>;
+};
+
+type PreparedTable = {
+  readonly resource: string;
+  readonly identifier: TableIdentifier;
+  readonly encode: RowEncoder;
+};
+
+const connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });
+
+/** Compiles local schemas and opens one Arrow Flight stream per bound resource. */
+export const layerArrowFlight = <const Source extends TableSource>(
+  config: ArrowFlightIngestorConfig<Source>,
+): Layer.Layer<Ingestor, ConnectorError, CatalogManager.CatalogManager> =>
+  Layer.effect(Ingestor)(
+    Effect.gen(function* () {
+      const manager = yield* CatalogManager.CatalogManager;
+      const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;
+
+      const prepared: Array<PreparedTable> = [];
+      for (const resource of config.connector.resources) {
+        const binding = bindings[resource.name];
+        if (!binding) continue;
+
+        const { namespace, name } = binding;
+        const identifier = { namespace, name };
+        const { schema } = yield* compileTable(resource.rowSchema).pipe(
+          Effect.mapError((cause) =>
+            connectorError(`Invalid schema for ${resource.name}: ${cause.message}`, cause),
+          ),
+        );
+
+        const encode = yield* makeRowEncoder(schema, resource.version);
+        prepared.push({ resource: resource.name, identifier, encode });
+      }
+
+      const entries = new Map<string, IngestorEntry>();
+      for (const { resource, identifier, encode } of prepared) {
+        const ingestor = yield* manager
+          .ingestor({
+            catalog: config.catalog,
+            namespace: identifier.namespace,
+            table: identifier.name,
+          })
+          .pipe(
+            Effect.mapError((cause) =>
+              connectorError(`Failed to open ingestor for ${resource}`, cause),
+            ),
+          );
+
+        entries.set(resource, {
+          encode,
+          push: (batch) =>
+            ingestor
+              .push(batch)
+              .pipe(
+                Effect.mapError((cause) =>
+                  connectorError(`Failed to ingest rows for ${resource}`, cause),
+                ),
+              ),
+        });
+      }
+
+      const service: IngestorService = {
+        resourceNames: prepared.map(({ resource }) => resource),
+        ingest: ({ resource, batch }) => {
+          const entry = entries.get(resource);
+
+          if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));
+          if (batch.rows.length === 0) return Effect.succeed({ rows: 0, bytes: 0 });
+
+          return entry
+            .encode(batch.rows)
+            .pipe(
+              Effect.flatMap((encoded) =>
+                entry
+                  .push(encoded)
+                  .pipe(Effect.as({ rows: batch.rows.length, bytes: encoded.data.byteLength })),
+              ),
+            );
+        },
+      };
+
+      return Ingestor.of(service);
+    }),
+  );
+
+/** Configures Arrow Flight ingestion from the hosted URL, catalog and table bindings. */
+export const layerArrowFlightConfig = (connector: TableSource) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const { catalog, uri, flightUri } = yield* Config.all({
+        catalog: RuntimeConfig.catalog,
+        uri: RuntimeConfig.engineUri,
+        flightUri: RuntimeConfig.flightUri,
+      }).pipe(
+        Effect.mapError((cause) => connectorError("Invalid Arrow Flight configuration", cause)),
+      );
+
+      if (uri.protocol !== "http:" && uri.protocol !== "https:") {
+        return yield* Effect.fail(connectorError("Invalid Arrow Flight URL"));
+      }
+      if (flightUri.protocol !== "http:" && flightUri.protocol !== "https:") {
+        return yield* Effect.fail(connectorError("Invalid Arrow Flight URL"));
+      }
+
+      const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(
+        Effect.mapError((cause) => connectorError(cause.message, cause)),
+      );
+      const manager = CatalogManager.layer({ engineUri: uri.href, flightUri: flightUri.href }).pipe(
+        Layer.provide(FetchHttpClient.layer),
+      );
+
+      return layerArrowFlight({ connector, catalog, tables }).pipe(Layer.provide(manager));
+    }),
+  );