@useairfoil/connector-kit 0.4.0 → 0.5.1

package/src/ingestion/engine.ts

@@ -1,203 +1,438 @@
-import { Effect, Queue, Ref, Stream } from "effect";
+import { DateTime, Effect, Layer, Queue } from "effect";
+import { HttpRouter, type HttpServer, HttpServerResponse } from "effect/unstable/http";
+import * as Observability from "effect/unstable/observability";
 
-import type { ConnectorError } from "../core/errors";
 import type {
-  Batch,
   ConnectorDefinition,
   Cursor,
-  EntityDefinition,
-  EntityRow,
-  EntitySchema,
-  EventDefinition,
-  IngestionState,
-  LiveSource,
-  Transform,
-  WebhookStream,
+  ResourceDefinition,
+  ResourceState,
+  WebhookRoute,
 } from "../core/types";
+import type { StateOperation, StateSource } from "../state-store/schema";
 
+import { ConnectorError } from "../errors";
+import * as Metrics from "../metrics";
+import { publishBatch } from "../publisher/instrumented";
 import { Publisher } from "../publisher/service";
-import { StateStore } from "./state-store";
+import { StateStore } from "../state-store";
+import { deriveSyncState, normalizeCursor } from "../state-store/state";
+import { statusRoute } from "../status";
+import { Attr, SpanName, annotateError } from "../telemetry";
+import { router, WebhookQueue, type QueuedWebhookBatch } from "../webhook/server";
 
-type TaggedBatch<T> = {
-  readonly source: "live" | "backfill";
-  readonly batch: Batch<T>;
+type RunBaseOptions = {
+  readonly initialCutoff?: Cursor.Value;
 };
 
-export const runConnector = (
-  connector: ConnectorDefinition,
-  initialCutoff: Cursor,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
-  Effect.gen(function* () {
-    // Start ingestion for every entity and event in parallel.
-    const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));
-    const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));
-    // main runner
-    yield* Effect.all([...entityRuns, ...eventRuns], {
+export type RunOptions = RunBaseOptions & {
+  readonly webhook?: {
+    readonly routes: ReadonlyArray<WebhookRoute>;
+    readonly healthPath?: HttpRouter.PathInput;
+    readonly metricsPath?: HttpRouter.PathInput | false;
+    readonly statusPath?: HttpRouter.PathInput | false;
+    readonly disableHttpLogger?: boolean;
+  };
+};
+
+type RunNoWebhookOptions = RunBaseOptions & {
+  readonly webhook?: undefined;
+};
+
+type RunWebhookOptions = RunOptions & {
+  readonly webhook: NonNullable<RunOptions["webhook"]>;
+};
+
+/**
+ * Runs all connector ingestion loops. Callers provide the Publisher and
+ * StateStore layers; webhook mode additionally requires an HttpServer layer.
+ */
+export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
+  connector: ConnectorDefinition<Resources>,
+  options?: RunNoWebhookOptions,
+): Effect.Effect<void, ConnectorError, StateStore | Publisher>;
+export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
+  connector: ConnectorDefinition<Resources>,
+  options: RunWebhookOptions,
+): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
+export function run(connector: ConnectorDefinition, options?: RunOptions) {
+  const queueLayer = makeWebhookQueueLayer(connector.name);
+  const runtimeLayer = options?.webhook
+    ? Layer.mergeAll(
+        queueLayer,
+        makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),
+      )
+    : Layer.empty;
+
+  return Effect.gen(function* () {
+    const initialCutoff =
+      options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+    yield* Effect.logInfo("Connector started").pipe(
+      Effect.annotateLogs({
+        [Attr.connectorName]: connector.name,
+        resources: connector.resources.length,
+      }),
+    );
+
+    const sourceRun = runIngestion(connector, initialCutoff);
+    const webhookRun = options?.webhook
+      ? runWebhookRuntime(connector, options.webhook.routes)
+      : Effect.void;
+
+    return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(
+      Effect.asVoid,
+    );
+  }).pipe(Effect.provide(runtimeLayer));
+}
+
+const makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>
+  Layer.effect(WebhookQueue)(
+    Effect.gen(function* () {
+      const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);
+      yield* Metrics.setWebhookQueueDepth(connector, 0);
+      return WebhookQueue.of({ queue });
+    }),
+  );
+
+const runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {
+  const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
+  return Effect.all(
+    [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],
+    {
       concurrency: "unbounded",
-    });
+    },
+  ).pipe(Effect.asVoid);
+};
+
+const runWebhookQueueConsumer = (connector: ConnectorDefinition) =>
+  Effect.gen(function* () {
+    const queue = yield* WebhookQueue;
+    while (true) {
+      const batch = yield* Queue.take(queue.queue);
+      const depth = yield* Queue.size(queue.queue);
+      yield* Metrics.setWebhookQueueDepth(connector.name, depth);
+      yield* publishBatch({
+        connector: connector.name,
+        resource: batch.resource,
+        source: "webhook",
+        batch: batch.batch,
+      });
+    }
   });
 
-const createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({
-  backfill: { cutoff },
-  live: { cutoff },
-});
+const makeWebhookServerLayer = (
+  connector: ConnectorDefinition,
+  options: {
+    readonly routes: ReadonlyArray<WebhookRoute>;
+    readonly healthPath?: HttpRouter.PathInput;
+    readonly metricsPath?: HttpRouter.PathInput | false;
+    readonly statusPath?: HttpRouter.PathInput | false;
+    readonly disableHttpLogger?: boolean;
+  },
+): Layer.Layer<never, never, HttpServer.HttpServer> => {
+  const healthPath: HttpRouter.PathInput = options.healthPath ?? "/health";
+  const metricsPath = options.metricsPath === undefined ? "/metrics" : options.metricsPath;
+  const statusPath = options.statusPath === undefined ? "/status" : options.statusPath;
+  const app = Layer.mergeAll(
+    router(options.routes, { connectorName: connector.name }),
+    HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))),
+    ...(metricsPath === false
+      ? []
+      : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),
+    ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),
+  );
 
-const makeStateRef = (
-  key: string,
-  initialCutoff: Cursor,
-): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>
-  Effect.gen(function* () {
-    // Load persisted state or initialize a new one for this stream.
-    const store = yield* StateStore;
-    const existing = yield* store.getState(key);
-    const initial = existing ?? createInitialState(initialCutoff);
-    return yield* Ref.make(initial);
+  return HttpRouter.serve(app, {
+    disableLogger: options.disableHttpLogger ?? true,
   });
+};
 
-const runEntity = <S extends EntitySchema>(
-  entity: EntityDefinition<S>,
-  initialCutoff: Cursor,
+const runIngestion = (
+  connector: ConnectorDefinition,
+  initialCutoff: Cursor.Value,
 ): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
-  Effect.gen(function* () {
-    type Row = EntityRow<S>;
-    const stateRef = yield* makeStateRef(entity.name, initialCutoff);
-    // Tracks which primary keys have already been emitted.
-    const seenRef = yield* Ref.make(new Set<string>());
-
-    const liveStream = resolveLiveStream(entity.live);
-    const tagLive = (batch: Batch<Row>) => ({
-      source: "live" as const,
-      batch,
-    });
-    const updateSeen = (rows: ReadonlyArray<Row>) =>
-      Ref.update(seenRef, (seen) => {
-        const next = new Set(seen);
-        for (const row of rows) {
-          const key = String(row[entity.primaryKey]);
-          next.add(key);
-        }
-        return next;
-      });
+  Effect.forEach(
+    connector.resources,
+    (resource) => runResourceSources(connector, resource, initialCutoff),
+    {
+      concurrency: "unbounded",
+    },
+  ).pipe(Effect.asVoid);
+
+const initializeResourceState = (
+  existing: ResourceState | undefined,
+  initialCutoff: Cursor.Value,
+): ResourceState => ({
+  ...existing,
+  changes: existing?.changes ?? { cursor: initialCutoff },
+  backfill: existing?.backfill ?? {
+    cutoff: initialCutoff,
+    completed: false,
+  },
+});
 
-    // Entities are upserts, so live and backfill can overlap. We keep an in-memory
-    // seen set (primary keys) so backfill does not re-emit rows already observed live.
-    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>
-      Ref.get(seenRef).pipe(
-        Effect.map((seen) => {
-          const filtered = batch.rows.filter((row) => {
-            const key = String(row[entity.primaryKey]);
-            return !seen.has(key);
-          });
-          return {
-            source: "backfill" as const,
-            batch: { cursor: batch.cursor, rows: filtered },
-          };
+const getInitializedState = Effect.fnUntraced(function* (
+  resource: string,
+  initialCutoff: Cursor.Value,
+) {
+  const store = yield* StateStore;
+  const existing = yield* store.getResourceState(resource);
+  return initializeResourceState(existing, initialCutoff);
+});
+
+const runResourceSources = (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) => {
+  const runs = [
+    resource.backfill
+      ? isolateSourceFailure(
+          runBackfill(connector, resource, initialCutoff),
+          connector,
+          resource,
+          "backfill",
+        )
+      : Effect.void,
+    resource.changes
+      ? isolateSourceFailure(
+          runChanges(connector, resource, initialCutoff),
+          connector,
+          resource,
+          "changes",
+        )
+      : Effect.void,
+  ];
+
+  return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(
+    Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })),
+    Effect.asVoid,
+    Effect.catch(() =>
+      Metrics.setSyncState({ connector: connector.name, resource: resource.name }, "error").pipe(
+        Effect.andThen(Effect.logError("Connector resource initialization failed")),
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resource: resource.name,
         }),
+        Effect.andThen(Effect.never),
       ),
-    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-
-    // For webhook live sources, we wait for the first live batch before starting
-    // backfill. That first batch establishes the cutoff timestamp and seeds the
-    // seen set so backfill can de-dupe correctly.
-    // Queue-backed streams are single-consumer; splitting with take/drop would
-    // consume and discard elements. Take the first element directly from the
-    // queue, then let Stream.fromQueue continue from element #2.
-    if (isWebhookStream(entity.live)) {
-      const firstBatch = yield* Queue.take(entity.live.queue);
-      yield* updateSeen(firstBatch.rows);
-      yield* processTaggedStream(
-        Stream.make({ source: "live" as const, batch: firstBatch }),
-        entity.name,
-        entity.transform,
-        stateRef,
-      );
-
-      // liveStream is Stream.fromQueue on the same queue, continues from element #2.
-      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(
-        Stream.tap(({ batch }) => updateSeen(batch.rows)),
-      );
-      const merged = Stream.merge(liveTailTagged, backfillTagged);
-      yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);
-      return;
-    }
+    ),
+  );
+};
 
-    // For pull-based live sources, we can merge immediately because there is no
-    // webhook cutoff gating and the live stream is not queue-backed.
-    const liveTagged = Stream.map(liveStream, tagLive).pipe(
-      Stream.tap(({ batch }) => updateSeen(batch.rows)),
+const recordResourceError =
+  (
+    connector: ConnectorDefinition,
+    resource: ResourceDefinition,
+    source: StateSource,
+    operation: StateOperation,
+  ) =>
+  <A, R>(
+    effect: Effect.Effect<A, ConnectorError, R>,
+  ): Effect.Effect<A, ConnectorError, R | StateStore> =>
+    effect.pipe(
+      Effect.tapError(() =>
+        Metrics.setSyncState({ connector: connector.name, resource: resource.name }, "error").pipe(
+          Effect.andThen(
+            StateStore.pipe(
+              Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),
+            ),
+          ),
+        ),
+      ),
     );
 
-    const merged = Stream.merge(liveTagged, backfillTagged);
-    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);
-  });
+const isolateSourceFailure = <A, R>(
+  effect: Effect.Effect<A, ConnectorError, R>,
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  source: StateSource,
+): Effect.Effect<A, never, R> =>
+  // Only typed source errors are isolated. Defects and interruption still stop the runtime.
+  effect.pipe(
+    Effect.catch(() =>
+      Effect.logError("Connector source parked").pipe(
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resource: resource.name,
+          source,
+        }),
+        Effect.andThen(Effect.never),
+      ),
+    ),
+  );
 
-const runEvent = <S extends EntitySchema>(
-  event: EventDefinition<S>,
-  initialCutoff: Cursor,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
-  Effect.gen(function* () {
-    type Row = EntityRow<S>;
-    const stateRef = yield* makeStateRef(event.name, initialCutoff);
-    const liveStream = resolveLiveStream<Row>(event.live);
-
-    // Events must backfill first to preserve ordering.
-    if (event.backfill) {
-      const backfillTagged = Stream.map(event.backfill, (batch) => ({
-        source: "backfill" as const,
-        batch,
-      }));
-      yield* processTaggedStream(backfillTagged, event.name, event.transform, stateRef);
+const refreshRuntimeStatus = Effect.fnUntraced(function* (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) {
+  const state = yield* getInitializedState(resource.name, initialCutoff);
+  yield* Metrics.setSyncState(
+    { connector: connector.name, resource: resource.name },
+    deriveSyncState(resource, state),
+  );
+  return state;
+});
+
+const initializeRuntimeStatus = Effect.fnUntraced(function* (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) {
+  const state = yield* getInitializedState(resource.name, initialCutoff);
+  yield* Metrics.setSyncState(
+    { connector: connector.name, resource: resource.name },
+    deriveSyncState(resource, state),
+  );
+  if (resource.backfill) {
+    yield* Metrics.setLastSuccessTimestamp(
+      { connector: connector.name, resource: resource.name, source: "backfill" },
+      state.backfill?.lastSuccessAt,
+    );
+  }
+  if (resource.changes) {
+    yield* Metrics.setLastSuccessTimestamp(
+      { connector: connector.name, resource: resource.name, source: "changes" },
+      state.changes?.lastSuccessAt,
+    );
+  }
+});
+
+const runBackfill = Effect.fnUntraced(function* (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) {
+  if (!resource.backfill) return;
+  const store = yield* StateStore;
+  let state = yield* getInitializedState(resource.name, initialCutoff);
+
+  if (state.backfill?.completed === true) {
+    // The checkpoint may have committed before the previous process cleared
+    // its error marker. It may also be hidden by a newer changes error.
+    yield* store
+      .clearResourceError(resource.name, "backfill")
+      .pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+    yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+    return;
+  }
+
+  while (state.backfill?.completed !== true) {
+    const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };
+    const clearsVisibleError = state.lastError?.source === "backfill";
+    const page = yield* resource.backfill
+      .fetch({
+        pageCursor: backfill.pageCursor,
+        cutoff: backfill.cutoff,
+      })
+      .pipe(recordResourceError(connector, resource, "backfill", "fetch"));
+
+    yield* publishBatch({
+      connector: connector.name,
+      resource: resource.name,
+      source: "backfill",
+      batch: {
+        cursor: page.nextPageCursor ?? backfill.cutoff,
+        mutations: page.mutations,
+      },
+    }).pipe(recordResourceError(connector, resource, "backfill", "publish"));
+
+    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+    state = {
+      ...state,
+      backfill: {
+        cutoff: backfill.cutoff,
+        pageCursor: page.nextPageCursor,
+        completed: !page.hasMore,
+        lastSuccessAt,
+      },
+    };
+
+    // Checkpoint only after publish succeeds so a crash cannot skip an
+    // unacknowledged page; replay therefore remains at-least-once.
+    yield* Effect.withSpan(
+      store
+        .setBackfillState(resource.name, {
+          cutoff: normalizeCursor(backfill.cutoff),
+          pageCursor:
+            page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),
+          completed: !page.hasMore,
+          lastSuccessAt,
+        })
+        .pipe(
+          Effect.andThen(
+            Metrics.setLastSuccessTimestamp(
+              { connector: connector.name, resource: resource.name, source: "backfill" },
+              lastSuccessAt,
+            ),
+          ),
+          Effect.andThen(store.clearResourceError(resource.name, "backfill")),
+          Effect.tapError((error) => annotateError("state_set", error)),
+        ),
+      SpanName.stateSet,
+      { attributes: { [Attr.stateKey]: resource.name } },
+    ).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+
+    // Completion or clearing an error can change the resource-wide sync state.
+    if (clearsVisibleError || !page.hasMore) {
+      state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
     }
+  }
+});
 
-    const liveTagged = Stream.map(liveStream, (batch) => ({
-      source: "live" as const,
-      batch,
-    }));
-    yield* processTaggedStream(liveTagged, event.name, event.transform, stateRef);
-  });
+const runChanges = Effect.fnUntraced(function* (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) {
+  if (!resource.changes) return;
+  const store = yield* StateStore;
 
-const updateState = (
-  state: IngestionState<Cursor>,
-  source: "live" | "backfill",
-  cursor: Cursor,
-): IngestionState<Cursor> =>
-  source === "live"
-    ? { ...state, live: { ...state.live, current: cursor } }
-    : { ...state, backfill: { ...state.backfill, current: cursor } };
-
-const resolveLiveStream = <T>(source: LiveSource<T>): Stream.Stream<Batch<T>, ConnectorError> =>
-  isWebhookStream(source) ? source.stream : source;
-
-const isWebhookStream = <T>(source: LiveSource<T>): source is WebhookStream<T> =>
-  typeof source === "object" && source !== null && "queue" in source && "stream" in source;
-
-const processTaggedStream = <T extends Record<string, unknown>>(
-  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,
-  name: string,
-  transform: Transform<T> | undefined,
-  stateRef: Ref.Ref<IngestionState<Cursor>>,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
-  Stream.runForEach(stream, ({ source, batch }) =>
-    Effect.gen(function* () {
-      // Optional per-row transformation.
-      const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;
-
-      // Publish before updating cursor state.
-      const publisher = yield* Publisher;
-      yield* publisher.publish({
-        name,
-        batch: {
-          cursor: batch.cursor,
-          rows,
-        },
-      });
+  while (true) {
+    const state = yield* getInitializedState(resource.name, initialCutoff);
+    const cursor = state.changes?.cursor ?? initialCutoff;
+    const clearsVisibleError = state.lastError?.source === "changes";
+    const page = yield* resource.changes
+      .fetch({ cursor })
+      .pipe(recordResourceError(connector, resource, "changes", "fetch"));
 
-      // Persist state only after publish succeeds.
-      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>
-        updateState(state, source, batch.cursor),
-      );
+    yield* publishBatch({
+      connector: connector.name,
+      resource: resource.name,
+      source: "changes",
+      batch: {
+        cursor: page.cursor,
+        mutations: page.mutations,
+      },
+    }).pipe(recordResourceError(connector, resource, "changes", "publish"));
 
-      const store = yield* StateStore;
-      yield* store.setState(name, nextState);
-    }),
-  );
+    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+    // Advance the durable cursor only after Wings accepts the page.
+    yield* Effect.withSpan(
+      store
+        .setChangesState(resource.name, {
+          cursor: normalizeCursor(page.cursor),
+          lastSuccessAt,
+        })
+        .pipe(
+          Effect.andThen(
+            Metrics.setLastSuccessTimestamp(
+              { connector: connector.name, resource: resource.name, source: "changes" },
+              lastSuccessAt,
+            ),
+          ),
+          Effect.andThen(store.clearResourceError(resource.name, "changes")),
+          Effect.tapError((error) => annotateError("state_set", error)),
+        ),
+      SpanName.stateSet,
+      { attributes: { [Attr.stateKey]: resource.name } },
+    ).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
+
+    if (clearsVisibleError) {
+      yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+    }
+
+    yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+  }
+});

package/src/ingestion/index.ts

@@ -0,0 +1 @@
+export { run, type RunOptions } from "./engine";