@useairfoil/connector-kit 0.5.0 → 0.5.1

package/src/ingestion/engine.ts

@@ -1,70 +1,36 @@
-import { DateTime, Effect, Layer, Metric, Queue } from "effect";
+import { DateTime, Effect, Layer, Queue } from "effect";
 import { HttpRouter, type HttpServer, HttpServerResponse } from "effect/unstable/http";
+import * as Observability from "effect/unstable/observability";
 
 import type {
   ConnectorDefinition,
   Cursor,
-  ResourceBatch,
   ResourceDefinition,
   ResourceState,
   WebhookRoute,
 } from "../core/types";
+import type { StateOperation, StateSource } from "../state-store/schema";
 
 import { ConnectorError } from "../errors";
+import * as Metrics from "../metrics";
+import { publishBatch } from "../publisher/instrumented";
 import { Publisher } from "../publisher/service";
 import { StateStore } from "../state-store";
-import {
-  Attr,
-  EventAttr,
-  EventName,
-  SpanName,
-  addCurrentSpanEvent,
-  annotateError,
-} from "../telemetry";
+import { deriveSyncState, normalizeCursor } from "../state-store/state";
+import { statusRoute } from "../status";
+import { Attr, SpanName, annotateError } from "../telemetry";
 import { router, WebhookQueue, type QueuedWebhookBatch } from "../webhook/server";
 
-const connectorBatchesTotal = Metric.counter("connector_batches_total", {
-  description: "Total resource batches attempted by connector sources",
-});
-
-const connectorMutationsTotal = Metric.counter("connector_mutations_total", {
-  description: "Total resource mutations attempted by connector sources",
-});
-
-const connectorBatchSize = Metric.histogram("connector_batch_size", {
-  description: "Distribution of resource mutation batch sizes",
-  boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],
-});
-
 type RunBaseOptions = {
   readonly initialCutoff?: Cursor.Value;
 };
 
-const normalizeCursor = (value: Cursor.Value): Cursor.Value =>
-  value instanceof Date ? value.toISOString() : value;
-
-const normalizeResourceState = (state: ResourceState): ResourceState => ({
-  backfill: state.backfill
-    ? {
-        cutoff: normalizeCursor(state.backfill.cutoff),
-        pageCursor:
-          state.backfill.pageCursor === undefined
-            ? undefined
-            : normalizeCursor(state.backfill.pageCursor),
-        completed: state.backfill.completed,
-      }
-    : undefined,
-  changes: state.changes
-    ? {
-        cursor: normalizeCursor(state.changes.cursor),
-      }
-    : undefined,
-});
-
 export type RunOptions = RunBaseOptions & {
   readonly webhook?: {
     readonly routes: ReadonlyArray<WebhookRoute>;
     readonly healthPath?: HttpRouter.PathInput;
+    readonly metricsPath?: HttpRouter.PathInput | false;
+    readonly statusPath?: HttpRouter.PathInput | false;
     readonly disableHttpLogger?: boolean;
   };
 };
@@ -77,6 +43,10 @@ type RunWebhookOptions = RunOptions & {
   readonly webhook: NonNullable<RunOptions["webhook"]>;
 };
 
+/**
+ * Runs all connector ingestion loops. Callers provide the Publisher and
+ * StateStore layers; webhook mode additionally requires an HttpServer layer.
+ */
 export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
   connector: ConnectorDefinition<Resources>,
   options?: RunNoWebhookOptions,
@@ -86,8 +56,12 @@ export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
   options: RunWebhookOptions,
 ): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
 export function run(connector: ConnectorDefinition, options?: RunOptions) {
+  const queueLayer = makeWebhookQueueLayer(connector.name);
   const runtimeLayer = options?.webhook
-    ? Layer.mergeAll(makeWebhookQueueLayer(), makeWebhookServerLayer(options.webhook))
+    ? Layer.mergeAll(
+        queueLayer,
+        makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),
+      )
     : Layer.empty;
 
   return Effect.gen(function* () {
@@ -101,7 +75,9 @@ export function run(connector: ConnectorDefinition, options?: RunOptions) {
     );
 
     const sourceRun = runIngestion(connector, initialCutoff);
-    const webhookRun = options?.webhook ? runWebhookRuntime(options.webhook.routes) : Effect.void;
+    const webhookRun = options?.webhook
+      ? runWebhookRuntime(connector, options.webhook.routes)
+      : Effect.void;
 
     return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(
       Effect.asVoid,
@@ -109,38 +85,61 @@ export function run(connector: ConnectorDefinition, options?: RunOptions) {
   }).pipe(Effect.provide(runtimeLayer));
 }
 
-const makeWebhookQueueLayer = (): Layer.Layer<WebhookQueue> =>
+const makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>
   Layer.effect(WebhookQueue)(
     Effect.gen(function* () {
       const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);
+      yield* Metrics.setWebhookQueueDepth(connector, 0);
       return WebhookQueue.of({ queue });
     }),
   );
 
-const runWebhookRuntime = (routes: ReadonlyArray<WebhookRoute>) => {
+const runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {
   const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
-  return Effect.all([hasAfterEnqueue ? runWebhookQueueConsumer : Effect.void, Effect.never], {
-    concurrency: "unbounded",
-  }).pipe(Effect.asVoid);
+  return Effect.all(
+    [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],
+    {
+      concurrency: "unbounded",
+    },
+  ).pipe(Effect.asVoid);
 };
 
-const runWebhookQueueConsumer = Effect.gen(function* () {
-  const queue = yield* WebhookQueue;
-  while (true) {
-    const batch = yield* Queue.take(queue.queue);
-    yield* publishBatch({ resource: batch.resource, source: "webhook", batch: batch.batch });
-  }
-});
+const runWebhookQueueConsumer = (connector: ConnectorDefinition) =>
+  Effect.gen(function* () {
+    const queue = yield* WebhookQueue;
+    while (true) {
+      const batch = yield* Queue.take(queue.queue);
+      const depth = yield* Queue.size(queue.queue);
+      yield* Metrics.setWebhookQueueDepth(connector.name, depth);
+      yield* publishBatch({
+        connector: connector.name,
+        resource: batch.resource,
+        source: "webhook",
+        batch: batch.batch,
+      });
+    }
+  });
 
-const makeWebhookServerLayer = (options: {
-  readonly routes: ReadonlyArray<WebhookRoute>;
-  readonly healthPath?: HttpRouter.PathInput;
-  readonly disableHttpLogger?: boolean;
-}): Layer.Layer<never, never, HttpServer.HttpServer> => {
+const makeWebhookServerLayer = (
+  connector: ConnectorDefinition,
+  options: {
+    readonly routes: ReadonlyArray<WebhookRoute>;
+    readonly healthPath?: HttpRouter.PathInput;
+    readonly metricsPath?: HttpRouter.PathInput | false;
+    readonly statusPath?: HttpRouter.PathInput | false;
+    readonly disableHttpLogger?: boolean;
+  },
+): Layer.Layer<never, never, HttpServer.HttpServer> => {
   const healthPath: HttpRouter.PathInput = options.healthPath ?? "/health";
+  const metricsPath = options.metricsPath === undefined ? "/metrics" : options.metricsPath;
+  const statusPath = options.statusPath === undefined ? "/status" : options.statusPath;
   const app = Layer.mergeAll(
-    router(options.routes),
+    router(options.routes, { connectorName: connector.name }),
     HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))),
+    ...(metricsPath === false
+      ? []
+      : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),
+    ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),
   );
 
   return HttpRouter.serve(app, {
@@ -164,6 +163,7 @@ const initializeResourceState = (
   existing: ResourceState | undefined,
   initialCutoff: Cursor.Value,
 ): ResourceState => ({
+  ...existing,
   changes: existing?.changes ?? { cursor: initialCutoff },
   backfill: existing?.backfill ?? {
     cutoff: initialCutoff,
@@ -186,13 +186,119 @@ const runResourceSources = (
   initialCutoff: Cursor.Value,
 ) => {
   const runs = [
-    resource.backfill ? runBackfill(connector, resource, initialCutoff) : Effect.void,
-    resource.changes ? runChanges(connector, resource, initialCutoff) : Effect.void,
+    resource.backfill
+      ? isolateSourceFailure(
+          runBackfill(connector, resource, initialCutoff),
+          connector,
+          resource,
+          "backfill",
+        )
+      : Effect.void,
+    resource.changes
+      ? isolateSourceFailure(
+          runChanges(connector, resource, initialCutoff),
+          connector,
+          resource,
+          "changes",
+        )
+      : Effect.void,
   ];
 
-  return Effect.all(runs, { concurrency: "unbounded" }).pipe(Effect.asVoid);
+  return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(
+    Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })),
+    Effect.asVoid,
+    Effect.catch(() =>
+      Metrics.setSyncState({ connector: connector.name, resource: resource.name }, "error").pipe(
+        Effect.andThen(Effect.logError("Connector resource initialization failed")),
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resource: resource.name,
+        }),
+        Effect.andThen(Effect.never),
+      ),
+    ),
+  );
 };
 
+const recordResourceError =
+  (
+    connector: ConnectorDefinition,
+    resource: ResourceDefinition,
+    source: StateSource,
+    operation: StateOperation,
+  ) =>
+  <A, R>(
+    effect: Effect.Effect<A, ConnectorError, R>,
+  ): Effect.Effect<A, ConnectorError, R | StateStore> =>
+    effect.pipe(
+      Effect.tapError(() =>
+        Metrics.setSyncState({ connector: connector.name, resource: resource.name }, "error").pipe(
+          Effect.andThen(
+            StateStore.pipe(
+              Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),
+            ),
+          ),
+        ),
+      ),
+    );
+
+const isolateSourceFailure = <A, R>(
+  effect: Effect.Effect<A, ConnectorError, R>,
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  source: StateSource,
+): Effect.Effect<A, never, R> =>
+  // Only typed source errors are isolated. Defects and interruption still stop the runtime.
+  effect.pipe(
+    Effect.catch(() =>
+      Effect.logError("Connector source parked").pipe(
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resource: resource.name,
+          source,
+        }),
+        Effect.andThen(Effect.never),
+      ),
+    ),
+  );
+
+const refreshRuntimeStatus = Effect.fnUntraced(function* (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) {
+  const state = yield* getInitializedState(resource.name, initialCutoff);
+  yield* Metrics.setSyncState(
+    { connector: connector.name, resource: resource.name },
+    deriveSyncState(resource, state),
+  );
+  return state;
+});
+
+const initializeRuntimeStatus = Effect.fnUntraced(function* (
+  connector: ConnectorDefinition,
+  resource: ResourceDefinition,
+  initialCutoff: Cursor.Value,
+) {
+  const state = yield* getInitializedState(resource.name, initialCutoff);
+  yield* Metrics.setSyncState(
+    { connector: connector.name, resource: resource.name },
+    deriveSyncState(resource, state),
+  );
+  if (resource.backfill) {
+    yield* Metrics.setLastSuccessTimestamp(
+      { connector: connector.name, resource: resource.name, source: "backfill" },
+      state.backfill?.lastSuccessAt,
+    );
+  }
+  if (resource.changes) {
+    yield* Metrics.setLastSuccessTimestamp(
+      { connector: connector.name, resource: resource.name, source: "changes" },
+      state.changes?.lastSuccessAt,
+    );
+  }
+});
+
 const runBackfill = Effect.fnUntraced(function* (
   connector: ConnectorDefinition,
   resource: ResourceDefinition,
@@ -202,39 +308,76 @@ const runBackfill = Effect.fnUntraced(function* (
   const store = yield* StateStore;
   let state = yield* getInitializedState(resource.name, initialCutoff);
 
+  if (state.backfill?.completed === true) {
+    // The checkpoint may have committed before the previous process cleared
+    // its error marker. It may also be hidden by a newer changes error.
+    yield* store
+      .clearResourceError(resource.name, "backfill")
+      .pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+    yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+    return;
+  }
+
   while (state.backfill?.completed !== true) {
     const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };
-    const page = yield* resource.backfill.fetch({
-      pageCursor: backfill.pageCursor,
-      cutoff: backfill.cutoff,
-    });
+    const clearsVisibleError = state.lastError?.source === "backfill";
+    const page = yield* resource.backfill
+      .fetch({
+        pageCursor: backfill.pageCursor,
+        cutoff: backfill.cutoff,
+      })
+      .pipe(recordResourceError(connector, resource, "backfill", "fetch"));
 
     yield* publishBatch({
-      connector,
+      connector: connector.name,
       resource: resource.name,
       source: "backfill",
       batch: {
         cursor: page.nextPageCursor ?? backfill.cutoff,
         mutations: page.mutations,
       },
-    });
+    }).pipe(recordResourceError(connector, resource, "backfill", "publish"));
 
+    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
     state = {
       ...state,
       backfill: {
         cutoff: backfill.cutoff,
         pageCursor: page.nextPageCursor,
         completed: !page.hasMore,
+        lastSuccessAt,
       },
     };
 
+    // Checkpoint only after publish succeeds so a crash cannot skip an
+    // unacknowledged page; replay therefore remains at-least-once.
     yield* Effect.withSpan(
       store
-        .setResourceState(resource.name, normalizeResourceState(state))
-        .pipe(Effect.tapError((error) => annotateError("state_set", error))),
+        .setBackfillState(resource.name, {
+          cutoff: normalizeCursor(backfill.cutoff),
+          pageCursor:
+            page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),
+          completed: !page.hasMore,
+          lastSuccessAt,
+        })
+        .pipe(
+          Effect.andThen(
+            Metrics.setLastSuccessTimestamp(
+              { connector: connector.name, resource: resource.name, source: "backfill" },
+              lastSuccessAt,
+            ),
+          ),
+          Effect.andThen(store.clearResourceError(resource.name, "backfill")),
+          Effect.tapError((error) => annotateError("state_set", error)),
+        ),
       SpanName.stateSet,
       { attributes: { [Attr.stateKey]: resource.name } },
-    );
+    ).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+
+    // Completion or clearing an error can change the resource-wide sync state.
+    if (clearsVisibleError || !page.hasMore) {
+      state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+    }
   }
 });
 
@@ -249,77 +392,47 @@ const runChanges = Effect.fnUntraced(function* (
   while (true) {
     const state = yield* getInitializedState(resource.name, initialCutoff);
     const cursor = state.changes?.cursor ?? initialCutoff;
-    const page = yield* resource.changes.fetch({ cursor });
+    const clearsVisibleError = state.lastError?.source === "changes";
+    const page = yield* resource.changes
+      .fetch({ cursor })
+      .pipe(recordResourceError(connector, resource, "changes", "fetch"));
 
     yield* publishBatch({
-      connector,
+      connector: connector.name,
       resource: resource.name,
       source: "changes",
       batch: {
         cursor: page.cursor,
         mutations: page.mutations,
       },
-    });
+    }).pipe(recordResourceError(connector, resource, "changes", "publish"));
 
+    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+    // Advance the durable cursor only after Wings accepts the page.
     yield* Effect.withSpan(
       store
-        .setResourceState(
-          resource.name,
-          normalizeResourceState({
-            ...state,
-            changes: { cursor: page.cursor },
-          }),
-        )
-        .pipe(Effect.tapError((error) => annotateError("state_set", error))),
+        .setChangesState(resource.name, {
+          cursor: normalizeCursor(page.cursor),
+          lastSuccessAt,
+        })
+        .pipe(
+          Effect.andThen(
+            Metrics.setLastSuccessTimestamp(
+              { connector: connector.name, resource: resource.name, source: "changes" },
+              lastSuccessAt,
+            ),
+          ),
+          Effect.andThen(store.clearResourceError(resource.name, "changes")),
+          Effect.tapError((error) => annotateError("state_set", error)),
+        ),
       SpanName.stateSet,
       { attributes: { [Attr.stateKey]: resource.name } },
-    );
+    ).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
 
-    yield* Effect.sleep(resource.changes.interval ?? "1 minute");
-  }
-});
+    if (clearsVisibleError) {
+      yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+    }
 
-const publishBatch = Effect.fnUntraced(function* (options: {
-  readonly connector?: ConnectorDefinition;
-  readonly resource: string;
-  readonly source: "backfill" | "changes" | "webhook";
-  readonly batch: ResourceBatch;
-}) {
-  const metric = {
-    connector: options.connector?.name ?? "unknown",
-    resource: options.resource,
-    source: options.source,
-  };
-  yield* Metric.update(Metric.withAttributes(connectorBatchesTotal, metric), 1);
-  yield* Metric.update(
-    Metric.withAttributes(connectorMutationsTotal, metric),
-    options.batch.mutations.length,
-  );
-  yield* Metric.update(
-    Metric.withAttributes(connectorBatchSize, metric),
-    options.batch.mutations.length,
-  );
-
-  const publisher = yield* Publisher;
-  const ack = yield* publisher.publish({
-    resource: options.resource,
-    source: options.source,
-    batch: options.batch,
-  });
-
-  yield* Effect.annotateCurrentSpan({ [Attr.publisherSuccess]: ack.status === "accepted" });
-  if (ack.status === "rejected") {
-    yield* Effect.annotateCurrentSpan({ [Attr.errorPhase]: "publish" });
-    return yield* Effect.fail(
-      new ConnectorError({
-        message: `Publisher rejected batch for ${options.resource}: ${ack.reason}`,
-      }),
-    );
-  }
-
-  if (options.batch.cursor !== undefined) {
-    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {
-      [EventAttr.batchCursor]: options.batch.cursor,
-    });
+    yield* Effect.sleep(resource.changes.interval ?? "1 minute");
   }
 });