@useairfoil/connector-kit 0.4.0 → 0.5.1

package/src/webhook/server.ts

@@ -1,51 +1,218 @@
-import { Cause, Data, Effect, Schema } from "effect";
+import { Cause, Context, Effect, Queue, Schema } from "effect";
 import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
 
-import type { WebhookRoute } from "./types";
+import type {
+  ResourceBatch,
+  ResourceDefinition,
+  ResourceMutation,
+  WebhookHandler,
+  WebhookRoute,
+  WebhookRouteContext,
+} from "../core/types";
 
-class InvalidWebhookPayloadError extends Data.TaggedError("InvalidWebhookPayloadError")<{
-  readonly message: string;
-}> {}
+import { ConnectorError } from "../errors";
+import * as Metrics from "../metrics";
+import { publishBatch } from "../publisher/instrumented";
+import { Attr, SpanName, annotateError } from "../telemetry";
 
-const makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>
+export type QueuedWebhookBatch = {
+  readonly resource: string;
+  readonly batch: ResourceBatch;
+};
+
+export class WebhookQueue extends Context.Service<
+  WebhookQueue,
+  {
+    readonly queue: Queue.Queue<QueuedWebhookBatch>;
+  }
+>()("@useairfoil/connector-kit/WebhookQueue") {}
+
+const resourceBatch = <Row extends object>(
+  mutations: ReadonlyArray<ResourceMutation<Row>>,
+): ResourceBatch<Row> => ({
+  mutations,
+});
+
+const decodeAndHandleWebhook = <Row extends object, Payload>(
+  resourceName: string,
+  webhook: WebhookHandler<Row, Payload>,
+  payload: unknown,
+) =>
+  Schema.decodeUnknownEffect(webhook.schema)(payload).pipe(
+    Effect.mapError(
+      (cause) =>
+        new ConnectorError({
+          message: `Invalid webhook payload for resource ${resourceName}`,
+          cause,
+        }),
+    ),
+    Effect.flatMap((decoded) => webhook.handler({ payload: decoded })),
+  );
+
+const makeRouteContext = (
+  request: HttpServerRequest.HttpServerRequest,
+  rawBody: Uint8Array,
+  payload: unknown,
+  batches: Array<QueuedWebhookBatch>,
+): WebhookRouteContext<ReadonlyArray<ResourceDefinition>, unknown> => ({
+  request,
+  rawBody,
+  payload,
+  to: (resource, payload) =>
+    Effect.gen(function* () {
+      if (!resource.webhook) {
+        return yield* Effect.fail(
+          new ConnectorError({
+            message: `Resource ${resource.name} does not define a webhook handler`,
+          }),
+        );
+      }
+
+      const mutations = yield* decodeAndHandleWebhook(resource.name, resource.webhook, payload);
+      batches.push({
+        resource: resource.name,
+        batch: resourceBatch(mutations),
+      });
+    }),
+});
+
+const compactBatches = (batches: ReadonlyArray<QueuedWebhookBatch>) => {
+  const grouped = new Map<string, ResourceMutation[]>();
+  for (const item of batches) {
+    const current = grouped.get(item.resource) ?? [];
+    grouped.set(item.resource, [...current, ...item.batch.mutations]);
+  }
+  return Array.from(grouped.entries()).map(([resource, mutations]) => ({
+    resource,
+    batch: resourceBatch(mutations),
+  }));
+};
+
+const publishBatches = (
+  batches: ReadonlyArray<QueuedWebhookBatch>,
+  options: { readonly connectorName: string },
+) =>
+  Effect.gen(function* () {
+    yield* Effect.forEach(
+      batches,
+      (batch) =>
+        publishBatch({
+          connector: options.connectorName,
+          resource: batch.resource,
+          source: "webhook",
+          batch: batch.batch,
+        }),
+      { concurrency: "unbounded" },
+    );
+  });
+
+const badRequest = (message: string) =>
+  HttpServerResponse.jsonUnsafe({ ok: false, error: message }, { status: 400 });
+
+const readRawBody = (request: HttpServerRequest.HttpServerRequest) =>
+  request.arrayBuffer.pipe(Effect.map((buffer) => new Uint8Array(buffer)));
+
+const parseJson = (rawBody: Uint8Array) =>
+  Effect.try({
+    try: () => JSON.parse(new TextDecoder().decode(rawBody)),
+    catch: (cause) => new ConnectorError({ message: "Invalid webhook JSON", cause }),
+  });
+
+const makeHandler = (route: WebhookRoute, options: { readonly connectorName: string }) =>
   Effect.gen(function* () {
     const request = yield* HttpServerRequest.HttpServerRequest;
-    const rawBuffer = yield* request.arrayBuffer.pipe(
-      Effect.mapError(
-        () =>
-          new InvalidWebhookPayloadError({
-            message: "Failed to read request body",
-          }),
-      ),
+    const collected: Array<QueuedWebhookBatch> = [];
+
+    const rawBodyResult = yield* readRawBody(request).pipe(
+      Effect.match({
+        onFailure: () => ({ _tag: "Error" as const, message: "Failed to read webhook body" }),
+        onSuccess: (value) => ({ _tag: "Success" as const, value }),
+      }),
     );
-    const rawBody = new Uint8Array(rawBuffer);
-    const rawText = new TextDecoder().decode(rawBody);
-    const rawJson = yield* Effect.try({
-      try: () => JSON.parse(rawText) as unknown,
-      catch: () => new InvalidWebhookPayloadError({ message: "Invalid JSON body" }),
-    });
-    const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(
-      Effect.mapError(
-        () =>
-          new InvalidWebhookPayloadError({
-            message: "Invalid webhook payload",
-          }),
+    if (rawBodyResult._tag === "Error") {
+      yield* Metrics.recordWebhookRequest({
+        connector: options.connectorName,
+        path: String(route.path),
+        outcome: "read_error",
+      });
+      return badRequest(rawBodyResult.message);
+    }
+
+    const jsonResult = yield* parseJson(rawBodyResult.value).pipe(
+      Effect.match({
+        onFailure: (error) => ({ _tag: "Error" as const, message: error.message }),
+        onSuccess: (value) => ({ _tag: "Success" as const, value }),
+      }),
+    );
+    if (jsonResult._tag === "Error") {
+      yield* Metrics.recordWebhookRequest({
+        connector: options.connectorName,
+        path: String(route.path),
+        outcome: "invalid_json",
+      });
+      return badRequest(jsonResult.message);
+    }
+
+    const payloadResult = yield* Schema.decodeUnknownEffect(route.schema)(jsonResult.value).pipe(
+      Effect.match({
+        onFailure: () => ({ _tag: "Error" as const, message: "Invalid webhook payload" }),
+        onSuccess: (value) => ({ _tag: "Success" as const, value }),
+      }),
+    );
+    if (payloadResult._tag === "Error") {
+      yield* Metrics.recordWebhookRequest({
+        connector: options.connectorName,
+        path: String(route.path),
+        outcome: "invalid_payload",
+      });
+      return badRequest(payloadResult.message);
+    }
+
+    const response = yield* Effect.withSpan(
+      route
+        .handler(makeRouteContext(request, rawBodyResult.value, payloadResult.value, collected))
+        .pipe(Effect.tapError((error) => annotateError("webhook_handle", error))),
+      SpanName.webhookHandle,
+      { attributes: { [Attr.webhookPath]: route.path } },
+    );
+
+    const batches = compactBatches(collected);
+
+    if (route.ackMode === "after-enqueue") {
+      const queue = yield* WebhookQueue;
+      yield* Queue.offerAll(queue.queue, batches);
+      const depth = yield* Queue.size(queue.queue);
+      yield* Metrics.setWebhookQueueDepth(options.connectorName, depth);
+      yield* Metrics.recordWebhookRequest({
+        connector: options.connectorName,
+        path: String(route.path),
+        outcome: response.status >= 400 ? "rejected" : "ok",
+      });
+      return response;
+    }
+
+    yield* Effect.withSpan(
+      publishBatches(batches, { connectorName: options.connectorName }).pipe(
+        Effect.tapError((error) => annotateError("webhook_publish", error)),
       ),
+      SpanName.webhookHandle,
+      { attributes: { [Attr.webhookPath]: route.path } },
     );
-    yield* route.handle(payload, request, rawBody);
-    // unsafeJson serializes synchronously — no Effect, no HttpBodyError
-    return HttpServerResponse.jsonUnsafe({ ok: true });
+
+    yield* Metrics.recordWebhookRequest({
+      connector: options.connectorName,
+      path: String(route.path),
+      outcome: response.status >= 400 ? "rejected" : "ok",
+    });
+    return response;
   }).pipe(
-    Effect.catchTag("InvalidWebhookPayloadError", () =>
-      Effect.succeed(
-        HttpServerResponse.jsonUnsafe(
-          { ok: false, error: "Invalid webhook payload" },
-          { status: 400 },
-        ),
-      ),
-    ),
     Effect.catchCause((cause) =>
-      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(
+      Metrics.recordWebhookRequest({
+        connector: options.connectorName,
+        path: String(route.path),
+        outcome: "handler_error",
+      }).pipe(
+        Effect.andThen(Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`)),
         Effect.andThen(
           Effect.succeed(
             HttpServerResponse.jsonUnsafe(
@@ -58,7 +225,10 @@ const makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>
     ),
   );
 
-export const buildWebhookRouter = <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) =>
+export const router = (
+  routes: ReadonlyArray<WebhookRoute>,
+  options: { readonly connectorName: string },
+) =>
   HttpRouter.addAll(
-    routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route))),
+    routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route, options))),
   );

package/src/webhook/types.ts

@@ -1,17 +1,6 @@
-/** biome-ignore-all lint/suspicious/noExplicitAny: Effect Schema encodes invariantly; `any` on the encoded type is intentional. */
-
-import type { Effect } from "effect";
-import type * as Schema from "effect/Schema";
-import type { HttpRouter, HttpServerRequest } from "effect/unstable/http";
-
-import type { ConnectorError } from "../core/errors";
-
-export type WebhookRoute<TPayload> = {
-  readonly path: HttpRouter.PathInput;
-  readonly schema: Schema.Schema<TPayload>;
-  readonly handle: (
-    payload: TPayload,
-    request: HttpServerRequest.HttpServerRequest,
-    rawBody?: Uint8Array,
-  ) => Effect.Effect<void, ConnectorError>;
-};
+export type {
+  WebhookAckMode,
+  WebhookRoute,
+  WebhookRouteContext,
+  WebhookRouteInput,
+} from "../core/types";

package/src/ingestion/state-store.ts

@@ -1,36 +0,0 @@
-import { Context, Effect, Layer, Ref } from "effect";
-
-import type { ConnectorError } from "../core/errors";
-import type { Cursor, IngestionState } from "../core/types";
-
-export class StateStore extends Context.Service<
-  StateStore,
-  {
-    readonly getState: (
-      key: string,
-    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;
-    readonly setState: (
-      key: string,
-      state: IngestionState<Cursor>,
-    ) => Effect.Effect<void, ConnectorError>;
-  }
->()("StateStore") {}
-
-export const StateStoreInMemory = Layer.effect(StateStore)(
-  Effect.gen(function* () {
-    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());
-
-    return {
-      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),
-      setState: (key, state) =>
-        Effect.map(
-          Ref.update(ref, (map) => {
-            const next = new Map(map);
-            next.set(key, state);
-            return next;
-          }),
-          () => undefined,
-        ),
-    };
-  }),
-);

package/src/streams/pull-stream.ts

@@ -1,54 +0,0 @@
-import { Effect, Stream } from "effect";
-
-import type { ConnectorError } from "../core/errors";
-import type { Batch, Cursor } from "../core/types";
-
-export type PullPage<T> = {
-  readonly cursor: Cursor;
-  readonly rows: ReadonlyArray<T>;
-  readonly hasMore: boolean;
-};
-
-export type PullFetcher<T, R = never> = (
-  cursor: Cursor | undefined,
-) => Effect.Effect<PullPage<T>, ConnectorError, R>;
-
-export type PullStreamOptions<T, R = never> = {
-  readonly initialCursor?: Cursor;
-  readonly fetchPage: PullFetcher<T, R>;
-};
-
-export const makePullStream = <T, R = never>(
-  options: PullStreamOptions<T, R>,
-): Stream.Stream<Batch<T>, ConnectorError, R> =>
-  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>
-    Effect.gen(function* () {
-      if (state.done) {
-        return undefined;
-      }
-
-      let nextCursor: Cursor | undefined = state.cursor;
-
-      while (true) {
-        const page = yield* options.fetchPage(nextCursor);
-
-        if (page.rows.length > 0) {
-          const batch: Batch<T> = {
-            cursor: page.cursor,
-            rows: page.rows,
-          };
-
-          return [
-            batch,
-            page.hasMore ? { cursor: page.cursor, done: false } : { cursor: undefined, done: true },
-          ] as const;
-        }
-
-        if (!page.hasMore) {
-          return undefined;
-        }
-
-        nextCursor = page.cursor;
-      }
-    }),
-  );

package/src/streams/webhook-queue.ts

@@ -1,14 +0,0 @@
-import { Effect, Queue, Stream } from "effect";
-
-import type { ConnectorError } from "../core/errors";
-import type { Batch, WebhookStream } from "../core/types";
-
-export const makeWebhookQueue = <T>(options?: {
-  readonly capacity?: number;
-}): Effect.Effect<WebhookStream<T>, never> =>
-  Effect.gen(function* () {
-    const capacity = options?.capacity ?? 1024;
-    const queue = yield* Queue.bounded<Batch<T>>(capacity);
-    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(queue);
-    return { queue, stream };
-  });