@useairfoil/connector-kit 0.3.0

package/src/core/types.ts

@@ -0,0 +1,102 @@
+import type { Effect, Queue, Schema, Stream } from "effect";
+import type { ConnectorError } from "./errors";
+
+export type Cursor = string | number | bigint | Date;
+
+export type Batch<T> = {
+  readonly cursor: Cursor;
+  readonly rows: ReadonlyArray<T>;
+};
+
+export type StreamState<C = Cursor> = {
+  readonly cutoff: C;
+  readonly current?: C;
+};
+
+export type IngestionState<C = Cursor> = {
+  readonly backfill: StreamState<C>;
+  readonly live: StreamState<C>;
+};
+
+export type Transform<T> = (row: T) => Effect.Effect<T, ConnectorError>;
+
+export type LiveStream<T> = Stream.Stream<Batch<T>, ConnectorError>;
+
+export type BackfillStream<T> = Stream.Stream<Batch<T>, ConnectorError>;
+
+export type WebhookStream<T> = {
+  /**
+   * The queue is used to store the batches that are received from the webhook.
+   */
+  readonly queue: Queue.Queue<Batch<T>>;
+  /**
+   * The stream is used to process the batches that are received from the webhook.
+   */
+  readonly stream: Stream.Stream<Batch<T>, ConnectorError>;
+};
+
+export type LiveSource<T> = LiveStream<T> | WebhookStream<T>;
+
+/** Schema type used by connector definitions. */
+export type EntitySchema = Schema.Schema<unknown>;
+/** Decoded row type produced by a schema. */
+export type EntityType<S extends EntitySchema> = Schema.Schema.Type<S>;
+/** Primary key type derived from the decoded schema shape. */
+export type EntityKey<S extends EntitySchema> =
+  EntityType<S> extends Record<string, unknown>
+    ? keyof EntityType<S> & string
+    : never;
+/** Row type constrained to object-like shapes for ingestion. */
+export type EntityRow<S extends EntitySchema> = EntityType<S> &
+  Record<string, unknown>;
+
+export type EntityDefinition<S extends EntitySchema> = {
+  readonly name: string;
+  readonly schema: S;
+  /**
+   * The primary key is used to identify the rows in the database.
+   */
+  readonly primaryKey: EntityKey<S>;
+  /**
+   * The live source is used to stream the live data from the database.
+   */
+  readonly live: LiveSource<EntityRow<S>>;
+  /**
+   * The backfill stream is used to stream the backfill data from the database.
+   */
+  readonly backfill: BackfillStream<EntityRow<S>>;
+  /**
+   * The transform is used to transform the rows before they are published.
+   */
+  readonly transform?: Transform<EntityRow<S>>;
+};
+
+export type EventDefinition<S extends EntitySchema> = {
+  readonly name: string;
+  readonly schema: S;
+  /**
+   * The live source is used to stream the live data from the database.
+   */
+  readonly live: LiveSource<EntityRow<S>>;
+  /**
+   * The backfill stream is used to stream the backfill data from the database.
+   */
+  readonly backfill?: BackfillStream<EntityRow<S>>;
+  /**
+   * The transform is used to transform the rows before they are published.
+   */
+  readonly transform?: Transform<EntityRow<S>>;
+};
+
+export type ConnectorDefinition<
+  Entities extends ReadonlyArray<
+    EntityDefinition<EntitySchema>
+  > = ReadonlyArray<EntityDefinition<EntitySchema>>,
+  Events extends ReadonlyArray<EventDefinition<EntitySchema>> = ReadonlyArray<
+    EventDefinition<EntitySchema>
+  >,
+> = {
+  readonly name: string;
+  readonly entities: Entities;
+  readonly events: Events;
+};

package/src/index.ts

@@ -0,0 +1,28 @@
+export { defineConnector, defineEntity, defineEvent } from "./core/builder";
+export { ConnectorError } from "./core/errors";
+export type {
+  BackfillStream,
+  Batch,
+  ConnectorDefinition,
+  Cursor,
+  EntityDefinition,
+  EntityKey,
+  EntityRow,
+  EntitySchema,
+  EntityType,
+  EventDefinition,
+  IngestionState,
+  LiveSource,
+  LiveStream,
+  StreamState,
+  Transform,
+  WebhookStream,
+} from "./core/types";
+export { runConnector } from "./ingestion/engine";
+export { StateStore, StateStoreInMemory } from "./ingestion/state-store";
+export { Publisher } from "./publisher/service";
+export { WingsPublisherLayer } from "./publisher/wings";
+export { makePullStream } from "./streams/pull-stream";
+export { makeWebhookQueue } from "./streams/webhook-queue";
+export { buildWebhookRouter } from "./webhook/server";
+export type { WebhookRoute } from "./webhook/types";

package/src/ingestion/engine.ts

@@ -0,0 +1,229 @@
+import { Effect, Queue, Ref, Stream } from "effect";
+import type { ConnectorError } from "../core/errors";
+import type {
+  Batch,
+  ConnectorDefinition,
+  Cursor,
+  EntityDefinition,
+  EntityRow,
+  EntitySchema,
+  EventDefinition,
+  IngestionState,
+  LiveSource,
+  Transform,
+  WebhookStream,
+} from "../core/types";
+import { Publisher } from "../publisher/service";
+import { StateStore } from "./state-store";
+
+type TaggedBatch<T> = {
+  readonly source: "live" | "backfill";
+  readonly batch: Batch<T>;
+};
+
+export const runConnector = (
+  connector: ConnectorDefinition,
+  initialCutoff: Cursor,
+): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
+  Effect.gen(function* () {
+    // Start ingestion for every entity and event in parallel.
+    const entityRuns = connector.entities.map((entity) =>
+      runEntity(entity, initialCutoff),
+    );
+    const eventRuns = connector.events.map((event) =>
+      runEvent(event, initialCutoff),
+    );
+    // main runner
+    yield* Effect.all([...entityRuns, ...eventRuns], {
+      concurrency: "unbounded",
+    });
+  });
+
+const createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({
+  backfill: { cutoff },
+  live: { cutoff },
+});
+
+const makeStateRef = (
+  key: string,
+  initialCutoff: Cursor,
+): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>
+  Effect.gen(function* () {
+    // Load persisted state or initialize a new one for this stream.
+    const store = yield* StateStore;
+    const existing = yield* store.getState(key);
+    const initial = existing ?? createInitialState(initialCutoff);
+    return yield* Ref.make(initial);
+  });
+
+const runEntity = <S extends EntitySchema>(
+  entity: EntityDefinition<S>,
+  initialCutoff: Cursor,
+): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
+  Effect.gen(function* () {
+    type Row = EntityRow<S>;
+    const stateRef = yield* makeStateRef(entity.name, initialCutoff);
+    // Tracks which primary keys have already been emitted.
+    const seenRef = yield* Ref.make(new Set<string>());
+
+    const liveStream = resolveLiveStream(entity.live);
+    const tagLive = (batch: Batch<Row>) => ({
+      source: "live" as const,
+      batch,
+    });
+    const updateSeen = (rows: ReadonlyArray<Row>) =>
+      Ref.update(seenRef, (seen) => {
+        const next = new Set(seen);
+        for (const row of rows) {
+          const key = String(row[entity.primaryKey]);
+          next.add(key);
+        }
+        return next;
+      });
+
+    // Entities are upserts, so live and backfill can overlap. We keep an in-memory
+    // seen set (primary keys) so backfill does not re-emit rows already observed live.
+    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>
+      Ref.get(seenRef).pipe(
+        Effect.map((seen) => {
+          const filtered = batch.rows.filter((row) => {
+            const key = String(row[entity.primaryKey]);
+            return !seen.has(key);
+          });
+          return {
+            source: "backfill" as const,
+            batch: { cursor: batch.cursor, rows: filtered },
+          };
+        }),
+      ),
+    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
+
+    // For webhook live sources, we wait for the first live batch before starting
+    // backfill. That first batch establishes the cutoff timestamp and seeds the
+    // seen set so backfill can de-dupe correctly.
+    // Queue-backed streams are single-consumer; splitting with take/drop would
+    // consume and discard elements. Take the first element directly from the
+    // queue, then let Stream.fromQueue continue from element #2.
+    if (isWebhookStream(entity.live)) {
+      const firstBatch = yield* Queue.take(entity.live.queue);
+      yield* updateSeen(firstBatch.rows);
+      yield* processTaggedStream(
+        Stream.make({ source: "live" as const, batch: firstBatch }),
+        entity.name,
+        entity.transform,
+        stateRef,
+      );
+
+      // liveStream is Stream.fromQueue on the same queue, continues from element #2.
+      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(
+        Stream.tap(({ batch }) => updateSeen(batch.rows)),
+      );
+      const merged = Stream.merge(liveTailTagged, backfillTagged);
+      yield* processTaggedStream(
+        merged,
+        entity.name,
+        entity.transform,
+        stateRef,
+      );
+      return;
+    }
+
+    // For pull-based live sources, we can merge immediately because there is no
+    // webhook cutoff gating and the live stream is not queue-backed.
+    const liveTagged = Stream.map(liveStream, tagLive).pipe(
+      Stream.tap(({ batch }) => updateSeen(batch.rows)),
+    );
+
+    const merged = Stream.merge(liveTagged, backfillTagged);
+    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);
+  });
+
+const runEvent = <S extends EntitySchema>(
+  event: EventDefinition<S>,
+  initialCutoff: Cursor,
+): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
+  Effect.gen(function* () {
+    type Row = EntityRow<S>;
+    const stateRef = yield* makeStateRef(event.name, initialCutoff);
+    const liveStream = resolveLiveStream<Row>(event.live);
+
+    // Events must backfill first to preserve ordering.
+    if (event.backfill) {
+      const backfillTagged = Stream.map(event.backfill, (batch) => ({
+        source: "backfill" as const,
+        batch,
+      }));
+      yield* processTaggedStream(
+        backfillTagged,
+        event.name,
+        event.transform,
+        stateRef,
+      );
+    }
+
+    const liveTagged = Stream.map(liveStream, (batch) => ({
+      source: "live" as const,
+      batch,
+    }));
+    yield* processTaggedStream(
+      liveTagged,
+      event.name,
+      event.transform,
+      stateRef,
+    );
+  });
+
+const updateState = (
+  state: IngestionState<Cursor>,
+  source: "live" | "backfill",
+  cursor: Cursor,
+): IngestionState<Cursor> =>
+  source === "live"
+    ? { ...state, live: { ...state.live, current: cursor } }
+    : { ...state, backfill: { ...state.backfill, current: cursor } };
+
+const resolveLiveStream = <T>(
+  source: LiveSource<T>,
+): Stream.Stream<Batch<T>, ConnectorError> =>
+  isWebhookStream(source) ? source.stream : source;
+
+const isWebhookStream = <T>(
+  source: LiveSource<T>,
+): source is WebhookStream<T> =>
+  typeof source === "object" &&
+  source !== null &&
+  "queue" in source &&
+  "stream" in source;
+
+const processTaggedStream = <T extends Record<string, unknown>>(
+  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,
+  name: string,
+  transform: Transform<T> | undefined,
+  stateRef: Ref.Ref<IngestionState<Cursor>>,
+): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
+  Stream.runForEach(stream, ({ source, batch }) =>
+    Effect.gen(function* () {
+      // Optional per-row transformation.
+      const rows = transform
+        ? yield* Effect.forEach(batch.rows, transform)
+        : batch.rows;
+
+      // Publish before updating cursor state.
+      const publisher = yield* Publisher;
+      yield* publisher.publish({
+        name,
+        batch: {
+          cursor: batch.cursor,
+          rows,
+        },
+      });
+
+      // Persist state only after publish succeeds.
+      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>
+        updateState(state, source, batch.cursor),
+      );
+
+      const store = yield* StateStore;
+      yield* store.setState(name, nextState);
+    }),
+  );

package/src/ingestion/state-store.ts

@@ -0,0 +1,36 @@
+import { Effect, Layer, Ref } from "effect";
+import * as ServiceMap from "effect/ServiceMap";
+import type { ConnectorError } from "../core/errors";
+import type { Cursor, IngestionState } from "../core/types";
+
+export class StateStore extends ServiceMap.Service<
+  StateStore,
+  {
+    readonly getState: (
+      key: string,
+    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;
+    readonly setState: (
+      key: string,
+      state: IngestionState<Cursor>,
+    ) => Effect.Effect<void, ConnectorError>;
+  }
+>()("StateStore") {}
+
+export const StateStoreInMemory = Layer.effect(StateStore)(
+  Effect.gen(function* () {
+    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());
+
+    return {
+      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),
+      setState: (key, state) =>
+        Effect.map(
+          Ref.update(ref, (map) => {
+            const next = new Map(map);
+            next.set(key, state);
+            return next;
+          }),
+          () => undefined,
+        ),
+    };
+  }),
+);

package/src/publisher/service.ts

@@ -0,0 +1,18 @@
+import type * as Effect from "effect/Effect";
+import * as ServiceMap from "effect/ServiceMap";
+import type { ConnectorError } from "../core/errors";
+import type { Batch } from "../core/types";
+
+export type PublishAck = {
+  readonly success: boolean;
+};
+
+export class Publisher extends ServiceMap.Service<
+  Publisher,
+  {
+    readonly publish: (options: {
+      readonly name: string;
+      readonly batch: Batch<Record<string, unknown>>;
+    }) => Effect.Effect<PublishAck, ConnectorError>;
+  }
+>()("Publisher") {}

package/src/publisher/wings.ts

@@ -0,0 +1,124 @@
+import type { PartitionValue } from "@useairfoil/wings";
+import * as Wings from "@useairfoil/wings";
+import { Effect, Layer } from "effect";
+import { ConnectorError } from "../core/errors";
+import type { ConnectorDefinition } from "../core/types";
+import { Publisher } from "./service";
+
+type Rows = Record<string, unknown>;
+
+export type WingsPublisherConfig = {
+  readonly connector: ConnectorDefinition;
+  /** Map of entity/event name to Wings topic. */
+  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;
+  /** per-stream partition value (key is entity/event name). */
+  readonly partitionValues?: Record<string, PartitionValue>;
+};
+
+/** Publisher entry for a single entity/event. */
+type PublisherEntry = {
+  /** Wings publisher for the entity/event. */
+  readonly publisher: Wings.WingsClient.Publisher;
+  /** Partition field name (if any). */
+  readonly partitionField?: string;
+  /** Partition value (if any). */
+  readonly partitionValue?: PartitionValue;
+};
+
+const buildRecordBatch = (rows: ReadonlyArray<Rows>) => {
+  // Convert JSON rows into an Arrow RecordBatch for Wings.
+  const table = Wings.tableFromJSON(Array.from(rows));
+  const [batch] = table.batches;
+
+  if (!batch) {
+    throw new ConnectorError({ message: "No rows to publish" });
+  }
+
+  return batch;
+};
+
+export const WingsPublisherLayer = (
+  config: WingsPublisherConfig,
+): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>
+  Layer.effect(Publisher)(
+    Effect.gen(function* () {
+      const entries = new Map<string, PublisherEntry>();
+
+      // create and store a wings publisher for each entity/event.
+      for (const def of [
+        ...config.connector.entities,
+        ...config.connector.events,
+      ]) {
+        const topic = config.topics[def.name];
+        if (!topic) {
+          return yield* Effect.fail(
+            new ConnectorError({ message: `Missing topic for ${def.name}` }),
+          );
+        }
+
+        const partitionIndex =
+          topic.partitionKey !== undefined
+            ? topic.schema.fields.findIndex(
+                (field) => field.id === topic.partitionKey,
+              )
+            : undefined;
+
+        const partitionField =
+          partitionIndex !== undefined && partitionIndex >= 0
+            ? topic.schema.fields[partitionIndex]?.name
+            : undefined;
+
+        const publisher = yield* Wings.WingsClient.publisher({
+          topic,
+          partitionValue: config.partitionValues?.[def.name],
+        }).pipe(
+          Effect.mapError(
+            (error) =>
+              new ConnectorError({ message: error.message, cause: error }),
+          ),
+        );
+
+        entries.set(def.name, {
+          publisher,
+          partitionField,
+          partitionValue: config.partitionValues?.[def.name],
+        });
+      }
+
+      return {
+        publish: ({ name, batch }) =>
+          Effect.gen(function* () {
+            const entry = entries.get(name);
+            if (!entry) {
+              return yield* Effect.fail(
+                new ConnectorError({ message: `Unknown stream ${name}` }),
+              );
+            }
+
+            if (batch.rows.length === 0) {
+              return { success: true };
+            }
+
+            const recordBatch = buildRecordBatch(batch.rows);
+            const result = yield* entry.publisher
+              .push({
+                batch: recordBatch,
+                partitionValue: entry.partitionValue,
+              })
+              .pipe(
+                Effect.mapError(
+                  (error) =>
+                    new ConnectorError({
+                      message: error.message,
+                      cause: error,
+                    }),
+                ),
+              );
+
+            return {
+              success: !!(result.result && result.result.$case === "accepted"),
+            };
+          }),
+      };
+    }),
+  );

package/src/streams/pull-stream.ts

@@ -0,0 +1,55 @@
+import { Effect, Stream } from "effect";
+import type { ConnectorError } from "../core/errors";
+import type { Batch, Cursor } from "../core/types";
+
+export type PullPage<T> = {
+  readonly cursor: Cursor;
+  readonly rows: ReadonlyArray<T>;
+  readonly hasMore: boolean;
+};
+
+export type PullFetcher<T, R = never> = (
+  cursor: Cursor | undefined,
+) => Effect.Effect<PullPage<T>, ConnectorError, R>;
+
+export type PullStreamOptions<T, R = never> = {
+  readonly initialCursor?: Cursor;
+  readonly fetchPage: PullFetcher<T, R>;
+};
+
+export const makePullStream = <T, R = never>(
+  options: PullStreamOptions<T, R>,
+): Stream.Stream<Batch<T>, ConnectorError, R> =>
+  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>
+    Effect.gen(function* () {
+      if (state.done) {
+        return undefined;
+      }
+
+      let nextCursor: Cursor | undefined = state.cursor;
+
+      while (true) {
+        const page = yield* options.fetchPage(nextCursor);
+
+        if (page.rows.length > 0) {
+          const batch: Batch<T> = {
+            cursor: page.cursor,
+            rows: page.rows,
+          };
+
+          return [
+            batch,
+            page.hasMore
+              ? { cursor: page.cursor, done: false }
+              : { cursor: undefined, done: true },
+          ] as const;
+        }
+
+        if (!page.hasMore) {
+          return undefined;
+        }
+
+        nextCursor = page.cursor;
+      }
+    }),
+  );

package/src/streams/webhook-queue.ts

@@ -0,0 +1,15 @@
+import { Effect, Queue, Stream } from "effect";
+import type { ConnectorError } from "../core/errors";
+import type { Batch, WebhookStream } from "../core/types";
+
+export const makeWebhookQueue = <T>(options?: {
+  readonly capacity?: number;
+}): Effect.Effect<WebhookStream<T>, never> =>
+  Effect.gen(function* () {
+    const capacity = options?.capacity ?? 1024;
+    const queue = yield* Queue.bounded<Batch<T>>(capacity);
+    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(
+      queue,
+    );
+    return { queue, stream };
+  });

package/src/webhook/server.ts

@@ -0,0 +1,76 @@
+import { Cause, Data, Effect, Schema } from "effect";
+import {
+  HttpRouter,
+  HttpServerRequest,
+  HttpServerResponse,
+} from "effect/unstable/http";
+import type { WebhookRoute } from "./types";
+
+class InvalidWebhookPayloadError extends Data.TaggedError(
+  "InvalidWebhookPayloadError",
+)<{
+  readonly message: string;
+}> {}
+
+const makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>
+  Effect.gen(function* () {
+    const request = yield* HttpServerRequest.HttpServerRequest;
+    const rawBuffer = yield* request.arrayBuffer.pipe(
+      Effect.mapError(
+        () =>
+          new InvalidWebhookPayloadError({
+            message: "Failed to read request body",
+          }),
+      ),
+    );
+    const rawBody = new Uint8Array(rawBuffer);
+    const rawText = new TextDecoder().decode(rawBody);
+    const rawJson = yield* Effect.try({
+      try: () => JSON.parse(rawText) as unknown,
+      catch: () =>
+        new InvalidWebhookPayloadError({ message: "Invalid JSON body" }),
+    });
+    const payload = yield* Schema.decodeUnknownEffect(route.schema)(
+      rawJson,
+    ).pipe(
+      Effect.mapError(
+        () =>
+          new InvalidWebhookPayloadError({
+            message: "Invalid webhook payload",
+          }),
+      ),
+    );
+    yield* route.handle(payload, request, rawBody);
+    // unsafeJson serializes synchronously — no Effect, no HttpBodyError
+    return HttpServerResponse.jsonUnsafe({ ok: true });
+  }).pipe(
+    Effect.catchTag("InvalidWebhookPayloadError", () =>
+      Effect.succeed(
+        HttpServerResponse.jsonUnsafe(
+          { ok: false, error: "Invalid webhook payload" },
+          { status: 400 },
+        ),
+      ),
+    ),
+    Effect.catchCause((cause) =>
+      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(
+        Effect.andThen(
+          Effect.succeed(
+            HttpServerResponse.jsonUnsafe(
+              { ok: false, error: "Webhook handler failed" },
+              { status: 500 },
+            ),
+          ),
+        ),
+      ),
+    ),
+  );
+
+export const buildWebhookRouter = <TPayload>(
+  routes: ReadonlyArray<WebhookRoute<TPayload>>,
+) =>
+  HttpRouter.addAll(
+    routes.map((route) =>
+      HttpRouter.route("POST", route.path, makeHandler(route)),
+    ),
+  );