@useairfoil/connector-kit 0.3.0 → 0.4.0

package/README.md

@@ -105,23 +105,19 @@ Connector-kit is designed around Effect services and Layers. Your application sh
 
 ### Testing with VCR
 
-VCR is provided via `@useairfoil/effect-http-client` as an Effect `HttpClient` layer. This keeps HTTP recording out of connector logic.
+VCR is provided via `@useairfoil/effect-vcr` as an Effect `HttpClient` layer. This keeps HTTP recording out of connector logic.
 
 ```ts
 import { FetchHttpClient } from "@effect/platform";
 import { NodeFileSystem } from "@effect/platform-node";
-import { CassetteStoreLive, VcrHttpClientLayer } from "@useairfoil/effect-http-client";
+import { FileSystemCassetteStore, VcrHttpClient } from "@useairfoil/effect-vcr";
 import { Layer } from "effect";
 
-const cassetteLayer = CassetteStoreLive.pipe(
-  Layer.provide(NodeFileSystem.layer),
-);
+const cassetteLayer = CassetteStoreLive.pipe(Layer.provide(NodeFileSystem.layer));
 
 const vcrLayer = VcrHttpClientLayer({
   cassetteDir: "cassettes",
   cassetteName: "example",
   mode: "auto",
-}).pipe(
-  Layer.provide(Layer.mergeAll(FetchHttpClient.layer, cassetteLayer)),
-);
+}).pipe(Layer.provide(Layer.mergeAll(FetchHttpClient.layer, cassetteLayer)));
 ```

package/dist/index.d.ts

@@ -1,5 +1,4 @@
-import { Effect, Layer, Queue, Schema, Stream } from "effect";
-import * as ServiceMap from "effect/ServiceMap";
+import { Context, Effect, Layer, Queue, Schema, Stream } from "effect";
 import * as Wings from "@useairfoil/wings";
 import { PartitionValue } from "@useairfoil/wings";
 import { HttpRouter, HttpServerRequest } from "effect/unstable/http";
@@ -10,7 +9,7 @@ import * as effect_Layer0 from "effect/Layer";
 import * as Schema$1 from "effect/Schema";
 
 //#region src/core/errors.d.ts
-declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.MatchRecord<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }, void, { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
+declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
   readonly _tag: "ConnectorError";
 } & Readonly<A>;
 declare class ConnectorError extends ConnectorError_base<{
@@ -105,7 +104,7 @@ declare const defineEvent: <S extends EntitySchema>(definition: EventDefinition<
 type PublishAck = {
   readonly success: boolean;
 };
-declare const Publisher_base: ServiceMap.ServiceClass<Publisher, "Publisher", {
+declare const Publisher_base: Context.ServiceClass<Publisher, "Publisher", {
   readonly publish: (options: {
     readonly name: string;
     readonly batch: Batch<Record<string, unknown>>;
@@ -114,7 +113,7 @@ declare const Publisher_base: ServiceMap.ServiceClass<Publisher, "Publisher", {
 declare class Publisher extends Publisher_base {}
 //#endregion
 //#region src/ingestion/state-store.d.ts
-declare const StateStore_base: ServiceMap.ServiceClass<StateStore, "StateStore", {
+declare const StateStore_base: Context.ServiceClass<StateStore, "StateStore", {
   readonly getState: (key: string) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;
   readonly setState: (key: string, state: IngestionState<Cursor>) => Effect.Effect<void, ConnectorError>;
 }>;

package/dist/index.js

@@ -1,5 +1,4 @@
-import { Cause, Data, Effect, Layer, Queue, Ref, Schema, Stream } from "effect";
-import * as ServiceMap from "effect/ServiceMap";
+import { Cause, Context, Data, Effect, Layer, Queue, Ref, Schema, Stream } from "effect";
 import * as Wings from "@useairfoil/wings";
 import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
 
@@ -14,11 +13,11 @@ var ConnectorError = class extends Data.TaggedError("ConnectorError") {};
 
 //#endregion
 //#region src/publisher/service.ts
-var Publisher = class extends ServiceMap.Service()("Publisher") {};
+var Publisher = class extends Context.Service()("Publisher") {};
 
 //#endregion
 //#region src/ingestion/state-store.ts
-var StateStore = class extends ServiceMap.Service()("StateStore") {};
+var StateStore = class extends Context.Service()("StateStore") {};
 const StateStoreInMemory = Layer.effect(StateStore)(Effect.gen(function* () {
 	const ref = yield* Ref.make(/* @__PURE__ */ new Map());
 	return {

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["nextCursor: Cursor | undefined"],"sources":["../src/core/builder.ts","../src/core/errors.ts","../src/publisher/service.ts","../src/ingestion/state-store.ts","../src/ingestion/engine.ts","../src/publisher/wings.ts","../src/streams/pull-stream.ts","../src/streams/webhook-queue.ts","../src/webhook/server.ts"],"sourcesContent":["import type {\n  ConnectorDefinition,\n  EntityDefinition,\n  EntitySchema,\n  EventDefinition,\n} from \"./types\";\n\nexport const defineConnector = <\n  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,\n  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,\n>(\n  definition: ConnectorDefinition<Entities, Events>,\n) => definition;\n\nexport const defineEntity = <S extends EntitySchema>(\n  definition: EntityDefinition<S>,\n) => definition;\n\nexport const defineEvent = <S extends EntitySchema>(\n  definition: EventDefinition<S>,\n) => definition;\n","import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n","import type * as Effect from \"effect/Effect\";\nimport * as ServiceMap from \"effect/ServiceMap\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch } from \"../core/types\";\n\nexport type PublishAck = {\n  readonly success: boolean;\n};\n\nexport class Publisher extends ServiceMap.Service<\n  Publisher,\n  {\n    readonly publish: (options: {\n      readonly name: string;\n      readonly batch: Batch<Record<string, unknown>>;\n    }) => Effect.Effect<PublishAck, ConnectorError>;\n  }\n>()(\"Publisher\") {}\n","import { Effect, Layer, Ref } from \"effect\";\nimport * as ServiceMap from \"effect/ServiceMap\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Cursor, IngestionState } from \"../core/types\";\n\nexport class StateStore extends ServiceMap.Service<\n  StateStore,\n  {\n    readonly getState: (\n      key: string,\n    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;\n    readonly setState: (\n      key: string,\n      state: IngestionState<Cursor>,\n    ) => Effect.Effect<void, ConnectorError>;\n  }\n>()(\"StateStore\") {}\n\nexport const StateStoreInMemory = Layer.effect(StateStore)(\n  Effect.gen(function* () {\n    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());\n\n    return {\n      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),\n      setState: (key, state) =>\n        Effect.map(\n          Ref.update(ref, (map) => {\n            const next = new Map(map);\n            next.set(key, state);\n            return next;\n          }),\n          () => undefined,\n        ),\n    };\n  }),\n);\n","import { Effect, Queue, Ref, Stream } from \"effect\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type {\n  Batch,\n  ConnectorDefinition,\n  Cursor,\n  EntityDefinition,\n  EntityRow,\n  EntitySchema,\n  EventDefinition,\n  IngestionState,\n  LiveSource,\n  Transform,\n  WebhookStream,\n} from \"../core/types\";\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"./state-store\";\n\ntype TaggedBatch<T> = {\n  readonly source: \"live\" | \"backfill\";\n  readonly batch: Batch<T>;\n};\n\nexport const runConnector = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    // Start ingestion for every entity and event in parallel.\n    const entityRuns = connector.entities.map((entity) =>\n      runEntity(entity, initialCutoff),\n    );\n    const eventRuns = connector.events.map((event) =>\n      runEvent(event, initialCutoff),\n    );\n    // main runner\n    yield* Effect.all([...entityRuns, ...eventRuns], {\n      concurrency: \"unbounded\",\n    });\n  });\n\nconst createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({\n  backfill: { cutoff },\n  live: { cutoff },\n});\n\nconst makeStateRef = (\n  key: string,\n  initialCutoff: Cursor,\n): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>\n  Effect.gen(function* () {\n    // Load persisted state or initialize a new one for this stream.\n    const store = yield* StateStore;\n    const existing = yield* store.getState(key);\n    const initial = existing ?? createInitialState(initialCutoff);\n    return yield* Ref.make(initial);\n  });\n\nconst runEntity = <S extends EntitySchema>(\n  entity: EntityDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(entity.name, initialCutoff);\n    // Tracks which primary keys have already been emitted.\n    const seenRef = yield* Ref.make(new Set<string>());\n\n    const liveStream = resolveLiveStream(entity.live);\n    const tagLive = (batch: Batch<Row>) => ({\n      source: \"live\" as const,\n      batch,\n    });\n    const updateSeen = (rows: ReadonlyArray<Row>) =>\n      Ref.update(seenRef, (seen) => {\n        const next = new Set(seen);\n        for (const row of rows) {\n          const key = String(row[entity.primaryKey]);\n          next.add(key);\n        }\n        return next;\n      });\n\n    // Entities are upserts, so live and backfill can overlap. We keep an in-memory\n    // seen set (primary keys) so backfill does not re-emit rows already observed live.\n    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>\n      Ref.get(seenRef).pipe(\n        Effect.map((seen) => {\n          const filtered = batch.rows.filter((row) => {\n            const key = String(row[entity.primaryKey]);\n            return !seen.has(key);\n          });\n          return {\n            source: \"backfill\" as const,\n            batch: { cursor: batch.cursor, rows: filtered },\n          };\n        }),\n      ),\n    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));\n\n    // For webhook live sources, we wait for the first live batch before starting\n    // backfill. That first batch establishes the cutoff timestamp and seeds the\n    // seen set so backfill can de-dupe correctly.\n    // Queue-backed streams are single-consumer; splitting with take/drop would\n    // consume and discard elements. Take the first element directly from the\n    // queue, then let Stream.fromQueue continue from element #2.\n    if (isWebhookStream(entity.live)) {\n      const firstBatch = yield* Queue.take(entity.live.queue);\n      yield* updateSeen(firstBatch.rows);\n      yield* processTaggedStream(\n        Stream.make({ source: \"live\" as const, batch: firstBatch }),\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n\n      // liveStream is Stream.fromQueue on the same queue, continues from element #2.\n      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(\n        Stream.tap(({ batch }) => updateSeen(batch.rows)),\n      );\n      const merged = Stream.merge(liveTailTagged, backfillTagged);\n      yield* processTaggedStream(\n        merged,\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n      return;\n    }\n\n    // For pull-based live sources, we can merge immediately because there is no\n    // webhook cutoff gating and the live stream is not queue-backed.\n    const liveTagged = Stream.map(liveStream, tagLive).pipe(\n      Stream.tap(({ batch }) => updateSeen(batch.rows)),\n    );\n\n    const merged = Stream.merge(liveTagged, backfillTagged);\n    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n  });\n\nconst runEvent = <S extends EntitySchema>(\n  event: EventDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(event.name, initialCutoff);\n    const liveStream = resolveLiveStream<Row>(event.live);\n\n    // Events must backfill first to preserve ordering.\n    if (event.backfill) {\n      const backfillTagged = Stream.map(event.backfill, (batch) => ({\n        source: \"backfill\" as const,\n        batch,\n      }));\n      yield* processTaggedStream(\n        backfillTagged,\n        event.name,\n        event.transform,\n        stateRef,\n      );\n    }\n\n    const liveTagged = Stream.map(liveStream, (batch) => ({\n      source: \"live\" as const,\n      batch,\n    }));\n    yield* processTaggedStream(\n      liveTagged,\n      event.name,\n      event.transform,\n      stateRef,\n    );\n  });\n\nconst updateState = (\n  state: IngestionState<Cursor>,\n  source: \"live\" | \"backfill\",\n  cursor: Cursor,\n): IngestionState<Cursor> =>\n  source === \"live\"\n    ? { ...state, live: { ...state.live, current: cursor } }\n    : { ...state, backfill: { ...state.backfill, current: cursor } };\n\nconst resolveLiveStream = <T>(\n  source: LiveSource<T>,\n): Stream.Stream<Batch<T>, ConnectorError> =>\n  isWebhookStream(source) ? source.stream : source;\n\nconst isWebhookStream = <T>(\n  source: LiveSource<T>,\n): source is WebhookStream<T> =>\n  typeof source === \"object\" &&\n  source !== null &&\n  \"queue\" in source &&\n  \"stream\" in source;\n\nconst processTaggedStream = <T extends Record<string, unknown>>(\n  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,\n  name: string,\n  transform: Transform<T> | undefined,\n  stateRef: Ref.Ref<IngestionState<Cursor>>,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Stream.runForEach(stream, ({ source, batch }) =>\n    Effect.gen(function* () {\n      // Optional per-row transformation.\n      const rows = transform\n        ? yield* Effect.forEach(batch.rows, transform)\n        : batch.rows;\n\n      // Publish before updating cursor state.\n      const publisher = yield* Publisher;\n      yield* publisher.publish({\n        name,\n        batch: {\n          cursor: batch.cursor,\n          rows,\n        },\n      });\n\n      // Persist state only after publish succeeds.\n      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>\n        updateState(state, source, batch.cursor),\n      );\n\n      const store = yield* StateStore;\n      yield* store.setState(name, nextState);\n    }),\n  );\n","import type { PartitionValue } from \"@useairfoil/wings\";\nimport * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\nimport { ConnectorError } from \"../core/errors\";\nimport type { ConnectorDefinition } from \"../core/types\";\nimport { Publisher } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\nexport type WingsPublisherConfig = {\n  readonly connector: ConnectorDefinition;\n  /** Map of entity/event name to Wings topic. */\n  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;\n  /** per-stream partition value (key is entity/event name). */\n  readonly partitionValues?: Record<string, PartitionValue>;\n};\n\n/** Publisher entry for a single entity/event. */\ntype PublisherEntry = {\n  /** Wings publisher for the entity/event. */\n  readonly publisher: Wings.WingsClient.Publisher;\n  /** Partition field name (if any). */\n  readonly partitionField?: string;\n  /** Partition value (if any). */\n  readonly partitionValue?: PartitionValue;\n};\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n  // Convert JSON rows into an Arrow RecordBatch for Wings.\n  const table = Wings.tableFromJSON(Array.from(rows));\n  const [batch] = table.batches;\n\n  if (!batch) {\n    throw new ConnectorError({ message: \"No rows to publish\" });\n  }\n\n  return batch;\n};\n\nexport const WingsPublisherLayer = (\n  config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n  Layer.effect(Publisher)(\n    Effect.gen(function* () {\n      const entries = new Map<string, PublisherEntry>();\n\n      // create and store a wings publisher for each entity/event.\n      for (const def of [\n        ...config.connector.entities,\n        ...config.connector.events,\n      ]) {\n        const topic = config.topics[def.name];\n        if (!topic) {\n          return yield* Effect.fail(\n            new ConnectorError({ message: `Missing topic for ${def.name}` }),\n          );\n        }\n\n        const partitionIndex =\n          topic.partitionKey !== undefined\n            ? topic.schema.fields.findIndex(\n                (field) => field.id === topic.partitionKey,\n              )\n            : undefined;\n\n        const partitionField =\n          partitionIndex !== undefined && partitionIndex >= 0\n            ? topic.schema.fields[partitionIndex]?.name\n            : undefined;\n\n        const publisher = yield* Wings.WingsClient.publisher({\n          topic,\n          partitionValue: config.partitionValues?.[def.name],\n        }).pipe(\n          Effect.mapError(\n            (error) =>\n              new ConnectorError({ message: error.message, cause: error }),\n          ),\n        );\n\n        entries.set(def.name, {\n          publisher,\n          partitionField,\n          partitionValue: config.partitionValues?.[def.name],\n        });\n      }\n\n      return {\n        publish: ({ name, batch }) =>\n          Effect.gen(function* () {\n            const entry = entries.get(name);\n            if (!entry) {\n              return yield* Effect.fail(\n                new ConnectorError({ message: `Unknown stream ${name}` }),\n              );\n            }\n\n            if (batch.rows.length === 0) {\n              return { success: true };\n            }\n\n            const recordBatch = buildRecordBatch(batch.rows);\n            const result = yield* entry.publisher\n              .push({\n                batch: recordBatch,\n                partitionValue: entry.partitionValue,\n              })\n              .pipe(\n                Effect.mapError(\n                  (error) =>\n                    new ConnectorError({\n                      message: error.message,\n                      cause: error,\n                    }),\n                ),\n              );\n\n            return {\n              success: !!(result.result && result.result.$case === \"accepted\"),\n            };\n          }),\n      };\n    }),\n  );\n","import { Effect, Stream } from \"effect\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, Cursor } from \"../core/types\";\n\nexport type PullPage<T> = {\n  readonly cursor: Cursor;\n  readonly rows: ReadonlyArray<T>;\n  readonly hasMore: boolean;\n};\n\nexport type PullFetcher<T, R = never> = (\n  cursor: Cursor | undefined,\n) => Effect.Effect<PullPage<T>, ConnectorError, R>;\n\nexport type PullStreamOptions<T, R = never> = {\n  readonly initialCursor?: Cursor;\n  readonly fetchPage: PullFetcher<T, R>;\n};\n\nexport const makePullStream = <T, R = never>(\n  options: PullStreamOptions<T, R>,\n): Stream.Stream<Batch<T>, ConnectorError, R> =>\n  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>\n    Effect.gen(function* () {\n      if (state.done) {\n        return undefined;\n      }\n\n      let nextCursor: Cursor | undefined = state.cursor;\n\n      while (true) {\n        const page = yield* options.fetchPage(nextCursor);\n\n        if (page.rows.length > 0) {\n          const batch: Batch<T> = {\n            cursor: page.cursor,\n            rows: page.rows,\n          };\n\n          return [\n            batch,\n            page.hasMore\n              ? { cursor: page.cursor, done: false }\n              : { cursor: undefined, done: true },\n          ] as const;\n        }\n\n        if (!page.hasMore) {\n          return undefined;\n        }\n\n        nextCursor = page.cursor;\n      }\n    }),\n  );\n","import { Effect, Queue, Stream } from \"effect\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, WebhookStream } from \"../core/types\";\n\nexport const makeWebhookQueue = <T>(options?: {\n  readonly capacity?: number;\n}): Effect.Effect<WebhookStream<T>, never> =>\n  Effect.gen(function* () {\n    const capacity = options?.capacity ?? 1024;\n    const queue = yield* Queue.bounded<Batch<T>>(capacity);\n    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(\n      queue,\n    );\n    return { queue, stream };\n  });\n","import { Cause, Data, Effect, Schema } from \"effect\";\nimport {\n  HttpRouter,\n  HttpServerRequest,\n  HttpServerResponse,\n} from \"effect/unstable/http\";\nimport type { WebhookRoute } from \"./types\";\n\nclass InvalidWebhookPayloadError extends Data.TaggedError(\n  \"InvalidWebhookPayloadError\",\n)<{\n  readonly message: string;\n}> {}\n\nconst makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>\n  Effect.gen(function* () {\n    const request = yield* HttpServerRequest.HttpServerRequest;\n    const rawBuffer = yield* request.arrayBuffer.pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Failed to read request body\",\n          }),\n      ),\n    );\n    const rawBody = new Uint8Array(rawBuffer);\n    const rawText = new TextDecoder().decode(rawBody);\n    const rawJson = yield* Effect.try({\n      try: () => JSON.parse(rawText) as unknown,\n      catch: () =>\n        new InvalidWebhookPayloadError({ message: \"Invalid JSON body\" }),\n    });\n    const payload = yield* Schema.decodeUnknownEffect(route.schema)(\n      rawJson,\n    ).pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Invalid webhook payload\",\n          }),\n      ),\n    );\n    yield* route.handle(payload, request, rawBody);\n    // unsafeJson serializes synchronously — no Effect, no HttpBodyError\n    return HttpServerResponse.jsonUnsafe({ ok: true });\n  }).pipe(\n    Effect.catchTag(\"InvalidWebhookPayloadError\", () =>\n      Effect.succeed(\n        HttpServerResponse.jsonUnsafe(\n          { ok: false, error: \"Invalid webhook payload\" },\n          { status: 400 },\n        ),\n      ),\n    ),\n    Effect.catchCause((cause) =>\n      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(\n        Effect.andThen(\n          Effect.succeed(\n            HttpServerResponse.jsonUnsafe(\n              { ok: false, error: \"Webhook handler failed\" },\n              { status: 500 },\n            ),\n          ),\n        ),\n      ),\n    ),\n  );\n\nexport const buildWebhookRouter = <TPayload>(\n  routes: ReadonlyArray<WebhookRoute<TPayload>>,\n) =>\n  HttpRouter.addAll(\n    routes.map((route) =>\n      HttpRouter.route(\"POST\", route.path, makeHandler(route)),\n    ),\n  );\n"],"mappings":";;;;;;AAOA,MAAa,mBAIX,eACG;AAEL,MAAa,gBACX,eACG;AAEL,MAAa,eACX,eACG;;;;AClBL,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE;;;;ACIH,IAAa,YAAb,cAA+B,WAAW,SAQvC,CAAC,YAAY,CAAC;;;;ACZjB,IAAa,aAAb,cAAgC,WAAW,SAWxC,CAAC,aAAa,CAAC;AAElB,MAAa,qBAAqB,MAAM,OAAO,WAAW,CACxD,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,IAAI,qBAAK,IAAI,KAAqC,CAAC;AAEtE,QAAO;EACL,WAAW,QAAQ,OAAO,IAAI,IAAI,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC;EAClE,WAAW,KAAK,UACd,OAAO,IACL,IAAI,OAAO,MAAM,QAAQ;GACvB,MAAM,OAAO,IAAI,IAAI,IAAI;AACzB,QAAK,IAAI,KAAK,MAAM;AACpB,UAAO;IACP,QACI,OACP;EACJ;EACD,CACH;;;;ACZD,MAAa,gBACX,WACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,aAAa,UAAU,SAAS,KAAK,WACzC,UAAU,QAAQ,cAAc,CACjC;CACD,MAAM,YAAY,UAAU,OAAO,KAAK,UACtC,SAAS,OAAO,cAAc,CAC/B;AAED,QAAO,OAAO,IAAI,CAAC,GAAG,YAAY,GAAG,UAAU,EAAE,EAC/C,aAAa,aACd,CAAC;EACF;AAEJ,MAAM,sBAAsB,YAA4C;CACtE,UAAU,EAAE,QAAQ;CACpB,MAAM,EAAE,QAAQ;CACjB;AAED,MAAM,gBACJ,KACA,kBAEA,OAAO,IAAI,aAAa;CAItB,MAAM,WADW,QADH,OAAO,YACS,SAAS,IAAI,KACf,mBAAmB,cAAc;AAC7D,QAAO,OAAO,IAAI,KAAK,QAAQ;EAC/B;AAEJ,MAAM,aACJ,QACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,OAAO,MAAM,cAAc;CAEhE,MAAM,UAAU,OAAO,IAAI,qBAAK,IAAI,KAAa,CAAC;CAElD,MAAM,aAAa,kBAAkB,OAAO,KAAK;CACjD,MAAM,WAAW,WAAuB;EACtC,QAAQ;EACR;EACD;CACD,MAAM,cAAc,SAClB,IAAI,OAAO,UAAU,SAAS;EAC5B,MAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,QAAK,IAAI,IAAI;;AAEf,SAAO;GACP;CAIJ,MAAM,iBAAiB,OAAO,UAAU,OAAO,WAAW,UACxD,IAAI,IAAI,QAAQ,CAAC,KACf,OAAO,KAAK,SAAS;EACnB,MAAM,WAAW,MAAM,KAAK,QAAQ,QAAQ;GAC1C,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,UAAO,CAAC,KAAK,IAAI,IAAI;IACrB;AACF,SAAO;GACL,QAAQ;GACR,OAAO;IAAE,QAAQ,MAAM;IAAQ,MAAM;IAAU;GAChD;GACD,CACH,CACF,CAAC,KAAK,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAAC;AAQzD,KAAI,gBAAgB,OAAO,KAAK,EAAE;EAChC,MAAM,aAAa,OAAO,MAAM,KAAK,OAAO,KAAK,MAAM;AACvD,SAAO,WAAW,WAAW,KAAK;AAClC,SAAO,oBACL,OAAO,KAAK;GAAE,QAAQ;GAAiB,OAAO;GAAY,CAAC,EAC3D,OAAO,MACP,OAAO,WACP,SACD;EAGD,MAAM,iBAAiB,OAAO,IAAI,YAAY,QAAQ,CAAC,KACrD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAED,SAAO,oBADQ,OAAO,MAAM,gBAAgB,eAAe,EAGzD,OAAO,MACP,OAAO,WACP,SACD;AACD;;CAKF,MAAM,aAAa,OAAO,IAAI,YAAY,QAAQ,CAAC,KACjD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAGD,QAAO,oBADQ,OAAO,MAAM,YAAY,eAAe,EACpB,OAAO,MAAM,OAAO,WAAW,SAAS;EAC3E;AAEJ,MAAM,YACJ,OACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,MAAM,MAAM,cAAc;CAC/D,MAAM,aAAa,kBAAuB,MAAM,KAAK;AAGrD,KAAI,MAAM,SAKR,QAAO,oBAJgB,OAAO,IAAI,MAAM,WAAW,WAAW;EAC5D,QAAQ;EACR;EACD,EAAE,EAGD,MAAM,MACN,MAAM,WACN,SACD;AAOH,QAAO,oBAJY,OAAO,IAAI,aAAa,WAAW;EACpD,QAAQ;EACR;EACD,EAAE,EAGD,MAAM,MACN,MAAM,WACN,SACD;EACD;AAEJ,MAAM,eACJ,OACA,QACA,WAEA,WAAW,SACP;CAAE,GAAG;CAAO,MAAM;EAAE,GAAG,MAAM;EAAM,SAAS;EAAQ;CAAE,GACtD;CAAE,GAAG;CAAO,UAAU;EAAE,GAAG,MAAM;EAAU,SAAS;EAAQ;CAAE;AAEpE,MAAM,qBACJ,WAEA,gBAAgB,OAAO,GAAG,OAAO,SAAS;AAE5C,MAAM,mBACJ,WAEA,OAAO,WAAW,YAClB,WAAW,QACX,WAAW,UACX,YAAY;AAEd,MAAM,uBACJ,QACA,MACA,WACA,aAEA,OAAO,WAAW,SAAS,EAAE,QAAQ,YACnC,OAAO,IAAI,aAAa;CAEtB,MAAM,OAAO,YACT,OAAO,OAAO,QAAQ,MAAM,MAAM,UAAU,GAC5C,MAAM;AAIV,SADkB,OAAO,WACR,QAAQ;EACvB;EACA,OAAO;GACL,QAAQ,MAAM;GACd;GACD;EACF,CAAC;CAGF,MAAM,YAAY,OAAO,IAAI,aAAa,WAAW,UACnD,YAAY,OAAO,QAAQ,MAAM,OAAO,CACzC;AAGD,SADc,OAAO,YACR,SAAS,MAAM,UAAU;EACtC,CACH;;;;ACzMH,MAAM,oBAAoB,SAA8B;CAGtD,MAAM,CAAC,SADO,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CAC7B;AAEtB,KAAI,CAAC,MACH,OAAM,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC;AAG7D,QAAO;;AAGT,MAAa,uBACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;AAGjD,MAAK,MAAM,OAAO,CAChB,GAAG,OAAO,UAAU,UACpB,GAAG,OAAO,UAAU,OACrB,EAAE;EACD,MAAM,QAAQ,OAAO,OAAO,IAAI;AAChC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,IAAI,QAAQ,CAAC,CACjE;EAGH,MAAM,iBACJ,MAAM,iBAAiB,SACnB,MAAM,OAAO,OAAO,WACjB,UAAU,MAAM,OAAO,MAAM,aAC/B,GACD;EAEN,MAAM,iBACJ,mBAAmB,UAAa,kBAAkB,IAC9C,MAAM,OAAO,OAAO,iBAAiB,OACrC;EAEN,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC,CAAC,KACD,OAAO,UACJ,UACC,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAC/D,CACF;AAED,UAAQ,IAAI,IAAI,MAAM;GACpB;GACA;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC;;AAGJ,QAAO,EACL,UAAU,EAAE,MAAM,YAChB,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,QAAQ,IAAI,KAAK;AAC/B,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,kBAAkB,QAAQ,CAAC,CAC1D;AAGH,MAAI,MAAM,KAAK,WAAW,EACxB,QAAO,EAAE,SAAS,MAAM;EAG1B,MAAM,cAAc,iBAAiB,MAAM,KAAK;EAChD,MAAM,SAAS,OAAO,MAAM,UACzB,KAAK;GACJ,OAAO;GACP,gBAAgB,MAAM;GACvB,CAAC,CACD,KACC,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,MAAM;GACf,OAAO;GACR,CAAC,CACL,CACF;AAEH,SAAO,EACL,SAAS,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,UAAU,aACtD;GACD,EACL;EACD,CACH;;;;ACxGH,MAAa,kBACX,YAEA,OAAO,OAAO;CAAE,QAAQ,QAAQ;CAAe,MAAM;CAAO,GAAG,UAC7D,OAAO,IAAI,aAAa;AACtB,KAAI,MAAM,KACR;CAGF,IAAIA,aAAiC,MAAM;AAE3C,QAAO,MAAM;EACX,MAAM,OAAO,OAAO,QAAQ,UAAU,WAAW;AAEjD,MAAI,KAAK,KAAK,SAAS,EAMrB,QAAO,CALiB;GACtB,QAAQ,KAAK;GACb,MAAM,KAAK;GACZ,EAIC,KAAK,UACD;GAAE,QAAQ,KAAK;GAAQ,MAAM;GAAO,GACpC;GAAE,QAAQ;GAAW,MAAM;GAAM,CACtC;AAGH,MAAI,CAAC,KAAK,QACR;AAGF,eAAa,KAAK;;EAEpB,CACH;;;;AClDH,MAAa,oBAAuB,YAGlC,OAAO,IAAI,aAAa;CACtB,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,QAAQ,OAAO,MAAM,QAAkB,SAAS;AAItD,QAAO;EAAE;EAAO,QAHwC,OAAO,UAC7D,MACD;EACuB;EACxB;;;;ACNJ,IAAM,6BAAN,cAAyC,KAAK,YAC5C,6BACD,CAEE;AAEH,MAAM,eAAyB,UAC7B,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,kBAAkB;CACzC,MAAM,YAAY,OAAO,QAAQ,YAAY,KAC3C,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,+BACV,CAAC,CACL,CACF;CACD,MAAM,UAAU,IAAI,WAAW,UAAU;CACzC,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,QAAQ;CACjD,MAAM,UAAU,OAAO,OAAO,IAAI;EAChC,WAAW,KAAK,MAAM,QAAQ;EAC9B,aACE,IAAI,2BAA2B,EAAE,SAAS,qBAAqB,CAAC;EACnE,CAAC;CACF,MAAM,UAAU,OAAO,OAAO,oBAAoB,MAAM,OAAO,CAC7D,QACD,CAAC,KACA,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,2BACV,CAAC,CACL,CACF;AACD,QAAO,MAAM,OAAO,SAAS,SAAS,QAAQ;AAE9C,QAAO,mBAAmB,WAAW,EAAE,IAAI,MAAM,CAAC;EAClD,CAAC,KACD,OAAO,SAAS,oCACd,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA2B,EAC/C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,EACD,OAAO,YAAY,UACjB,OAAO,WAAW,0BAA0B,MAAM,OAAO,MAAM,GAAG,CAAC,KACjE,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA0B,EAC9C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,CACF,CACF,CACF;AAEH,MAAa,sBACX,WAEA,WAAW,OACT,OAAO,KAAK,UACV,WAAW,MAAM,QAAQ,MAAM,MAAM,YAAY,MAAM,CAAC,CACzD,CACF"}
+{"version":3,"file":"index.js","names":["nextCursor: Cursor | undefined"],"sources":["../src/core/builder.ts","../src/core/errors.ts","../src/publisher/service.ts","../src/ingestion/state-store.ts","../src/ingestion/engine.ts","../src/publisher/wings.ts","../src/streams/pull-stream.ts","../src/streams/webhook-queue.ts","../src/webhook/server.ts"],"sourcesContent":["import type { ConnectorDefinition, EntityDefinition, EntitySchema, EventDefinition } from \"./types\";\n\nexport const defineConnector = <\n  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,\n  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,\n>(\n  definition: ConnectorDefinition<Entities, Events>,\n) => definition;\n\nexport const defineEntity = <S extends EntitySchema>(definition: EntityDefinition<S>) => definition;\n\nexport const defineEvent = <S extends EntitySchema>(definition: EventDefinition<S>) => definition;\n","import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n","import type * as Effect from \"effect/Effect\";\n\nimport { Context } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch } from \"../core/types\";\n\nexport type PublishAck = {\n  readonly success: boolean;\n};\n\nexport class Publisher extends Context.Service<\n  Publisher,\n  {\n    readonly publish: (options: {\n      readonly name: string;\n      readonly batch: Batch<Record<string, unknown>>;\n    }) => Effect.Effect<PublishAck, ConnectorError>;\n  }\n>()(\"Publisher\") {}\n","import { Context, Effect, Layer, Ref } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Cursor, IngestionState } from \"../core/types\";\n\nexport class StateStore extends Context.Service<\n  StateStore,\n  {\n    readonly getState: (\n      key: string,\n    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;\n    readonly setState: (\n      key: string,\n      state: IngestionState<Cursor>,\n    ) => Effect.Effect<void, ConnectorError>;\n  }\n>()(\"StateStore\") {}\n\nexport const StateStoreInMemory = Layer.effect(StateStore)(\n  Effect.gen(function* () {\n    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());\n\n    return {\n      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),\n      setState: (key, state) =>\n        Effect.map(\n          Ref.update(ref, (map) => {\n            const next = new Map(map);\n            next.set(key, state);\n            return next;\n          }),\n          () => undefined,\n        ),\n    };\n  }),\n);\n","import { Effect, Queue, Ref, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type {\n  Batch,\n  ConnectorDefinition,\n  Cursor,\n  EntityDefinition,\n  EntityRow,\n  EntitySchema,\n  EventDefinition,\n  IngestionState,\n  LiveSource,\n  Transform,\n  WebhookStream,\n} from \"../core/types\";\n\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"./state-store\";\n\ntype TaggedBatch<T> = {\n  readonly source: \"live\" | \"backfill\";\n  readonly batch: Batch<T>;\n};\n\nexport const runConnector = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    // Start ingestion for every entity and event in parallel.\n    const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));\n    const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));\n    // main runner\n    yield* Effect.all([...entityRuns, ...eventRuns], {\n      concurrency: \"unbounded\",\n    });\n  });\n\nconst createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({\n  backfill: { cutoff },\n  live: { cutoff },\n});\n\nconst makeStateRef = (\n  key: string,\n  initialCutoff: Cursor,\n): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>\n  Effect.gen(function* () {\n    // Load persisted state or initialize a new one for this stream.\n    const store = yield* StateStore;\n    const existing = yield* store.getState(key);\n    const initial = existing ?? createInitialState(initialCutoff);\n    return yield* Ref.make(initial);\n  });\n\nconst runEntity = <S extends EntitySchema>(\n  entity: EntityDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(entity.name, initialCutoff);\n    // Tracks which primary keys have already been emitted.\n    const seenRef = yield* Ref.make(new Set<string>());\n\n    const liveStream = resolveLiveStream(entity.live);\n    const tagLive = (batch: Batch<Row>) => ({\n      source: \"live\" as const,\n      batch,\n    });\n    const updateSeen = (rows: ReadonlyArray<Row>) =>\n      Ref.update(seenRef, (seen) => {\n        const next = new Set(seen);\n        for (const row of rows) {\n          const key = String(row[entity.primaryKey]);\n          next.add(key);\n        }\n        return next;\n      });\n\n    // Entities are upserts, so live and backfill can overlap. We keep an in-memory\n    // seen set (primary keys) so backfill does not re-emit rows already observed live.\n    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>\n      Ref.get(seenRef).pipe(\n        Effect.map((seen) => {\n          const filtered = batch.rows.filter((row) => {\n            const key = String(row[entity.primaryKey]);\n            return !seen.has(key);\n          });\n          return {\n            source: \"backfill\" as const,\n            batch: { cursor: batch.cursor, rows: filtered },\n          };\n        }),\n      ),\n    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));\n\n    // For webhook live sources, we wait for the first live batch before starting\n    // backfill. That first batch establishes the cutoff timestamp and seeds the\n    // seen set so backfill can de-dupe correctly.\n    // Queue-backed streams are single-consumer; splitting with take/drop would\n    // consume and discard elements. Take the first element directly from the\n    // queue, then let Stream.fromQueue continue from element #2.\n    if (isWebhookStream(entity.live)) {\n      const firstBatch = yield* Queue.take(entity.live.queue);\n      yield* updateSeen(firstBatch.rows);\n      yield* processTaggedStream(\n        Stream.make({ source: \"live\" as const, batch: firstBatch }),\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n\n      // liveStream is Stream.fromQueue on the same queue, continues from element #2.\n      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(\n        Stream.tap(({ batch }) => updateSeen(batch.rows)),\n      );\n      const merged = Stream.merge(liveTailTagged, backfillTagged);\n      yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n      return;\n    }\n\n    // For pull-based live sources, we can merge immediately because there is no\n    // webhook cutoff gating and the live stream is not queue-backed.\n    const liveTagged = Stream.map(liveStream, tagLive).pipe(\n      Stream.tap(({ batch }) => updateSeen(batch.rows)),\n    );\n\n    const merged = Stream.merge(liveTagged, backfillTagged);\n    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n  });\n\nconst runEvent = <S extends EntitySchema>(\n  event: EventDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(event.name, initialCutoff);\n    const liveStream = resolveLiveStream<Row>(event.live);\n\n    // Events must backfill first to preserve ordering.\n    if (event.backfill) {\n      const backfillTagged = Stream.map(event.backfill, (batch) => ({\n        source: \"backfill\" as const,\n        batch,\n      }));\n      yield* processTaggedStream(backfillTagged, event.name, event.transform, stateRef);\n    }\n\n    const liveTagged = Stream.map(liveStream, (batch) => ({\n      source: \"live\" as const,\n      batch,\n    }));\n    yield* processTaggedStream(liveTagged, event.name, event.transform, stateRef);\n  });\n\nconst updateState = (\n  state: IngestionState<Cursor>,\n  source: \"live\" | \"backfill\",\n  cursor: Cursor,\n): IngestionState<Cursor> =>\n  source === \"live\"\n    ? { ...state, live: { ...state.live, current: cursor } }\n    : { ...state, backfill: { ...state.backfill, current: cursor } };\n\nconst resolveLiveStream = <T>(source: LiveSource<T>): Stream.Stream<Batch<T>, ConnectorError> =>\n  isWebhookStream(source) ? source.stream : source;\n\nconst isWebhookStream = <T>(source: LiveSource<T>): source is WebhookStream<T> =>\n  typeof source === \"object\" && source !== null && \"queue\" in source && \"stream\" in source;\n\nconst processTaggedStream = <T extends Record<string, unknown>>(\n  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,\n  name: string,\n  transform: Transform<T> | undefined,\n  stateRef: Ref.Ref<IngestionState<Cursor>>,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Stream.runForEach(stream, ({ source, batch }) =>\n    Effect.gen(function* () {\n      // Optional per-row transformation.\n      const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;\n\n      // Publish before updating cursor state.\n      const publisher = yield* Publisher;\n      yield* publisher.publish({\n        name,\n        batch: {\n          cursor: batch.cursor,\n          rows,\n        },\n      });\n\n      // Persist state only after publish succeeds.\n      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>\n        updateState(state, source, batch.cursor),\n      );\n\n      const store = yield* StateStore;\n      yield* store.setState(name, nextState);\n    }),\n  );\n","import type { PartitionValue } from \"@useairfoil/wings\";\n\nimport * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\n\nimport type { ConnectorDefinition } from \"../core/types\";\n\nimport { ConnectorError } from \"../core/errors\";\nimport { Publisher } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\nexport type WingsPublisherConfig = {\n  readonly connector: ConnectorDefinition;\n  /** Map of entity/event name to Wings topic. */\n  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;\n  /** per-stream partition value (key is entity/event name). */\n  readonly partitionValues?: Record<string, PartitionValue>;\n};\n\n/** Publisher entry for a single entity/event. */\ntype PublisherEntry = {\n  /** Wings publisher for the entity/event. */\n  readonly publisher: Wings.WingsClient.Publisher;\n  /** Partition field name (if any). */\n  readonly partitionField?: string;\n  /** Partition value (if any). */\n  readonly partitionValue?: PartitionValue;\n};\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n  // Convert JSON rows into an Arrow RecordBatch for Wings.\n  const table = Wings.tableFromJSON(Array.from(rows));\n  const [batch] = table.batches;\n\n  if (!batch) {\n    throw new ConnectorError({ message: \"No rows to publish\" });\n  }\n\n  return batch;\n};\n\nexport const WingsPublisherLayer = (\n  config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n  Layer.effect(Publisher)(\n    Effect.gen(function* () {\n      const entries = new Map<string, PublisherEntry>();\n\n      // create and store a wings publisher for each entity/event.\n      for (const def of [...config.connector.entities, ...config.connector.events]) {\n        const topic = config.topics[def.name];\n        if (!topic) {\n          return yield* Effect.fail(\n            new ConnectorError({ message: `Missing topic for ${def.name}` }),\n          );\n        }\n\n        const partitionIndex =\n          topic.partitionKey !== undefined\n            ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey)\n            : undefined;\n\n        const partitionField =\n          partitionIndex !== undefined && partitionIndex >= 0\n            ? topic.schema.fields[partitionIndex]?.name\n            : undefined;\n\n        const publisher = yield* Wings.WingsClient.publisher({\n          topic,\n          partitionValue: config.partitionValues?.[def.name],\n        }).pipe(\n          Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),\n        );\n\n        entries.set(def.name, {\n          publisher,\n          partitionField,\n          partitionValue: config.partitionValues?.[def.name],\n        });\n      }\n\n      return {\n        publish: ({ name, batch }) =>\n          Effect.gen(function* () {\n            const entry = entries.get(name);\n            if (!entry) {\n              return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));\n            }\n\n            if (batch.rows.length === 0) {\n              return { success: true };\n            }\n\n            const recordBatch = buildRecordBatch(batch.rows);\n            const result = yield* entry.publisher\n              .push({\n                batch: recordBatch,\n                partitionValue: entry.partitionValue,\n              })\n              .pipe(\n                Effect.mapError(\n                  (error) =>\n                    new ConnectorError({\n                      message: error.message,\n                      cause: error,\n                    }),\n                ),\n              );\n\n            return {\n              success: !!(result.result && result.result.$case === \"accepted\"),\n            };\n          }),\n      };\n    }),\n  );\n","import { Effect, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, Cursor } from \"../core/types\";\n\nexport type PullPage<T> = {\n  readonly cursor: Cursor;\n  readonly rows: ReadonlyArray<T>;\n  readonly hasMore: boolean;\n};\n\nexport type PullFetcher<T, R = never> = (\n  cursor: Cursor | undefined,\n) => Effect.Effect<PullPage<T>, ConnectorError, R>;\n\nexport type PullStreamOptions<T, R = never> = {\n  readonly initialCursor?: Cursor;\n  readonly fetchPage: PullFetcher<T, R>;\n};\n\nexport const makePullStream = <T, R = never>(\n  options: PullStreamOptions<T, R>,\n): Stream.Stream<Batch<T>, ConnectorError, R> =>\n  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>\n    Effect.gen(function* () {\n      if (state.done) {\n        return undefined;\n      }\n\n      let nextCursor: Cursor | undefined = state.cursor;\n\n      while (true) {\n        const page = yield* options.fetchPage(nextCursor);\n\n        if (page.rows.length > 0) {\n          const batch: Batch<T> = {\n            cursor: page.cursor,\n            rows: page.rows,\n          };\n\n          return [\n            batch,\n            page.hasMore ? { cursor: page.cursor, done: false } : { cursor: undefined, done: true },\n          ] as const;\n        }\n\n        if (!page.hasMore) {\n          return undefined;\n        }\n\n        nextCursor = page.cursor;\n      }\n    }),\n  );\n","import { Effect, Queue, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, WebhookStream } from \"../core/types\";\n\nexport const makeWebhookQueue = <T>(options?: {\n  readonly capacity?: number;\n}): Effect.Effect<WebhookStream<T>, never> =>\n  Effect.gen(function* () {\n    const capacity = options?.capacity ?? 1024;\n    const queue = yield* Queue.bounded<Batch<T>>(capacity);\n    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(queue);\n    return { queue, stream };\n  });\n","import { Cause, Data, Effect, Schema } from \"effect\";\nimport { HttpRouter, HttpServerRequest, HttpServerResponse } from \"effect/unstable/http\";\n\nimport type { WebhookRoute } from \"./types\";\n\nclass InvalidWebhookPayloadError extends Data.TaggedError(\"InvalidWebhookPayloadError\")<{\n  readonly message: string;\n}> {}\n\nconst makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>\n  Effect.gen(function* () {\n    const request = yield* HttpServerRequest.HttpServerRequest;\n    const rawBuffer = yield* request.arrayBuffer.pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Failed to read request body\",\n          }),\n      ),\n    );\n    const rawBody = new Uint8Array(rawBuffer);\n    const rawText = new TextDecoder().decode(rawBody);\n    const rawJson = yield* Effect.try({\n      try: () => JSON.parse(rawText) as unknown,\n      catch: () => new InvalidWebhookPayloadError({ message: \"Invalid JSON body\" }),\n    });\n    const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Invalid webhook payload\",\n          }),\n      ),\n    );\n    yield* route.handle(payload, request, rawBody);\n    // unsafeJson serializes synchronously — no Effect, no HttpBodyError\n    return HttpServerResponse.jsonUnsafe({ ok: true });\n  }).pipe(\n    Effect.catchTag(\"InvalidWebhookPayloadError\", () =>\n      Effect.succeed(\n        HttpServerResponse.jsonUnsafe(\n          { ok: false, error: \"Invalid webhook payload\" },\n          { status: 400 },\n        ),\n      ),\n    ),\n    Effect.catchCause((cause) =>\n      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(\n        Effect.andThen(\n          Effect.succeed(\n            HttpServerResponse.jsonUnsafe(\n              { ok: false, error: \"Webhook handler failed\" },\n              { status: 500 },\n            ),\n          ),\n        ),\n      ),\n    ),\n  );\n\nexport const buildWebhookRouter = <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) =>\n  HttpRouter.addAll(\n    routes.map((route) => HttpRouter.route(\"POST\", route.path, makeHandler(route))),\n  );\n"],"mappings":";;;;;AAEA,MAAa,mBAIX,eACG;AAEL,MAAa,gBAAwC,eAAoC;AAEzF,MAAa,eAAuC,eAAmC;;;;ACTvF,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE;;;;ACMH,IAAa,YAAb,cAA+B,QAAQ,SAQpC,CAAC,YAAY,CAAC;;;;ACdjB,IAAa,aAAb,cAAgC,QAAQ,SAWrC,CAAC,aAAa,CAAC;AAElB,MAAa,qBAAqB,MAAM,OAAO,WAAW,CACxD,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,IAAI,qBAAK,IAAI,KAAqC,CAAC;AAEtE,QAAO;EACL,WAAW,QAAQ,OAAO,IAAI,IAAI,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC;EAClE,WAAW,KAAK,UACd,OAAO,IACL,IAAI,OAAO,MAAM,QAAQ;GACvB,MAAM,OAAO,IAAI,IAAI,IAAI;AACzB,QAAK,IAAI,KAAK,MAAM;AACpB,UAAO;IACP,QACI,OACP;EACJ;EACD,CACH;;;;ACVD,MAAa,gBACX,WACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,aAAa,UAAU,SAAS,KAAK,WAAW,UAAU,QAAQ,cAAc,CAAC;CACvF,MAAM,YAAY,UAAU,OAAO,KAAK,UAAU,SAAS,OAAO,cAAc,CAAC;AAEjF,QAAO,OAAO,IAAI,CAAC,GAAG,YAAY,GAAG,UAAU,EAAE,EAC/C,aAAa,aACd,CAAC;EACF;AAEJ,MAAM,sBAAsB,YAA4C;CACtE,UAAU,EAAE,QAAQ;CACpB,MAAM,EAAE,QAAQ;CACjB;AAED,MAAM,gBACJ,KACA,kBAEA,OAAO,IAAI,aAAa;CAItB,MAAM,WADW,QADH,OAAO,YACS,SAAS,IAAI,KACf,mBAAmB,cAAc;AAC7D,QAAO,OAAO,IAAI,KAAK,QAAQ;EAC/B;AAEJ,MAAM,aACJ,QACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,OAAO,MAAM,cAAc;CAEhE,MAAM,UAAU,OAAO,IAAI,qBAAK,IAAI,KAAa,CAAC;CAElD,MAAM,aAAa,kBAAkB,OAAO,KAAK;CACjD,MAAM,WAAW,WAAuB;EACtC,QAAQ;EACR;EACD;CACD,MAAM,cAAc,SAClB,IAAI,OAAO,UAAU,SAAS;EAC5B,MAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,QAAK,IAAI,IAAI;;AAEf,SAAO;GACP;CAIJ,MAAM,iBAAiB,OAAO,UAAU,OAAO,WAAW,UACxD,IAAI,IAAI,QAAQ,CAAC,KACf,OAAO,KAAK,SAAS;EACnB,MAAM,WAAW,MAAM,KAAK,QAAQ,QAAQ;GAC1C,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,UAAO,CAAC,KAAK,IAAI,IAAI;IACrB;AACF,SAAO;GACL,QAAQ;GACR,OAAO;IAAE,QAAQ,MAAM;IAAQ,MAAM;IAAU;GAChD;GACD,CACH,CACF,CAAC,KAAK,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAAC;AAQzD,KAAI,gBAAgB,OAAO,KAAK,EAAE;EAChC,MAAM,aAAa,OAAO,MAAM,KAAK,OAAO,KAAK,MAAM;AACvD,SAAO,WAAW,WAAW,KAAK;AAClC,SAAO,oBACL,OAAO,KAAK;GAAE,QAAQ;GAAiB,OAAO;GAAY,CAAC,EAC3D,OAAO,MACP,OAAO,WACP,SACD;EAGD,MAAM,iBAAiB,OAAO,IAAI,YAAY,QAAQ,CAAC,KACrD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAED,SAAO,oBADQ,OAAO,MAAM,gBAAgB,eAAe,EACxB,OAAO,MAAM,OAAO,WAAW,SAAS;AAC3E;;CAKF,MAAM,aAAa,OAAO,IAAI,YAAY,QAAQ,CAAC,KACjD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAGD,QAAO,oBADQ,OAAO,MAAM,YAAY,eAAe,EACpB,OAAO,MAAM,OAAO,WAAW,SAAS;EAC3E;AAEJ,MAAM,YACJ,OACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,MAAM,MAAM,cAAc;CAC/D,MAAM,aAAa,kBAAuB,MAAM,KAAK;AAGrD,KAAI,MAAM,SAKR,QAAO,oBAJgB,OAAO,IAAI,MAAM,WAAW,WAAW;EAC5D,QAAQ;EACR;EACD,EAAE,EACwC,MAAM,MAAM,MAAM,WAAW,SAAS;AAOnF,QAAO,oBAJY,OAAO,IAAI,aAAa,WAAW;EACpD,QAAQ;EACR;EACD,EAAE,EACoC,MAAM,MAAM,MAAM,WAAW,SAAS;EAC7E;AAEJ,MAAM,eACJ,OACA,QACA,WAEA,WAAW,SACP;CAAE,GAAG;CAAO,MAAM;EAAE,GAAG,MAAM;EAAM,SAAS;EAAQ;CAAE,GACtD;CAAE,GAAG;CAAO,UAAU;EAAE,GAAG,MAAM;EAAU,SAAS;EAAQ;CAAE;AAEpE,MAAM,qBAAwB,WAC5B,gBAAgB,OAAO,GAAG,OAAO,SAAS;AAE5C,MAAM,mBAAsB,WAC1B,OAAO,WAAW,YAAY,WAAW,QAAQ,WAAW,UAAU,YAAY;AAEpF,MAAM,uBACJ,QACA,MACA,WACA,aAEA,OAAO,WAAW,SAAS,EAAE,QAAQ,YACnC,OAAO,IAAI,aAAa;CAEtB,MAAM,OAAO,YAAY,OAAO,OAAO,QAAQ,MAAM,MAAM,UAAU,GAAG,MAAM;AAI9E,SADkB,OAAO,WACR,QAAQ;EACvB;EACA,OAAO;GACL,QAAQ,MAAM;GACd;GACD;EACF,CAAC;CAGF,MAAM,YAAY,OAAO,IAAI,aAAa,WAAW,UACnD,YAAY,OAAO,QAAQ,MAAM,OAAO,CACzC;AAGD,SADc,OAAO,YACR,SAAS,MAAM,UAAU;EACtC,CACH;;;;AC5KH,MAAM,oBAAoB,SAA8B;CAGtD,MAAM,CAAC,SADO,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CAC7B;AAEtB,KAAI,CAAC,MACH,OAAM,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC;AAG7D,QAAO;;AAGT,MAAa,uBACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;AAGjD,MAAK,MAAM,OAAO,CAAC,GAAG,OAAO,UAAU,UAAU,GAAG,OAAO,UAAU,OAAO,EAAE;EAC5E,MAAM,QAAQ,OAAO,OAAO,IAAI;AAChC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,IAAI,QAAQ,CAAC,CACjE;EAGH,MAAM,iBACJ,MAAM,iBAAiB,SACnB,MAAM,OAAO,OAAO,WAAW,UAAU,MAAM,OAAO,MAAM,aAAa,GACzE;EAEN,MAAM,iBACJ,mBAAmB,UAAa,kBAAkB,IAC9C,MAAM,OAAO,OAAO,iBAAiB,OACrC;EAEN,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAAC,CACzF;AAED,UAAQ,IAAI,IAAI,MAAM;GACpB;GACA;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC;;AAGJ,QAAO,EACL,UAAU,EAAE,MAAM,YAChB,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,QAAQ,IAAI,KAAK;AAC/B,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,kBAAkB,QAAQ,CAAC,CAAC;AAGtF,MAAI,MAAM,KAAK,WAAW,EACxB,QAAO,EAAE,SAAS,MAAM;EAG1B,MAAM,cAAc,iBAAiB,MAAM,KAAK;EAChD,MAAM,SAAS,OAAO,MAAM,UACzB,KAAK;GACJ,OAAO;GACP,gBAAgB,MAAM;GACvB,CAAC,CACD,KACC,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,MAAM;GACf,OAAO;GACR,CAAC,CACL,CACF;AAEH,SAAO,EACL,SAAS,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,UAAU,aACtD;GACD,EACL;EACD,CACH;;;;AChGH,MAAa,kBACX,YAEA,OAAO,OAAO;CAAE,QAAQ,QAAQ;CAAe,MAAM;CAAO,GAAG,UAC7D,OAAO,IAAI,aAAa;AACtB,KAAI,MAAM,KACR;CAGF,IAAIA,aAAiC,MAAM;AAE3C,QAAO,MAAM;EACX,MAAM,OAAO,OAAO,QAAQ,UAAU,WAAW;AAEjD,MAAI,KAAK,KAAK,SAAS,EAMrB,QAAO,CALiB;GACtB,QAAQ,KAAK;GACb,MAAM,KAAK;GACZ,EAIC,KAAK,UAAU;GAAE,QAAQ,KAAK;GAAQ,MAAM;GAAO,GAAG;GAAE,QAAQ;GAAW,MAAM;GAAM,CACxF;AAGH,MAAI,CAAC,KAAK,QACR;AAGF,eAAa,KAAK;;EAEpB,CACH;;;;AChDH,MAAa,oBAAuB,YAGlC,OAAO,IAAI,aAAa;CACtB,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,QAAQ,OAAO,MAAM,QAAkB,SAAS;AAEtD,QAAO;EAAE;EAAO,QADwC,OAAO,UAAU,MAAM;EACvD;EACxB;;;;ACRJ,IAAM,6BAAN,cAAyC,KAAK,YAAY,6BAA6B,CAEpF;AAEH,MAAM,eAAyB,UAC7B,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,kBAAkB;CACzC,MAAM,YAAY,OAAO,QAAQ,YAAY,KAC3C,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,+BACV,CAAC,CACL,CACF;CACD,MAAM,UAAU,IAAI,WAAW,UAAU;CACzC,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,QAAQ;CACjD,MAAM,UAAU,OAAO,OAAO,IAAI;EAChC,WAAW,KAAK,MAAM,QAAQ;EAC9B,aAAa,IAAI,2BAA2B,EAAE,SAAS,qBAAqB,CAAC;EAC9E,CAAC;CACF,MAAM,UAAU,OAAO,OAAO,oBAAoB,MAAM,OAAO,CAAC,QAAQ,CAAC,KACvE,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,2BACV,CAAC,CACL,CACF;AACD,QAAO,MAAM,OAAO,SAAS,SAAS,QAAQ;AAE9C,QAAO,mBAAmB,WAAW,EAAE,IAAI,MAAM,CAAC;EAClD,CAAC,KACD,OAAO,SAAS,oCACd,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA2B,EAC/C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,EACD,OAAO,YAAY,UACjB,OAAO,WAAW,0BAA0B,MAAM,OAAO,MAAM,GAAG,CAAC,KACjE,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA0B,EAC9C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,CACF,CACF,CACF;AAEH,MAAa,sBAAgC,WAC3C,WAAW,OACT,OAAO,KAAK,UAAU,WAAW,MAAM,QAAQ,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC,CAChF"}

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@useairfoil/connector-kit",
-  "version": "0.3.0",
-  "type": "module",
+  "version": "0.4.0",
+  "repository": "useairfoil/typescript-sdk",
   "files": [
     "dist",
     "src",
     "README.md"
   ],
+  "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "exports": {
@@ -20,25 +21,22 @@
     "build": "tsdown",
     "test": "vitest",
     "test:ci": "vitest run",
-    "typecheck": "tsc --noEmit",
-    "lint": "biome check .",
-    "lint:fix": "biome check . --write",
-    "format": "biome format --write ."
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@useairfoil/flight": "0.4.0",
+    "@useairfoil/wings": "0.4.0",
+    "consola": "^3.4.2",
+    "json-stable-stringify": "^1.3.0"
   },
   "devDependencies": {
-    "@effect/vitest": "4.0.0-beta.25",
+    "@effect/vitest": "4.0.0-beta.54",
     "@types/node": "^24.2.0",
     "tsdown": "^0.15.6",
     "vitest": "^3.2.4"
   },
-  "dependencies": {
-    "@useairfoil/flight": "0.2.0",
-    "@useairfoil/wings": "0.2.0",
-    "consola": "^3.4.2",
-    "json-stable-stringify": "^1.3.0"
-  },
   "peerDependencies": {
-    "effect": "4.0.0-beta.25"
+    "effect": "4.0.0-beta.54"
   },
   "peerDependenciesMeta": {
     "effect": {

package/src/core/builder.ts

@@ -1,9 +1,4 @@
-import type {
-  ConnectorDefinition,
-  EntityDefinition,
-  EntitySchema,
-  EventDefinition,
-} from "./types";
+import type { ConnectorDefinition, EntityDefinition, EntitySchema, EventDefinition } from "./types";
 
 export const defineConnector = <
   const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,
@@ -12,10 +7,6 @@ export const defineConnector = <
   definition: ConnectorDefinition<Entities, Events>,
 ) => definition;
 
-export const defineEntity = <S extends EntitySchema>(
-  definition: EntityDefinition<S>,
-) => definition;
+export const defineEntity = <S extends EntitySchema>(definition: EntityDefinition<S>) => definition;
 
-export const defineEvent = <S extends EntitySchema>(
-  definition: EventDefinition<S>,
-) => definition;
+export const defineEvent = <S extends EntitySchema>(definition: EventDefinition<S>) => definition;

package/src/core/types.ts

@@ -1,4 +1,5 @@
 import type { Effect, Queue, Schema, Stream } from "effect";
+
 import type { ConnectorError } from "./errors";
 
 export type Cursor = string | number | bigint | Date;
@@ -43,12 +44,9 @@ export type EntitySchema = Schema.Schema<unknown>;
 export type EntityType<S extends EntitySchema> = Schema.Schema.Type<S>;
 /** Primary key type derived from the decoded schema shape. */
 export type EntityKey<S extends EntitySchema> =
-  EntityType<S> extends Record<string, unknown>
-    ? keyof EntityType<S> & string
-    : never;
+  EntityType<S> extends Record<string, unknown> ? keyof EntityType<S> & string : never;
 /** Row type constrained to object-like shapes for ingestion. */
-export type EntityRow<S extends EntitySchema> = EntityType<S> &
-  Record<string, unknown>;
+export type EntityRow<S extends EntitySchema> = EntityType<S> & Record<string, unknown>;
 
 export type EntityDefinition<S extends EntitySchema> = {
   readonly name: string;
@@ -89,9 +87,9 @@ export type EventDefinition<S extends EntitySchema> = {
 };
 
 export type ConnectorDefinition<
-  Entities extends ReadonlyArray<
+  Entities extends ReadonlyArray<EntityDefinition<EntitySchema>> = ReadonlyArray<
     EntityDefinition<EntitySchema>
-  > = ReadonlyArray<EntityDefinition<EntitySchema>>,
+  >,
   Events extends ReadonlyArray<EventDefinition<EntitySchema>> = ReadonlyArray<
     EventDefinition<EntitySchema>
   >,

package/src/ingestion/engine.ts

@@ -1,4 +1,5 @@
 import { Effect, Queue, Ref, Stream } from "effect";
+
 import type { ConnectorError } from "../core/errors";
 import type {
   Batch,
@@ -13,6 +14,7 @@ import type {
   Transform,
   WebhookStream,
 } from "../core/types";
+
 import { Publisher } from "../publisher/service";
 import { StateStore } from "./state-store";
 
@@ -27,12 +29,8 @@ export const runConnector = (
 ): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
   Effect.gen(function* () {
     // Start ingestion for every entity and event in parallel.
-    const entityRuns = connector.entities.map((entity) =>
-      runEntity(entity, initialCutoff),
-    );
-    const eventRuns = connector.events.map((event) =>
-      runEvent(event, initialCutoff),
-    );
+    const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));
+    const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));
     // main runner
     yield* Effect.all([...entityRuns, ...eventRuns], {
       concurrency: "unbounded",
@@ -119,12 +117,7 @@ const runEntity = <S extends EntitySchema>(
         Stream.tap(({ batch }) => updateSeen(batch.rows)),
       );
       const merged = Stream.merge(liveTailTagged, backfillTagged);
-      yield* processTaggedStream(
-        merged,
-        entity.name,
-        entity.transform,
-        stateRef,
-      );
+      yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);
       return;
     }
 
@@ -153,24 +146,14 @@ const runEvent = <S extends EntitySchema>(
         source: "backfill" as const,
         batch,
       }));
-      yield* processTaggedStream(
-        backfillTagged,
-        event.name,
-        event.transform,
-        stateRef,
-      );
+      yield* processTaggedStream(backfillTagged, event.name, event.transform, stateRef);
     }
 
     const liveTagged = Stream.map(liveStream, (batch) => ({
       source: "live" as const,
       batch,
     }));
-    yield* processTaggedStream(
-      liveTagged,
-      event.name,
-      event.transform,
-      stateRef,
-    );
+    yield* processTaggedStream(liveTagged, event.name, event.transform, stateRef);
   });
 
 const updateState = (
@@ -182,18 +165,11 @@ const updateState = (
     ? { ...state, live: { ...state.live, current: cursor } }
     : { ...state, backfill: { ...state.backfill, current: cursor } };
 
-const resolveLiveStream = <T>(
-  source: LiveSource<T>,
-): Stream.Stream<Batch<T>, ConnectorError> =>
+const resolveLiveStream = <T>(source: LiveSource<T>): Stream.Stream<Batch<T>, ConnectorError> =>
   isWebhookStream(source) ? source.stream : source;
 
-const isWebhookStream = <T>(
-  source: LiveSource<T>,
-): source is WebhookStream<T> =>
-  typeof source === "object" &&
-  source !== null &&
-  "queue" in source &&
-  "stream" in source;
+const isWebhookStream = <T>(source: LiveSource<T>): source is WebhookStream<T> =>
+  typeof source === "object" && source !== null && "queue" in source && "stream" in source;
 
 const processTaggedStream = <T extends Record<string, unknown>>(
   stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,
@@ -204,9 +180,7 @@ const processTaggedStream = <T extends Record<string, unknown>>(
   Stream.runForEach(stream, ({ source, batch }) =>
     Effect.gen(function* () {
       // Optional per-row transformation.
-      const rows = transform
-        ? yield* Effect.forEach(batch.rows, transform)
-        : batch.rows;
+      const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;
 
       // Publish before updating cursor state.
       const publisher = yield* Publisher;

package/src/ingestion/state-store.ts

@@ -1,9 +1,9 @@
-import { Effect, Layer, Ref } from "effect";
-import * as ServiceMap from "effect/ServiceMap";
+import { Context, Effect, Layer, Ref } from "effect";
+
 import type { ConnectorError } from "../core/errors";
 import type { Cursor, IngestionState } from "../core/types";
 
-export class StateStore extends ServiceMap.Service<
+export class StateStore extends Context.Service<
   StateStore,
   {
     readonly getState: (

package/src/publisher/service.ts

@@ -1,5 +1,7 @@
 import type * as Effect from "effect/Effect";
-import * as ServiceMap from "effect/ServiceMap";
+
+import { Context } from "effect";
+
 import type { ConnectorError } from "../core/errors";
 import type { Batch } from "../core/types";
 
@@ -7,7 +9,7 @@ export type PublishAck = {
   readonly success: boolean;
 };
 
-export class Publisher extends ServiceMap.Service<
+export class Publisher extends Context.Service<
   Publisher,
   {
     readonly publish: (options: {

package/src/publisher/wings.ts

@@ -1,8 +1,11 @@
 import type { PartitionValue } from "@useairfoil/wings";
+
 import * as Wings from "@useairfoil/wings";
 import { Effect, Layer } from "effect";
-import { ConnectorError } from "../core/errors";
+
 import type { ConnectorDefinition } from "../core/types";
+
+import { ConnectorError } from "../core/errors";
 import { Publisher } from "./service";
 
 type Rows = Record<string, unknown>;
@@ -45,10 +48,7 @@ export const WingsPublisherLayer = (
       const entries = new Map<string, PublisherEntry>();
 
       // create and store a wings publisher for each entity/event.
-      for (const def of [
-        ...config.connector.entities,
-        ...config.connector.events,
-      ]) {
+      for (const def of [...config.connector.entities, ...config.connector.events]) {
         const topic = config.topics[def.name];
         if (!topic) {
           return yield* Effect.fail(
@@ -58,9 +58,7 @@ export const WingsPublisherLayer = (
 
         const partitionIndex =
           topic.partitionKey !== undefined
-            ? topic.schema.fields.findIndex(
-                (field) => field.id === topic.partitionKey,
-              )
+            ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey)
             : undefined;
 
         const partitionField =
@@ -72,10 +70,7 @@ export const WingsPublisherLayer = (
           topic,
           partitionValue: config.partitionValues?.[def.name],
         }).pipe(
-          Effect.mapError(
-            (error) =>
-              new ConnectorError({ message: error.message, cause: error }),
-          ),
+          Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),
         );
 
         entries.set(def.name, {
@@ -90,9 +85,7 @@ export const WingsPublisherLayer = (
           Effect.gen(function* () {
             const entry = entries.get(name);
             if (!entry) {
-              return yield* Effect.fail(
-                new ConnectorError({ message: `Unknown stream ${name}` }),
-              );
+              return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));
             }
 
             if (batch.rows.length === 0) {

package/src/streams/pull-stream.ts

@@ -1,4 +1,5 @@
 import { Effect, Stream } from "effect";
+
 import type { ConnectorError } from "../core/errors";
 import type { Batch, Cursor } from "../core/types";
 
@@ -39,9 +40,7 @@ export const makePullStream = <T, R = never>(
 
           return [
             batch,
-            page.hasMore
-              ? { cursor: page.cursor, done: false }
-              : { cursor: undefined, done: true },
+            page.hasMore ? { cursor: page.cursor, done: false } : { cursor: undefined, done: true },
           ] as const;
         }
 

package/src/streams/webhook-queue.ts

@@ -1,4 +1,5 @@
 import { Effect, Queue, Stream } from "effect";
+
 import type { ConnectorError } from "../core/errors";
 import type { Batch, WebhookStream } from "../core/types";
 
@@ -8,8 +9,6 @@ export const makeWebhookQueue = <T>(options?: {
   Effect.gen(function* () {
     const capacity = options?.capacity ?? 1024;
     const queue = yield* Queue.bounded<Batch<T>>(capacity);
-    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(
-      queue,
-    );
+    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(queue);
     return { queue, stream };
   });

package/src/webhook/server.ts

@@ -1,14 +1,9 @@
 import { Cause, Data, Effect, Schema } from "effect";
-import {
-  HttpRouter,
-  HttpServerRequest,
-  HttpServerResponse,
-} from "effect/unstable/http";
+import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+
 import type { WebhookRoute } from "./types";
 
-class InvalidWebhookPayloadError extends Data.TaggedError(
-  "InvalidWebhookPayloadError",
-)<{
+class InvalidWebhookPayloadError extends Data.TaggedError("InvalidWebhookPayloadError")<{
   readonly message: string;
 }> {}
 
@@ -27,12 +22,9 @@ const makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>
     const rawText = new TextDecoder().decode(rawBody);
     const rawJson = yield* Effect.try({
       try: () => JSON.parse(rawText) as unknown,
-      catch: () =>
-        new InvalidWebhookPayloadError({ message: "Invalid JSON body" }),
+      catch: () => new InvalidWebhookPayloadError({ message: "Invalid JSON body" }),
     });
-    const payload = yield* Schema.decodeUnknownEffect(route.schema)(
-      rawJson,
-    ).pipe(
+    const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(
       Effect.mapError(
         () =>
           new InvalidWebhookPayloadError({
@@ -66,11 +58,7 @@ const makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>
     ),
   );
 
-export const buildWebhookRouter = <TPayload>(
-  routes: ReadonlyArray<WebhookRoute<TPayload>>,
-) =>
+export const buildWebhookRouter = <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) =>
   HttpRouter.addAll(
-    routes.map((route) =>
-      HttpRouter.route("POST", route.path, makeHandler(route)),
-    ),
+    routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route))),
   );

package/src/webhook/types.ts

@@ -3,6 +3,7 @@
 import type { Effect } from "effect";
 import type * as Schema from "effect/Schema";
 import type { HttpRouter, HttpServerRequest } from "effect/unstable/http";
+
 import type { ConnectorError } from "../core/errors";
 
 export type WebhookRoute<TPayload> = {