@useairfoil/connector-kit 0.3.0

package/README.md

@@ -0,0 +1,127 @@
+# @useairfoil/connector-kit
+
+Toolkit for building Airfoil connectors with Effect.
+
+---
+
+## User guide (build a connector)
+
+This section is for connector authors who want to build and run a connector.
+
+### Install
+
+```bash
+bun add @useairfoil/connector-kit
+```
+
+### Minimal example
+
+```ts
+import { Schema, Effect, Layer, Stream } from "effect";
+import {
+  defineConnector,
+  defineEntity,
+  Publisher,
+  runConnector,
+  StateStoreInMemory,
+  WebhookServerLayer,
+  makeWebhookQueue,
+} from "@useairfoil/connector-kit";
+
+const Customer = Schema.Struct({
+  id: Schema.String,
+  email: Schema.String,
+  created_at: Schema.Date,
+});
+
+const program = Effect.gen(function* () {
+  const webhook = yield* makeWebhookQueue<Schema.Schema.Type<typeof Customer>>();
+
+  const connector = defineConnector({
+    name: "producer-example",
+    entities: [
+      defineEntity({
+        name: "customers",
+        schema: Customer,
+        primaryKey: "id",
+        live: webhook.queue,
+        backfill: Stream.empty,
+      }),
+    ],
+    events: [],
+  });
+
+  yield* runConnector(connector, new Date());
+}).pipe(
+  Effect.provide(StateStoreInMemory),
+  Effect.provide(
+    Layer.succeed(Publisher, {
+      publish: () => Effect.succeed({ requestId: 1n }),
+    }),
+  ),
+  Effect.provide(
+    WebhookServerLayer({
+      port: 8080,
+      routes: [
+        {
+          path: "/webhook/customers",
+          schema: Customer,
+          dispatch: (payload) =>
+            Effect.succeed([
+              {
+                queue: webhook.queue,
+                batch: { cursor: new Date(), rows: [payload] },
+              },
+            ]),
+        },
+      ],
+    }),
+  ),
+);
+
+Effect.runPromise(program);
+```
+
+---
+
+## Development (concepts and architecture)
+
+### Core concepts
+
+- `defineConnector` describes your connector and its entities.
+- `defineEntity` wires live and backfill streams for each entity.
+- `Publisher` is the output boundary (where batches go).
+- `StateStore` tracks cursors and backfill state.
+- `WebhookServerLayer` turns webhook routes into an HTTP server.
+
+### Layers and Effect services
+
+Connector-kit is designed around Effect services and Layers. Your application should provide:
+
+- a `Publisher` Layer
+- a `StateStore` Layer
+- an HTTP server Layer (if you use webhooks)
+- any custom services your connector needs (API clients, Effect Config)
+
+### Testing with VCR
+
+VCR is provided via `@useairfoil/effect-http-client` as an Effect `HttpClient` layer. This keeps HTTP recording out of connector logic.
+
+```ts
+import { FetchHttpClient } from "@effect/platform";
+import { NodeFileSystem } from "@effect/platform-node";
+import { CassetteStoreLive, VcrHttpClientLayer } from "@useairfoil/effect-http-client";
+import { Layer } from "effect";
+
+const cassetteLayer = CassetteStoreLive.pipe(
+  Layer.provide(NodeFileSystem.layer),
+);
+
+const vcrLayer = VcrHttpClientLayer({
+  cassetteDir: "cassettes",
+  cassetteName: "example",
+  mode: "auto",
+}).pipe(
+  Layer.provide(Layer.mergeAll(FetchHttpClient.layer, cassetteLayer)),
+);
+```

package/dist/index.d.ts

@@ -0,0 +1,166 @@
+import { Effect, Layer, Queue, Schema, Stream } from "effect";
+import * as ServiceMap from "effect/ServiceMap";
+import * as Wings from "@useairfoil/wings";
+import { PartitionValue } from "@useairfoil/wings";
+import { HttpRouter, HttpServerRequest } from "effect/unstable/http";
+import * as effect_Types0 from "effect/Types";
+import * as effect_Cause0 from "effect/Cause";
+import * as Effect$1 from "effect/Effect";
+import * as effect_Layer0 from "effect/Layer";
+import * as Schema$1 from "effect/Schema";
+
+//#region src/core/errors.d.ts
+declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.MatchRecord<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }, void, { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
+  readonly _tag: "ConnectorError";
+} & Readonly<A>;
+declare class ConnectorError extends ConnectorError_base<{
+  readonly message: string;
+  readonly cause?: unknown;
+}> {}
+//#endregion
+//#region src/core/types.d.ts
+type Cursor = string | number | bigint | Date;
+type Batch<T> = {
+  readonly cursor: Cursor;
+  readonly rows: ReadonlyArray<T>;
+};
+type StreamState<C = Cursor> = {
+  readonly cutoff: C;
+  readonly current?: C;
+};
+type IngestionState<C = Cursor> = {
+  readonly backfill: StreamState<C>;
+  readonly live: StreamState<C>;
+};
+type Transform<T> = (row: T) => Effect.Effect<T, ConnectorError>;
+type LiveStream<T> = Stream.Stream<Batch<T>, ConnectorError>;
+type BackfillStream<T> = Stream.Stream<Batch<T>, ConnectorError>;
+type WebhookStream<T> = {
+  /**
+   * The queue is used to store the batches that are received from the webhook.
+   */
+  readonly queue: Queue.Queue<Batch<T>>;
+  /**
+   * The stream is used to process the batches that are received from the webhook.
+   */
+  readonly stream: Stream.Stream<Batch<T>, ConnectorError>;
+};
+type LiveSource<T> = LiveStream<T> | WebhookStream<T>;
+/** Schema type used by connector definitions. */
+type EntitySchema = Schema.Schema<unknown>;
+/** Decoded row type produced by a schema. */
+type EntityType<S extends EntitySchema> = Schema.Schema.Type<S>;
+/** Primary key type derived from the decoded schema shape. */
+type EntityKey<S extends EntitySchema> = EntityType<S> extends Record<string, unknown> ? keyof EntityType<S> & string : never;
+/** Row type constrained to object-like shapes for ingestion. */
+type EntityRow<S extends EntitySchema> = EntityType<S> & Record<string, unknown>;
+type EntityDefinition<S extends EntitySchema> = {
+  readonly name: string;
+  readonly schema: S;
+  /**
+   * The primary key is used to identify the rows in the database.
+   */
+  readonly primaryKey: EntityKey<S>;
+  /**
+   * The live source is used to stream the live data from the database.
+   */
+  readonly live: LiveSource<EntityRow<S>>;
+  /**
+   * The backfill stream is used to stream the backfill data from the database.
+   */
+  readonly backfill: BackfillStream<EntityRow<S>>;
+  /**
+   * The transform is used to transform the rows before they are published.
+   */
+  readonly transform?: Transform<EntityRow<S>>;
+};
+type EventDefinition<S extends EntitySchema> = {
+  readonly name: string;
+  readonly schema: S;
+  /**
+   * The live source is used to stream the live data from the database.
+   */
+  readonly live: LiveSource<EntityRow<S>>;
+  /**
+   * The backfill stream is used to stream the backfill data from the database.
+   */
+  readonly backfill?: BackfillStream<EntityRow<S>>;
+  /**
+   * The transform is used to transform the rows before they are published.
+   */
+  readonly transform?: Transform<EntityRow<S>>;
+};
+type ConnectorDefinition<Entities extends ReadonlyArray<EntityDefinition<EntitySchema>> = ReadonlyArray<EntityDefinition<EntitySchema>>, Events extends ReadonlyArray<EventDefinition<EntitySchema>> = ReadonlyArray<EventDefinition<EntitySchema>>> = {
+  readonly name: string;
+  readonly entities: Entities;
+  readonly events: Events;
+};
+//#endregion
+//#region src/core/builder.d.ts
+declare const defineConnector: <const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>, const Events extends ReadonlyArray<EventDefinition<EntitySchema>>>(definition: ConnectorDefinition<Entities, Events>) => ConnectorDefinition<Entities, Events>;
+declare const defineEntity: <S extends EntitySchema>(definition: EntityDefinition<S>) => EntityDefinition<S>;
+declare const defineEvent: <S extends EntitySchema>(definition: EventDefinition<S>) => EventDefinition<S>;
+//#endregion
+//#region src/publisher/service.d.ts
+type PublishAck = {
+  readonly success: boolean;
+};
+declare const Publisher_base: ServiceMap.ServiceClass<Publisher, "Publisher", {
+  readonly publish: (options: {
+    readonly name: string;
+    readonly batch: Batch<Record<string, unknown>>;
+  }) => Effect$1.Effect<PublishAck, ConnectorError>;
+}>;
+declare class Publisher extends Publisher_base {}
+//#endregion
+//#region src/ingestion/state-store.d.ts
+declare const StateStore_base: ServiceMap.ServiceClass<StateStore, "StateStore", {
+  readonly getState: (key: string) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;
+  readonly setState: (key: string, state: IngestionState<Cursor>) => Effect.Effect<void, ConnectorError>;
+}>;
+declare class StateStore extends StateStore_base {}
+declare const StateStoreInMemory: Layer.Layer<StateStore, never, never>;
+//#endregion
+//#region src/ingestion/engine.d.ts
+declare const runConnector: (connector: ConnectorDefinition, initialCutoff: Cursor) => Effect.Effect<void, ConnectorError, StateStore | Publisher>;
+//#endregion
+//#region src/publisher/wings.d.ts
+type WingsPublisherConfig = {
+  readonly connector: ConnectorDefinition;
+  /** Map of entity/event name to Wings topic. */
+  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;
+  /** per-stream partition value (key is entity/event name). */
+  readonly partitionValues?: Record<string, PartitionValue>;
+};
+declare const WingsPublisherLayer: (config: WingsPublisherConfig) => Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient>;
+//#endregion
+//#region src/streams/pull-stream.d.ts
+type PullPage<T> = {
+  readonly cursor: Cursor;
+  readonly rows: ReadonlyArray<T>;
+  readonly hasMore: boolean;
+};
+type PullFetcher<T, R = never> = (cursor: Cursor | undefined) => Effect.Effect<PullPage<T>, ConnectorError, R>;
+type PullStreamOptions<T, R = never> = {
+  readonly initialCursor?: Cursor;
+  readonly fetchPage: PullFetcher<T, R>;
+};
+declare const makePullStream: <T, R = never>(options: PullStreamOptions<T, R>) => Stream.Stream<Batch<T>, ConnectorError, R>;
+//#endregion
+//#region src/streams/webhook-queue.d.ts
+declare const makeWebhookQueue: <T>(options?: {
+  readonly capacity?: number;
+}) => Effect.Effect<WebhookStream<T>, never>;
+//#endregion
+//#region src/webhook/types.d.ts
+type WebhookRoute<TPayload> = {
+  readonly path: HttpRouter.PathInput;
+  readonly schema: Schema$1.Schema<TPayload>;
+  readonly handle: (payload: TPayload, request: HttpServerRequest.HttpServerRequest, rawBody?: Uint8Array) => Effect.Effect<void, ConnectorError>;
+};
+//#endregion
+//#region src/webhook/server.d.ts
+declare const buildWebhookRouter: <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) => effect_Layer0.Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", unknown>>;
+//#endregion
+export { type BackfillStream, type Batch, type ConnectorDefinition, ConnectorError, type Cursor, type EntityDefinition, type EntityKey, type EntityRow, type EntitySchema, type EntityType, type EventDefinition, type IngestionState, type LiveSource, type LiveStream, Publisher, StateStore, StateStoreInMemory, type StreamState, type Transform, type WebhookRoute, type WebhookStream, WingsPublisherLayer, buildWebhookRouter, defineConnector, defineEntity, defineEvent, makePullStream, makeWebhookQueue, runConnector };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,237 @@
+import { Cause, Data, Effect, Layer, Queue, Ref, Schema, Stream } from "effect";
+import * as ServiceMap from "effect/ServiceMap";
+import * as Wings from "@useairfoil/wings";
+import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+
+//#region src/core/builder.ts
+const defineConnector = (definition) => definition;
+const defineEntity = (definition) => definition;
+const defineEvent = (definition) => definition;
+
+//#endregion
+//#region src/core/errors.ts
+var ConnectorError = class extends Data.TaggedError("ConnectorError") {};
+
+//#endregion
+//#region src/publisher/service.ts
+var Publisher = class extends ServiceMap.Service()("Publisher") {};
+
+//#endregion
+//#region src/ingestion/state-store.ts
+var StateStore = class extends ServiceMap.Service()("StateStore") {};
+const StateStoreInMemory = Layer.effect(StateStore)(Effect.gen(function* () {
+	const ref = yield* Ref.make(/* @__PURE__ */ new Map());
+	return {
+		getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),
+		setState: (key, state) => Effect.map(Ref.update(ref, (map) => {
+			const next = new Map(map);
+			next.set(key, state);
+			return next;
+		}), () => void 0)
+	};
+}));
+
+//#endregion
+//#region src/ingestion/engine.ts
+const runConnector = (connector, initialCutoff) => Effect.gen(function* () {
+	const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));
+	const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));
+	yield* Effect.all([...entityRuns, ...eventRuns], { concurrency: "unbounded" });
+});
+const createInitialState = (cutoff) => ({
+	backfill: { cutoff },
+	live: { cutoff }
+});
+const makeStateRef = (key, initialCutoff) => Effect.gen(function* () {
+	const initial = (yield* (yield* StateStore).getState(key)) ?? createInitialState(initialCutoff);
+	return yield* Ref.make(initial);
+});
+const runEntity = (entity, initialCutoff) => Effect.gen(function* () {
+	const stateRef = yield* makeStateRef(entity.name, initialCutoff);
+	const seenRef = yield* Ref.make(/* @__PURE__ */ new Set());
+	const liveStream = resolveLiveStream(entity.live);
+	const tagLive = (batch) => ({
+		source: "live",
+		batch
+	});
+	const updateSeen = (rows) => Ref.update(seenRef, (seen) => {
+		const next = new Set(seen);
+		for (const row of rows) {
+			const key = String(row[entity.primaryKey]);
+			next.add(key);
+		}
+		return next;
+	});
+	const backfillTagged = Stream.mapEffect(entity.backfill, (batch) => Ref.get(seenRef).pipe(Effect.map((seen) => {
+		const filtered = batch.rows.filter((row) => {
+			const key = String(row[entity.primaryKey]);
+			return !seen.has(key);
+		});
+		return {
+			source: "backfill",
+			batch: {
+				cursor: batch.cursor,
+				rows: filtered
+			}
+		};
+	}))).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
+	if (isWebhookStream(entity.live)) {
+		const firstBatch = yield* Queue.take(entity.live.queue);
+		yield* updateSeen(firstBatch.rows);
+		yield* processTaggedStream(Stream.make({
+			source: "live",
+			batch: firstBatch
+		}), entity.name, entity.transform, stateRef);
+		const liveTailTagged = Stream.map(liveStream, tagLive).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
+		yield* processTaggedStream(Stream.merge(liveTailTagged, backfillTagged), entity.name, entity.transform, stateRef);
+		return;
+	}
+	const liveTagged = Stream.map(liveStream, tagLive).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
+	yield* processTaggedStream(Stream.merge(liveTagged, backfillTagged), entity.name, entity.transform, stateRef);
+});
+const runEvent = (event, initialCutoff) => Effect.gen(function* () {
+	const stateRef = yield* makeStateRef(event.name, initialCutoff);
+	const liveStream = resolveLiveStream(event.live);
+	if (event.backfill) yield* processTaggedStream(Stream.map(event.backfill, (batch) => ({
+		source: "backfill",
+		batch
+	})), event.name, event.transform, stateRef);
+	yield* processTaggedStream(Stream.map(liveStream, (batch) => ({
+		source: "live",
+		batch
+	})), event.name, event.transform, stateRef);
+});
+const updateState = (state, source, cursor) => source === "live" ? {
+	...state,
+	live: {
+		...state.live,
+		current: cursor
+	}
+} : {
+	...state,
+	backfill: {
+		...state.backfill,
+		current: cursor
+	}
+};
+const resolveLiveStream = (source) => isWebhookStream(source) ? source.stream : source;
+const isWebhookStream = (source) => typeof source === "object" && source !== null && "queue" in source && "stream" in source;
+const processTaggedStream = (stream, name, transform, stateRef) => Stream.runForEach(stream, ({ source, batch }) => Effect.gen(function* () {
+	const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;
+	yield* (yield* Publisher).publish({
+		name,
+		batch: {
+			cursor: batch.cursor,
+			rows
+		}
+	});
+	const nextState = yield* Ref.updateAndGet(stateRef, (state) => updateState(state, source, batch.cursor));
+	yield* (yield* StateStore).setState(name, nextState);
+}));
+
+//#endregion
+//#region src/publisher/wings.ts
+const buildRecordBatch = (rows) => {
+	const [batch] = Wings.tableFromJSON(Array.from(rows)).batches;
+	if (!batch) throw new ConnectorError({ message: "No rows to publish" });
+	return batch;
+};
+const WingsPublisherLayer = (config) => Layer.effect(Publisher)(Effect.gen(function* () {
+	const entries = /* @__PURE__ */ new Map();
+	for (const def of [...config.connector.entities, ...config.connector.events]) {
+		const topic = config.topics[def.name];
+		if (!topic) return yield* Effect.fail(new ConnectorError({ message: `Missing topic for ${def.name}` }));
+		const partitionIndex = topic.partitionKey !== void 0 ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey) : void 0;
+		const partitionField = partitionIndex !== void 0 && partitionIndex >= 0 ? topic.schema.fields[partitionIndex]?.name : void 0;
+		const publisher = yield* Wings.WingsClient.publisher({
+			topic,
+			partitionValue: config.partitionValues?.[def.name]
+		}).pipe(Effect.mapError((error) => new ConnectorError({
+			message: error.message,
+			cause: error
+		})));
+		entries.set(def.name, {
+			publisher,
+			partitionField,
+			partitionValue: config.partitionValues?.[def.name]
+		});
+	}
+	return { publish: ({ name, batch }) => Effect.gen(function* () {
+		const entry = entries.get(name);
+		if (!entry) return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));
+		if (batch.rows.length === 0) return { success: true };
+		const recordBatch = buildRecordBatch(batch.rows);
+		const result = yield* entry.publisher.push({
+			batch: recordBatch,
+			partitionValue: entry.partitionValue
+		}).pipe(Effect.mapError((error) => new ConnectorError({
+			message: error.message,
+			cause: error
+		})));
+		return { success: !!(result.result && result.result.$case === "accepted") };
+	}) };
+}));
+
+//#endregion
+//#region src/streams/pull-stream.ts
+const makePullStream = (options) => Stream.unfold({
+	cursor: options.initialCursor,
+	done: false
+}, (state) => Effect.gen(function* () {
+	if (state.done) return;
+	let nextCursor = state.cursor;
+	while (true) {
+		const page = yield* options.fetchPage(nextCursor);
+		if (page.rows.length > 0) return [{
+			cursor: page.cursor,
+			rows: page.rows
+		}, page.hasMore ? {
+			cursor: page.cursor,
+			done: false
+		} : {
+			cursor: void 0,
+			done: true
+		}];
+		if (!page.hasMore) return;
+		nextCursor = page.cursor;
+	}
+}));
+
+//#endregion
+//#region src/streams/webhook-queue.ts
+const makeWebhookQueue = (options) => Effect.gen(function* () {
+	const capacity = options?.capacity ?? 1024;
+	const queue = yield* Queue.bounded(capacity);
+	return {
+		queue,
+		stream: Stream.fromQueue(queue)
+	};
+});
+
+//#endregion
+//#region src/webhook/server.ts
+var InvalidWebhookPayloadError = class extends Data.TaggedError("InvalidWebhookPayloadError") {};
+const makeHandler = (route) => Effect.gen(function* () {
+	const request = yield* HttpServerRequest.HttpServerRequest;
+	const rawBuffer = yield* request.arrayBuffer.pipe(Effect.mapError(() => new InvalidWebhookPayloadError({ message: "Failed to read request body" })));
+	const rawBody = new Uint8Array(rawBuffer);
+	const rawText = new TextDecoder().decode(rawBody);
+	const rawJson = yield* Effect.try({
+		try: () => JSON.parse(rawText),
+		catch: () => new InvalidWebhookPayloadError({ message: "Invalid JSON body" })
+	});
+	const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(Effect.mapError(() => new InvalidWebhookPayloadError({ message: "Invalid webhook payload" })));
+	yield* route.handle(payload, request, rawBody);
+	return HttpServerResponse.jsonUnsafe({ ok: true });
+}).pipe(Effect.catchTag("InvalidWebhookPayloadError", () => Effect.succeed(HttpServerResponse.jsonUnsafe({
+	ok: false,
+	error: "Invalid webhook payload"
+}, { status: 400 }))), Effect.catchCause((cause) => Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(Effect.andThen(Effect.succeed(HttpServerResponse.jsonUnsafe({
+	ok: false,
+	error: "Webhook handler failed"
+}, { status: 500 }))))));
+const buildWebhookRouter = (routes) => HttpRouter.addAll(routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route))));
+
+//#endregion
+export { ConnectorError, Publisher, StateStore, StateStoreInMemory, WingsPublisherLayer, buildWebhookRouter, defineConnector, defineEntity, defineEvent, makePullStream, makeWebhookQueue, runConnector };
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["nextCursor: Cursor | undefined"],"sources":["../src/core/builder.ts","../src/core/errors.ts","../src/publisher/service.ts","../src/ingestion/state-store.ts","../src/ingestion/engine.ts","../src/publisher/wings.ts","../src/streams/pull-stream.ts","../src/streams/webhook-queue.ts","../src/webhook/server.ts"],"sourcesContent":["import type {\n  ConnectorDefinition,\n  EntityDefinition,\n  EntitySchema,\n  EventDefinition,\n} from \"./types\";\n\nexport const defineConnector = <\n  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,\n  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,\n>(\n  definition: ConnectorDefinition<Entities, Events>,\n) => definition;\n\nexport const defineEntity = <S extends EntitySchema>(\n  definition: EntityDefinition<S>,\n) => definition;\n\nexport const defineEvent = <S extends EntitySchema>(\n  definition: EventDefinition<S>,\n) => definition;\n","import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n","import type * as Effect from \"effect/Effect\";\nimport * as ServiceMap from \"effect/ServiceMap\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch } from \"../core/types\";\n\nexport type PublishAck = {\n  readonly success: boolean;\n};\n\nexport class Publisher extends ServiceMap.Service<\n  Publisher,\n  {\n    readonly publish: (options: {\n      readonly name: string;\n      readonly batch: Batch<Record<string, unknown>>;\n    }) => Effect.Effect<PublishAck, ConnectorError>;\n  }\n>()(\"Publisher\") {}\n","import { Effect, Layer, Ref } from \"effect\";\nimport * as ServiceMap from \"effect/ServiceMap\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Cursor, IngestionState } from \"../core/types\";\n\nexport class StateStore extends ServiceMap.Service<\n  StateStore,\n  {\n    readonly getState: (\n      key: string,\n    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;\n    readonly setState: (\n      key: string,\n      state: IngestionState<Cursor>,\n    ) => Effect.Effect<void, ConnectorError>;\n  }\n>()(\"StateStore\") {}\n\nexport const StateStoreInMemory = Layer.effect(StateStore)(\n  Effect.gen(function* () {\n    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());\n\n    return {\n      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),\n      setState: (key, state) =>\n        Effect.map(\n          Ref.update(ref, (map) => {\n            const next = new Map(map);\n            next.set(key, state);\n            return next;\n          }),\n          () => undefined,\n        ),\n    };\n  }),\n);\n","import { Effect, Queue, Ref, Stream } from \"effect\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type {\n  Batch,\n  ConnectorDefinition,\n  Cursor,\n  EntityDefinition,\n  EntityRow,\n  EntitySchema,\n  EventDefinition,\n  IngestionState,\n  LiveSource,\n  Transform,\n  WebhookStream,\n} from \"../core/types\";\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"./state-store\";\n\ntype TaggedBatch<T> = {\n  readonly source: \"live\" | \"backfill\";\n  readonly batch: Batch<T>;\n};\n\nexport const runConnector = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    // Start ingestion for every entity and event in parallel.\n    const entityRuns = connector.entities.map((entity) =>\n      runEntity(entity, initialCutoff),\n    );\n    const eventRuns = connector.events.map((event) =>\n      runEvent(event, initialCutoff),\n    );\n    // main runner\n    yield* Effect.all([...entityRuns, ...eventRuns], {\n      concurrency: \"unbounded\",\n    });\n  });\n\nconst createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({\n  backfill: { cutoff },\n  live: { cutoff },\n});\n\nconst makeStateRef = (\n  key: string,\n  initialCutoff: Cursor,\n): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>\n  Effect.gen(function* () {\n    // Load persisted state or initialize a new one for this stream.\n    const store = yield* StateStore;\n    const existing = yield* store.getState(key);\n    const initial = existing ?? createInitialState(initialCutoff);\n    return yield* Ref.make(initial);\n  });\n\nconst runEntity = <S extends EntitySchema>(\n  entity: EntityDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(entity.name, initialCutoff);\n    // Tracks which primary keys have already been emitted.\n    const seenRef = yield* Ref.make(new Set<string>());\n\n    const liveStream = resolveLiveStream(entity.live);\n    const tagLive = (batch: Batch<Row>) => ({\n      source: \"live\" as const,\n      batch,\n    });\n    const updateSeen = (rows: ReadonlyArray<Row>) =>\n      Ref.update(seenRef, (seen) => {\n        const next = new Set(seen);\n        for (const row of rows) {\n          const key = String(row[entity.primaryKey]);\n          next.add(key);\n        }\n        return next;\n      });\n\n    // Entities are upserts, so live and backfill can overlap. We keep an in-memory\n    // seen set (primary keys) so backfill does not re-emit rows already observed live.\n    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>\n      Ref.get(seenRef).pipe(\n        Effect.map((seen) => {\n          const filtered = batch.rows.filter((row) => {\n            const key = String(row[entity.primaryKey]);\n            return !seen.has(key);\n          });\n          return {\n            source: \"backfill\" as const,\n            batch: { cursor: batch.cursor, rows: filtered },\n          };\n        }),\n      ),\n    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));\n\n    // For webhook live sources, we wait for the first live batch before starting\n    // backfill. That first batch establishes the cutoff timestamp and seeds the\n    // seen set so backfill can de-dupe correctly.\n    // Queue-backed streams are single-consumer; splitting with take/drop would\n    // consume and discard elements. Take the first element directly from the\n    // queue, then let Stream.fromQueue continue from element #2.\n    if (isWebhookStream(entity.live)) {\n      const firstBatch = yield* Queue.take(entity.live.queue);\n      yield* updateSeen(firstBatch.rows);\n      yield* processTaggedStream(\n        Stream.make({ source: \"live\" as const, batch: firstBatch }),\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n\n      // liveStream is Stream.fromQueue on the same queue, continues from element #2.\n      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(\n        Stream.tap(({ batch }) => updateSeen(batch.rows)),\n      );\n      const merged = Stream.merge(liveTailTagged, backfillTagged);\n      yield* processTaggedStream(\n        merged,\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n      return;\n    }\n\n    // For pull-based live sources, we can merge immediately because there is no\n    // webhook cutoff gating and the live stream is not queue-backed.\n    const liveTagged = Stream.map(liveStream, tagLive).pipe(\n      Stream.tap(({ batch }) => updateSeen(batch.rows)),\n    );\n\n    const merged = Stream.merge(liveTagged, backfillTagged);\n    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n  });\n\nconst runEvent = <S extends EntitySchema>(\n  event: EventDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(event.name, initialCutoff);\n    const liveStream = resolveLiveStream<Row>(event.live);\n\n    // Events must backfill first to preserve ordering.\n    if (event.backfill) {\n      const backfillTagged = Stream.map(event.backfill, (batch) => ({\n        source: \"backfill\" as const,\n        batch,\n      }));\n      yield* processTaggedStream(\n        backfillTagged,\n        event.name,\n        event.transform,\n        stateRef,\n      );\n    }\n\n    const liveTagged = Stream.map(liveStream, (batch) => ({\n      source: \"live\" as const,\n      batch,\n    }));\n    yield* processTaggedStream(\n      liveTagged,\n      event.name,\n      event.transform,\n      stateRef,\n    );\n  });\n\nconst updateState = (\n  state: IngestionState<Cursor>,\n  source: \"live\" | \"backfill\",\n  cursor: Cursor,\n): IngestionState<Cursor> =>\n  source === \"live\"\n    ? { ...state, live: { ...state.live, current: cursor } }\n    : { ...state, backfill: { ...state.backfill, current: cursor } };\n\nconst resolveLiveStream = <T>(\n  source: LiveSource<T>,\n): Stream.Stream<Batch<T>, ConnectorError> =>\n  isWebhookStream(source) ? source.stream : source;\n\nconst isWebhookStream = <T>(\n  source: LiveSource<T>,\n): source is WebhookStream<T> =>\n  typeof source === \"object\" &&\n  source !== null &&\n  \"queue\" in source &&\n  \"stream\" in source;\n\nconst processTaggedStream = <T extends Record<string, unknown>>(\n  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,\n  name: string,\n  transform: Transform<T> | undefined,\n  stateRef: Ref.Ref<IngestionState<Cursor>>,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Stream.runForEach(stream, ({ source, batch }) =>\n    Effect.gen(function* () {\n      // Optional per-row transformation.\n      const rows = transform\n        ? yield* Effect.forEach(batch.rows, transform)\n        : batch.rows;\n\n      // Publish before updating cursor state.\n      const publisher = yield* Publisher;\n      yield* publisher.publish({\n        name,\n        batch: {\n          cursor: batch.cursor,\n          rows,\n        },\n      });\n\n      // Persist state only after publish succeeds.\n      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>\n        updateState(state, source, batch.cursor),\n      );\n\n      const store = yield* StateStore;\n      yield* store.setState(name, nextState);\n    }),\n  );\n","import type { PartitionValue } from \"@useairfoil/wings\";\nimport * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\nimport { ConnectorError } from \"../core/errors\";\nimport type { ConnectorDefinition } from \"../core/types\";\nimport { Publisher } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\nexport type WingsPublisherConfig = {\n  readonly connector: ConnectorDefinition;\n  /** Map of entity/event name to Wings topic. */\n  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;\n  /** per-stream partition value (key is entity/event name). */\n  readonly partitionValues?: Record<string, PartitionValue>;\n};\n\n/** Publisher entry for a single entity/event. */\ntype PublisherEntry = {\n  /** Wings publisher for the entity/event. */\n  readonly publisher: Wings.WingsClient.Publisher;\n  /** Partition field name (if any). */\n  readonly partitionField?: string;\n  /** Partition value (if any). */\n  readonly partitionValue?: PartitionValue;\n};\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n  // Convert JSON rows into an Arrow RecordBatch for Wings.\n  const table = Wings.tableFromJSON(Array.from(rows));\n  const [batch] = table.batches;\n\n  if (!batch) {\n    throw new ConnectorError({ message: \"No rows to publish\" });\n  }\n\n  return batch;\n};\n\nexport const WingsPublisherLayer = (\n  config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n  Layer.effect(Publisher)(\n    Effect.gen(function* () {\n      const entries = new Map<string, PublisherEntry>();\n\n      // create and store a wings publisher for each entity/event.\n      for (const def of [\n        ...config.connector.entities,\n        ...config.connector.events,\n      ]) {\n        const topic = config.topics[def.name];\n        if (!topic) {\n          return yield* Effect.fail(\n            new ConnectorError({ message: `Missing topic for ${def.name}` }),\n          );\n        }\n\n        const partitionIndex =\n          topic.partitionKey !== undefined\n            ? topic.schema.fields.findIndex(\n                (field) => field.id === topic.partitionKey,\n              )\n            : undefined;\n\n        const partitionField =\n          partitionIndex !== undefined && partitionIndex >= 0\n            ? topic.schema.fields[partitionIndex]?.name\n            : undefined;\n\n        const publisher = yield* Wings.WingsClient.publisher({\n          topic,\n          partitionValue: config.partitionValues?.[def.name],\n        }).pipe(\n          Effect.mapError(\n            (error) =>\n              new ConnectorError({ message: error.message, cause: error }),\n          ),\n        );\n\n        entries.set(def.name, {\n          publisher,\n          partitionField,\n          partitionValue: config.partitionValues?.[def.name],\n        });\n      }\n\n      return {\n        publish: ({ name, batch }) =>\n          Effect.gen(function* () {\n            const entry = entries.get(name);\n            if (!entry) {\n              return yield* Effect.fail(\n                new ConnectorError({ message: `Unknown stream ${name}` }),\n              );\n            }\n\n            if (batch.rows.length === 0) {\n              return { success: true };\n            }\n\n            const recordBatch = buildRecordBatch(batch.rows);\n            const result = yield* entry.publisher\n              .push({\n                batch: recordBatch,\n                partitionValue: entry.partitionValue,\n              })\n              .pipe(\n                Effect.mapError(\n                  (error) =>\n                    new ConnectorError({\n                      message: error.message,\n                      cause: error,\n                    }),\n                ),\n              );\n\n            return {\n              success: !!(result.result && result.result.$case === \"accepted\"),\n            };\n          }),\n      };\n    }),\n  );\n","import { Effect, Stream } from \"effect\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, Cursor } from \"../core/types\";\n\nexport type PullPage<T> = {\n  readonly cursor: Cursor;\n  readonly rows: ReadonlyArray<T>;\n  readonly hasMore: boolean;\n};\n\nexport type PullFetcher<T, R = never> = (\n  cursor: Cursor | undefined,\n) => Effect.Effect<PullPage<T>, ConnectorError, R>;\n\nexport type PullStreamOptions<T, R = never> = {\n  readonly initialCursor?: Cursor;\n  readonly fetchPage: PullFetcher<T, R>;\n};\n\nexport const makePullStream = <T, R = never>(\n  options: PullStreamOptions<T, R>,\n): Stream.Stream<Batch<T>, ConnectorError, R> =>\n  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>\n    Effect.gen(function* () {\n      if (state.done) {\n        return undefined;\n      }\n\n      let nextCursor: Cursor | undefined = state.cursor;\n\n      while (true) {\n        const page = yield* options.fetchPage(nextCursor);\n\n        if (page.rows.length > 0) {\n          const batch: Batch<T> = {\n            cursor: page.cursor,\n            rows: page.rows,\n          };\n\n          return [\n            batch,\n            page.hasMore\n              ? { cursor: page.cursor, done: false }\n              : { cursor: undefined, done: true },\n          ] as const;\n        }\n\n        if (!page.hasMore) {\n          return undefined;\n        }\n\n        nextCursor = page.cursor;\n      }\n    }),\n  );\n","import { Effect, Queue, Stream } from \"effect\";\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, WebhookStream } from \"../core/types\";\n\nexport const makeWebhookQueue = <T>(options?: {\n  readonly capacity?: number;\n}): Effect.Effect<WebhookStream<T>, never> =>\n  Effect.gen(function* () {\n    const capacity = options?.capacity ?? 1024;\n    const queue = yield* Queue.bounded<Batch<T>>(capacity);\n    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(\n      queue,\n    );\n    return { queue, stream };\n  });\n","import { Cause, Data, Effect, Schema } from \"effect\";\nimport {\n  HttpRouter,\n  HttpServerRequest,\n  HttpServerResponse,\n} from \"effect/unstable/http\";\nimport type { WebhookRoute } from \"./types\";\n\nclass InvalidWebhookPayloadError extends Data.TaggedError(\n  \"InvalidWebhookPayloadError\",\n)<{\n  readonly message: string;\n}> {}\n\nconst makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>\n  Effect.gen(function* () {\n    const request = yield* HttpServerRequest.HttpServerRequest;\n    const rawBuffer = yield* request.arrayBuffer.pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Failed to read request body\",\n          }),\n      ),\n    );\n    const rawBody = new Uint8Array(rawBuffer);\n    const rawText = new TextDecoder().decode(rawBody);\n    const rawJson = yield* Effect.try({\n      try: () => JSON.parse(rawText) as unknown,\n      catch: () =>\n        new InvalidWebhookPayloadError({ message: \"Invalid JSON body\" }),\n    });\n    const payload = yield* Schema.decodeUnknownEffect(route.schema)(\n      rawJson,\n    ).pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Invalid webhook payload\",\n          }),\n      ),\n    );\n    yield* route.handle(payload, request, rawBody);\n    // unsafeJson serializes synchronously — no Effect, no HttpBodyError\n    return HttpServerResponse.jsonUnsafe({ ok: true });\n  }).pipe(\n    Effect.catchTag(\"InvalidWebhookPayloadError\", () =>\n      Effect.succeed(\n        HttpServerResponse.jsonUnsafe(\n          { ok: false, error: \"Invalid webhook payload\" },\n          { status: 400 },\n        ),\n      ),\n    ),\n    Effect.catchCause((cause) =>\n      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(\n        Effect.andThen(\n          Effect.succeed(\n            HttpServerResponse.jsonUnsafe(\n              { ok: false, error: \"Webhook handler failed\" },\n              { status: 500 },\n            ),\n          ),\n        ),\n      ),\n    ),\n  );\n\nexport const buildWebhookRouter = <TPayload>(\n  routes: ReadonlyArray<WebhookRoute<TPayload>>,\n) =>\n  HttpRouter.addAll(\n    routes.map((route) =>\n      HttpRouter.route(\"POST\", route.path, makeHandler(route)),\n    ),\n  );\n"],"mappings":";;;;;;AAOA,MAAa,mBAIX,eACG;AAEL,MAAa,gBACX,eACG;AAEL,MAAa,eACX,eACG;;;;AClBL,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE;;;;ACIH,IAAa,YAAb,cAA+B,WAAW,SAQvC,CAAC,YAAY,CAAC;;;;ACZjB,IAAa,aAAb,cAAgC,WAAW,SAWxC,CAAC,aAAa,CAAC;AAElB,MAAa,qBAAqB,MAAM,OAAO,WAAW,CACxD,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,IAAI,qBAAK,IAAI,KAAqC,CAAC;AAEtE,QAAO;EACL,WAAW,QAAQ,OAAO,IAAI,IAAI,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC;EAClE,WAAW,KAAK,UACd,OAAO,IACL,IAAI,OAAO,MAAM,QAAQ;GACvB,MAAM,OAAO,IAAI,IAAI,IAAI;AACzB,QAAK,IAAI,KAAK,MAAM;AACpB,UAAO;IACP,QACI,OACP;EACJ;EACD,CACH;;;;ACZD,MAAa,gBACX,WACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,aAAa,UAAU,SAAS,KAAK,WACzC,UAAU,QAAQ,cAAc,CACjC;CACD,MAAM,YAAY,UAAU,OAAO,KAAK,UACtC,SAAS,OAAO,cAAc,CAC/B;AAED,QAAO,OAAO,IAAI,CAAC,GAAG,YAAY,GAAG,UAAU,EAAE,EAC/C,aAAa,aACd,CAAC;EACF;AAEJ,MAAM,sBAAsB,YAA4C;CACtE,UAAU,EAAE,QAAQ;CACpB,MAAM,EAAE,QAAQ;CACjB;AAED,MAAM,gBACJ,KACA,kBAEA,OAAO,IAAI,aAAa;CAItB,MAAM,WADW,QADH,OAAO,YACS,SAAS,IAAI,KACf,mBAAmB,cAAc;AAC7D,QAAO,OAAO,IAAI,KAAK,QAAQ;EAC/B;AAEJ,MAAM,aACJ,QACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,OAAO,MAAM,cAAc;CAEhE,MAAM,UAAU,OAAO,IAAI,qBAAK,IAAI,KAAa,CAAC;CAElD,MAAM,aAAa,kBAAkB,OAAO,KAAK;CACjD,MAAM,WAAW,WAAuB;EACtC,QAAQ;EACR;EACD;CACD,MAAM,cAAc,SAClB,IAAI,OAAO,UAAU,SAAS;EAC5B,MAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,QAAK,IAAI,IAAI;;AAEf,SAAO;GACP;CAIJ,MAAM,iBAAiB,OAAO,UAAU,OAAO,WAAW,UACxD,IAAI,IAAI,QAAQ,CAAC,KACf,OAAO,KAAK,SAAS;EACnB,MAAM,WAAW,MAAM,KAAK,QAAQ,QAAQ;GAC1C,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,UAAO,CAAC,KAAK,IAAI,IAAI;IACrB;AACF,SAAO;GACL,QAAQ;GACR,OAAO;IAAE,QAAQ,MAAM;IAAQ,MAAM;IAAU;GAChD;GACD,CACH,CACF,CAAC,KAAK,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAAC;AAQzD,KAAI,gBAAgB,OAAO,KAAK,EAAE;EAChC,MAAM,aAAa,OAAO,MAAM,KAAK,OAAO,KAAK,MAAM;AACvD,SAAO,WAAW,WAAW,KAAK;AAClC,SAAO,oBACL,OAAO,KAAK;GAAE,QAAQ;GAAiB,OAAO;GAAY,CAAC,EAC3D,OAAO,MACP,OAAO,WACP,SACD;EAGD,MAAM,iBAAiB,OAAO,IAAI,YAAY,QAAQ,CAAC,KACrD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAED,SAAO,oBADQ,OAAO,MAAM,gBAAgB,eAAe,EAGzD,OAAO,MACP,OAAO,WACP,SACD;AACD;;CAKF,MAAM,aAAa,OAAO,IAAI,YAAY,QAAQ,CAAC,KACjD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAGD,QAAO,oBADQ,OAAO,MAAM,YAAY,eAAe,EACpB,OAAO,MAAM,OAAO,WAAW,SAAS;EAC3E;AAEJ,MAAM,YACJ,OACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,MAAM,MAAM,cAAc;CAC/D,MAAM,aAAa,kBAAuB,MAAM,KAAK;AAGrD,KAAI,MAAM,SAKR,QAAO,oBAJgB,OAAO,IAAI,MAAM,WAAW,WAAW;EAC5D,QAAQ;EACR;EACD,EAAE,EAGD,MAAM,MACN,MAAM,WACN,SACD;AAOH,QAAO,oBAJY,OAAO,IAAI,aAAa,WAAW;EACpD,QAAQ;EACR;EACD,EAAE,EAGD,MAAM,MACN,MAAM,WACN,SACD;EACD;AAEJ,MAAM,eACJ,OACA,QACA,WAEA,WAAW,SACP;CAAE,GAAG;CAAO,MAAM;EAAE,GAAG,MAAM;EAAM,SAAS;EAAQ;CAAE,GACtD;CAAE,GAAG;CAAO,UAAU;EAAE,GAAG,MAAM;EAAU,SAAS;EAAQ;CAAE;AAEpE,MAAM,qBACJ,WAEA,gBAAgB,OAAO,GAAG,OAAO,SAAS;AAE5C,MAAM,mBACJ,WAEA,OAAO,WAAW,YAClB,WAAW,QACX,WAAW,UACX,YAAY;AAEd,MAAM,uBACJ,QACA,MACA,WACA,aAEA,OAAO,WAAW,SAAS,EAAE,QAAQ,YACnC,OAAO,IAAI,aAAa;CAEtB,MAAM,OAAO,YACT,OAAO,OAAO,QAAQ,MAAM,MAAM,UAAU,GAC5C,MAAM;AAIV,SADkB,OAAO,WACR,QAAQ;EACvB;EACA,OAAO;GACL,QAAQ,MAAM;GACd;GACD;EACF,CAAC;CAGF,MAAM,YAAY,OAAO,IAAI,aAAa,WAAW,UACnD,YAAY,OAAO,QAAQ,MAAM,OAAO,CACzC;AAGD,SADc,OAAO,YACR,SAAS,MAAM,UAAU;EACtC,CACH;;;;ACzMH,MAAM,oBAAoB,SAA8B;CAGtD,MAAM,CAAC,SADO,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CAC7B;AAEtB,KAAI,CAAC,MACH,OAAM,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC;AAG7D,QAAO;;AAGT,MAAa,uBACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;AAGjD,MAAK,MAAM,OAAO,CAChB,GAAG,OAAO,UAAU,UACpB,GAAG,OAAO,UAAU,OACrB,EAAE;EACD,MAAM,QAAQ,OAAO,OAAO,IAAI;AAChC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,IAAI,QAAQ,CAAC,CACjE;EAGH,MAAM,iBACJ,MAAM,iBAAiB,SACnB,MAAM,OAAO,OAAO,WACjB,UAAU,MAAM,OAAO,MAAM,aAC/B,GACD;EAEN,MAAM,iBACJ,mBAAmB,UAAa,kBAAkB,IAC9C,MAAM,OAAO,OAAO,iBAAiB,OACrC;EAEN,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC,CAAC,KACD,OAAO,UACJ,UACC,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAC/D,CACF;AAED,UAAQ,IAAI,IAAI,MAAM;GACpB;GACA;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC;;AAGJ,QAAO,EACL,UAAU,EAAE,MAAM,YAChB,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,QAAQ,IAAI,KAAK;AAC/B,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,kBAAkB,QAAQ,CAAC,CAC1D;AAGH,MAAI,MAAM,KAAK,WAAW,EACxB,QAAO,EAAE,SAAS,MAAM;EAG1B,MAAM,cAAc,iBAAiB,MAAM,KAAK;EAChD,MAAM,SAAS,OAAO,MAAM,UACzB,KAAK;GACJ,OAAO;GACP,gBAAgB,MAAM;GACvB,CAAC,CACD,KACC,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,MAAM;GACf,OAAO;GACR,CAAC,CACL,CACF;AAEH,SAAO,EACL,SAAS,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,UAAU,aACtD;GACD,EACL;EACD,CACH;;;;ACxGH,MAAa,kBACX,YAEA,OAAO,OAAO;CAAE,QAAQ,QAAQ;CAAe,MAAM;CAAO,GAAG,UAC7D,OAAO,IAAI,aAAa;AACtB,KAAI,MAAM,KACR;CAGF,IAAIA,aAAiC,MAAM;AAE3C,QAAO,MAAM;EACX,MAAM,OAAO,OAAO,QAAQ,UAAU,WAAW;AAEjD,MAAI,KAAK,KAAK,SAAS,EAMrB,QAAO,CALiB;GACtB,QAAQ,KAAK;GACb,MAAM,KAAK;GACZ,EAIC,KAAK,UACD;GAAE,QAAQ,KAAK;GAAQ,MAAM;GAAO,GACpC;GAAE,QAAQ;GAAW,MAAM;GAAM,CACtC;AAGH,MAAI,CAAC,KAAK,QACR;AAGF,eAAa,KAAK;;EAEpB,CACH;;;;AClDH,MAAa,oBAAuB,YAGlC,OAAO,IAAI,aAAa;CACtB,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,QAAQ,OAAO,MAAM,QAAkB,SAAS;AAItD,QAAO;EAAE;EAAO,QAHwC,OAAO,UAC7D,MACD;EACuB;EACxB;;;;ACNJ,IAAM,6BAAN,cAAyC,KAAK,YAC5C,6BACD,CAEE;AAEH,MAAM,eAAyB,UAC7B,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,kBAAkB;CACzC,MAAM,YAAY,OAAO,QAAQ,YAAY,KAC3C,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,+BACV,CAAC,CACL,CACF;CACD,MAAM,UAAU,IAAI,WAAW,UAAU;CACzC,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,QAAQ;CACjD,MAAM,UAAU,OAAO,OAAO,IAAI;EAChC,WAAW,KAAK,MAAM,QAAQ;EAC9B,aACE,IAAI,2BAA2B,EAAE,SAAS,qBAAqB,CAAC;EACnE,CAAC;CACF,MAAM,UAAU,OAAO,OAAO,oBAAoB,MAAM,OAAO,CAC7D,QACD,CAAC,KACA,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,2BACV,CAAC,CACL,CACF;AACD,QAAO,MAAM,OAAO,SAAS,SAAS,QAAQ;AAE9C,QAAO,mBAAmB,WAAW,EAAE,IAAI,MAAM,CAAC;EAClD,CAAC,KACD,OAAO,SAAS,oCACd,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA2B,EAC/C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,EACD,OAAO,YAAY,UACjB,OAAO,WAAW,0BAA0B,MAAM,OAAO,MAAM,GAAG,CAAC,KACjE,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA0B,EAC9C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,CACF,CACF,CACF;AAEH,MAAa,sBACX,WAEA,WAAW,OACT,OAAO,KAAK,UACV,WAAW,MAAM,QAAQ,MAAM,MAAM,YAAY,MAAM,CAAC,CACzD,CACF"}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@useairfoil/connector-kit",
+  "version": "0.3.0",
+  "type": "module",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest",
+    "test:ci": "vitest run",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check .",
+    "lint:fix": "biome check . --write",
+    "format": "biome format --write ."
+  },
+  "devDependencies": {
+    "@effect/vitest": "4.0.0-beta.25",
+    "@types/node": "^24.2.0",
+    "tsdown": "^0.15.6",
+    "vitest": "^3.2.4"
+  },
+  "dependencies": {
+    "@useairfoil/flight": "0.2.0",
+    "@useairfoil/wings": "0.2.0",
+    "consola": "^3.4.2",
+    "json-stable-stringify": "^1.3.0"
+  },
+  "peerDependencies": {
+    "effect": "4.0.0-beta.25"
+  },
+  "peerDependenciesMeta": {
+    "effect": {
+      "optional": false
+    }
+  }
+}

package/src/core/builder.ts

@@ -0,0 +1,21 @@
+import type {
+  ConnectorDefinition,
+  EntityDefinition,
+  EntitySchema,
+  EventDefinition,
+} from "./types";
+
+export const defineConnector = <
+  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,
+  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,
+>(
+  definition: ConnectorDefinition<Entities, Events>,
+) => definition;
+
+export const defineEntity = <S extends EntitySchema>(
+  definition: EntityDefinition<S>,
+) => definition;
+
+export const defineEvent = <S extends EntitySchema>(
+  definition: EventDefinition<S>,
+) => definition;

package/src/core/errors.ts

@@ -0,0 +1,6 @@
+import { Data } from "effect";
+
+export class ConnectorError extends Data.TaggedError("ConnectorError")<{
+  readonly message: string;
+  readonly cause?: unknown;
+}> {}