@useairfoil/connector-kit 0.4.0 → 0.5.0

package/dist/index-CQJsFJs3.d.ts

@@ -0,0 +1,23 @@
+import { _ as WebhookAckMode, b as WebhookRouteContext, c as ResourceBatch, l as ResourceDefinition, x as WebhookRouteInput, y as WebhookRoute } from "./types-ZNrA-Dn0.js";
+import { i as Publisher } from "./service-B7g_lHMI.js";
+import { Context, Queue } from "effect";
+import { HttpRouter } from "effect/unstable/http";
+import * as effect_Layer0 from "effect/Layer";
+
+//#region src/webhook/server.d.ts
+type QueuedWebhookBatch = {
+  readonly resource: string;
+  readonly batch: ResourceBatch;
+};
+declare const WebhookQueue_base: Context.ServiceClass<WebhookQueue, "@useairfoil/connector-kit/WebhookQueue", {
+  readonly queue: Queue.Queue<QueuedWebhookBatch>;
+}>;
+declare class WebhookQueue extends WebhookQueue_base {}
+declare const router: (routes: ReadonlyArray<WebhookRoute>) => effect_Layer0.Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", Publisher> | HttpRouter.Request<"Requires", WebhookQueue>>;
+declare namespace index_d_exports {
+  export { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput, route, router };
+}
+declare const route: <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(definition: WebhookRouteInput<Resources, Payload>) => WebhookRoute<Resources, Payload>;
+//#endregion
+export { route as n, router as r, index_d_exports as t };
+//# sourceMappingURL=index-CQJsFJs3.d.ts.map

package/dist/index-DK9YcR6W.d.ts

@@ -0,0 +1,25 @@
+import { t as ConnectorError } from "./index-DyuT2jzP.js";
+import { c as ResourceBatch, n as ConnectorDefinition } from "./types-ZNrA-Dn0.js";
+import { a as PublisherService, i as Publisher, n as PublishOptions, r as PublishSource, t as PublishAck } from "./service-B7g_lHMI.js";
+import { Layer } from "effect";
+import * as Wings from "@useairfoil/wings";
+
+//#region src/publisher/wings.d.ts
+type WingsTableMapping = {
+  readonly name: string;
+  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
+};
+type WingsPublisherConfig = {
+  readonly connector: ConnectorDefinition;
+  readonly tables: Record<string, string | WingsTableMapping>;
+};
+declare const layerWings: (config: WingsPublisherConfig) => Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient>;
+//#endregion
+//#region src/publisher/console.d.ts
+declare const layerConsole: Layer.Layer<Publisher, never, never>;
+declare namespace index_d_exports {
+  export { PublishAck, PublishOptions, PublishSource, Publisher, PublisherService, ResourceBatch, WingsPublisherConfig, layerConsole, layerWings };
+}
+//#endregion
+export { layerWings as i, layerConsole as n, WingsPublisherConfig as r, index_d_exports as t };
+//# sourceMappingURL=index-DK9YcR6W.d.ts.map

package/dist/index-DaNW63Mv.d.ts

@@ -0,0 +1,21 @@
+import { t as ConnectorError } from "./index-DyuT2jzP.js";
+import { g as ResourceState } from "./types-ZNrA-Dn0.js";
+import { Context, Effect, Layer } from "effect";
+
+//#region src/state-store/service.d.ts
+interface StateStoreService {
+  readonly getResourceState: (resource: string) => Effect.Effect<ResourceState | undefined, ConnectorError>;
+  readonly setResourceState: (resource: string, state: ResourceState) => Effect.Effect<void, ConnectorError>;
+}
+declare const StateStore_base: Context.ServiceClass<StateStore, "@useairfoil/connector-kit/StateStore", StateStoreService>;
+declare class StateStore extends StateStore_base {}
+//#endregion
+//#region src/state-store/memory.d.ts
+/** In-memory resource state store backed by a `Ref<Map>`. Suitable for development and tests. */
+declare const layerMemory: Layer.Layer<StateStore>;
+declare namespace index_d_exports {
+  export { StateStore, StateStoreService, layerMemory };
+}
+//#endregion
+export { StateStoreService as i, layerMemory as n, StateStore as r, index_d_exports as t };
+//# sourceMappingURL=index-DaNW63Mv.d.ts.map

package/dist/index-DyuT2jzP.d.ts

@@ -0,0 +1,14 @@
+import * as effect_Types0 from "effect/Types";
+import * as effect_Cause0 from "effect/Cause";
+
+//#region src/errors/connector-error.d.ts
+declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
+  readonly _tag: "ConnectorError";
+} & Readonly<A>;
+declare class ConnectorError extends ConnectorError_base<{
+  readonly message: string;
+  readonly cause?: unknown;
+}> {}
+//#endregion
+export { ConnectorError as t };
+//# sourceMappingURL=index-DyuT2jzP.d.ts.map

package/dist/index.d.ts

@@ -1,165 +1,9 @@
-import { Context, Effect, Layer, Queue, Schema, Stream } from "effect";
-import * as Wings from "@useairfoil/wings";
-import { PartitionValue } from "@useairfoil/wings";
-import { HttpRouter, HttpServerRequest } from "effect/unstable/http";
-import * as effect_Types0 from "effect/Types";
-import * as effect_Cause0 from "effect/Cause";
-import * as Effect$1 from "effect/Effect";
-import * as effect_Layer0 from "effect/Layer";
-import * as Schema$1 from "effect/Schema";
-
-//#region src/core/errors.d.ts
-declare const ConnectorError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
-  readonly _tag: "ConnectorError";
-} & Readonly<A>;
-declare class ConnectorError extends ConnectorError_base<{
-  readonly message: string;
-  readonly cause?: unknown;
-}> {}
-//#endregion
-//#region src/core/types.d.ts
-type Cursor = string | number | bigint | Date;
-type Batch<T> = {
-  readonly cursor: Cursor;
-  readonly rows: ReadonlyArray<T>;
-};
-type StreamState<C = Cursor> = {
-  readonly cutoff: C;
-  readonly current?: C;
-};
-type IngestionState<C = Cursor> = {
-  readonly backfill: StreamState<C>;
-  readonly live: StreamState<C>;
-};
-type Transform<T> = (row: T) => Effect.Effect<T, ConnectorError>;
-type LiveStream<T> = Stream.Stream<Batch<T>, ConnectorError>;
-type BackfillStream<T> = Stream.Stream<Batch<T>, ConnectorError>;
-type WebhookStream<T> = {
-  /**
-   * The queue is used to store the batches that are received from the webhook.
-   */
-  readonly queue: Queue.Queue<Batch<T>>;
-  /**
-   * The stream is used to process the batches that are received from the webhook.
-   */
-  readonly stream: Stream.Stream<Batch<T>, ConnectorError>;
-};
-type LiveSource<T> = LiveStream<T> | WebhookStream<T>;
-/** Schema type used by connector definitions. */
-type EntitySchema = Schema.Schema<unknown>;
-/** Decoded row type produced by a schema. */
-type EntityType<S extends EntitySchema> = Schema.Schema.Type<S>;
-/** Primary key type derived from the decoded schema shape. */
-type EntityKey<S extends EntitySchema> = EntityType<S> extends Record<string, unknown> ? keyof EntityType<S> & string : never;
-/** Row type constrained to object-like shapes for ingestion. */
-type EntityRow<S extends EntitySchema> = EntityType<S> & Record<string, unknown>;
-type EntityDefinition<S extends EntitySchema> = {
-  readonly name: string;
-  readonly schema: S;
-  /**
-   * The primary key is used to identify the rows in the database.
-   */
-  readonly primaryKey: EntityKey<S>;
-  /**
-   * The live source is used to stream the live data from the database.
-   */
-  readonly live: LiveSource<EntityRow<S>>;
-  /**
-   * The backfill stream is used to stream the backfill data from the database.
-   */
-  readonly backfill: BackfillStream<EntityRow<S>>;
-  /**
-   * The transform is used to transform the rows before they are published.
-   */
-  readonly transform?: Transform<EntityRow<S>>;
-};
-type EventDefinition<S extends EntitySchema> = {
-  readonly name: string;
-  readonly schema: S;
-  /**
-   * The live source is used to stream the live data from the database.
-   */
-  readonly live: LiveSource<EntityRow<S>>;
-  /**
-   * The backfill stream is used to stream the backfill data from the database.
-   */
-  readonly backfill?: BackfillStream<EntityRow<S>>;
-  /**
-   * The transform is used to transform the rows before they are published.
-   */
-  readonly transform?: Transform<EntityRow<S>>;
-};
-type ConnectorDefinition<Entities extends ReadonlyArray<EntityDefinition<EntitySchema>> = ReadonlyArray<EntityDefinition<EntitySchema>>, Events extends ReadonlyArray<EventDefinition<EntitySchema>> = ReadonlyArray<EventDefinition<EntitySchema>>> = {
-  readonly name: string;
-  readonly entities: Entities;
-  readonly events: Events;
-};
-//#endregion
-//#region src/core/builder.d.ts
-declare const defineConnector: <const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>, const Events extends ReadonlyArray<EventDefinition<EntitySchema>>>(definition: ConnectorDefinition<Entities, Events>) => ConnectorDefinition<Entities, Events>;
-declare const defineEntity: <S extends EntitySchema>(definition: EntityDefinition<S>) => EntityDefinition<S>;
-declare const defineEvent: <S extends EntitySchema>(definition: EventDefinition<S>) => EventDefinition<S>;
-//#endregion
-//#region src/publisher/service.d.ts
-type PublishAck = {
-  readonly success: boolean;
-};
-declare const Publisher_base: Context.ServiceClass<Publisher, "Publisher", {
-  readonly publish: (options: {
-    readonly name: string;
-    readonly batch: Batch<Record<string, unknown>>;
-  }) => Effect$1.Effect<PublishAck, ConnectorError>;
-}>;
-declare class Publisher extends Publisher_base {}
-//#endregion
-//#region src/ingestion/state-store.d.ts
-declare const StateStore_base: Context.ServiceClass<StateStore, "StateStore", {
-  readonly getState: (key: string) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;
-  readonly setState: (key: string, state: IngestionState<Cursor>) => Effect.Effect<void, ConnectorError>;
-}>;
-declare class StateStore extends StateStore_base {}
-declare const StateStoreInMemory: Layer.Layer<StateStore, never, never>;
-//#endregion
-//#region src/ingestion/engine.d.ts
-declare const runConnector: (connector: ConnectorDefinition, initialCutoff: Cursor) => Effect.Effect<void, ConnectorError, StateStore | Publisher>;
-//#endregion
-//#region src/publisher/wings.d.ts
-type WingsPublisherConfig = {
-  readonly connector: ConnectorDefinition;
-  /** Map of entity/event name to Wings topic. */
-  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;
-  /** per-stream partition value (key is entity/event name). */
-  readonly partitionValues?: Record<string, PartitionValue>;
-};
-declare const WingsPublisherLayer: (config: WingsPublisherConfig) => Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient>;
-//#endregion
-//#region src/streams/pull-stream.d.ts
-type PullPage<T> = {
-  readonly cursor: Cursor;
-  readonly rows: ReadonlyArray<T>;
-  readonly hasMore: boolean;
-};
-type PullFetcher<T, R = never> = (cursor: Cursor | undefined) => Effect.Effect<PullPage<T>, ConnectorError, R>;
-type PullStreamOptions<T, R = never> = {
-  readonly initialCursor?: Cursor;
-  readonly fetchPage: PullFetcher<T, R>;
-};
-declare const makePullStream: <T, R = never>(options: PullStreamOptions<T, R>) => Stream.Stream<Batch<T>, ConnectorError, R>;
-//#endregion
-//#region src/streams/webhook-queue.d.ts
-declare const makeWebhookQueue: <T>(options?: {
-  readonly capacity?: number;
-}) => Effect.Effect<WebhookStream<T>, never>;
-//#endregion
-//#region src/webhook/types.d.ts
-type WebhookRoute<TPayload> = {
-  readonly path: HttpRouter.PathInput;
-  readonly schema: Schema$1.Schema<TPayload>;
-  readonly handle: (payload: TPayload, request: HttpServerRequest.HttpServerRequest, rawBody?: Uint8Array) => Effect.Effect<void, ConnectorError>;
-};
-//#endregion
-//#region src/webhook/server.d.ts
-declare const buildWebhookRouter: <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) => effect_Layer0.Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", unknown>>;
-//#endregion
-export { type BackfillStream, type Batch, type ConnectorDefinition, ConnectorError, type Cursor, type EntityDefinition, type EntityKey, type EntityRow, type EntitySchema, type EntityType, type EventDefinition, type IngestionState, type LiveSource, type LiveStream, Publisher, StateStore, StateStoreInMemory, type StreamState, type Transform, type WebhookRoute, type WebhookStream, WingsPublisherLayer, buildWebhookRouter, defineConnector, defineEntity, defineEvent, makePullStream, makeWebhookQueue, runConnector };
-//# sourceMappingURL=index.d.ts.map
+import { t as ConnectorError } from "./index-DyuT2jzP.js";
+import { _ as WebhookAckMode, a as FetchChangesResult, b as WebhookRouteContext, c as ResourceBatch, d as ResourceMutation, f as ResourcePayload, g as ResourceState, h as ResourceSchema, l as ResourceDefinition, m as ResourceRows, n as ConnectorDefinition, o as FetchPageResult, p as ResourceRow, r as Cursor$1, s as PageFetch, t as ChangesFetch, u as ResourceField, v as WebhookHandler, y as WebhookRoute } from "./types-ZNrA-Dn0.js";
+import { a as telemetry_d_exports, c as Fetch, l as Resource, o as Connector, s as Cursor, t as index_d_exports } from "./index-BlhJCpEk.js";
+import "./service-B7g_lHMI.js";
+import { t as index_d_exports$3 } from "./index-DaNW63Mv.js";
+import { t as index_d_exports$1 } from "./index-8KnVpoZp.js";
+import { t as index_d_exports$2 } from "./index-DK9YcR6W.js";
+import { t as index_d_exports$4 } from "./index-CQJsFJs3.js";
+export { ChangesFetch, Connector, index_d_exports as ConnectorApp, ConnectorDefinition, ConnectorError, Cursor, Cursor$1 as CursorTypes, Fetch, FetchChangesResult, FetchPageResult, index_d_exports$1 as Ingestion, PageFetch, index_d_exports$2 as Publisher, Resource, ResourceBatch, ResourceDefinition, ResourceField, ResourceMutation, ResourcePayload, ResourceRow, ResourceRows, ResourceSchema, ResourceState, index_d_exports$3 as StateStore, telemetry_d_exports as Telemetry, index_d_exports$4 as Webhook, WebhookAckMode, WebhookHandler, WebhookRoute, WebhookRouteContext };

package/dist/index.js

@@ -1,236 +1,52 @@
-import { Cause, Context, Data, Effect, Layer, Queue, Ref, Schema, Stream } from "effect";
-import * as Wings from "@useairfoil/wings";
-import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+import { t as ConnectorError } from "./errors-DHWylqRZ.js";
+import { s as telemetry_exports } from "./telemetry-p0dE5Mdu.js";
+import { t as state_store_exports } from "./state-store-DG_4QpWG.js";
+import "./server-DAeFWT4o.js";
+import { t as ingestion_exports } from "./ingestion-CMCroK7Y.js";
+import { t as connector_app_exports } from "./connector-app-CvqbfN6R.js";
+import { t as publisher_exports } from "./publisher-DusgYvaV.js";
+import { n as webhook_exports } from "./webhook-BSA-tXtU.js";
+import { DateTime, Effect } from "effect";
 
 //#region src/core/builder.ts
-const defineConnector = (definition) => definition;
-const defineEntity = (definition) => definition;
-const defineEvent = (definition) => definition;
-
-//#endregion
-//#region src/core/errors.ts
-var ConnectorError = class extends Data.TaggedError("ConnectorError") {};
-
-//#endregion
-//#region src/publisher/service.ts
-var Publisher = class extends Context.Service()("Publisher") {};
-
-//#endregion
-//#region src/ingestion/state-store.ts
-var StateStore = class extends Context.Service()("StateStore") {};
-const StateStoreInMemory = Layer.effect(StateStore)(Effect.gen(function* () {
-	const ref = yield* Ref.make(/* @__PURE__ */ new Map());
-	return {
-		getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),
-		setState: (key, state) => Effect.map(Ref.update(ref, (map) => {
-			const next = new Map(map);
-			next.set(key, state);
-			return next;
-		}), () => void 0)
-	};
-}));
-
-//#endregion
-//#region src/ingestion/engine.ts
-const runConnector = (connector, initialCutoff) => Effect.gen(function* () {
-	const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));
-	const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));
-	yield* Effect.all([...entityRuns, ...eventRuns], { concurrency: "unbounded" });
-});
-const createInitialState = (cutoff) => ({
-	backfill: { cutoff },
-	live: { cutoff }
-});
-const makeStateRef = (key, initialCutoff) => Effect.gen(function* () {
-	const initial = (yield* (yield* StateStore).getState(key)) ?? createInitialState(initialCutoff);
-	return yield* Ref.make(initial);
-});
-const runEntity = (entity, initialCutoff) => Effect.gen(function* () {
-	const stateRef = yield* makeStateRef(entity.name, initialCutoff);
-	const seenRef = yield* Ref.make(/* @__PURE__ */ new Set());
-	const liveStream = resolveLiveStream(entity.live);
-	const tagLive = (batch) => ({
-		source: "live",
-		batch
-	});
-	const updateSeen = (rows) => Ref.update(seenRef, (seen) => {
-		const next = new Set(seen);
-		for (const row of rows) {
-			const key = String(row[entity.primaryKey]);
-			next.add(key);
-		}
-		return next;
-	});
-	const backfillTagged = Stream.mapEffect(entity.backfill, (batch) => Ref.get(seenRef).pipe(Effect.map((seen) => {
-		const filtered = batch.rows.filter((row) => {
-			const key = String(row[entity.primaryKey]);
-			return !seen.has(key);
-		});
-		return {
-			source: "backfill",
-			batch: {
-				cursor: batch.cursor,
-				rows: filtered
-			}
-		};
-	}))).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-	if (isWebhookStream(entity.live)) {
-		const firstBatch = yield* Queue.take(entity.live.queue);
-		yield* updateSeen(firstBatch.rows);
-		yield* processTaggedStream(Stream.make({
-			source: "live",
-			batch: firstBatch
-		}), entity.name, entity.transform, stateRef);
-		const liveTailTagged = Stream.map(liveStream, tagLive).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-		yield* processTaggedStream(Stream.merge(liveTailTagged, backfillTagged), entity.name, entity.transform, stateRef);
-		return;
-	}
-	const liveTagged = Stream.map(liveStream, tagLive).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-	yield* processTaggedStream(Stream.merge(liveTagged, backfillTagged), entity.name, entity.transform, stateRef);
-});
-const runEvent = (event, initialCutoff) => Effect.gen(function* () {
-	const stateRef = yield* makeStateRef(event.name, initialCutoff);
-	const liveStream = resolveLiveStream(event.live);
-	if (event.backfill) yield* processTaggedStream(Stream.map(event.backfill, (batch) => ({
-		source: "backfill",
-		batch
-	})), event.name, event.transform, stateRef);
-	yield* processTaggedStream(Stream.map(liveStream, (batch) => ({
-		source: "live",
-		batch
-	})), event.name, event.transform, stateRef);
-});
-const updateState = (state, source, cursor) => source === "live" ? {
-	...state,
-	live: {
-		...state.live,
-		current: cursor
-	}
-} : {
-	...state,
-	backfill: {
-		...state.backfill,
-		current: cursor
-	}
+const Connector = { define: (definition) => definition };
+const Resource = {
+	entity: (definition) => definition,
+	webhook: (definition) => definition,
+	upsert: (row) => ({
+		op: "upsert",
+		row
+	}),
+	delete: (options) => ({
+		op: "delete",
+		key: options.key,
+		version: options.version
+	})
 };
-const resolveLiveStream = (source) => isWebhookStream(source) ? source.stream : source;
-const isWebhookStream = (source) => typeof source === "object" && source !== null && "queue" in source && "stream" in source;
-const processTaggedStream = (stream, name, transform, stateRef) => Stream.runForEach(stream, ({ source, batch }) => Effect.gen(function* () {
-	const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;
-	yield* (yield* Publisher).publish({
-		name,
-		batch: {
-			cursor: batch.cursor,
-			rows
+const Fetch = {
+	page: (definition) => definition,
+	changes: (definition) => definition
+};
+const decodeFailure = (kind, value) => new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });
+const Cursor = {
+	string: () => ({
+		kind: "string",
+		decode: (value) => typeof value === "string" ? Effect.succeed(value) : Effect.fail(decodeFailure("string", value))
+	}),
+	number: () => ({
+		kind: "number",
+		decode: (value) => typeof value === "number" && Number.isFinite(value) ? Effect.succeed(value) : Effect.fail(decodeFailure("number", value))
+	}),
+	isoDateTime: () => ({
+		kind: "isoDateTime",
+		decode: (value) => {
+			if (typeof value !== "string" || Number.isNaN(Date.parse(value))) return Effect.fail(decodeFailure("ISO datetime", value));
+			return Effect.succeed(value);
 		}
-	});
-	const nextState = yield* Ref.updateAndGet(stateRef, (state) => updateState(state, source, batch.cursor));
-	yield* (yield* StateStore).setState(name, nextState);
-}));
-
-//#endregion
-//#region src/publisher/wings.ts
-const buildRecordBatch = (rows) => {
-	const [batch] = Wings.tableFromJSON(Array.from(rows)).batches;
-	if (!batch) throw new ConnectorError({ message: "No rows to publish" });
-	return batch;
+	}),
+	nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso)
 };
-const WingsPublisherLayer = (config) => Layer.effect(Publisher)(Effect.gen(function* () {
-	const entries = /* @__PURE__ */ new Map();
-	for (const def of [...config.connector.entities, ...config.connector.events]) {
-		const topic = config.topics[def.name];
-		if (!topic) return yield* Effect.fail(new ConnectorError({ message: `Missing topic for ${def.name}` }));
-		const partitionIndex = topic.partitionKey !== void 0 ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey) : void 0;
-		const partitionField = partitionIndex !== void 0 && partitionIndex >= 0 ? topic.schema.fields[partitionIndex]?.name : void 0;
-		const publisher = yield* Wings.WingsClient.publisher({
-			topic,
-			partitionValue: config.partitionValues?.[def.name]
-		}).pipe(Effect.mapError((error) => new ConnectorError({
-			message: error.message,
-			cause: error
-		})));
-		entries.set(def.name, {
-			publisher,
-			partitionField,
-			partitionValue: config.partitionValues?.[def.name]
-		});
-	}
-	return { publish: ({ name, batch }) => Effect.gen(function* () {
-		const entry = entries.get(name);
-		if (!entry) return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));
-		if (batch.rows.length === 0) return { success: true };
-		const recordBatch = buildRecordBatch(batch.rows);
-		const result = yield* entry.publisher.push({
-			batch: recordBatch,
-			partitionValue: entry.partitionValue
-		}).pipe(Effect.mapError((error) => new ConnectorError({
-			message: error.message,
-			cause: error
-		})));
-		return { success: !!(result.result && result.result.$case === "accepted") };
-	}) };
-}));
-
-//#endregion
-//#region src/streams/pull-stream.ts
-const makePullStream = (options) => Stream.unfold({
-	cursor: options.initialCursor,
-	done: false
-}, (state) => Effect.gen(function* () {
-	if (state.done) return;
-	let nextCursor = state.cursor;
-	while (true) {
-		const page = yield* options.fetchPage(nextCursor);
-		if (page.rows.length > 0) return [{
-			cursor: page.cursor,
-			rows: page.rows
-		}, page.hasMore ? {
-			cursor: page.cursor,
-			done: false
-		} : {
-			cursor: void 0,
-			done: true
-		}];
-		if (!page.hasMore) return;
-		nextCursor = page.cursor;
-	}
-}));
-
-//#endregion
-//#region src/streams/webhook-queue.ts
-const makeWebhookQueue = (options) => Effect.gen(function* () {
-	const capacity = options?.capacity ?? 1024;
-	const queue = yield* Queue.bounded(capacity);
-	return {
-		queue,
-		stream: Stream.fromQueue(queue)
-	};
-});
-
-//#endregion
-//#region src/webhook/server.ts
-var InvalidWebhookPayloadError = class extends Data.TaggedError("InvalidWebhookPayloadError") {};
-const makeHandler = (route) => Effect.gen(function* () {
-	const request = yield* HttpServerRequest.HttpServerRequest;
-	const rawBuffer = yield* request.arrayBuffer.pipe(Effect.mapError(() => new InvalidWebhookPayloadError({ message: "Failed to read request body" })));
-	const rawBody = new Uint8Array(rawBuffer);
-	const rawText = new TextDecoder().decode(rawBody);
-	const rawJson = yield* Effect.try({
-		try: () => JSON.parse(rawText),
-		catch: () => new InvalidWebhookPayloadError({ message: "Invalid JSON body" })
-	});
-	const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(Effect.mapError(() => new InvalidWebhookPayloadError({ message: "Invalid webhook payload" })));
-	yield* route.handle(payload, request, rawBody);
-	return HttpServerResponse.jsonUnsafe({ ok: true });
-}).pipe(Effect.catchTag("InvalidWebhookPayloadError", () => Effect.succeed(HttpServerResponse.jsonUnsafe({
-	ok: false,
-	error: "Invalid webhook payload"
-}, { status: 400 }))), Effect.catchCause((cause) => Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(Effect.andThen(Effect.succeed(HttpServerResponse.jsonUnsafe({
-	ok: false,
-	error: "Webhook handler failed"
-}, { status: 500 }))))));
-const buildWebhookRouter = (routes) => HttpRouter.addAll(routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route))));
 
 //#endregion
-export { ConnectorError, Publisher, StateStore, StateStoreInMemory, WingsPublisherLayer, buildWebhookRouter, defineConnector, defineEntity, defineEvent, makePullStream, makeWebhookQueue, runConnector };
+export { Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, Fetch, ingestion_exports as Ingestion, publisher_exports as Publisher, Resource, state_store_exports as StateStore, telemetry_exports as Telemetry, webhook_exports as Webhook };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["nextCursor: Cursor | undefined"],"sources":["../src/core/builder.ts","../src/core/errors.ts","../src/publisher/service.ts","../src/ingestion/state-store.ts","../src/ingestion/engine.ts","../src/publisher/wings.ts","../src/streams/pull-stream.ts","../src/streams/webhook-queue.ts","../src/webhook/server.ts"],"sourcesContent":["import type { ConnectorDefinition, EntityDefinition, EntitySchema, EventDefinition } from \"./types\";\n\nexport const defineConnector = <\n  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,\n  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,\n>(\n  definition: ConnectorDefinition<Entities, Events>,\n) => definition;\n\nexport const defineEntity = <S extends EntitySchema>(definition: EntityDefinition<S>) => definition;\n\nexport const defineEvent = <S extends EntitySchema>(definition: EventDefinition<S>) => definition;\n","import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n","import type * as Effect from \"effect/Effect\";\n\nimport { Context } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch } from \"../core/types\";\n\nexport type PublishAck = {\n  readonly success: boolean;\n};\n\nexport class Publisher extends Context.Service<\n  Publisher,\n  {\n    readonly publish: (options: {\n      readonly name: string;\n      readonly batch: Batch<Record<string, unknown>>;\n    }) => Effect.Effect<PublishAck, ConnectorError>;\n  }\n>()(\"Publisher\") {}\n","import { Context, Effect, Layer, Ref } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Cursor, IngestionState } from \"../core/types\";\n\nexport class StateStore extends Context.Service<\n  StateStore,\n  {\n    readonly getState: (\n      key: string,\n    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;\n    readonly setState: (\n      key: string,\n      state: IngestionState<Cursor>,\n    ) => Effect.Effect<void, ConnectorError>;\n  }\n>()(\"StateStore\") {}\n\nexport const StateStoreInMemory = Layer.effect(StateStore)(\n  Effect.gen(function* () {\n    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());\n\n    return {\n      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),\n      setState: (key, state) =>\n        Effect.map(\n          Ref.update(ref, (map) => {\n            const next = new Map(map);\n            next.set(key, state);\n            return next;\n          }),\n          () => undefined,\n        ),\n    };\n  }),\n);\n","import { Effect, Queue, Ref, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type {\n  Batch,\n  ConnectorDefinition,\n  Cursor,\n  EntityDefinition,\n  EntityRow,\n  EntitySchema,\n  EventDefinition,\n  IngestionState,\n  LiveSource,\n  Transform,\n  WebhookStream,\n} from \"../core/types\";\n\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"./state-store\";\n\ntype TaggedBatch<T> = {\n  readonly source: \"live\" | \"backfill\";\n  readonly batch: Batch<T>;\n};\n\nexport const runConnector = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    // Start ingestion for every entity and event in parallel.\n    const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));\n    const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));\n    // main runner\n    yield* Effect.all([...entityRuns, ...eventRuns], {\n      concurrency: \"unbounded\",\n    });\n  });\n\nconst createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({\n  backfill: { cutoff },\n  live: { cutoff },\n});\n\nconst makeStateRef = (\n  key: string,\n  initialCutoff: Cursor,\n): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>\n  Effect.gen(function* () {\n    // Load persisted state or initialize a new one for this stream.\n    const store = yield* StateStore;\n    const existing = yield* store.getState(key);\n    const initial = existing ?? createInitialState(initialCutoff);\n    return yield* Ref.make(initial);\n  });\n\nconst runEntity = <S extends EntitySchema>(\n  entity: EntityDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(entity.name, initialCutoff);\n    // Tracks which primary keys have already been emitted.\n    const seenRef = yield* Ref.make(new Set<string>());\n\n    const liveStream = resolveLiveStream(entity.live);\n    const tagLive = (batch: Batch<Row>) => ({\n      source: \"live\" as const,\n      batch,\n    });\n    const updateSeen = (rows: ReadonlyArray<Row>) =>\n      Ref.update(seenRef, (seen) => {\n        const next = new Set(seen);\n        for (const row of rows) {\n          const key = String(row[entity.primaryKey]);\n          next.add(key);\n        }\n        return next;\n      });\n\n    // Entities are upserts, so live and backfill can overlap. We keep an in-memory\n    // seen set (primary keys) so backfill does not re-emit rows already observed live.\n    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>\n      Ref.get(seenRef).pipe(\n        Effect.map((seen) => {\n          const filtered = batch.rows.filter((row) => {\n            const key = String(row[entity.primaryKey]);\n            return !seen.has(key);\n          });\n          return {\n            source: \"backfill\" as const,\n            batch: { cursor: batch.cursor, rows: filtered },\n          };\n        }),\n      ),\n    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));\n\n    // For webhook live sources, we wait for the first live batch before starting\n    // backfill. That first batch establishes the cutoff timestamp and seeds the\n    // seen set so backfill can de-dupe correctly.\n    // Queue-backed streams are single-consumer; splitting with take/drop would\n    // consume and discard elements. Take the first element directly from the\n    // queue, then let Stream.fromQueue continue from element #2.\n    if (isWebhookStream(entity.live)) {\n      const firstBatch = yield* Queue.take(entity.live.queue);\n      yield* updateSeen(firstBatch.rows);\n      yield* processTaggedStream(\n        Stream.make({ source: \"live\" as const, batch: firstBatch }),\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n\n      // liveStream is Stream.fromQueue on the same queue, continues from element #2.\n      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(\n        Stream.tap(({ batch }) => updateSeen(batch.rows)),\n      );\n      const merged = Stream.merge(liveTailTagged, backfillTagged);\n      yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n      return;\n    }\n\n    // For pull-based live sources, we can merge immediately because there is no\n    // webhook cutoff gating and the live stream is not queue-backed.\n    const liveTagged = Stream.map(liveStream, tagLive).pipe(\n      Stream.tap(({ batch }) => updateSeen(batch.rows)),\n    );\n\n    const merged = Stream.merge(liveTagged, backfillTagged);\n    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n  });\n\nconst runEvent = <S extends EntitySchema>(\n  event: EventDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(event.name, initialCutoff);\n    const liveStream = resolveLiveStream<Row>(event.live);\n\n    // Events must backfill first to preserve ordering.\n    if (event.backfill) {\n      const backfillTagged = Stream.map(event.backfill, (batch) => ({\n        source: \"backfill\" as const,\n        batch,\n      }));\n      yield* processTaggedStream(backfillTagged, event.name, event.transform, stateRef);\n    }\n\n    const liveTagged = Stream.map(liveStream, (batch) => ({\n      source: \"live\" as const,\n      batch,\n    }));\n    yield* processTaggedStream(liveTagged, event.name, event.transform, stateRef);\n  });\n\nconst updateState = (\n  state: IngestionState<Cursor>,\n  source: \"live\" | \"backfill\",\n  cursor: Cursor,\n): IngestionState<Cursor> =>\n  source === \"live\"\n    ? { ...state, live: { ...state.live, current: cursor } }\n    : { ...state, backfill: { ...state.backfill, current: cursor } };\n\nconst resolveLiveStream = <T>(source: LiveSource<T>): Stream.Stream<Batch<T>, ConnectorError> =>\n  isWebhookStream(source) ? source.stream : source;\n\nconst isWebhookStream = <T>(source: LiveSource<T>): source is WebhookStream<T> =>\n  typeof source === \"object\" && source !== null && \"queue\" in source && \"stream\" in source;\n\nconst processTaggedStream = <T extends Record<string, unknown>>(\n  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,\n  name: string,\n  transform: Transform<T> | undefined,\n  stateRef: Ref.Ref<IngestionState<Cursor>>,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Stream.runForEach(stream, ({ source, batch }) =>\n    Effect.gen(function* () {\n      // Optional per-row transformation.\n      const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;\n\n      // Publish before updating cursor state.\n      const publisher = yield* Publisher;\n      yield* publisher.publish({\n        name,\n        batch: {\n          cursor: batch.cursor,\n          rows,\n        },\n      });\n\n      // Persist state only after publish succeeds.\n      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>\n        updateState(state, source, batch.cursor),\n      );\n\n      const store = yield* StateStore;\n      yield* store.setState(name, nextState);\n    }),\n  );\n","import type { PartitionValue } from \"@useairfoil/wings\";\n\nimport * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\n\nimport type { ConnectorDefinition } from \"../core/types\";\n\nimport { ConnectorError } from \"../core/errors\";\nimport { Publisher } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\nexport type WingsPublisherConfig = {\n  readonly connector: ConnectorDefinition;\n  /** Map of entity/event name to Wings topic. */\n  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;\n  /** per-stream partition value (key is entity/event name). */\n  readonly partitionValues?: Record<string, PartitionValue>;\n};\n\n/** Publisher entry for a single entity/event. */\ntype PublisherEntry = {\n  /** Wings publisher for the entity/event. */\n  readonly publisher: Wings.WingsClient.Publisher;\n  /** Partition field name (if any). */\n  readonly partitionField?: string;\n  /** Partition value (if any). */\n  readonly partitionValue?: PartitionValue;\n};\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n  // Convert JSON rows into an Arrow RecordBatch for Wings.\n  const table = Wings.tableFromJSON(Array.from(rows));\n  const [batch] = table.batches;\n\n  if (!batch) {\n    throw new ConnectorError({ message: \"No rows to publish\" });\n  }\n\n  return batch;\n};\n\nexport const WingsPublisherLayer = (\n  config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n  Layer.effect(Publisher)(\n    Effect.gen(function* () {\n      const entries = new Map<string, PublisherEntry>();\n\n      // create and store a wings publisher for each entity/event.\n      for (const def of [...config.connector.entities, ...config.connector.events]) {\n        const topic = config.topics[def.name];\n        if (!topic) {\n          return yield* Effect.fail(\n            new ConnectorError({ message: `Missing topic for ${def.name}` }),\n          );\n        }\n\n        const partitionIndex =\n          topic.partitionKey !== undefined\n            ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey)\n            : undefined;\n\n        const partitionField =\n          partitionIndex !== undefined && partitionIndex >= 0\n            ? topic.schema.fields[partitionIndex]?.name\n            : undefined;\n\n        const publisher = yield* Wings.WingsClient.publisher({\n          topic,\n          partitionValue: config.partitionValues?.[def.name],\n        }).pipe(\n          Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),\n        );\n\n        entries.set(def.name, {\n          publisher,\n          partitionField,\n          partitionValue: config.partitionValues?.[def.name],\n        });\n      }\n\n      return {\n        publish: ({ name, batch }) =>\n          Effect.gen(function* () {\n            const entry = entries.get(name);\n            if (!entry) {\n              return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));\n            }\n\n            if (batch.rows.length === 0) {\n              return { success: true };\n            }\n\n            const recordBatch = buildRecordBatch(batch.rows);\n            const result = yield* entry.publisher\n              .push({\n                batch: recordBatch,\n                partitionValue: entry.partitionValue,\n              })\n              .pipe(\n                Effect.mapError(\n                  (error) =>\n                    new ConnectorError({\n                      message: error.message,\n                      cause: error,\n                    }),\n                ),\n              );\n\n            return {\n              success: !!(result.result && result.result.$case === \"accepted\"),\n            };\n          }),\n      };\n    }),\n  );\n","import { Effect, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, Cursor } from \"../core/types\";\n\nexport type PullPage<T> = {\n  readonly cursor: Cursor;\n  readonly rows: ReadonlyArray<T>;\n  readonly hasMore: boolean;\n};\n\nexport type PullFetcher<T, R = never> = (\n  cursor: Cursor | undefined,\n) => Effect.Effect<PullPage<T>, ConnectorError, R>;\n\nexport type PullStreamOptions<T, R = never> = {\n  readonly initialCursor?: Cursor;\n  readonly fetchPage: PullFetcher<T, R>;\n};\n\nexport const makePullStream = <T, R = never>(\n  options: PullStreamOptions<T, R>,\n): Stream.Stream<Batch<T>, ConnectorError, R> =>\n  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>\n    Effect.gen(function* () {\n      if (state.done) {\n        return undefined;\n      }\n\n      let nextCursor: Cursor | undefined = state.cursor;\n\n      while (true) {\n        const page = yield* options.fetchPage(nextCursor);\n\n        if (page.rows.length > 0) {\n          const batch: Batch<T> = {\n            cursor: page.cursor,\n            rows: page.rows,\n          };\n\n          return [\n            batch,\n            page.hasMore ? { cursor: page.cursor, done: false } : { cursor: undefined, done: true },\n          ] as const;\n        }\n\n        if (!page.hasMore) {\n          return undefined;\n        }\n\n        nextCursor = page.cursor;\n      }\n    }),\n  );\n","import { Effect, Queue, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, WebhookStream } from \"../core/types\";\n\nexport const makeWebhookQueue = <T>(options?: {\n  readonly capacity?: number;\n}): Effect.Effect<WebhookStream<T>, never> =>\n  Effect.gen(function* () {\n    const capacity = options?.capacity ?? 1024;\n    const queue = yield* Queue.bounded<Batch<T>>(capacity);\n    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(queue);\n    return { queue, stream };\n  });\n","import { Cause, Data, Effect, Schema } from \"effect\";\nimport { HttpRouter, HttpServerRequest, HttpServerResponse } from \"effect/unstable/http\";\n\nimport type { WebhookRoute } from \"./types\";\n\nclass InvalidWebhookPayloadError extends Data.TaggedError(\"InvalidWebhookPayloadError\")<{\n  readonly message: string;\n}> {}\n\nconst makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>\n  Effect.gen(function* () {\n    const request = yield* HttpServerRequest.HttpServerRequest;\n    const rawBuffer = yield* request.arrayBuffer.pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Failed to read request body\",\n          }),\n      ),\n    );\n    const rawBody = new Uint8Array(rawBuffer);\n    const rawText = new TextDecoder().decode(rawBody);\n    const rawJson = yield* Effect.try({\n      try: () => JSON.parse(rawText) as unknown,\n      catch: () => new InvalidWebhookPayloadError({ message: \"Invalid JSON body\" }),\n    });\n    const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Invalid webhook payload\",\n          }),\n      ),\n    );\n    yield* route.handle(payload, request, rawBody);\n    // unsafeJson serializes synchronously — no Effect, no HttpBodyError\n    return HttpServerResponse.jsonUnsafe({ ok: true });\n  }).pipe(\n    Effect.catchTag(\"InvalidWebhookPayloadError\", () =>\n      Effect.succeed(\n        HttpServerResponse.jsonUnsafe(\n          { ok: false, error: \"Invalid webhook payload\" },\n          { status: 400 },\n        ),\n      ),\n    ),\n    Effect.catchCause((cause) =>\n      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(\n        Effect.andThen(\n          Effect.succeed(\n            HttpServerResponse.jsonUnsafe(\n              { ok: false, error: \"Webhook handler failed\" },\n              { status: 500 },\n            ),\n          ),\n        ),\n      ),\n    ),\n  );\n\nexport const buildWebhookRouter = <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) =>\n  HttpRouter.addAll(\n    routes.map((route) => HttpRouter.route(\"POST\", route.path, makeHandler(route))),\n  );\n"],"mappings":";;;;;AAEA,MAAa,mBAIX,eACG;AAEL,MAAa,gBAAwC,eAAoC;AAEzF,MAAa,eAAuC,eAAmC;;;;ACTvF,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE;;;;ACMH,IAAa,YAAb,cAA+B,QAAQ,SAQpC,CAAC,YAAY,CAAC;;;;ACdjB,IAAa,aAAb,cAAgC,QAAQ,SAWrC,CAAC,aAAa,CAAC;AAElB,MAAa,qBAAqB,MAAM,OAAO,WAAW,CACxD,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,IAAI,qBAAK,IAAI,KAAqC,CAAC;AAEtE,QAAO;EACL,WAAW,QAAQ,OAAO,IAAI,IAAI,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC;EAClE,WAAW,KAAK,UACd,OAAO,IACL,IAAI,OAAO,MAAM,QAAQ;GACvB,MAAM,OAAO,IAAI,IAAI,IAAI;AACzB,QAAK,IAAI,KAAK,MAAM;AACpB,UAAO;IACP,QACI,OACP;EACJ;EACD,CACH;;;;ACVD,MAAa,gBACX,WACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,aAAa,UAAU,SAAS,KAAK,WAAW,UAAU,QAAQ,cAAc,CAAC;CACvF,MAAM,YAAY,UAAU,OAAO,KAAK,UAAU,SAAS,OAAO,cAAc,CAAC;AAEjF,QAAO,OAAO,IAAI,CAAC,GAAG,YAAY,GAAG,UAAU,EAAE,EAC/C,aAAa,aACd,CAAC;EACF;AAEJ,MAAM,sBAAsB,YAA4C;CACtE,UAAU,EAAE,QAAQ;CACpB,MAAM,EAAE,QAAQ;CACjB;AAED,MAAM,gBACJ,KACA,kBAEA,OAAO,IAAI,aAAa;CAItB,MAAM,WADW,QADH,OAAO,YACS,SAAS,IAAI,KACf,mBAAmB,cAAc;AAC7D,QAAO,OAAO,IAAI,KAAK,QAAQ;EAC/B;AAEJ,MAAM,aACJ,QACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,OAAO,MAAM,cAAc;CAEhE,MAAM,UAAU,OAAO,IAAI,qBAAK,IAAI,KAAa,CAAC;CAElD,MAAM,aAAa,kBAAkB,OAAO,KAAK;CACjD,MAAM,WAAW,WAAuB;EACtC,QAAQ;EACR;EACD;CACD,MAAM,cAAc,SAClB,IAAI,OAAO,UAAU,SAAS;EAC5B,MAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,QAAK,IAAI,IAAI;;AAEf,SAAO;GACP;CAIJ,MAAM,iBAAiB,OAAO,UAAU,OAAO,WAAW,UACxD,IAAI,IAAI,QAAQ,CAAC,KACf,OAAO,KAAK,SAAS;EACnB,MAAM,WAAW,MAAM,KAAK,QAAQ,QAAQ;GAC1C,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,UAAO,CAAC,KAAK,IAAI,IAAI;IACrB;AACF,SAAO;GACL,QAAQ;GACR,OAAO;IAAE,QAAQ,MAAM;IAAQ,MAAM;IAAU;GAChD;GACD,CACH,CACF,CAAC,KAAK,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAAC;AAQzD,KAAI,gBAAgB,OAAO,KAAK,EAAE;EAChC,MAAM,aAAa,OAAO,MAAM,KAAK,OAAO,KAAK,MAAM;AACvD,SAAO,WAAW,WAAW,KAAK;AAClC,SAAO,oBACL,OAAO,KAAK;GAAE,QAAQ;GAAiB,OAAO;GAAY,CAAC,EAC3D,OAAO,MACP,OAAO,WACP,SACD;EAGD,MAAM,iBAAiB,OAAO,IAAI,YAAY,QAAQ,CAAC,KACrD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAED,SAAO,oBADQ,OAAO,MAAM,gBAAgB,eAAe,EACxB,OAAO,MAAM,OAAO,WAAW,SAAS;AAC3E;;CAKF,MAAM,aAAa,OAAO,IAAI,YAAY,QAAQ,CAAC,KACjD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAGD,QAAO,oBADQ,OAAO,MAAM,YAAY,eAAe,EACpB,OAAO,MAAM,OAAO,WAAW,SAAS;EAC3E;AAEJ,MAAM,YACJ,OACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,MAAM,MAAM,cAAc;CAC/D,MAAM,aAAa,kBAAuB,MAAM,KAAK;AAGrD,KAAI,MAAM,SAKR,QAAO,oBAJgB,OAAO,IAAI,MAAM,WAAW,WAAW;EAC5D,QAAQ;EACR;EACD,EAAE,EACwC,MAAM,MAAM,MAAM,WAAW,SAAS;AAOnF,QAAO,oBAJY,OAAO,IAAI,aAAa,WAAW;EACpD,QAAQ;EACR;EACD,EAAE,EACoC,MAAM,MAAM,MAAM,WAAW,SAAS;EAC7E;AAEJ,MAAM,eACJ,OACA,QACA,WAEA,WAAW,SACP;CAAE,GAAG;CAAO,MAAM;EAAE,GAAG,MAAM;EAAM,SAAS;EAAQ;CAAE,GACtD;CAAE,GAAG;CAAO,UAAU;EAAE,GAAG,MAAM;EAAU,SAAS;EAAQ;CAAE;AAEpE,MAAM,qBAAwB,WAC5B,gBAAgB,OAAO,GAAG,OAAO,SAAS;AAE5C,MAAM,mBAAsB,WAC1B,OAAO,WAAW,YAAY,WAAW,QAAQ,WAAW,UAAU,YAAY;AAEpF,MAAM,uBACJ,QACA,MACA,WACA,aAEA,OAAO,WAAW,SAAS,EAAE,QAAQ,YACnC,OAAO,IAAI,aAAa;CAEtB,MAAM,OAAO,YAAY,OAAO,OAAO,QAAQ,MAAM,MAAM,UAAU,GAAG,MAAM;AAI9E,SADkB,OAAO,WACR,QAAQ;EACvB;EACA,OAAO;GACL,QAAQ,MAAM;GACd;GACD;EACF,CAAC;CAGF,MAAM,YAAY,OAAO,IAAI,aAAa,WAAW,UACnD,YAAY,OAAO,QAAQ,MAAM,OAAO,CACzC;AAGD,SADc,OAAO,YACR,SAAS,MAAM,UAAU;EACtC,CACH;;;;AC5KH,MAAM,oBAAoB,SAA8B;CAGtD,MAAM,CAAC,SADO,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CAC7B;AAEtB,KAAI,CAAC,MACH,OAAM,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC;AAG7D,QAAO;;AAGT,MAAa,uBACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;AAGjD,MAAK,MAAM,OAAO,CAAC,GAAG,OAAO,UAAU,UAAU,GAAG,OAAO,UAAU,OAAO,EAAE;EAC5E,MAAM,QAAQ,OAAO,OAAO,IAAI;AAChC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,IAAI,QAAQ,CAAC,CACjE;EAGH,MAAM,iBACJ,MAAM,iBAAiB,SACnB,MAAM,OAAO,OAAO,WAAW,UAAU,MAAM,OAAO,MAAM,aAAa,GACzE;EAEN,MAAM,iBACJ,mBAAmB,UAAa,kBAAkB,IAC9C,MAAM,OAAO,OAAO,iBAAiB,OACrC;EAEN,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAAC,CACzF;AAED,UAAQ,IAAI,IAAI,MAAM;GACpB;GACA;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC;;AAGJ,QAAO,EACL,UAAU,EAAE,MAAM,YAChB,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,QAAQ,IAAI,KAAK;AAC/B,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,kBAAkB,QAAQ,CAAC,CAAC;AAGtF,MAAI,MAAM,KAAK,WAAW,EACxB,QAAO,EAAE,SAAS,MAAM;EAG1B,MAAM,cAAc,iBAAiB,MAAM,KAAK;EAChD,MAAM,SAAS,OAAO,MAAM,UACzB,KAAK;GACJ,OAAO;GACP,gBAAgB,MAAM;GACvB,CAAC,CACD,KACC,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,MAAM;GACf,OAAO;GACR,CAAC,CACL,CACF;AAEH,SAAO,EACL,SAAS,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,UAAU,aACtD;GACD,EACL;EACD,CACH;;;;AChGH,MAAa,kBACX,YAEA,OAAO,OAAO;CAAE,QAAQ,QAAQ;CAAe,MAAM;CAAO,GAAG,UAC7D,OAAO,IAAI,aAAa;AACtB,KAAI,MAAM,KACR;CAGF,IAAIA,aAAiC,MAAM;AAE3C,QAAO,MAAM;EACX,MAAM,OAAO,OAAO,QAAQ,UAAU,WAAW;AAEjD,MAAI,KAAK,KAAK,SAAS,EAMrB,QAAO,CALiB;GACtB,QAAQ,KAAK;GACb,MAAM,KAAK;GACZ,EAIC,KAAK,UAAU;GAAE,QAAQ,KAAK;GAAQ,MAAM;GAAO,GAAG;GAAE,QAAQ;GAAW,MAAM;GAAM,CACxF;AAGH,MAAI,CAAC,KAAK,QACR;AAGF,eAAa,KAAK;;EAEpB,CACH;;;;AChDH,MAAa,oBAAuB,YAGlC,OAAO,IAAI,aAAa;CACtB,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,QAAQ,OAAO,MAAM,QAAkB,SAAS;AAEtD,QAAO;EAAE;EAAO,QADwC,OAAO,UAAU,MAAM;EACvD;EACxB;;;;ACRJ,IAAM,6BAAN,cAAyC,KAAK,YAAY,6BAA6B,CAEpF;AAEH,MAAM,eAAyB,UAC7B,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,kBAAkB;CACzC,MAAM,YAAY,OAAO,QAAQ,YAAY,KAC3C,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,+BACV,CAAC,CACL,CACF;CACD,MAAM,UAAU,IAAI,WAAW,UAAU;CACzC,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,QAAQ;CACjD,MAAM,UAAU,OAAO,OAAO,IAAI;EAChC,WAAW,KAAK,MAAM,QAAQ;EAC9B,aAAa,IAAI,2BAA2B,EAAE,SAAS,qBAAqB,CAAC;EAC9E,CAAC;CACF,MAAM,UAAU,OAAO,OAAO,oBAAoB,MAAM,OAAO,CAAC,QAAQ,CAAC,KACvE,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,2BACV,CAAC,CACL,CACF;AACD,QAAO,MAAM,OAAO,SAAS,SAAS,QAAQ;AAE9C,QAAO,mBAAmB,WAAW,EAAE,IAAI,MAAM,CAAC;EAClD,CAAC,KACD,OAAO,SAAS,oCACd,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA2B,EAC/C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,EACD,OAAO,YAAY,UACjB,OAAO,WAAW,0BAA0B,MAAM,OAAO,MAAM,GAAG,CAAC,KACjE,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA0B,EAC9C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,CACF,CACF,CACF;AAEH,MAAa,sBAAgC,WAC3C,WAAW,OACT,OAAO,KAAK,UAAU,WAAW,MAAM,QAAQ,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC,CAChF"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect } from \"effect\";\n\nimport type {\n  ChangesFetch,\n  ConnectorDefinition,\n  Cursor as CursorType,\n  DeleteValue,\n  PageFetch,\n  ResourceDefinition,\n  ResourceMutation,\n  ResourceSchema,\n  WebhookHandler,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(\n    definition: ConnectorDefinition<Resources>,\n  ) => definition,\n};\n\nexport const Resource = {\n  entity: <S extends ResourceSchema, Payload = never, R = never>(\n    definition: ResourceDefinition<S, Payload, R>,\n  ): ResourceDefinition<S, Payload, R> => definition,\n\n  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>\n    definition,\n\n  upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: \"upsert\", row }),\n\n  delete: (options: {\n    readonly key: DeleteValue;\n    readonly version: DeleteValue;\n  }): ResourceMutation<never> => ({\n    op: \"delete\",\n    key: options.key,\n    version: options.version,\n  }),\n};\n\nexport const Fetch = {\n  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n  string: (): CursorType.Definition<string> => ({\n    kind: \"string\",\n    decode: (value) =>\n      typeof value === \"string\"\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"string\", value)),\n  }),\n\n  number: (): CursorType.Definition<number> => ({\n    kind: \"number\",\n    decode: (value) =>\n      typeof value === \"number\" && Number.isFinite(value)\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"number\", value)),\n  }),\n\n  isoDateTime: (): CursorType.Definition<string> => ({\n    kind: \"isoDateTime\",\n    decode: (value) => {\n      if (typeof value !== \"string\" || Number.isNaN(Date.parse(value))) {\n        return Effect.fail(decodeFailure(\"ISO datetime\", value));\n      }\n      return Effect.succeed(value);\n    },\n  }),\n\n  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;AAgBA,MAAa,YAAY,EACvB,SACE,eACG,YACN;AAED,MAAa,WAAW;CACtB,SACE,eACsC;CAExC,UAA+C,eAC7C;CAEF,SAA6B,SAAqC;EAAE,IAAI;EAAU;EAAK;CAEvF,SAAS,aAGuB;EAC9B,IAAI;EACJ,KAAK,QAAQ;EACb,SAAS,QAAQ;EAClB;CACF;AAED,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACvF;AAED,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,MAAM,IAAI,CAAC;AAEnF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAC/C,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;AACjB,OAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,MAAM,MAAM,CAAC,CAC9D,QAAO,OAAO,KAAK,cAAc,gBAAgB,MAAM,CAAC;AAE1D,UAAO,OAAO,QAAQ,MAAM;;EAE/B;CAED,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,UAAU;CAC7D"}

package/dist/ingestion/index.d.ts

@@ -0,0 +1,6 @@
+import "../index-DyuT2jzP.js";
+import "../types-ZNrA-Dn0.js";
+import "../service-B7g_lHMI.js";
+import "../index-DaNW63Mv.js";
+import { n as RunOptions, r as run } from "../index-8KnVpoZp.js";
+export { RunOptions, run };

package/dist/ingestion/index.js

@@ -0,0 +1,7 @@
+import "../errors-DHWylqRZ.js";
+import "../telemetry-p0dE5Mdu.js";
+import "../state-store-DG_4QpWG.js";
+import "../server-DAeFWT4o.js";
+import { n as run } from "../ingestion-CMCroK7Y.js";
+
+export { run };