@useairfoil/connector-kit 0.4.0 → 0.5.1

package/dist/index.js

@@ -1,236 +1,58 @@
-import { Cause, Context, Data, Effect, Layer, Queue, Ref, Schema, Stream } from "effect";
-import * as Wings from "@useairfoil/wings";
-import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+import { t as ConnectorError } from "./errors-DHWylqRZ.js";
+import { t as auth_exports } from "./auth-DvU_CZ7C.js";
+import { t as connector_app_exports } from "./connector-app-d7Z-SZLQ.js";
+import { s as metrics_exports } from "./metrics-CWPy2Aqc.js";
+import "./constants-BRhIX_rz.js";
+import { c as telemetry_exports } from "./service-DuFiQwid.js";
+import "./server-CXTQ7odl.js";
+import { t as state_store_exports } from "./state-store-yl6LiTQ0.js";
+import { t as status_exports } from "./status-B-oZBc33.js";
+import { t as ingestion_exports } from "./ingestion-BQo4cbGw.js";
+import { t as manifest_exports } from "./manifest-ByE6YPG0.js";
+import { t as publisher_exports } from "./publisher-BYT3RLVO.js";
+import { t as runtime_config_exports } from "./runtime-config-BpB-d_bF.js";
+import { n as webhook_exports } from "./webhook-DI-Afxkg.js";
+import { DateTime, Effect } from "effect";
 
 //#region src/core/builder.ts
-const defineConnector = (definition) => definition;
-const defineEntity = (definition) => definition;
-const defineEvent = (definition) => definition;
-
-//#endregion
-//#region src/core/errors.ts
-var ConnectorError = class extends Data.TaggedError("ConnectorError") {};
-
-//#endregion
-//#region src/publisher/service.ts
-var Publisher = class extends Context.Service()("Publisher") {};
-
-//#endregion
-//#region src/ingestion/state-store.ts
-var StateStore = class extends Context.Service()("StateStore") {};
-const StateStoreInMemory = Layer.effect(StateStore)(Effect.gen(function* () {
-	const ref = yield* Ref.make(/* @__PURE__ */ new Map());
-	return {
-		getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),
-		setState: (key, state) => Effect.map(Ref.update(ref, (map) => {
-			const next = new Map(map);
-			next.set(key, state);
-			return next;
-		}), () => void 0)
-	};
-}));
-
-//#endregion
-//#region src/ingestion/engine.ts
-const runConnector = (connector, initialCutoff) => Effect.gen(function* () {
-	const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));
-	const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));
-	yield* Effect.all([...entityRuns, ...eventRuns], { concurrency: "unbounded" });
-});
-const createInitialState = (cutoff) => ({
-	backfill: { cutoff },
-	live: { cutoff }
-});
-const makeStateRef = (key, initialCutoff) => Effect.gen(function* () {
-	const initial = (yield* (yield* StateStore).getState(key)) ?? createInitialState(initialCutoff);
-	return yield* Ref.make(initial);
-});
-const runEntity = (entity, initialCutoff) => Effect.gen(function* () {
-	const stateRef = yield* makeStateRef(entity.name, initialCutoff);
-	const seenRef = yield* Ref.make(/* @__PURE__ */ new Set());
-	const liveStream = resolveLiveStream(entity.live);
-	const tagLive = (batch) => ({
-		source: "live",
-		batch
-	});
-	const updateSeen = (rows) => Ref.update(seenRef, (seen) => {
-		const next = new Set(seen);
-		for (const row of rows) {
-			const key = String(row[entity.primaryKey]);
-			next.add(key);
-		}
-		return next;
-	});
-	const backfillTagged = Stream.mapEffect(entity.backfill, (batch) => Ref.get(seenRef).pipe(Effect.map((seen) => {
-		const filtered = batch.rows.filter((row) => {
-			const key = String(row[entity.primaryKey]);
-			return !seen.has(key);
-		});
-		return {
-			source: "backfill",
-			batch: {
-				cursor: batch.cursor,
-				rows: filtered
-			}
-		};
-	}))).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-	if (isWebhookStream(entity.live)) {
-		const firstBatch = yield* Queue.take(entity.live.queue);
-		yield* updateSeen(firstBatch.rows);
-		yield* processTaggedStream(Stream.make({
-			source: "live",
-			batch: firstBatch
-		}), entity.name, entity.transform, stateRef);
-		const liveTailTagged = Stream.map(liveStream, tagLive).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-		yield* processTaggedStream(Stream.merge(liveTailTagged, backfillTagged), entity.name, entity.transform, stateRef);
-		return;
-	}
-	const liveTagged = Stream.map(liveStream, tagLive).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));
-	yield* processTaggedStream(Stream.merge(liveTagged, backfillTagged), entity.name, entity.transform, stateRef);
-});
-const runEvent = (event, initialCutoff) => Effect.gen(function* () {
-	const stateRef = yield* makeStateRef(event.name, initialCutoff);
-	const liveStream = resolveLiveStream(event.live);
-	if (event.backfill) yield* processTaggedStream(Stream.map(event.backfill, (batch) => ({
-		source: "backfill",
-		batch
-	})), event.name, event.transform, stateRef);
-	yield* processTaggedStream(Stream.map(liveStream, (batch) => ({
-		source: "live",
-		batch
-	})), event.name, event.transform, stateRef);
-});
-const updateState = (state, source, cursor) => source === "live" ? {
-	...state,
-	live: {
-		...state.live,
-		current: cursor
-	}
-} : {
-	...state,
-	backfill: {
-		...state.backfill,
-		current: cursor
-	}
+const Connector = { define: (definition) => definition };
+const Resource = {
+	entity: (definition) => definition,
+	webhook: (definition) => definition,
+	upsert: (row) => ({
+		op: "upsert",
+		row
+	}),
+	delete: (options) => ({
+		op: "delete",
+		key: options.key,
+		version: options.version
+	})
 };
-const resolveLiveStream = (source) => isWebhookStream(source) ? source.stream : source;
-const isWebhookStream = (source) => typeof source === "object" && source !== null && "queue" in source && "stream" in source;
-const processTaggedStream = (stream, name, transform, stateRef) => Stream.runForEach(stream, ({ source, batch }) => Effect.gen(function* () {
-	const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;
-	yield* (yield* Publisher).publish({
-		name,
-		batch: {
-			cursor: batch.cursor,
-			rows
+const Fetch = {
+	page: (definition) => definition,
+	changes: (definition) => definition
+};
+const decodeFailure = (kind, value) => new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });
+const Cursor = {
+	string: () => ({
+		kind: "string",
+		decode: (value) => typeof value === "string" ? Effect.succeed(value) : Effect.fail(decodeFailure("string", value))
+	}),
+	number: () => ({
+		kind: "number",
+		decode: (value) => typeof value === "number" && Number.isFinite(value) ? Effect.succeed(value) : Effect.fail(decodeFailure("number", value))
+	}),
+	isoDateTime: () => ({
+		kind: "isoDateTime",
+		decode: (value) => {
+			if (typeof value !== "string" || Number.isNaN(Date.parse(value))) return Effect.fail(decodeFailure("ISO datetime", value));
+			return Effect.succeed(value);
 		}
-	});
-	const nextState = yield* Ref.updateAndGet(stateRef, (state) => updateState(state, source, batch.cursor));
-	yield* (yield* StateStore).setState(name, nextState);
-}));
-
-//#endregion
-//#region src/publisher/wings.ts
-const buildRecordBatch = (rows) => {
-	const [batch] = Wings.tableFromJSON(Array.from(rows)).batches;
-	if (!batch) throw new ConnectorError({ message: "No rows to publish" });
-	return batch;
+	}),
+	nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso)
 };
-const WingsPublisherLayer = (config) => Layer.effect(Publisher)(Effect.gen(function* () {
-	const entries = /* @__PURE__ */ new Map();
-	for (const def of [...config.connector.entities, ...config.connector.events]) {
-		const topic = config.topics[def.name];
-		if (!topic) return yield* Effect.fail(new ConnectorError({ message: `Missing topic for ${def.name}` }));
-		const partitionIndex = topic.partitionKey !== void 0 ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey) : void 0;
-		const partitionField = partitionIndex !== void 0 && partitionIndex >= 0 ? topic.schema.fields[partitionIndex]?.name : void 0;
-		const publisher = yield* Wings.WingsClient.publisher({
-			topic,
-			partitionValue: config.partitionValues?.[def.name]
-		}).pipe(Effect.mapError((error) => new ConnectorError({
-			message: error.message,
-			cause: error
-		})));
-		entries.set(def.name, {
-			publisher,
-			partitionField,
-			partitionValue: config.partitionValues?.[def.name]
-		});
-	}
-	return { publish: ({ name, batch }) => Effect.gen(function* () {
-		const entry = entries.get(name);
-		if (!entry) return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));
-		if (batch.rows.length === 0) return { success: true };
-		const recordBatch = buildRecordBatch(batch.rows);
-		const result = yield* entry.publisher.push({
-			batch: recordBatch,
-			partitionValue: entry.partitionValue
-		}).pipe(Effect.mapError((error) => new ConnectorError({
-			message: error.message,
-			cause: error
-		})));
-		return { success: !!(result.result && result.result.$case === "accepted") };
-	}) };
-}));
-
-//#endregion
-//#region src/streams/pull-stream.ts
-const makePullStream = (options) => Stream.unfold({
-	cursor: options.initialCursor,
-	done: false
-}, (state) => Effect.gen(function* () {
-	if (state.done) return;
-	let nextCursor = state.cursor;
-	while (true) {
-		const page = yield* options.fetchPage(nextCursor);
-		if (page.rows.length > 0) return [{
-			cursor: page.cursor,
-			rows: page.rows
-		}, page.hasMore ? {
-			cursor: page.cursor,
-			done: false
-		} : {
-			cursor: void 0,
-			done: true
-		}];
-		if (!page.hasMore) return;
-		nextCursor = page.cursor;
-	}
-}));
-
-//#endregion
-//#region src/streams/webhook-queue.ts
-const makeWebhookQueue = (options) => Effect.gen(function* () {
-	const capacity = options?.capacity ?? 1024;
-	const queue = yield* Queue.bounded(capacity);
-	return {
-		queue,
-		stream: Stream.fromQueue(queue)
-	};
-});
-
-//#endregion
-//#region src/webhook/server.ts
-var InvalidWebhookPayloadError = class extends Data.TaggedError("InvalidWebhookPayloadError") {};
-const makeHandler = (route) => Effect.gen(function* () {
-	const request = yield* HttpServerRequest.HttpServerRequest;
-	const rawBuffer = yield* request.arrayBuffer.pipe(Effect.mapError(() => new InvalidWebhookPayloadError({ message: "Failed to read request body" })));
-	const rawBody = new Uint8Array(rawBuffer);
-	const rawText = new TextDecoder().decode(rawBody);
-	const rawJson = yield* Effect.try({
-		try: () => JSON.parse(rawText),
-		catch: () => new InvalidWebhookPayloadError({ message: "Invalid JSON body" })
-	});
-	const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(Effect.mapError(() => new InvalidWebhookPayloadError({ message: "Invalid webhook payload" })));
-	yield* route.handle(payload, request, rawBody);
-	return HttpServerResponse.jsonUnsafe({ ok: true });
-}).pipe(Effect.catchTag("InvalidWebhookPayloadError", () => Effect.succeed(HttpServerResponse.jsonUnsafe({
-	ok: false,
-	error: "Invalid webhook payload"
-}, { status: 400 }))), Effect.catchCause((cause) => Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(Effect.andThen(Effect.succeed(HttpServerResponse.jsonUnsafe({
-	ok: false,
-	error: "Webhook handler failed"
-}, { status: 500 }))))));
-const buildWebhookRouter = (routes) => HttpRouter.addAll(routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route))));
 
 //#endregion
-export { ConnectorError, Publisher, StateStore, StateStoreInMemory, WingsPublisherLayer, buildWebhookRouter, defineConnector, defineEntity, defineEvent, makePullStream, makeWebhookQueue, runConnector };
+export { auth_exports as Auth, Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, Fetch, ingestion_exports as Ingestion, manifest_exports as Manifest, metrics_exports as Metrics, publisher_exports as Publisher, Resource, runtime_config_exports as RuntimeConfig, state_store_exports as StateStore, status_exports as Status, telemetry_exports as Telemetry, webhook_exports as Webhook };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["nextCursor: Cursor | undefined"],"sources":["../src/core/builder.ts","../src/core/errors.ts","../src/publisher/service.ts","../src/ingestion/state-store.ts","../src/ingestion/engine.ts","../src/publisher/wings.ts","../src/streams/pull-stream.ts","../src/streams/webhook-queue.ts","../src/webhook/server.ts"],"sourcesContent":["import type { ConnectorDefinition, EntityDefinition, EntitySchema, EventDefinition } from \"./types\";\n\nexport const defineConnector = <\n  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,\n  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,\n>(\n  definition: ConnectorDefinition<Entities, Events>,\n) => definition;\n\nexport const defineEntity = <S extends EntitySchema>(definition: EntityDefinition<S>) => definition;\n\nexport const defineEvent = <S extends EntitySchema>(definition: EventDefinition<S>) => definition;\n","import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n","import type * as Effect from \"effect/Effect\";\n\nimport { Context } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch } from \"../core/types\";\n\nexport type PublishAck = {\n  readonly success: boolean;\n};\n\nexport class Publisher extends Context.Service<\n  Publisher,\n  {\n    readonly publish: (options: {\n      readonly name: string;\n      readonly batch: Batch<Record<string, unknown>>;\n    }) => Effect.Effect<PublishAck, ConnectorError>;\n  }\n>()(\"Publisher\") {}\n","import { Context, Effect, Layer, Ref } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Cursor, IngestionState } from \"../core/types\";\n\nexport class StateStore extends Context.Service<\n  StateStore,\n  {\n    readonly getState: (\n      key: string,\n    ) => Effect.Effect<IngestionState<Cursor> | undefined, ConnectorError>;\n    readonly setState: (\n      key: string,\n      state: IngestionState<Cursor>,\n    ) => Effect.Effect<void, ConnectorError>;\n  }\n>()(\"StateStore\") {}\n\nexport const StateStoreInMemory = Layer.effect(StateStore)(\n  Effect.gen(function* () {\n    const ref = yield* Ref.make(new Map<string, IngestionState<Cursor>>());\n\n    return {\n      getState: (key) => Effect.map(Ref.get(ref), (map) => map.get(key)),\n      setState: (key, state) =>\n        Effect.map(\n          Ref.update(ref, (map) => {\n            const next = new Map(map);\n            next.set(key, state);\n            return next;\n          }),\n          () => undefined,\n        ),\n    };\n  }),\n);\n","import { Effect, Queue, Ref, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type {\n  Batch,\n  ConnectorDefinition,\n  Cursor,\n  EntityDefinition,\n  EntityRow,\n  EntitySchema,\n  EventDefinition,\n  IngestionState,\n  LiveSource,\n  Transform,\n  WebhookStream,\n} from \"../core/types\";\n\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"./state-store\";\n\ntype TaggedBatch<T> = {\n  readonly source: \"live\" | \"backfill\";\n  readonly batch: Batch<T>;\n};\n\nexport const runConnector = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    // Start ingestion for every entity and event in parallel.\n    const entityRuns = connector.entities.map((entity) => runEntity(entity, initialCutoff));\n    const eventRuns = connector.events.map((event) => runEvent(event, initialCutoff));\n    // main runner\n    yield* Effect.all([...entityRuns, ...eventRuns], {\n      concurrency: \"unbounded\",\n    });\n  });\n\nconst createInitialState = (cutoff: Cursor): IngestionState<Cursor> => ({\n  backfill: { cutoff },\n  live: { cutoff },\n});\n\nconst makeStateRef = (\n  key: string,\n  initialCutoff: Cursor,\n): Effect.Effect<Ref.Ref<IngestionState<Cursor>>, ConnectorError, StateStore> =>\n  Effect.gen(function* () {\n    // Load persisted state or initialize a new one for this stream.\n    const store = yield* StateStore;\n    const existing = yield* store.getState(key);\n    const initial = existing ?? createInitialState(initialCutoff);\n    return yield* Ref.make(initial);\n  });\n\nconst runEntity = <S extends EntitySchema>(\n  entity: EntityDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(entity.name, initialCutoff);\n    // Tracks which primary keys have already been emitted.\n    const seenRef = yield* Ref.make(new Set<string>());\n\n    const liveStream = resolveLiveStream(entity.live);\n    const tagLive = (batch: Batch<Row>) => ({\n      source: \"live\" as const,\n      batch,\n    });\n    const updateSeen = (rows: ReadonlyArray<Row>) =>\n      Ref.update(seenRef, (seen) => {\n        const next = new Set(seen);\n        for (const row of rows) {\n          const key = String(row[entity.primaryKey]);\n          next.add(key);\n        }\n        return next;\n      });\n\n    // Entities are upserts, so live and backfill can overlap. We keep an in-memory\n    // seen set (primary keys) so backfill does not re-emit rows already observed live.\n    const backfillTagged = Stream.mapEffect(entity.backfill, (batch) =>\n      Ref.get(seenRef).pipe(\n        Effect.map((seen) => {\n          const filtered = batch.rows.filter((row) => {\n            const key = String(row[entity.primaryKey]);\n            return !seen.has(key);\n          });\n          return {\n            source: \"backfill\" as const,\n            batch: { cursor: batch.cursor, rows: filtered },\n          };\n        }),\n      ),\n    ).pipe(Stream.tap(({ batch }) => updateSeen(batch.rows)));\n\n    // For webhook live sources, we wait for the first live batch before starting\n    // backfill. That first batch establishes the cutoff timestamp and seeds the\n    // seen set so backfill can de-dupe correctly.\n    // Queue-backed streams are single-consumer; splitting with take/drop would\n    // consume and discard elements. Take the first element directly from the\n    // queue, then let Stream.fromQueue continue from element #2.\n    if (isWebhookStream(entity.live)) {\n      const firstBatch = yield* Queue.take(entity.live.queue);\n      yield* updateSeen(firstBatch.rows);\n      yield* processTaggedStream(\n        Stream.make({ source: \"live\" as const, batch: firstBatch }),\n        entity.name,\n        entity.transform,\n        stateRef,\n      );\n\n      // liveStream is Stream.fromQueue on the same queue, continues from element #2.\n      const liveTailTagged = Stream.map(liveStream, tagLive).pipe(\n        Stream.tap(({ batch }) => updateSeen(batch.rows)),\n      );\n      const merged = Stream.merge(liveTailTagged, backfillTagged);\n      yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n      return;\n    }\n\n    // For pull-based live sources, we can merge immediately because there is no\n    // webhook cutoff gating and the live stream is not queue-backed.\n    const liveTagged = Stream.map(liveStream, tagLive).pipe(\n      Stream.tap(({ batch }) => updateSeen(batch.rows)),\n    );\n\n    const merged = Stream.merge(liveTagged, backfillTagged);\n    yield* processTaggedStream(merged, entity.name, entity.transform, stateRef);\n  });\n\nconst runEvent = <S extends EntitySchema>(\n  event: EventDefinition<S>,\n  initialCutoff: Cursor,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.gen(function* () {\n    type Row = EntityRow<S>;\n    const stateRef = yield* makeStateRef(event.name, initialCutoff);\n    const liveStream = resolveLiveStream<Row>(event.live);\n\n    // Events must backfill first to preserve ordering.\n    if (event.backfill) {\n      const backfillTagged = Stream.map(event.backfill, (batch) => ({\n        source: \"backfill\" as const,\n        batch,\n      }));\n      yield* processTaggedStream(backfillTagged, event.name, event.transform, stateRef);\n    }\n\n    const liveTagged = Stream.map(liveStream, (batch) => ({\n      source: \"live\" as const,\n      batch,\n    }));\n    yield* processTaggedStream(liveTagged, event.name, event.transform, stateRef);\n  });\n\nconst updateState = (\n  state: IngestionState<Cursor>,\n  source: \"live\" | \"backfill\",\n  cursor: Cursor,\n): IngestionState<Cursor> =>\n  source === \"live\"\n    ? { ...state, live: { ...state.live, current: cursor } }\n    : { ...state, backfill: { ...state.backfill, current: cursor } };\n\nconst resolveLiveStream = <T>(source: LiveSource<T>): Stream.Stream<Batch<T>, ConnectorError> =>\n  isWebhookStream(source) ? source.stream : source;\n\nconst isWebhookStream = <T>(source: LiveSource<T>): source is WebhookStream<T> =>\n  typeof source === \"object\" && source !== null && \"queue\" in source && \"stream\" in source;\n\nconst processTaggedStream = <T extends Record<string, unknown>>(\n  stream: Stream.Stream<TaggedBatch<T>, ConnectorError>,\n  name: string,\n  transform: Transform<T> | undefined,\n  stateRef: Ref.Ref<IngestionState<Cursor>>,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Stream.runForEach(stream, ({ source, batch }) =>\n    Effect.gen(function* () {\n      // Optional per-row transformation.\n      const rows = transform ? yield* Effect.forEach(batch.rows, transform) : batch.rows;\n\n      // Publish before updating cursor state.\n      const publisher = yield* Publisher;\n      yield* publisher.publish({\n        name,\n        batch: {\n          cursor: batch.cursor,\n          rows,\n        },\n      });\n\n      // Persist state only after publish succeeds.\n      const nextState = yield* Ref.updateAndGet(stateRef, (state) =>\n        updateState(state, source, batch.cursor),\n      );\n\n      const store = yield* StateStore;\n      yield* store.setState(name, nextState);\n    }),\n  );\n","import type { PartitionValue } from \"@useairfoil/wings\";\n\nimport * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\n\nimport type { ConnectorDefinition } from \"../core/types\";\n\nimport { ConnectorError } from \"../core/errors\";\nimport { Publisher } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\nexport type WingsPublisherConfig = {\n  readonly connector: ConnectorDefinition;\n  /** Map of entity/event name to Wings topic. */\n  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;\n  /** per-stream partition value (key is entity/event name). */\n  readonly partitionValues?: Record<string, PartitionValue>;\n};\n\n/** Publisher entry for a single entity/event. */\ntype PublisherEntry = {\n  /** Wings publisher for the entity/event. */\n  readonly publisher: Wings.WingsClient.Publisher;\n  /** Partition field name (if any). */\n  readonly partitionField?: string;\n  /** Partition value (if any). */\n  readonly partitionValue?: PartitionValue;\n};\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n  // Convert JSON rows into an Arrow RecordBatch for Wings.\n  const table = Wings.tableFromJSON(Array.from(rows));\n  const [batch] = table.batches;\n\n  if (!batch) {\n    throw new ConnectorError({ message: \"No rows to publish\" });\n  }\n\n  return batch;\n};\n\nexport const WingsPublisherLayer = (\n  config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n  Layer.effect(Publisher)(\n    Effect.gen(function* () {\n      const entries = new Map<string, PublisherEntry>();\n\n      // create and store a wings publisher for each entity/event.\n      for (const def of [...config.connector.entities, ...config.connector.events]) {\n        const topic = config.topics[def.name];\n        if (!topic) {\n          return yield* Effect.fail(\n            new ConnectorError({ message: `Missing topic for ${def.name}` }),\n          );\n        }\n\n        const partitionIndex =\n          topic.partitionKey !== undefined\n            ? topic.schema.fields.findIndex((field) => field.id === topic.partitionKey)\n            : undefined;\n\n        const partitionField =\n          partitionIndex !== undefined && partitionIndex >= 0\n            ? topic.schema.fields[partitionIndex]?.name\n            : undefined;\n\n        const publisher = yield* Wings.WingsClient.publisher({\n          topic,\n          partitionValue: config.partitionValues?.[def.name],\n        }).pipe(\n          Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),\n        );\n\n        entries.set(def.name, {\n          publisher,\n          partitionField,\n          partitionValue: config.partitionValues?.[def.name],\n        });\n      }\n\n      return {\n        publish: ({ name, batch }) =>\n          Effect.gen(function* () {\n            const entry = entries.get(name);\n            if (!entry) {\n              return yield* Effect.fail(new ConnectorError({ message: `Unknown stream ${name}` }));\n            }\n\n            if (batch.rows.length === 0) {\n              return { success: true };\n            }\n\n            const recordBatch = buildRecordBatch(batch.rows);\n            const result = yield* entry.publisher\n              .push({\n                batch: recordBatch,\n                partitionValue: entry.partitionValue,\n              })\n              .pipe(\n                Effect.mapError(\n                  (error) =>\n                    new ConnectorError({\n                      message: error.message,\n                      cause: error,\n                    }),\n                ),\n              );\n\n            return {\n              success: !!(result.result && result.result.$case === \"accepted\"),\n            };\n          }),\n      };\n    }),\n  );\n","import { Effect, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, Cursor } from \"../core/types\";\n\nexport type PullPage<T> = {\n  readonly cursor: Cursor;\n  readonly rows: ReadonlyArray<T>;\n  readonly hasMore: boolean;\n};\n\nexport type PullFetcher<T, R = never> = (\n  cursor: Cursor | undefined,\n) => Effect.Effect<PullPage<T>, ConnectorError, R>;\n\nexport type PullStreamOptions<T, R = never> = {\n  readonly initialCursor?: Cursor;\n  readonly fetchPage: PullFetcher<T, R>;\n};\n\nexport const makePullStream = <T, R = never>(\n  options: PullStreamOptions<T, R>,\n): Stream.Stream<Batch<T>, ConnectorError, R> =>\n  Stream.unfold({ cursor: options.initialCursor, done: false }, (state) =>\n    Effect.gen(function* () {\n      if (state.done) {\n        return undefined;\n      }\n\n      let nextCursor: Cursor | undefined = state.cursor;\n\n      while (true) {\n        const page = yield* options.fetchPage(nextCursor);\n\n        if (page.rows.length > 0) {\n          const batch: Batch<T> = {\n            cursor: page.cursor,\n            rows: page.rows,\n          };\n\n          return [\n            batch,\n            page.hasMore ? { cursor: page.cursor, done: false } : { cursor: undefined, done: true },\n          ] as const;\n        }\n\n        if (!page.hasMore) {\n          return undefined;\n        }\n\n        nextCursor = page.cursor;\n      }\n    }),\n  );\n","import { Effect, Queue, Stream } from \"effect\";\n\nimport type { ConnectorError } from \"../core/errors\";\nimport type { Batch, WebhookStream } from \"../core/types\";\n\nexport const makeWebhookQueue = <T>(options?: {\n  readonly capacity?: number;\n}): Effect.Effect<WebhookStream<T>, never> =>\n  Effect.gen(function* () {\n    const capacity = options?.capacity ?? 1024;\n    const queue = yield* Queue.bounded<Batch<T>>(capacity);\n    const stream: Stream.Stream<Batch<T>, ConnectorError> = Stream.fromQueue(queue);\n    return { queue, stream };\n  });\n","import { Cause, Data, Effect, Schema } from \"effect\";\nimport { HttpRouter, HttpServerRequest, HttpServerResponse } from \"effect/unstable/http\";\n\nimport type { WebhookRoute } from \"./types\";\n\nclass InvalidWebhookPayloadError extends Data.TaggedError(\"InvalidWebhookPayloadError\")<{\n  readonly message: string;\n}> {}\n\nconst makeHandler = <TPayload>(route: WebhookRoute<TPayload>) =>\n  Effect.gen(function* () {\n    const request = yield* HttpServerRequest.HttpServerRequest;\n    const rawBuffer = yield* request.arrayBuffer.pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Failed to read request body\",\n          }),\n      ),\n    );\n    const rawBody = new Uint8Array(rawBuffer);\n    const rawText = new TextDecoder().decode(rawBody);\n    const rawJson = yield* Effect.try({\n      try: () => JSON.parse(rawText) as unknown,\n      catch: () => new InvalidWebhookPayloadError({ message: \"Invalid JSON body\" }),\n    });\n    const payload = yield* Schema.decodeUnknownEffect(route.schema)(rawJson).pipe(\n      Effect.mapError(\n        () =>\n          new InvalidWebhookPayloadError({\n            message: \"Invalid webhook payload\",\n          }),\n      ),\n    );\n    yield* route.handle(payload, request, rawBody);\n    // unsafeJson serializes synchronously — no Effect, no HttpBodyError\n    return HttpServerResponse.jsonUnsafe({ ok: true });\n  }).pipe(\n    Effect.catchTag(\"InvalidWebhookPayloadError\", () =>\n      Effect.succeed(\n        HttpServerResponse.jsonUnsafe(\n          { ok: false, error: \"Invalid webhook payload\" },\n          { status: 400 },\n        ),\n      ),\n    ),\n    Effect.catchCause((cause) =>\n      Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`).pipe(\n        Effect.andThen(\n          Effect.succeed(\n            HttpServerResponse.jsonUnsafe(\n              { ok: false, error: \"Webhook handler failed\" },\n              { status: 500 },\n            ),\n          ),\n        ),\n      ),\n    ),\n  );\n\nexport const buildWebhookRouter = <TPayload>(routes: ReadonlyArray<WebhookRoute<TPayload>>) =>\n  HttpRouter.addAll(\n    routes.map((route) => HttpRouter.route(\"POST\", route.path, makeHandler(route))),\n  );\n"],"mappings":";;;;;AAEA,MAAa,mBAIX,eACG;AAEL,MAAa,gBAAwC,eAAoC;AAEzF,MAAa,eAAuC,eAAmC;;;;ACTvF,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE;;;;ACMH,IAAa,YAAb,cAA+B,QAAQ,SAQpC,CAAC,YAAY,CAAC;;;;ACdjB,IAAa,aAAb,cAAgC,QAAQ,SAWrC,CAAC,aAAa,CAAC;AAElB,MAAa,qBAAqB,MAAM,OAAO,WAAW,CACxD,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,IAAI,qBAAK,IAAI,KAAqC,CAAC;AAEtE,QAAO;EACL,WAAW,QAAQ,OAAO,IAAI,IAAI,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI,IAAI,CAAC;EAClE,WAAW,KAAK,UACd,OAAO,IACL,IAAI,OAAO,MAAM,QAAQ;GACvB,MAAM,OAAO,IAAI,IAAI,IAAI;AACzB,QAAK,IAAI,KAAK,MAAM;AACpB,UAAO;IACP,QACI,OACP;EACJ;EACD,CACH;;;;ACVD,MAAa,gBACX,WACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,aAAa,UAAU,SAAS,KAAK,WAAW,UAAU,QAAQ,cAAc,CAAC;CACvF,MAAM,YAAY,UAAU,OAAO,KAAK,UAAU,SAAS,OAAO,cAAc,CAAC;AAEjF,QAAO,OAAO,IAAI,CAAC,GAAG,YAAY,GAAG,UAAU,EAAE,EAC/C,aAAa,aACd,CAAC;EACF;AAEJ,MAAM,sBAAsB,YAA4C;CACtE,UAAU,EAAE,QAAQ;CACpB,MAAM,EAAE,QAAQ;CACjB;AAED,MAAM,gBACJ,KACA,kBAEA,OAAO,IAAI,aAAa;CAItB,MAAM,WADW,QADH,OAAO,YACS,SAAS,IAAI,KACf,mBAAmB,cAAc;AAC7D,QAAO,OAAO,IAAI,KAAK,QAAQ;EAC/B;AAEJ,MAAM,aACJ,QACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,OAAO,MAAM,cAAc;CAEhE,MAAM,UAAU,OAAO,IAAI,qBAAK,IAAI,KAAa,CAAC;CAElD,MAAM,aAAa,kBAAkB,OAAO,KAAK;CACjD,MAAM,WAAW,WAAuB;EACtC,QAAQ;EACR;EACD;CACD,MAAM,cAAc,SAClB,IAAI,OAAO,UAAU,SAAS;EAC5B,MAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,QAAK,IAAI,IAAI;;AAEf,SAAO;GACP;CAIJ,MAAM,iBAAiB,OAAO,UAAU,OAAO,WAAW,UACxD,IAAI,IAAI,QAAQ,CAAC,KACf,OAAO,KAAK,SAAS;EACnB,MAAM,WAAW,MAAM,KAAK,QAAQ,QAAQ;GAC1C,MAAM,MAAM,OAAO,IAAI,OAAO,YAAY;AAC1C,UAAO,CAAC,KAAK,IAAI,IAAI;IACrB;AACF,SAAO;GACL,QAAQ;GACR,OAAO;IAAE,QAAQ,MAAM;IAAQ,MAAM;IAAU;GAChD;GACD,CACH,CACF,CAAC,KAAK,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAAC;AAQzD,KAAI,gBAAgB,OAAO,KAAK,EAAE;EAChC,MAAM,aAAa,OAAO,MAAM,KAAK,OAAO,KAAK,MAAM;AACvD,SAAO,WAAW,WAAW,KAAK;AAClC,SAAO,oBACL,OAAO,KAAK;GAAE,QAAQ;GAAiB,OAAO;GAAY,CAAC,EAC3D,OAAO,MACP,OAAO,WACP,SACD;EAGD,MAAM,iBAAiB,OAAO,IAAI,YAAY,QAAQ,CAAC,KACrD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAED,SAAO,oBADQ,OAAO,MAAM,gBAAgB,eAAe,EACxB,OAAO,MAAM,OAAO,WAAW,SAAS;AAC3E;;CAKF,MAAM,aAAa,OAAO,IAAI,YAAY,QAAQ,CAAC,KACjD,OAAO,KAAK,EAAE,YAAY,WAAW,MAAM,KAAK,CAAC,CAClD;AAGD,QAAO,oBADQ,OAAO,MAAM,YAAY,eAAe,EACpB,OAAO,MAAM,OAAO,WAAW,SAAS;EAC3E;AAEJ,MAAM,YACJ,OACA,kBAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,WAAW,OAAO,aAAa,MAAM,MAAM,cAAc;CAC/D,MAAM,aAAa,kBAAuB,MAAM,KAAK;AAGrD,KAAI,MAAM,SAKR,QAAO,oBAJgB,OAAO,IAAI,MAAM,WAAW,WAAW;EAC5D,QAAQ;EACR;EACD,EAAE,EACwC,MAAM,MAAM,MAAM,WAAW,SAAS;AAOnF,QAAO,oBAJY,OAAO,IAAI,aAAa,WAAW;EACpD,QAAQ;EACR;EACD,EAAE,EACoC,MAAM,MAAM,MAAM,WAAW,SAAS;EAC7E;AAEJ,MAAM,eACJ,OACA,QACA,WAEA,WAAW,SACP;CAAE,GAAG;CAAO,MAAM;EAAE,GAAG,MAAM;EAAM,SAAS;EAAQ;CAAE,GACtD;CAAE,GAAG;CAAO,UAAU;EAAE,GAAG,MAAM;EAAU,SAAS;EAAQ;CAAE;AAEpE,MAAM,qBAAwB,WAC5B,gBAAgB,OAAO,GAAG,OAAO,SAAS;AAE5C,MAAM,mBAAsB,WAC1B,OAAO,WAAW,YAAY,WAAW,QAAQ,WAAW,UAAU,YAAY;AAEpF,MAAM,uBACJ,QACA,MACA,WACA,aAEA,OAAO,WAAW,SAAS,EAAE,QAAQ,YACnC,OAAO,IAAI,aAAa;CAEtB,MAAM,OAAO,YAAY,OAAO,OAAO,QAAQ,MAAM,MAAM,UAAU,GAAG,MAAM;AAI9E,SADkB,OAAO,WACR,QAAQ;EACvB;EACA,OAAO;GACL,QAAQ,MAAM;GACd;GACD;EACF,CAAC;CAGF,MAAM,YAAY,OAAO,IAAI,aAAa,WAAW,UACnD,YAAY,OAAO,QAAQ,MAAM,OAAO,CACzC;AAGD,SADc,OAAO,YACR,SAAS,MAAM,UAAU;EACtC,CACH;;;;AC5KH,MAAM,oBAAoB,SAA8B;CAGtD,MAAM,CAAC,SADO,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CAC7B;AAEtB,KAAI,CAAC,MACH,OAAM,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC;AAG7D,QAAO;;AAGT,MAAa,uBACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;AAGjD,MAAK,MAAM,OAAO,CAAC,GAAG,OAAO,UAAU,UAAU,GAAG,OAAO,UAAU,OAAO,EAAE;EAC5E,MAAM,QAAQ,OAAO,OAAO,IAAI;AAChC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,IAAI,QAAQ,CAAC,CACjE;EAGH,MAAM,iBACJ,MAAM,iBAAiB,SACnB,MAAM,OAAO,OAAO,WAAW,UAAU,MAAM,OAAO,MAAM,aAAa,GACzE;EAEN,MAAM,iBACJ,mBAAmB,UAAa,kBAAkB,IAC9C,MAAM,OAAO,OAAO,iBAAiB,OACrC;EAEN,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAAC,CACzF;AAED,UAAQ,IAAI,IAAI,MAAM;GACpB;GACA;GACA,gBAAgB,OAAO,kBAAkB,IAAI;GAC9C,CAAC;;AAGJ,QAAO,EACL,UAAU,EAAE,MAAM,YAChB,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,QAAQ,IAAI,KAAK;AAC/B,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,kBAAkB,QAAQ,CAAC,CAAC;AAGtF,MAAI,MAAM,KAAK,WAAW,EACxB,QAAO,EAAE,SAAS,MAAM;EAG1B,MAAM,cAAc,iBAAiB,MAAM,KAAK;EAChD,MAAM,SAAS,OAAO,MAAM,UACzB,KAAK;GACJ,OAAO;GACP,gBAAgB,MAAM;GACvB,CAAC,CACD,KACC,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,MAAM;GACf,OAAO;GACR,CAAC,CACL,CACF;AAEH,SAAO,EACL,SAAS,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,UAAU,aACtD;GACD,EACL;EACD,CACH;;;;AChGH,MAAa,kBACX,YAEA,OAAO,OAAO;CAAE,QAAQ,QAAQ;CAAe,MAAM;CAAO,GAAG,UAC7D,OAAO,IAAI,aAAa;AACtB,KAAI,MAAM,KACR;CAGF,IAAIA,aAAiC,MAAM;AAE3C,QAAO,MAAM;EACX,MAAM,OAAO,OAAO,QAAQ,UAAU,WAAW;AAEjD,MAAI,KAAK,KAAK,SAAS,EAMrB,QAAO,CALiB;GACtB,QAAQ,KAAK;GACb,MAAM,KAAK;GACZ,EAIC,KAAK,UAAU;GAAE,QAAQ,KAAK;GAAQ,MAAM;GAAO,GAAG;GAAE,QAAQ;GAAW,MAAM;GAAM,CACxF;AAGH,MAAI,CAAC,KAAK,QACR;AAGF,eAAa,KAAK;;EAEpB,CACH;;;;AChDH,MAAa,oBAAuB,YAGlC,OAAO,IAAI,aAAa;CACtB,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,QAAQ,OAAO,MAAM,QAAkB,SAAS;AAEtD,QAAO;EAAE;EAAO,QADwC,OAAO,UAAU,MAAM;EACvD;EACxB;;;;ACRJ,IAAM,6BAAN,cAAyC,KAAK,YAAY,6BAA6B,CAEpF;AAEH,MAAM,eAAyB,UAC7B,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,kBAAkB;CACzC,MAAM,YAAY,OAAO,QAAQ,YAAY,KAC3C,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,+BACV,CAAC,CACL,CACF;CACD,MAAM,UAAU,IAAI,WAAW,UAAU;CACzC,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,QAAQ;CACjD,MAAM,UAAU,OAAO,OAAO,IAAI;EAChC,WAAW,KAAK,MAAM,QAAQ;EAC9B,aAAa,IAAI,2BAA2B,EAAE,SAAS,qBAAqB,CAAC;EAC9E,CAAC;CACF,MAAM,UAAU,OAAO,OAAO,oBAAoB,MAAM,OAAO,CAAC,QAAQ,CAAC,KACvE,OAAO,eAEH,IAAI,2BAA2B,EAC7B,SAAS,2BACV,CAAC,CACL,CACF;AACD,QAAO,MAAM,OAAO,SAAS,SAAS,QAAQ;AAE9C,QAAO,mBAAmB,WAAW,EAAE,IAAI,MAAM,CAAC;EAClD,CAAC,KACD,OAAO,SAAS,oCACd,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA2B,EAC/C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,EACD,OAAO,YAAY,UACjB,OAAO,WAAW,0BAA0B,MAAM,OAAO,MAAM,GAAG,CAAC,KACjE,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;CAA0B,EAC9C,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,CACF,CACF,CACF;AAEH,MAAa,sBAAgC,WAC3C,WAAW,OACT,OAAO,KAAK,UAAU,WAAW,MAAM,QAAQ,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC,CAChF"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect } from \"effect\";\n\nimport type {\n  ChangesFetch,\n  ConnectorDefinition,\n  Cursor as CursorType,\n  DeleteValue,\n  PageFetch,\n  ResourceDefinition,\n  ResourceMutation,\n  ResourceSchema,\n  WebhookHandler,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(\n    definition: ConnectorDefinition<Resources>,\n  ) => definition,\n};\n\nexport const Resource = {\n  entity: <\n    S extends ResourceSchema,\n    Payload = never,\n    R = never,\n    const Name extends string = string,\n  >(\n    definition: ResourceDefinition<S, Payload, R, Name>,\n  ): ResourceDefinition<S, Payload, R, Name> => definition,\n\n  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>\n    definition,\n\n  upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: \"upsert\", row }),\n\n  delete: (options: {\n    readonly key: DeleteValue;\n    readonly version: DeleteValue;\n  }): ResourceMutation<never> => ({\n    op: \"delete\",\n    key: options.key,\n    version: options.version,\n  }),\n};\n\nexport const Fetch = {\n  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n  string: (): CursorType.Definition<string> => ({\n    kind: \"string\",\n    decode: (value) =>\n      typeof value === \"string\"\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"string\", value)),\n  }),\n\n  number: (): CursorType.Definition<number> => ({\n    kind: \"number\",\n    decode: (value) =>\n      typeof value === \"number\" && Number.isFinite(value)\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"number\", value)),\n  }),\n\n  isoDateTime: (): CursorType.Definition<string> => ({\n    kind: \"isoDateTime\",\n    decode: (value) => {\n      if (typeof value !== \"string\" || Number.isNaN(Date.parse(value))) {\n        return Effect.fail(decodeFailure(\"ISO datetime\", value));\n      }\n      return Effect.succeed(value);\n    },\n  }),\n\n  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;;;;;;;AAgBA,MAAa,YAAY,EACvB,SACE,eACG,YACN;AAED,MAAa,WAAW;CACtB,SAME,eAC4C;CAE9C,UAA+C,eAC7C;CAEF,SAA6B,SAAqC;EAAE,IAAI;EAAU;EAAK;CAEvF,SAAS,aAGuB;EAC9B,IAAI;EACJ,KAAK,QAAQ;EACb,SAAS,QAAQ;EAClB;CACF;AAED,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACvF;AAED,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,MAAM,IAAI,CAAC;AAEnF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAC/C,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;AACjB,OAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,MAAM,MAAM,CAAC,CAC9D,QAAO,OAAO,KAAK,cAAc,gBAAgB,MAAM,CAAC;AAE1D,UAAO,OAAO,QAAQ,MAAM;;EAE/B;CAED,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,UAAU;CAC7D"}

package/dist/ingestion/index.d.ts

@@ -0,0 +1,6 @@
+import "../index-D54ySAKn.js";
+import "../types-B35J5YPu.js";
+import "../service-IE5qaf9x.js";
+import "../index-Coh_SkBs.js";
+import { n as RunOptions, r as run } from "../index-J42AZaXU.js";
+export { RunOptions, run };

package/dist/ingestion/index.js

@@ -0,0 +1,10 @@
+import "../errors-DHWylqRZ.js";
+import "../metrics-CWPy2Aqc.js";
+import "../constants-BRhIX_rz.js";
+import "../service-DuFiQwid.js";
+import "../server-CXTQ7odl.js";
+import "../state-store-yl6LiTQ0.js";
+import "../status-B-oZBc33.js";
+import { n as run } from "../ingestion-BQo4cbGw.js";
+
+export { run };

package/dist/ingestion-BQo4cbGw.js

@@ -0,0 +1,200 @@
+import { t as __export } from "./chunk-Bp6m_JJh.js";
+import { d as setLastSuccessTimestamp, f as setSyncState, p as setWebhookQueueDepth } from "./metrics-CWPy2Aqc.js";
+import { a as SpanName, n as Attr, s as annotateError } from "./service-DuFiQwid.js";
+import { n as router, r as publishBatch, t as WebhookQueue } from "./server-CXTQ7odl.js";
+import { o as StateStore } from "./state-store-yl6LiTQ0.js";
+import { c as deriveSyncState, l as normalizeCursor, s as statusRoute } from "./status-B-oZBc33.js";
+import { DateTime, Effect, Layer, Queue } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+import * as Observability from "effect/unstable/observability";
+
+//#region src/ingestion/engine.ts
+function run(connector, options) {
+	const queueLayer = makeWebhookQueueLayer(connector.name);
+	const runtimeLayer = options?.webhook ? Layer.mergeAll(queueLayer, makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer))) : Layer.empty;
+	return Effect.gen(function* () {
+		const initialCutoff = options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+		yield* Effect.logInfo("Connector started").pipe(Effect.annotateLogs({
+			[Attr.connectorName]: connector.name,
+			resources: connector.resources.length
+		}));
+		const sourceRun = runIngestion(connector, initialCutoff);
+		const webhookRun = options?.webhook ? runWebhookRuntime(connector, options.webhook.routes) : Effect.void;
+		return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+	}).pipe(Effect.provide(runtimeLayer));
+}
+const makeWebhookQueueLayer = (connector) => Layer.effect(WebhookQueue)(Effect.gen(function* () {
+	const queue = yield* Queue.bounded(1024);
+	yield* setWebhookQueueDepth(connector, 0);
+	return WebhookQueue.of({ queue });
+}));
+const runWebhookRuntime = (connector, routes) => {
+	const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
+	return Effect.all([hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+};
+const runWebhookQueueConsumer = (connector) => Effect.gen(function* () {
+	const queue = yield* WebhookQueue;
+	while (true) {
+		const batch = yield* Queue.take(queue.queue);
+		const depth = yield* Queue.size(queue.queue);
+		yield* setWebhookQueueDepth(connector.name, depth);
+		yield* publishBatch({
+			connector: connector.name,
+			resource: batch.resource,
+			source: "webhook",
+			batch: batch.batch
+		});
+	}
+});
+const makeWebhookServerLayer = (connector, options) => {
+	const healthPath = options.healthPath ?? "/health";
+	const metricsPath = options.metricsPath === void 0 ? "/metrics" : options.metricsPath;
+	const statusPath = options.statusPath === void 0 ? "/status" : options.statusPath;
+	const app = Layer.mergeAll(router(options.routes, { connectorName: connector.name }), HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))), ...metricsPath === false ? [] : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })], ...statusPath === false ? [] : [statusRoute(connector, statusPath)]);
+	return HttpRouter.serve(app, { disableLogger: options.disableHttpLogger ?? true });
+};
+const runIngestion = (connector, initialCutoff) => Effect.forEach(connector.resources, (resource) => runResourceSources(connector, resource, initialCutoff), { concurrency: "unbounded" }).pipe(Effect.asVoid);
+const initializeResourceState = (existing, initialCutoff) => ({
+	...existing,
+	changes: existing?.changes ?? { cursor: initialCutoff },
+	backfill: existing?.backfill ?? {
+		cutoff: initialCutoff,
+		completed: false
+	}
+});
+const getInitializedState = Effect.fnUntraced(function* (resource, initialCutoff) {
+	return initializeResourceState(yield* (yield* StateStore).getResourceState(resource), initialCutoff);
+});
+const runResourceSources = (connector, resource, initialCutoff) => {
+	const runs = [resource.backfill ? isolateSourceFailure(runBackfill(connector, resource, initialCutoff), connector, resource, "backfill") : Effect.void, resource.changes ? isolateSourceFailure(runChanges(connector, resource, initialCutoff), connector, resource, "changes") : Effect.void];
+	return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })), Effect.asVoid, Effect.catch(() => setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, "error").pipe(Effect.andThen(Effect.logError("Connector resource initialization failed")), Effect.annotateLogs({
+		[Attr.connectorName]: connector.name,
+		resource: resource.name
+	}), Effect.andThen(Effect.never))));
+};
+const recordResourceError = (connector, resource, source, operation) => (effect) => effect.pipe(Effect.tapError(() => setSyncState({
+	connector: connector.name,
+	resource: resource.name
+}, "error").pipe(Effect.andThen(StateStore.pipe(Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)))))));
+const isolateSourceFailure = (effect, connector, resource, source) => effect.pipe(Effect.catch(() => Effect.logError("Connector source parked").pipe(Effect.annotateLogs({
+	[Attr.connectorName]: connector.name,
+	resource: resource.name,
+	source
+}), Effect.andThen(Effect.never))));
+const refreshRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(resource, state));
+	return state;
+});
+const initializeRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(resource, state));
+	if (resource.backfill) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "backfill"
+	}, state.backfill?.lastSuccessAt);
+	if (resource.changes) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "changes"
+	}, state.changes?.lastSuccessAt);
+});
+const runBackfill = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.backfill) return;
+	const store = yield* StateStore;
+	let state = yield* getInitializedState(resource.name, initialCutoff);
+	if (state.backfill?.completed === true) {
+		yield* store.clearResourceError(resource.name, "backfill").pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		return;
+	}
+	while (state.backfill?.completed !== true) {
+		const backfill = state.backfill ?? {
+			cutoff: initialCutoff,
+			completed: false
+		};
+		const clearsVisibleError = state.lastError?.source === "backfill";
+		const page = yield* resource.backfill.fetch({
+			pageCursor: backfill.pageCursor,
+			cutoff: backfill.cutoff
+		}).pipe(recordResourceError(connector, resource, "backfill", "fetch"));
+		yield* publishBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill",
+			batch: {
+				cursor: page.nextPageCursor ?? backfill.cutoff,
+				mutations: page.mutations
+			}
+		}).pipe(recordResourceError(connector, resource, "backfill", "publish"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		state = {
+			...state,
+			backfill: {
+				cutoff: backfill.cutoff,
+				pageCursor: page.nextPageCursor,
+				completed: !page.hasMore,
+				lastSuccessAt
+			}
+		};
+		yield* Effect.withSpan(store.setBackfillState(resource.name, {
+			cutoff: normalizeCursor(backfill.cutoff),
+			pageCursor: page.nextPageCursor === void 0 ? void 0 : normalizeCursor(page.nextPageCursor),
+			completed: !page.hasMore,
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "backfill")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		if (clearsVisibleError || !page.hasMore) state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+	}
+});
+const runChanges = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.changes) return;
+	const store = yield* StateStore;
+	while (true) {
+		const state = yield* getInitializedState(resource.name, initialCutoff);
+		const cursor = state.changes?.cursor ?? initialCutoff;
+		const clearsVisibleError = state.lastError?.source === "changes";
+		const page = yield* resource.changes.fetch({ cursor }).pipe(recordResourceError(connector, resource, "changes", "fetch"));
+		yield* publishBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes",
+			batch: {
+				cursor: page.cursor,
+				mutations: page.mutations
+			}
+		}).pipe(recordResourceError(connector, resource, "changes", "publish"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		yield* Effect.withSpan(store.setChangesState(resource.name, {
+			cursor: normalizeCursor(page.cursor),
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
+		if (clearsVisibleError) yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+	}
+});
+
+//#endregion
+//#region src/ingestion/index.ts
+var ingestion_exports = /* @__PURE__ */ __export({ run: () => run });
+
+//#endregion
+export { run as n, ingestion_exports as t };
+//# sourceMappingURL=ingestion-BQo4cbGw.js.map

package/dist/ingestion-BQo4cbGw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestion-BQo4cbGw.js","names":["Metrics.setWebhookQueueDepth","healthPath: HttpRouter.PathInput","Metrics.setSyncState","Metrics.setLastSuccessTimestamp"],"sources":["../src/ingestion/engine.ts","../src/ingestion/index.ts"],"sourcesContent":["import { DateTime, Effect, Layer, Queue } from \"effect\";\nimport { HttpRouter, type HttpServer, HttpServerResponse } from \"effect/unstable/http\";\nimport * as Observability from \"effect/unstable/observability\";\n\nimport type {\n  ConnectorDefinition,\n  Cursor,\n  ResourceDefinition,\n  ResourceState,\n  WebhookRoute,\n} from \"../core/types\";\nimport type { StateOperation, StateSource } from \"../state-store/schema\";\n\nimport { ConnectorError } from \"../errors\";\nimport * as Metrics from \"../metrics\";\nimport { publishBatch } from \"../publisher/instrumented\";\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, normalizeCursor } from \"../state-store/state\";\nimport { statusRoute } from \"../status\";\nimport { Attr, SpanName, annotateError } from \"../telemetry\";\nimport { router, WebhookQueue, type QueuedWebhookBatch } from \"../webhook/server\";\n\ntype RunBaseOptions = {\n  readonly initialCutoff?: Cursor.Value;\n};\n\nexport type RunOptions = RunBaseOptions & {\n  readonly webhook?: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  };\n};\n\ntype RunNoWebhookOptions = RunBaseOptions & {\n  readonly webhook?: undefined;\n};\n\ntype RunWebhookOptions = RunOptions & {\n  readonly webhook: NonNullable<RunOptions[\"webhook\"]>;\n};\n\n/**\n * Runs all connector ingestion loops. Callers provide the Publisher and\n * StateStore layers; webhook mode additionally requires an HttpServer layer.\n */\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options?: RunNoWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher>;\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options: RunWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;\nexport function run(connector: ConnectorDefinition, options?: RunOptions) {\n  const queueLayer = makeWebhookQueueLayer(connector.name);\n  const runtimeLayer = options?.webhook\n    ? Layer.mergeAll(\n        queueLayer,\n        makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),\n      )\n    : Layer.empty;\n\n  return Effect.gen(function* () {\n    const initialCutoff =\n      options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    yield* Effect.logInfo(\"Connector started\").pipe(\n      Effect.annotateLogs({\n        [Attr.connectorName]: connector.name,\n        resources: connector.resources.length,\n      }),\n    );\n\n    const sourceRun = runIngestion(connector, initialCutoff);\n    const webhookRun = options?.webhook\n      ? runWebhookRuntime(connector, options.webhook.routes)\n      : Effect.void;\n\n    return yield* Effect.all([sourceRun, webhookRun], { concurrency: \"unbounded\" }).pipe(\n      Effect.asVoid,\n    );\n  }).pipe(Effect.provide(runtimeLayer));\n}\n\nconst makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>\n  Layer.effect(WebhookQueue)(\n    Effect.gen(function* () {\n      const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);\n      yield* Metrics.setWebhookQueueDepth(connector, 0);\n      return WebhookQueue.of({ queue });\n    }),\n  );\n\nconst runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {\n  const hasAfterEnqueue = routes.some((route) => route.ackMode === \"after-enqueue\");\n  return Effect.all(\n    [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],\n    {\n      concurrency: \"unbounded\",\n    },\n  ).pipe(Effect.asVoid);\n};\n\nconst runWebhookQueueConsumer = (connector: ConnectorDefinition) =>\n  Effect.gen(function* () {\n    const queue = yield* WebhookQueue;\n    while (true) {\n      const batch = yield* Queue.take(queue.queue);\n      const depth = yield* Queue.size(queue.queue);\n      yield* Metrics.setWebhookQueueDepth(connector.name, depth);\n      yield* publishBatch({\n        connector: connector.name,\n        resource: batch.resource,\n        source: \"webhook\",\n        batch: batch.batch,\n      });\n    }\n  });\n\nconst makeWebhookServerLayer = (\n  connector: ConnectorDefinition,\n  options: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  },\n): Layer.Layer<never, never, HttpServer.HttpServer> => {\n  const healthPath: HttpRouter.PathInput = options.healthPath ?? \"/health\";\n  const metricsPath = options.metricsPath === undefined ? \"/metrics\" : options.metricsPath;\n  const statusPath = options.statusPath === undefined ? \"/status\" : options.statusPath;\n  const app = Layer.mergeAll(\n    router(options.routes, { connectorName: connector.name }),\n    HttpRouter.add(\"GET\", healthPath, Effect.succeed(HttpServerResponse.text(\"ok\"))),\n    ...(metricsPath === false\n      ? []\n      : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),\n    ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),\n  );\n\n  return HttpRouter.serve(app, {\n    disableLogger: options.disableHttpLogger ?? true,\n  });\n};\n\nconst runIngestion = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor.Value,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.forEach(\n    connector.resources,\n    (resource) => runResourceSources(connector, resource, initialCutoff),\n    {\n      concurrency: \"unbounded\",\n    },\n  ).pipe(Effect.asVoid);\n\nconst initializeResourceState = (\n  existing: ResourceState | undefined,\n  initialCutoff: Cursor.Value,\n): ResourceState => ({\n  ...existing,\n  changes: existing?.changes ?? { cursor: initialCutoff },\n  backfill: existing?.backfill ?? {\n    cutoff: initialCutoff,\n    completed: false,\n  },\n});\n\nconst getInitializedState = Effect.fnUntraced(function* (\n  resource: string,\n  initialCutoff: Cursor.Value,\n) {\n  const store = yield* StateStore;\n  const existing = yield* store.getResourceState(resource);\n  return initializeResourceState(existing, initialCutoff);\n});\n\nconst runResourceSources = (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) => {\n  const runs = [\n    resource.backfill\n      ? isolateSourceFailure(\n          runBackfill(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"backfill\",\n        )\n      : Effect.void,\n    resource.changes\n      ? isolateSourceFailure(\n          runChanges(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"changes\",\n        )\n      : Effect.void,\n  ];\n\n  return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(\n    Effect.andThen(Effect.all(runs, { concurrency: \"unbounded\" })),\n    Effect.asVoid,\n    Effect.catch(() =>\n      Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n        Effect.andThen(Effect.logError(\"Connector resource initialization failed\")),\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n};\n\nconst recordResourceError =\n  (\n    connector: ConnectorDefinition,\n    resource: ResourceDefinition,\n    source: StateSource,\n    operation: StateOperation,\n  ) =>\n  <A, R>(\n    effect: Effect.Effect<A, ConnectorError, R>,\n  ): Effect.Effect<A, ConnectorError, R | StateStore> =>\n    effect.pipe(\n      Effect.tapError(() =>\n        Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n          Effect.andThen(\n            StateStore.pipe(\n              Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),\n            ),\n          ),\n        ),\n      ),\n    );\n\nconst isolateSourceFailure = <A, R>(\n  effect: Effect.Effect<A, ConnectorError, R>,\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  source: StateSource,\n): Effect.Effect<A, never, R> =>\n  // Only typed source errors are isolated. Defects and interruption still stop the runtime.\n  effect.pipe(\n    Effect.catch(() =>\n      Effect.logError(\"Connector source parked\").pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n          source,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n\nconst refreshRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(resource, state),\n  );\n  return state;\n});\n\nconst initializeRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(resource, state),\n  );\n  if (resource.backfill) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"backfill\" },\n      state.backfill?.lastSuccessAt,\n    );\n  }\n  if (resource.changes) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"changes\" },\n      state.changes?.lastSuccessAt,\n    );\n  }\n});\n\nconst runBackfill = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.backfill) return;\n  const store = yield* StateStore;\n  let state = yield* getInitializedState(resource.name, initialCutoff);\n\n  if (state.backfill?.completed === true) {\n    // The checkpoint may have committed before the previous process cleared\n    // its error marker. It may also be hidden by a newer changes error.\n    yield* store\n      .clearResourceError(resource.name, \"backfill\")\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n    yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    return;\n  }\n\n  while (state.backfill?.completed !== true) {\n    const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };\n    const clearsVisibleError = state.lastError?.source === \"backfill\";\n    const page = yield* resource.backfill\n      .fetch({\n        pageCursor: backfill.pageCursor,\n        cutoff: backfill.cutoff,\n      })\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"fetch\"));\n\n    yield* publishBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"backfill\",\n      batch: {\n        cursor: page.nextPageCursor ?? backfill.cutoff,\n        mutations: page.mutations,\n      },\n    }).pipe(recordResourceError(connector, resource, \"backfill\", \"publish\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    state = {\n      ...state,\n      backfill: {\n        cutoff: backfill.cutoff,\n        pageCursor: page.nextPageCursor,\n        completed: !page.hasMore,\n        lastSuccessAt,\n      },\n    };\n\n    // Checkpoint only after publish succeeds so a crash cannot skip an\n    // unacknowledged page; replay therefore remains at-least-once.\n    yield* Effect.withSpan(\n      store\n        .setBackfillState(resource.name, {\n          cutoff: normalizeCursor(backfill.cutoff),\n          pageCursor:\n            page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),\n          completed: !page.hasMore,\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"backfill\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"backfill\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n\n    // Completion or clearing an error can change the resource-wide sync state.\n    if (clearsVisibleError || !page.hasMore) {\n      state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n  }\n});\n\nconst runChanges = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.changes) return;\n  const store = yield* StateStore;\n\n  while (true) {\n    const state = yield* getInitializedState(resource.name, initialCutoff);\n    const cursor = state.changes?.cursor ?? initialCutoff;\n    const clearsVisibleError = state.lastError?.source === \"changes\";\n    const page = yield* resource.changes\n      .fetch({ cursor })\n      .pipe(recordResourceError(connector, resource, \"changes\", \"fetch\"));\n\n    yield* publishBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"changes\",\n      batch: {\n        cursor: page.cursor,\n        mutations: page.mutations,\n      },\n    }).pipe(recordResourceError(connector, resource, \"changes\", \"publish\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    // Advance the durable cursor only after Wings accepts the page.\n    yield* Effect.withSpan(\n      store\n        .setChangesState(resource.name, {\n          cursor: normalizeCursor(page.cursor),\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"changes\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"changes\", \"checkpoint\"));\n\n    if (clearsVisibleError) {\n      yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n\n    yield* Effect.sleep(resource.changes.interval ?? \"1 minute\");\n  }\n});\n","export { run, type RunOptions } from \"./engine\";\n"],"mappings":";;;;;;;;;;;AAyDA,SAAgB,IAAI,WAAgC,SAAsB;CACxE,MAAM,aAAa,sBAAsB,UAAU,KAAK;CACxD,MAAM,eAAe,SAAS,UAC1B,MAAM,SACJ,YACA,uBAAuB,WAAW,QAAQ,QAAQ,CAAC,KAAK,MAAM,QAAQ,WAAW,CAAC,CACnF,GACD,MAAM;AAEV,QAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,gBACJ,SAAS,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AACrF,SAAO,OAAO,QAAQ,oBAAoB,CAAC,KACzC,OAAO,aAAa;IACjB,KAAK,gBAAgB,UAAU;GAChC,WAAW,UAAU,UAAU;GAChC,CAAC,CACH;EAED,MAAM,YAAY,aAAa,WAAW,cAAc;EACxD,MAAM,aAAa,SAAS,UACxB,kBAAkB,WAAW,QAAQ,QAAQ,OAAO,GACpD,OAAO;AAEX,SAAO,OAAO,OAAO,IAAI,CAAC,WAAW,WAAW,EAAE,EAAE,aAAa,aAAa,CAAC,CAAC,KAC9E,OAAO,OACR;GACD,CAAC,KAAK,OAAO,QAAQ,aAAa,CAAC;;AAGvC,MAAM,yBAAyB,cAC7B,MAAM,OAAO,aAAa,CACxB,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,MAAM,QAA4B,KAAK;AAC5D,QAAOA,qBAA6B,WAAW,EAAE;AACjD,QAAO,aAAa,GAAG,EAAE,OAAO,CAAC;EACjC,CACH;AAEH,MAAM,qBAAqB,WAAgC,WAAwC;CACjG,MAAM,kBAAkB,OAAO,MAAM,UAAU,MAAM,YAAY,gBAAgB;AACjF,QAAO,OAAO,IACZ,CAAC,kBAAkB,wBAAwB,UAAU,GAAG,OAAO,MAAM,OAAO,MAAM,EAClF,EACE,aAAa,aACd,CACF,CAAC,KAAK,OAAO,OAAO;;AAGvB,MAAM,2BAA2B,cAC/B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;AACrB,QAAO,MAAM;EACX,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,MAAM;EAC5C,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,MAAM;AAC5C,SAAOA,qBAA6B,UAAU,MAAM,MAAM;AAC1D,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,MAAM;GACd,CAAC;;EAEJ;AAEJ,MAAM,0BACJ,WACA,YAOqD;CACrD,MAAMC,aAAmC,QAAQ,cAAc;CAC/D,MAAM,cAAc,QAAQ,gBAAgB,SAAY,aAAa,QAAQ;CAC7E,MAAM,aAAa,QAAQ,eAAe,SAAY,YAAY,QAAQ;CAC1E,MAAM,MAAM,MAAM,SAChB,OAAO,QAAQ,QAAQ,EAAE,eAAe,UAAU,MAAM,CAAC,EACzD,WAAW,IAAI,OAAO,YAAY,OAAO,QAAQ,mBAAmB,KAAK,KAAK,CAAC,CAAC,EAChF,GAAI,gBAAgB,QAChB,EAAE,GACF,CAAC,cAAc,kBAAkB,UAAU,EAAE,MAAM,aAAa,CAAC,CAAC,EACtE,GAAI,eAAe,QAAQ,EAAE,GAAG,CAAC,YAAY,WAAW,WAAW,CAAC,CACrE;AAED,QAAO,WAAW,MAAM,KAAK,EAC3B,eAAe,QAAQ,qBAAqB,MAC7C,CAAC;;AAGJ,MAAM,gBACJ,WACA,kBAEA,OAAO,QACL,UAAU,YACT,aAAa,mBAAmB,WAAW,UAAU,cAAc,EACpE,EACE,aAAa,aACd,CACF,CAAC,KAAK,OAAO,OAAO;AAEvB,MAAM,2BACJ,UACA,mBACmB;CACnB,GAAG;CACH,SAAS,UAAU,WAAW,EAAE,QAAQ,eAAe;CACvD,UAAU,UAAU,YAAY;EAC9B,QAAQ;EACR,WAAW;EACZ;CACF;AAED,MAAM,sBAAsB,OAAO,WAAW,WAC5C,UACA,eACA;AAGA,QAAO,wBADU,QADH,OAAO,YACS,iBAAiB,SAAS,EACf,cAAc;EACvD;AAEF,MAAM,sBACJ,WACA,UACA,kBACG;CACH,MAAM,OAAO,CACX,SAAS,WACL,qBACE,YAAY,WAAW,UAAU,cAAc,EAC/C,WACA,UACA,WACD,GACD,OAAO,MACX,SAAS,UACL,qBACE,WAAW,WAAW,UAAU,cAAc,EAC9C,WACA,UACA,UACD,GACD,OAAO,KACZ;AAED,QAAO,wBAAwB,WAAW,UAAU,cAAc,CAAC,KACjE,OAAO,QAAQ,OAAO,IAAI,MAAM,EAAE,aAAa,aAAa,CAAC,CAAC,EAC9D,OAAO,QACP,OAAO,YACLC,aAAqB;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EAAE,QAAQ,CAAC,KACpF,OAAO,QAAQ,OAAO,SAAS,2CAA2C,CAAC,EAC3E,OAAO,aAAa;GACjB,KAAK,gBAAgB,UAAU;EAChC,UAAU,SAAS;EACpB,CAAC,EACF,OAAO,QAAQ,OAAO,MAAM,CAC7B,CACF,CACF;;AAGH,MAAM,uBAEF,WACA,UACA,QACA,eAGA,WAEA,OAAO,KACL,OAAO,eACLA,aAAqB;CAAE,WAAW,UAAU;CAAM,UAAU,SAAS;CAAM,EAAE,QAAQ,CAAC,KACpF,OAAO,QACL,WAAW,KACT,OAAO,SAAS,UAAU,MAAM,iBAAiB,SAAS,MAAM,QAAQ,UAAU,CAAC,CACpF,CACF,CACF,CACF,CACF;AAEL,MAAM,wBACJ,QACA,WACA,UACA,WAGA,OAAO,KACL,OAAO,YACL,OAAO,SAAS,0BAA0B,CAAC,KACzC,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,UAAU,SAAS;CACnB;CACD,CAAC,EACF,OAAO,QAAQ,OAAO,MAAM,CAC7B,CACF,CACF;AAEH,MAAM,uBAAuB,OAAO,WAAW,WAC7C,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AACtE,QAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EACtD,gBAAgB,UAAU,MAAM,CACjC;AACD,QAAO;EACP;AAEF,MAAM,0BAA0B,OAAO,WAAW,WAChD,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AACtE,QAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EACtD,gBAAgB,UAAU,MAAM,CACjC;AACD,KAAI,SAAS,SACX,QAAOC,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;EAAY,EAC1E,MAAM,UAAU,cACjB;AAEH,KAAI,SAAS,QACX,QAAOA,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;EAAW,EACzE,MAAM,SAAS,cAChB;EAEH;AAEF,MAAM,cAAc,OAAO,WAAW,WACpC,WACA,UACA,eACA;AACA,KAAI,CAAC,SAAS,SAAU;CACxB,MAAM,QAAQ,OAAO;CACrB,IAAI,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AAEpE,KAAI,MAAM,UAAU,cAAc,MAAM;AAGtC,SAAO,MACJ,mBAAmB,SAAS,MAAM,WAAW,CAC7C,KAAK,oBAAoB,WAAW,UAAU,YAAY,aAAa,CAAC;AAC3E,SAAO,qBAAqB,WAAW,UAAU,cAAc;AAC/D;;AAGF,QAAO,MAAM,UAAU,cAAc,MAAM;EACzC,MAAM,WAAW,MAAM,YAAY;GAAE,QAAQ;GAAe,WAAW;GAAO;EAC9E,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,SAC1B,MAAM;GACL,YAAY,SAAS;GACrB,QAAQ,SAAS;GAClB,CAAC,CACD,KAAK,oBAAoB,WAAW,UAAU,YAAY,QAAQ,CAAC;AAEtE,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK,kBAAkB,SAAS;IACxC,WAAW,KAAK;IACjB;GACF,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,UAAU,CAAC;EAExE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AAC9E,UAAQ;GACN,GAAG;GACH,UAAU;IACR,QAAQ,SAAS;IACjB,YAAY,KAAK;IACjB,WAAW,CAAC,KAAK;IACjB;IACD;GACF;AAID,SAAO,OAAO,SACZ,MACG,iBAAiB,SAAS,MAAM;GAC/B,QAAQ,gBAAgB,SAAS,OAAO;GACxC,YACE,KAAK,mBAAmB,SAAY,SAAY,gBAAgB,KAAK,eAAe;GACtF,WAAW,CAAC,KAAK;GACjB;GACD,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;GAAY,EAC1E,cACD,CACF,EACD,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,WAAW,CAAC,EACnE,OAAO,UAAU,UAAU,cAAc,aAAa,MAAM,CAAC,CAC9D,EACH,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,MAAM,EAAE,CACnD,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,aAAa,CAAC;AAG1E,MAAI,sBAAsB,CAAC,KAAK,QAC9B,SAAQ,OAAO,qBAAqB,WAAW,UAAU,cAAc;;EAG3E;AAEF,MAAM,aAAa,OAAO,WAAW,WACnC,WACA,UACA,eACA;AACA,KAAI,CAAC,SAAS,QAAS;CACvB,MAAM,QAAQ,OAAO;AAErB,QAAO,MAAM;EACX,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;EACtE,MAAM,SAAS,MAAM,SAAS,UAAU;EACxC,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,QAC1B,MAAM,EAAE,QAAQ,CAAC,CACjB,KAAK,oBAAoB,WAAW,UAAU,WAAW,QAAQ,CAAC;AAErE,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK;IACb,WAAW,KAAK;IACjB;GACF,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,UAAU,CAAC;EAEvE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AAE9E,SAAO,OAAO,SACZ,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,KAAK,OAAO;GACpC;GACD,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;GAAW,EACzE,cACD,CACF,EACD,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,UAAU,CAAC,EAClE,OAAO,UAAU,UAAU,cAAc,aAAa,MAAM,CAAC,CAC9D,EACH,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,MAAM,EAAE,CACnD,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,aAAa,CAAC;AAEzE,MAAI,mBACF,QAAO,qBAAqB,WAAW,UAAU,cAAc;AAGjE,SAAO,OAAO,MAAM,SAAS,QAAQ,YAAY,WAAW;;EAE9D"}

package/dist/manifest/index.d.ts

@@ -0,0 +1,2 @@
+import { _ as number, a as configSchema, b as select, c as ConnectorManifest, d as ConfigField, f as ConfigFieldSpec, g as define$1, h as boolean, i as RuntimeConfigDocument, l as ResourceCapability, m as ConnectorConfigDef, n as ConnectorConfigSchema, o as decodeConfig, p as ConfigValuesOf, r as ConnectorConfigValues, s as toRuntimeDocument, u as define, v as optional, x as string, y as secret } from "../index-CBT-JX-T.js";
+export { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef, ConnectorConfigSchema, ConnectorConfigValues, ConnectorManifest, ResourceCapability, RuntimeConfigDocument, boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };

package/dist/manifest/index.js

@@ -0,0 +1,4 @@
+import "../constants-BRhIX_rz.js";
+import { a as define, c as number, d as select, f as string, i as toRuntimeDocument, l as optional, n as configSchema, o as boolean, r as decodeConfig, s as define$1, u as secret } from "../manifest-ByE6YPG0.js";
+
+export { boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };