@useairfoil/connector-kit 0.5.0 → 0.5.2

package/dist/index.d.ts

@@ -1,9 +1,14 @@
-import { t as ConnectorError } from "./index-DyuT2jzP.js";
-import { _ as WebhookAckMode, a as FetchChangesResult, b as WebhookRouteContext, c as ResourceBatch, d as ResourceMutation, f as ResourcePayload, g as ResourceState, h as ResourceSchema, l as ResourceDefinition, m as ResourceRows, n as ConnectorDefinition, o as FetchPageResult, p as ResourceRow, r as Cursor$1, s as PageFetch, t as ChangesFetch, u as ResourceField, v as WebhookHandler, y as WebhookRoute } from "./types-ZNrA-Dn0.js";
-import { a as telemetry_d_exports, c as Fetch, l as Resource, o as Connector, s as Cursor, t as index_d_exports } from "./index-BlhJCpEk.js";
-import "./service-B7g_lHMI.js";
-import { t as index_d_exports$3 } from "./index-DaNW63Mv.js";
-import { t as index_d_exports$1 } from "./index-8KnVpoZp.js";
-import { t as index_d_exports$2 } from "./index-DK9YcR6W.js";
-import { t as index_d_exports$4 } from "./index-CQJsFJs3.js";
-export { ChangesFetch, Connector, index_d_exports as ConnectorApp, ConnectorDefinition, ConnectorError, Cursor, Cursor$1 as CursorTypes, Fetch, FetchChangesResult, FetchPageResult, index_d_exports$1 as Ingestion, PageFetch, index_d_exports$2 as Publisher, Resource, ResourceBatch, ResourceDefinition, ResourceField, ResourceMutation, ResourcePayload, ResourceRow, ResourceRows, ResourceSchema, ResourceState, index_d_exports$3 as StateStore, telemetry_d_exports as Telemetry, index_d_exports$4 as Webhook, WebhookAckMode, WebhookHandler, WebhookRoute, WebhookRouteContext };
+import { t as index_d_exports } from "./index-CTHVudvQ.js";
+import { t as ConnectorError } from "./index-D54ySAKn.js";
+import { S as WebhookRouteContext, _ as ResourceState, a as FetchChangesResult, b as WebhookHandler, c as ResourceBatch, d as ResourceMutation, g as ResourceSchema, h as ResourceRows, l as ResourceDefinition, m as ResourceRow, n as ConnectorDefinition, o as FetchPageResult, p as ResourcePayload, r as Cursor$1, s as PageFetch, t as ChangesFetch, u as ResourceField, x as WebhookRoute, y as WebhookAckMode } from "./types-B35J5YPu.js";
+import { a as telemetry_d_exports, c as Fetch, l as Resource, o as Connector, s as Cursor, t as index_d_exports$1 } from "./index-DtJX_tic.js";
+import "./service-IE5qaf9x.js";
+import { t as index_d_exports$6 } from "./index-Coh_SkBs.js";
+import { t as index_d_exports$2 } from "./index-J42AZaXU.js";
+import { t as index_d_exports$3 } from "./index-CBT-JX-T.js";
+import { g as metrics_d_exports } from "./metrics-CVAk1ALR.js";
+import { t as index_d_exports$4 } from "./index-Byr0Lf0C.js";
+import { t as index_d_exports$5 } from "./index-o1-Z3n7V.js";
+import { t as index_d_exports$7 } from "./index-Cq2IzY7T.js";
+import { t as index_d_exports$8 } from "./index-DKWIh6M1.js";
+export { index_d_exports as Auth, ChangesFetch, Connector, index_d_exports$1 as ConnectorApp, ConnectorDefinition, ConnectorError, Cursor, Cursor$1 as CursorTypes, Fetch, FetchChangesResult, FetchPageResult, index_d_exports$2 as Ingestion, index_d_exports$3 as Manifest, metrics_d_exports as Metrics, PageFetch, index_d_exports$4 as Publisher, Resource, ResourceBatch, ResourceDefinition, ResourceField, ResourceMutation, ResourcePayload, ResourceRow, ResourceRows, ResourceSchema, ResourceState, index_d_exports$5 as RuntimeConfig, index_d_exports$6 as StateStore, index_d_exports$7 as Status, telemetry_d_exports as Telemetry, index_d_exports$8 as Webhook, WebhookAckMode, WebhookHandler, WebhookRoute, WebhookRouteContext };

package/dist/index.js

@@ -1,11 +1,17 @@
 import { t as ConnectorError } from "./errors-DHWylqRZ.js";
-import { s as telemetry_exports } from "./telemetry-p0dE5Mdu.js";
-import { t as state_store_exports } from "./state-store-DG_4QpWG.js";
-import "./server-DAeFWT4o.js";
-import { t as ingestion_exports } from "./ingestion-CMCroK7Y.js";
-import { t as connector_app_exports } from "./connector-app-CvqbfN6R.js";
-import { t as publisher_exports } from "./publisher-DusgYvaV.js";
-import { n as webhook_exports } from "./webhook-BSA-tXtU.js";
+import { t as auth_exports } from "./auth-DvU_CZ7C.js";
+import { t as connector_app_exports } from "./connector-app-d7Z-SZLQ.js";
+import { s as metrics_exports } from "./metrics-CWPy2Aqc.js";
+import "./constants-BRhIX_rz.js";
+import { c as telemetry_exports } from "./service-DuFiQwid.js";
+import "./server-CXTQ7odl.js";
+import { t as state_store_exports } from "./state-store-yl6LiTQ0.js";
+import { t as status_exports } from "./status-B-oZBc33.js";
+import { t as ingestion_exports } from "./ingestion-BQo4cbGw.js";
+import { t as manifest_exports } from "./manifest-ByE6YPG0.js";
+import { t as publisher_exports } from "./publisher-BYT3RLVO.js";
+import { t as runtime_config_exports } from "./runtime-config-BpB-d_bF.js";
+import { n as webhook_exports } from "./webhook-DI-Afxkg.js";
 import { DateTime, Effect } from "effect";
 
 //#region src/core/builder.ts
@@ -48,5 +54,5 @@ const Cursor = {
 };
 
 //#endregion
-export { Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, Fetch, ingestion_exports as Ingestion, publisher_exports as Publisher, Resource, state_store_exports as StateStore, telemetry_exports as Telemetry, webhook_exports as Webhook };
+export { auth_exports as Auth, Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, Fetch, ingestion_exports as Ingestion, manifest_exports as Manifest, metrics_exports as Metrics, publisher_exports as Publisher, Resource, runtime_config_exports as RuntimeConfig, state_store_exports as StateStore, status_exports as Status, telemetry_exports as Telemetry, webhook_exports as Webhook };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect } from \"effect\";\n\nimport type {\n  ChangesFetch,\n  ConnectorDefinition,\n  Cursor as CursorType,\n  DeleteValue,\n  PageFetch,\n  ResourceDefinition,\n  ResourceMutation,\n  ResourceSchema,\n  WebhookHandler,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(\n    definition: ConnectorDefinition<Resources>,\n  ) => definition,\n};\n\nexport const Resource = {\n  entity: <S extends ResourceSchema, Payload = never, R = never>(\n    definition: ResourceDefinition<S, Payload, R>,\n  ): ResourceDefinition<S, Payload, R> => definition,\n\n  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>\n    definition,\n\n  upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: \"upsert\", row }),\n\n  delete: (options: {\n    readonly key: DeleteValue;\n    readonly version: DeleteValue;\n  }): ResourceMutation<never> => ({\n    op: \"delete\",\n    key: options.key,\n    version: options.version,\n  }),\n};\n\nexport const Fetch = {\n  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n  string: (): CursorType.Definition<string> => ({\n    kind: \"string\",\n    decode: (value) =>\n      typeof value === \"string\"\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"string\", value)),\n  }),\n\n  number: (): CursorType.Definition<number> => ({\n    kind: \"number\",\n    decode: (value) =>\n      typeof value === \"number\" && Number.isFinite(value)\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"number\", value)),\n  }),\n\n  isoDateTime: (): CursorType.Definition<string> => ({\n    kind: \"isoDateTime\",\n    decode: (value) => {\n      if (typeof value !== \"string\" || Number.isNaN(Date.parse(value))) {\n        return Effect.fail(decodeFailure(\"ISO datetime\", value));\n      }\n      return Effect.succeed(value);\n    },\n  }),\n\n  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;AAgBA,MAAa,YAAY,EACvB,SACE,eACG,YACN;AAED,MAAa,WAAW;CACtB,SACE,eACsC;CAExC,UAA+C,eAC7C;CAEF,SAA6B,SAAqC;EAAE,IAAI;EAAU;EAAK;CAEvF,SAAS,aAGuB;EAC9B,IAAI;EACJ,KAAK,QAAQ;EACb,SAAS,QAAQ;EAClB;CACF;AAED,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACvF;AAED,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,MAAM,IAAI,CAAC;AAEnF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAC/C,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;AACjB,OAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,MAAM,MAAM,CAAC,CAC9D,QAAO,OAAO,KAAK,cAAc,gBAAgB,MAAM,CAAC;AAE1D,UAAO,OAAO,QAAQ,MAAM;;EAE/B;CAED,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,UAAU;CAC7D"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect } from \"effect\";\n\nimport type {\n  ChangesFetch,\n  ConnectorDefinition,\n  Cursor as CursorType,\n  DeleteValue,\n  PageFetch,\n  ResourceDefinition,\n  ResourceMutation,\n  ResourceSchema,\n  WebhookHandler,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(\n    definition: ConnectorDefinition<Resources>,\n  ) => definition,\n};\n\nexport const Resource = {\n  entity: <\n    S extends ResourceSchema,\n    Payload = never,\n    R = never,\n    const Name extends string = string,\n  >(\n    definition: ResourceDefinition<S, Payload, R, Name>,\n  ): ResourceDefinition<S, Payload, R, Name> => definition,\n\n  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>\n    definition,\n\n  upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: \"upsert\", row }),\n\n  delete: (options: {\n    readonly key: DeleteValue;\n    readonly version: DeleteValue;\n  }): ResourceMutation<never> => ({\n    op: \"delete\",\n    key: options.key,\n    version: options.version,\n  }),\n};\n\nexport const Fetch = {\n  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n  string: (): CursorType.Definition<string> => ({\n    kind: \"string\",\n    decode: (value) =>\n      typeof value === \"string\"\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"string\", value)),\n  }),\n\n  number: (): CursorType.Definition<number> => ({\n    kind: \"number\",\n    decode: (value) =>\n      typeof value === \"number\" && Number.isFinite(value)\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"number\", value)),\n  }),\n\n  isoDateTime: (): CursorType.Definition<string> => ({\n    kind: \"isoDateTime\",\n    decode: (value) => {\n      if (typeof value !== \"string\" || Number.isNaN(Date.parse(value))) {\n        return Effect.fail(decodeFailure(\"ISO datetime\", value));\n      }\n      return Effect.succeed(value);\n    },\n  }),\n\n  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;;;;;;;AAgBA,MAAa,YAAY,EACvB,SACE,eACG,YACN;AAED,MAAa,WAAW;CACtB,SAME,eAC4C;CAE9C,UAA+C,eAC7C;CAEF,SAA6B,SAAqC;EAAE,IAAI;EAAU;EAAK;CAEvF,SAAS,aAGuB;EAC9B,IAAI;EACJ,KAAK,QAAQ;EACb,SAAS,QAAQ;EAClB;CACF;AAED,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACvF;AAED,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,MAAM,IAAI,CAAC;AAEnF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAC/C,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;AACjB,OAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,MAAM,MAAM,CAAC,CAC9D,QAAO,OAAO,KAAK,cAAc,gBAAgB,MAAM,CAAC;AAE1D,UAAO,OAAO,QAAQ,MAAM;;EAE/B;CAED,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,UAAU;CAC7D"}

package/dist/ingestion/index.d.ts

@@ -1,6 +1,6 @@
-import "../index-DyuT2jzP.js";
-import "../types-ZNrA-Dn0.js";
-import "../service-B7g_lHMI.js";
-import "../index-DaNW63Mv.js";
-import { n as RunOptions, r as run } from "../index-8KnVpoZp.js";
+import "../index-D54ySAKn.js";
+import "../types-B35J5YPu.js";
+import "../service-IE5qaf9x.js";
+import "../index-Coh_SkBs.js";
+import { n as RunOptions, r as run } from "../index-J42AZaXU.js";
 export { RunOptions, run };

package/dist/ingestion/index.js

@@ -1,7 +1,10 @@
 import "../errors-DHWylqRZ.js";
-import "../telemetry-p0dE5Mdu.js";
-import "../state-store-DG_4QpWG.js";
-import "../server-DAeFWT4o.js";
-import { n as run } from "../ingestion-CMCroK7Y.js";
+import "../metrics-CWPy2Aqc.js";
+import "../constants-BRhIX_rz.js";
+import "../service-DuFiQwid.js";
+import "../server-CXTQ7odl.js";
+import "../state-store-yl6LiTQ0.js";
+import "../status-B-oZBc33.js";
+import { n as run } from "../ingestion-BQo4cbGw.js";
 
 export { run };

package/dist/ingestion-BQo4cbGw.js

@@ -0,0 +1,200 @@
+import { t as __export } from "./chunk-Bp6m_JJh.js";
+import { d as setLastSuccessTimestamp, f as setSyncState, p as setWebhookQueueDepth } from "./metrics-CWPy2Aqc.js";
+import { a as SpanName, n as Attr, s as annotateError } from "./service-DuFiQwid.js";
+import { n as router, r as publishBatch, t as WebhookQueue } from "./server-CXTQ7odl.js";
+import { o as StateStore } from "./state-store-yl6LiTQ0.js";
+import { c as deriveSyncState, l as normalizeCursor, s as statusRoute } from "./status-B-oZBc33.js";
+import { DateTime, Effect, Layer, Queue } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+import * as Observability from "effect/unstable/observability";
+
+//#region src/ingestion/engine.ts
+function run(connector, options) {
+	const queueLayer = makeWebhookQueueLayer(connector.name);
+	const runtimeLayer = options?.webhook ? Layer.mergeAll(queueLayer, makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer))) : Layer.empty;
+	return Effect.gen(function* () {
+		const initialCutoff = options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+		yield* Effect.logInfo("Connector started").pipe(Effect.annotateLogs({
+			[Attr.connectorName]: connector.name,
+			resources: connector.resources.length
+		}));
+		const sourceRun = runIngestion(connector, initialCutoff);
+		const webhookRun = options?.webhook ? runWebhookRuntime(connector, options.webhook.routes) : Effect.void;
+		return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+	}).pipe(Effect.provide(runtimeLayer));
+}
+const makeWebhookQueueLayer = (connector) => Layer.effect(WebhookQueue)(Effect.gen(function* () {
+	const queue = yield* Queue.bounded(1024);
+	yield* setWebhookQueueDepth(connector, 0);
+	return WebhookQueue.of({ queue });
+}));
+const runWebhookRuntime = (connector, routes) => {
+	const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
+	return Effect.all([hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+};
+const runWebhookQueueConsumer = (connector) => Effect.gen(function* () {
+	const queue = yield* WebhookQueue;
+	while (true) {
+		const batch = yield* Queue.take(queue.queue);
+		const depth = yield* Queue.size(queue.queue);
+		yield* setWebhookQueueDepth(connector.name, depth);
+		yield* publishBatch({
+			connector: connector.name,
+			resource: batch.resource,
+			source: "webhook",
+			batch: batch.batch
+		});
+	}
+});
+const makeWebhookServerLayer = (connector, options) => {
+	const healthPath = options.healthPath ?? "/health";
+	const metricsPath = options.metricsPath === void 0 ? "/metrics" : options.metricsPath;
+	const statusPath = options.statusPath === void 0 ? "/status" : options.statusPath;
+	const app = Layer.mergeAll(router(options.routes, { connectorName: connector.name }), HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))), ...metricsPath === false ? [] : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })], ...statusPath === false ? [] : [statusRoute(connector, statusPath)]);
+	return HttpRouter.serve(app, { disableLogger: options.disableHttpLogger ?? true });
+};
+const runIngestion = (connector, initialCutoff) => Effect.forEach(connector.resources, (resource) => runResourceSources(connector, resource, initialCutoff), { concurrency: "unbounded" }).pipe(Effect.asVoid);
+const initializeResourceState = (existing, initialCutoff) => ({
+	...existing,
+	changes: existing?.changes ?? { cursor: initialCutoff },
+	backfill: existing?.backfill ?? {
+		cutoff: initialCutoff,
+		completed: false
+	}
+});
+const getInitializedState = Effect.fnUntraced(function* (resource, initialCutoff) {
+	return initializeResourceState(yield* (yield* StateStore).getResourceState(resource), initialCutoff);
+});
+const runResourceSources = (connector, resource, initialCutoff) => {
+	const runs = [resource.backfill ? isolateSourceFailure(runBackfill(connector, resource, initialCutoff), connector, resource, "backfill") : Effect.void, resource.changes ? isolateSourceFailure(runChanges(connector, resource, initialCutoff), connector, resource, "changes") : Effect.void];
+	return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })), Effect.asVoid, Effect.catch(() => setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, "error").pipe(Effect.andThen(Effect.logError("Connector resource initialization failed")), Effect.annotateLogs({
+		[Attr.connectorName]: connector.name,
+		resource: resource.name
+	}), Effect.andThen(Effect.never))));
+};
+const recordResourceError = (connector, resource, source, operation) => (effect) => effect.pipe(Effect.tapError(() => setSyncState({
+	connector: connector.name,
+	resource: resource.name
+}, "error").pipe(Effect.andThen(StateStore.pipe(Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)))))));
+const isolateSourceFailure = (effect, connector, resource, source) => effect.pipe(Effect.catch(() => Effect.logError("Connector source parked").pipe(Effect.annotateLogs({
+	[Attr.connectorName]: connector.name,
+	resource: resource.name,
+	source
+}), Effect.andThen(Effect.never))));
+const refreshRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(resource, state));
+	return state;
+});
+const initializeRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(resource, state));
+	if (resource.backfill) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "backfill"
+	}, state.backfill?.lastSuccessAt);
+	if (resource.changes) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "changes"
+	}, state.changes?.lastSuccessAt);
+});
+const runBackfill = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.backfill) return;
+	const store = yield* StateStore;
+	let state = yield* getInitializedState(resource.name, initialCutoff);
+	if (state.backfill?.completed === true) {
+		yield* store.clearResourceError(resource.name, "backfill").pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		return;
+	}
+	while (state.backfill?.completed !== true) {
+		const backfill = state.backfill ?? {
+			cutoff: initialCutoff,
+			completed: false
+		};
+		const clearsVisibleError = state.lastError?.source === "backfill";
+		const page = yield* resource.backfill.fetch({
+			pageCursor: backfill.pageCursor,
+			cutoff: backfill.cutoff
+		}).pipe(recordResourceError(connector, resource, "backfill", "fetch"));
+		yield* publishBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill",
+			batch: {
+				cursor: page.nextPageCursor ?? backfill.cutoff,
+				mutations: page.mutations
+			}
+		}).pipe(recordResourceError(connector, resource, "backfill", "publish"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		state = {
+			...state,
+			backfill: {
+				cutoff: backfill.cutoff,
+				pageCursor: page.nextPageCursor,
+				completed: !page.hasMore,
+				lastSuccessAt
+			}
+		};
+		yield* Effect.withSpan(store.setBackfillState(resource.name, {
+			cutoff: normalizeCursor(backfill.cutoff),
+			pageCursor: page.nextPageCursor === void 0 ? void 0 : normalizeCursor(page.nextPageCursor),
+			completed: !page.hasMore,
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "backfill")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		if (clearsVisibleError || !page.hasMore) state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+	}
+});
+const runChanges = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.changes) return;
+	const store = yield* StateStore;
+	while (true) {
+		const state = yield* getInitializedState(resource.name, initialCutoff);
+		const cursor = state.changes?.cursor ?? initialCutoff;
+		const clearsVisibleError = state.lastError?.source === "changes";
+		const page = yield* resource.changes.fetch({ cursor }).pipe(recordResourceError(connector, resource, "changes", "fetch"));
+		yield* publishBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes",
+			batch: {
+				cursor: page.cursor,
+				mutations: page.mutations
+			}
+		}).pipe(recordResourceError(connector, resource, "changes", "publish"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		yield* Effect.withSpan(store.setChangesState(resource.name, {
+			cursor: normalizeCursor(page.cursor),
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
+		if (clearsVisibleError) yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+	}
+});
+
+//#endregion
+//#region src/ingestion/index.ts
+var ingestion_exports = /* @__PURE__ */ __export({ run: () => run });
+
+//#endregion
+export { run as n, ingestion_exports as t };
+//# sourceMappingURL=ingestion-BQo4cbGw.js.map

package/dist/ingestion-BQo4cbGw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestion-BQo4cbGw.js","names":["Metrics.setWebhookQueueDepth","healthPath: HttpRouter.PathInput","Metrics.setSyncState","Metrics.setLastSuccessTimestamp"],"sources":["../src/ingestion/engine.ts","../src/ingestion/index.ts"],"sourcesContent":["import { DateTime, Effect, Layer, Queue } from \"effect\";\nimport { HttpRouter, type HttpServer, HttpServerResponse } from \"effect/unstable/http\";\nimport * as Observability from \"effect/unstable/observability\";\n\nimport type {\n  ConnectorDefinition,\n  Cursor,\n  ResourceDefinition,\n  ResourceState,\n  WebhookRoute,\n} from \"../core/types\";\nimport type { StateOperation, StateSource } from \"../state-store/schema\";\n\nimport { ConnectorError } from \"../errors\";\nimport * as Metrics from \"../metrics\";\nimport { publishBatch } from \"../publisher/instrumented\";\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, normalizeCursor } from \"../state-store/state\";\nimport { statusRoute } from \"../status\";\nimport { Attr, SpanName, annotateError } from \"../telemetry\";\nimport { router, WebhookQueue, type QueuedWebhookBatch } from \"../webhook/server\";\n\ntype RunBaseOptions = {\n  readonly initialCutoff?: Cursor.Value;\n};\n\nexport type RunOptions = RunBaseOptions & {\n  readonly webhook?: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  };\n};\n\ntype RunNoWebhookOptions = RunBaseOptions & {\n  readonly webhook?: undefined;\n};\n\ntype RunWebhookOptions = RunOptions & {\n  readonly webhook: NonNullable<RunOptions[\"webhook\"]>;\n};\n\n/**\n * Runs all connector ingestion loops. Callers provide the Publisher and\n * StateStore layers; webhook mode additionally requires an HttpServer layer.\n */\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options?: RunNoWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher>;\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options: RunWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;\nexport function run(connector: ConnectorDefinition, options?: RunOptions) {\n  const queueLayer = makeWebhookQueueLayer(connector.name);\n  const runtimeLayer = options?.webhook\n    ? Layer.mergeAll(\n        queueLayer,\n        makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),\n      )\n    : Layer.empty;\n\n  return Effect.gen(function* () {\n    const initialCutoff =\n      options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    yield* Effect.logInfo(\"Connector started\").pipe(\n      Effect.annotateLogs({\n        [Attr.connectorName]: connector.name,\n        resources: connector.resources.length,\n      }),\n    );\n\n    const sourceRun = runIngestion(connector, initialCutoff);\n    const webhookRun = options?.webhook\n      ? runWebhookRuntime(connector, options.webhook.routes)\n      : Effect.void;\n\n    return yield* Effect.all([sourceRun, webhookRun], { concurrency: \"unbounded\" }).pipe(\n      Effect.asVoid,\n    );\n  }).pipe(Effect.provide(runtimeLayer));\n}\n\nconst makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>\n  Layer.effect(WebhookQueue)(\n    Effect.gen(function* () {\n      const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);\n      yield* Metrics.setWebhookQueueDepth(connector, 0);\n      return WebhookQueue.of({ queue });\n    }),\n  );\n\nconst runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {\n  const hasAfterEnqueue = routes.some((route) => route.ackMode === \"after-enqueue\");\n  return Effect.all(\n    [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],\n    {\n      concurrency: \"unbounded\",\n    },\n  ).pipe(Effect.asVoid);\n};\n\nconst runWebhookQueueConsumer = (connector: ConnectorDefinition) =>\n  Effect.gen(function* () {\n    const queue = yield* WebhookQueue;\n    while (true) {\n      const batch = yield* Queue.take(queue.queue);\n      const depth = yield* Queue.size(queue.queue);\n      yield* Metrics.setWebhookQueueDepth(connector.name, depth);\n      yield* publishBatch({\n        connector: connector.name,\n        resource: batch.resource,\n        source: \"webhook\",\n        batch: batch.batch,\n      });\n    }\n  });\n\nconst makeWebhookServerLayer = (\n  connector: ConnectorDefinition,\n  options: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  },\n): Layer.Layer<never, never, HttpServer.HttpServer> => {\n  const healthPath: HttpRouter.PathInput = options.healthPath ?? \"/health\";\n  const metricsPath = options.metricsPath === undefined ? \"/metrics\" : options.metricsPath;\n  const statusPath = options.statusPath === undefined ? \"/status\" : options.statusPath;\n  const app = Layer.mergeAll(\n    router(options.routes, { connectorName: connector.name }),\n    HttpRouter.add(\"GET\", healthPath, Effect.succeed(HttpServerResponse.text(\"ok\"))),\n    ...(metricsPath === false\n      ? []\n      : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),\n    ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),\n  );\n\n  return HttpRouter.serve(app, {\n    disableLogger: options.disableHttpLogger ?? true,\n  });\n};\n\nconst runIngestion = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor.Value,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n  Effect.forEach(\n    connector.resources,\n    (resource) => runResourceSources(connector, resource, initialCutoff),\n    {\n      concurrency: \"unbounded\",\n    },\n  ).pipe(Effect.asVoid);\n\nconst initializeResourceState = (\n  existing: ResourceState | undefined,\n  initialCutoff: Cursor.Value,\n): ResourceState => ({\n  ...existing,\n  changes: existing?.changes ?? { cursor: initialCutoff },\n  backfill: existing?.backfill ?? {\n    cutoff: initialCutoff,\n    completed: false,\n  },\n});\n\nconst getInitializedState = Effect.fnUntraced(function* (\n  resource: string,\n  initialCutoff: Cursor.Value,\n) {\n  const store = yield* StateStore;\n  const existing = yield* store.getResourceState(resource);\n  return initializeResourceState(existing, initialCutoff);\n});\n\nconst runResourceSources = (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) => {\n  const runs = [\n    resource.backfill\n      ? isolateSourceFailure(\n          runBackfill(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"backfill\",\n        )\n      : Effect.void,\n    resource.changes\n      ? isolateSourceFailure(\n          runChanges(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"changes\",\n        )\n      : Effect.void,\n  ];\n\n  return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(\n    Effect.andThen(Effect.all(runs, { concurrency: \"unbounded\" })),\n    Effect.asVoid,\n    Effect.catch(() =>\n      Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n        Effect.andThen(Effect.logError(\"Connector resource initialization failed\")),\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n};\n\nconst recordResourceError =\n  (\n    connector: ConnectorDefinition,\n    resource: ResourceDefinition,\n    source: StateSource,\n    operation: StateOperation,\n  ) =>\n  <A, R>(\n    effect: Effect.Effect<A, ConnectorError, R>,\n  ): Effect.Effect<A, ConnectorError, R | StateStore> =>\n    effect.pipe(\n      Effect.tapError(() =>\n        Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n          Effect.andThen(\n            StateStore.pipe(\n              Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),\n            ),\n          ),\n        ),\n      ),\n    );\n\nconst isolateSourceFailure = <A, R>(\n  effect: Effect.Effect<A, ConnectorError, R>,\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  source: StateSource,\n): Effect.Effect<A, never, R> =>\n  // Only typed source errors are isolated. Defects and interruption still stop the runtime.\n  effect.pipe(\n    Effect.catch(() =>\n      Effect.logError(\"Connector source parked\").pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n          source,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n\nconst refreshRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(resource, state),\n  );\n  return state;\n});\n\nconst initializeRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(resource, state),\n  );\n  if (resource.backfill) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"backfill\" },\n      state.backfill?.lastSuccessAt,\n    );\n  }\n  if (resource.changes) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"changes\" },\n      state.changes?.lastSuccessAt,\n    );\n  }\n});\n\nconst runBackfill = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.backfill) return;\n  const store = yield* StateStore;\n  let state = yield* getInitializedState(resource.name, initialCutoff);\n\n  if (state.backfill?.completed === true) {\n    // The checkpoint may have committed before the previous process cleared\n    // its error marker. It may also be hidden by a newer changes error.\n    yield* store\n      .clearResourceError(resource.name, \"backfill\")\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n    yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    return;\n  }\n\n  while (state.backfill?.completed !== true) {\n    const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };\n    const clearsVisibleError = state.lastError?.source === \"backfill\";\n    const page = yield* resource.backfill\n      .fetch({\n        pageCursor: backfill.pageCursor,\n        cutoff: backfill.cutoff,\n      })\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"fetch\"));\n\n    yield* publishBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"backfill\",\n      batch: {\n        cursor: page.nextPageCursor ?? backfill.cutoff,\n        mutations: page.mutations,\n      },\n    }).pipe(recordResourceError(connector, resource, \"backfill\", \"publish\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    state = {\n      ...state,\n      backfill: {\n        cutoff: backfill.cutoff,\n        pageCursor: page.nextPageCursor,\n        completed: !page.hasMore,\n        lastSuccessAt,\n      },\n    };\n\n    // Checkpoint only after publish succeeds so a crash cannot skip an\n    // unacknowledged page; replay therefore remains at-least-once.\n    yield* Effect.withSpan(\n      store\n        .setBackfillState(resource.name, {\n          cutoff: normalizeCursor(backfill.cutoff),\n          pageCursor:\n            page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),\n          completed: !page.hasMore,\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"backfill\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"backfill\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n\n    // Completion or clearing an error can change the resource-wide sync state.\n    if (clearsVisibleError || !page.hasMore) {\n      state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n  }\n});\n\nconst runChanges = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.changes) return;\n  const store = yield* StateStore;\n\n  while (true) {\n    const state = yield* getInitializedState(resource.name, initialCutoff);\n    const cursor = state.changes?.cursor ?? initialCutoff;\n    const clearsVisibleError = state.lastError?.source === \"changes\";\n    const page = yield* resource.changes\n      .fetch({ cursor })\n      .pipe(recordResourceError(connector, resource, \"changes\", \"fetch\"));\n\n    yield* publishBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"changes\",\n      batch: {\n        cursor: page.cursor,\n        mutations: page.mutations,\n      },\n    }).pipe(recordResourceError(connector, resource, \"changes\", \"publish\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    // Advance the durable cursor only after Wings accepts the page.\n    yield* Effect.withSpan(\n      store\n        .setChangesState(resource.name, {\n          cursor: normalizeCursor(page.cursor),\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"changes\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"changes\", \"checkpoint\"));\n\n    if (clearsVisibleError) {\n      yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n\n    yield* Effect.sleep(resource.changes.interval ?? \"1 minute\");\n  }\n});\n","export { run, type RunOptions } from \"./engine\";\n"],"mappings":";;;;;;;;;;;AAyDA,SAAgB,IAAI,WAAgC,SAAsB;CACxE,MAAM,aAAa,sBAAsB,UAAU,KAAK;CACxD,MAAM,eAAe,SAAS,UAC1B,MAAM,SACJ,YACA,uBAAuB,WAAW,QAAQ,QAAQ,CAAC,KAAK,MAAM,QAAQ,WAAW,CAAC,CACnF,GACD,MAAM;AAEV,QAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,gBACJ,SAAS,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AACrF,SAAO,OAAO,QAAQ,oBAAoB,CAAC,KACzC,OAAO,aAAa;IACjB,KAAK,gBAAgB,UAAU;GAChC,WAAW,UAAU,UAAU;GAChC,CAAC,CACH;EAED,MAAM,YAAY,aAAa,WAAW,cAAc;EACxD,MAAM,aAAa,SAAS,UACxB,kBAAkB,WAAW,QAAQ,QAAQ,OAAO,GACpD,OAAO;AAEX,SAAO,OAAO,OAAO,IAAI,CAAC,WAAW,WAAW,EAAE,EAAE,aAAa,aAAa,CAAC,CAAC,KAC9E,OAAO,OACR;GACD,CAAC,KAAK,OAAO,QAAQ,aAAa,CAAC;;AAGvC,MAAM,yBAAyB,cAC7B,MAAM,OAAO,aAAa,CACxB,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,MAAM,QAA4B,KAAK;AAC5D,QAAOA,qBAA6B,WAAW,EAAE;AACjD,QAAO,aAAa,GAAG,EAAE,OAAO,CAAC;EACjC,CACH;AAEH,MAAM,qBAAqB,WAAgC,WAAwC;CACjG,MAAM,kBAAkB,OAAO,MAAM,UAAU,MAAM,YAAY,gBAAgB;AACjF,QAAO,OAAO,IACZ,CAAC,kBAAkB,wBAAwB,UAAU,GAAG,OAAO,MAAM,OAAO,MAAM,EAClF,EACE,aAAa,aACd,CACF,CAAC,KAAK,OAAO,OAAO;;AAGvB,MAAM,2BAA2B,cAC/B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;AACrB,QAAO,MAAM;EACX,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,MAAM;EAC5C,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,MAAM;AAC5C,SAAOA,qBAA6B,UAAU,MAAM,MAAM;AAC1D,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,MAAM;GACd,CAAC;;EAEJ;AAEJ,MAAM,0BACJ,WACA,YAOqD;CACrD,MAAMC,aAAmC,QAAQ,cAAc;CAC/D,MAAM,cAAc,QAAQ,gBAAgB,SAAY,aAAa,QAAQ;CAC7E,MAAM,aAAa,QAAQ,eAAe,SAAY,YAAY,QAAQ;CAC1E,MAAM,MAAM,MAAM,SAChB,OAAO,QAAQ,QAAQ,EAAE,eAAe,UAAU,MAAM,CAAC,EACzD,WAAW,IAAI,OAAO,YAAY,OAAO,QAAQ,mBAAmB,KAAK,KAAK,CAAC,CAAC,EAChF,GAAI,gBAAgB,QAChB,EAAE,GACF,CAAC,cAAc,kBAAkB,UAAU,EAAE,MAAM,aAAa,CAAC,CAAC,EACtE,GAAI,eAAe,QAAQ,EAAE,GAAG,CAAC,YAAY,WAAW,WAAW,CAAC,CACrE;AAED,QAAO,WAAW,MAAM,KAAK,EAC3B,eAAe,QAAQ,qBAAqB,MAC7C,CAAC;;AAGJ,MAAM,gBACJ,WACA,kBAEA,OAAO,QACL,UAAU,YACT,aAAa,mBAAmB,WAAW,UAAU,cAAc,EACpE,EACE,aAAa,aACd,CACF,CAAC,KAAK,OAAO,OAAO;AAEvB,MAAM,2BACJ,UACA,mBACmB;CACnB,GAAG;CACH,SAAS,UAAU,WAAW,EAAE,QAAQ,eAAe;CACvD,UAAU,UAAU,YAAY;EAC9B,QAAQ;EACR,WAAW;EACZ;CACF;AAED,MAAM,sBAAsB,OAAO,WAAW,WAC5C,UACA,eACA;AAGA,QAAO,wBADU,QADH,OAAO,YACS,iBAAiB,SAAS,EACf,cAAc;EACvD;AAEF,MAAM,sBACJ,WACA,UACA,kBACG;CACH,MAAM,OAAO,CACX,SAAS,WACL,qBACE,YAAY,WAAW,UAAU,cAAc,EAC/C,WACA,UACA,WACD,GACD,OAAO,MACX,SAAS,UACL,qBACE,WAAW,WAAW,UAAU,cAAc,EAC9C,WACA,UACA,UACD,GACD,OAAO,KACZ;AAED,QAAO,wBAAwB,WAAW,UAAU,cAAc,CAAC,KACjE,OAAO,QAAQ,OAAO,IAAI,MAAM,EAAE,aAAa,aAAa,CAAC,CAAC,EAC9D,OAAO,QACP,OAAO,YACLC,aAAqB;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EAAE,QAAQ,CAAC,KACpF,OAAO,QAAQ,OAAO,SAAS,2CAA2C,CAAC,EAC3E,OAAO,aAAa;GACjB,KAAK,gBAAgB,UAAU;EAChC,UAAU,SAAS;EACpB,CAAC,EACF,OAAO,QAAQ,OAAO,MAAM,CAC7B,CACF,CACF;;AAGH,MAAM,uBAEF,WACA,UACA,QACA,eAGA,WAEA,OAAO,KACL,OAAO,eACLA,aAAqB;CAAE,WAAW,UAAU;CAAM,UAAU,SAAS;CAAM,EAAE,QAAQ,CAAC,KACpF,OAAO,QACL,WAAW,KACT,OAAO,SAAS,UAAU,MAAM,iBAAiB,SAAS,MAAM,QAAQ,UAAU,CAAC,CACpF,CACF,CACF,CACF,CACF;AAEL,MAAM,wBACJ,QACA,WACA,UACA,WAGA,OAAO,KACL,OAAO,YACL,OAAO,SAAS,0BAA0B,CAAC,KACzC,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,UAAU,SAAS;CACnB;CACD,CAAC,EACF,OAAO,QAAQ,OAAO,MAAM,CAC7B,CACF,CACF;AAEH,MAAM,uBAAuB,OAAO,WAAW,WAC7C,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AACtE,QAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EACtD,gBAAgB,UAAU,MAAM,CACjC;AACD,QAAO;EACP;AAEF,MAAM,0BAA0B,OAAO,WAAW,WAChD,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AACtE,QAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EACtD,gBAAgB,UAAU,MAAM,CACjC;AACD,KAAI,SAAS,SACX,QAAOC,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;EAAY,EAC1E,MAAM,UAAU,cACjB;AAEH,KAAI,SAAS,QACX,QAAOA,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;EAAW,EACzE,MAAM,SAAS,cAChB;EAEH;AAEF,MAAM,cAAc,OAAO,WAAW,WACpC,WACA,UACA,eACA;AACA,KAAI,CAAC,SAAS,SAAU;CACxB,MAAM,QAAQ,OAAO;CACrB,IAAI,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AAEpE,KAAI,MAAM,UAAU,cAAc,MAAM;AAGtC,SAAO,MACJ,mBAAmB,SAAS,MAAM,WAAW,CAC7C,KAAK,oBAAoB,WAAW,UAAU,YAAY,aAAa,CAAC;AAC3E,SAAO,qBAAqB,WAAW,UAAU,cAAc;AAC/D;;AAGF,QAAO,MAAM,UAAU,cAAc,MAAM;EACzC,MAAM,WAAW,MAAM,YAAY;GAAE,QAAQ;GAAe,WAAW;GAAO;EAC9E,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,SAC1B,MAAM;GACL,YAAY,SAAS;GACrB,QAAQ,SAAS;GAClB,CAAC,CACD,KAAK,oBAAoB,WAAW,UAAU,YAAY,QAAQ,CAAC;AAEtE,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK,kBAAkB,SAAS;IACxC,WAAW,KAAK;IACjB;GACF,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,UAAU,CAAC;EAExE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AAC9E,UAAQ;GACN,GAAG;GACH,UAAU;IACR,QAAQ,SAAS;IACjB,YAAY,KAAK;IACjB,WAAW,CAAC,KAAK;IACjB;IACD;GACF;AAID,SAAO,OAAO,SACZ,MACG,iBAAiB,SAAS,MAAM;GAC/B,QAAQ,gBAAgB,SAAS,OAAO;GACxC,YACE,KAAK,mBAAmB,SAAY,SAAY,gBAAgB,KAAK,eAAe;GACtF,WAAW,CAAC,KAAK;GACjB;GACD,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;GAAY,EAC1E,cACD,CACF,EACD,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,WAAW,CAAC,EACnE,OAAO,UAAU,UAAU,cAAc,aAAa,MAAM,CAAC,CAC9D,EACH,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,MAAM,EAAE,CACnD,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,aAAa,CAAC;AAG1E,MAAI,sBAAsB,CAAC,KAAK,QAC9B,SAAQ,OAAO,qBAAqB,WAAW,UAAU,cAAc;;EAG3E;AAEF,MAAM,aAAa,OAAO,WAAW,WACnC,WACA,UACA,eACA;AACA,KAAI,CAAC,SAAS,QAAS;CACvB,MAAM,QAAQ,OAAO;AAErB,QAAO,MAAM;EACX,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;EACtE,MAAM,SAAS,MAAM,SAAS,UAAU;EACxC,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,QAC1B,MAAM,EAAE,QAAQ,CAAC,CACjB,KAAK,oBAAoB,WAAW,UAAU,WAAW,QAAQ,CAAC;AAErE,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK;IACb,WAAW,KAAK;IACjB;GACF,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,UAAU,CAAC;EAEvE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AAE9E,SAAO,OAAO,SACZ,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,KAAK,OAAO;GACpC;GACD,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;GAAW,EACzE,cACD,CACF,EACD,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,UAAU,CAAC,EAClE,OAAO,UAAU,UAAU,cAAc,aAAa,MAAM,CAAC,CAC9D,EACH,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,MAAM,EAAE,CACnD,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,aAAa,CAAC;AAEzE,MAAI,mBACF,QAAO,qBAAqB,WAAW,UAAU,cAAc;AAGjE,SAAO,OAAO,MAAM,SAAS,QAAQ,YAAY,WAAW;;EAE9D"}

package/dist/manifest/index.d.ts

@@ -0,0 +1,2 @@
+import { _ as number, a as configSchema, b as select, c as ConnectorManifest, d as ConfigField, f as ConfigFieldSpec, g as define$1, h as boolean, i as RuntimeConfigDocument, l as ResourceCapability, m as ConnectorConfigDef, n as ConnectorConfigSchema, o as decodeConfig, p as ConfigValuesOf, r as ConnectorConfigValues, s as toRuntimeDocument, u as define, v as optional, x as string, y as secret } from "../index-CBT-JX-T.js";
+export { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef, ConnectorConfigSchema, ConnectorConfigValues, ConnectorManifest, ResourceCapability, RuntimeConfigDocument, boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };

package/dist/manifest/index.js

@@ -0,0 +1,4 @@
+import "../constants-BRhIX_rz.js";
+import { a as define, c as number, d as select, f as string, i as toRuntimeDocument, l as optional, n as configSchema, o as boolean, r as decodeConfig, s as define$1, u as secret } from "../manifest-ByE6YPG0.js";
+
+export { boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };

package/dist/manifest-ByE6YPG0.js

@@ -0,0 +1,221 @@
+import { t as __export } from "./chunk-Bp6m_JJh.js";
+import { r as isPlatformRuntimeKey } from "./constants-BRhIX_rz.js";
+import { Config, Option, Predicate, Redacted, Schema, SchemaTransformation, Struct } from "effect";
+
+//#region src/manifest/number-schema.ts
+const makeNumberSchema = (constraints) => {
+	if (constraints.integer) return constraints.minimum === void 0 ? Schema.Int : Schema.Int.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
+	return constraints.minimum === void 0 ? Schema.Finite : Schema.Finite.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
+};
+
+//#endregion
+//#region src/manifest/config.ts
+const configFieldToConfig = Struct.lambda((field) => field.config);
+const withDefault = (config, defaultValue) => defaultValue === void 0 ? config : Config.withDefault(config, defaultValue);
+/** Defines a required non-empty string, optionally with a default. */
+const string = (options) => ({
+	config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
+	spec: {
+		runtimeKey: options.runtimeKey,
+		type: "string",
+		required: options.required ?? true,
+		secret: false,
+		...options.description !== void 0 ? { description: options.description } : {},
+		...options.default !== void 0 ? { default: options.default } : {}
+	}
+});
+/** Defines a finite number with optional integer, minimum, and default constraints. */
+const number = (options) => {
+	const schema = makeNumberSchema(options);
+	const defaultValue = options.default === void 0 ? void 0 : Schema.decodeUnknownSync(schema)(options.default);
+	return {
+		config: withDefault(Config.schema(schema, options.runtimeKey), defaultValue),
+		spec: {
+			runtimeKey: options.runtimeKey,
+			type: "number",
+			required: options.required ?? true,
+			secret: false,
+			...options.description !== void 0 ? { description: options.description } : {},
+			...defaultValue !== void 0 ? { default: defaultValue } : {},
+			...options.integer !== void 0 ? { integer: options.integer } : {},
+			...options.minimum !== void 0 ? { minimum: options.minimum } : {}
+		}
+	};
+};
+/** Defines a boolean, optionally with a default. */
+const boolean = (options) => ({
+	config: withDefault(Config.boolean(options.runtimeKey), options.default),
+	spec: {
+		runtimeKey: options.runtimeKey,
+		type: "boolean",
+		required: options.required ?? true,
+		secret: false,
+		...options.description !== void 0 ? { description: options.description } : {},
+		...options.default !== void 0 ? { default: options.default } : {}
+	}
+});
+/** Defines a string literal choice whose values are preserved in the inferred type. */
+const select = (options) => ({
+	config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
+	spec: {
+		runtimeKey: options.runtimeKey,
+		type: "select",
+		required: options.required ?? true,
+		secret: false,
+		values: options.values,
+		...options.description !== void 0 ? { description: options.description } : {},
+		...options.default !== void 0 ? { default: options.default } : {}
+	}
+});
+/** Defines a required, non-empty secret decoded as `Redacted<string>`. */
+const secret = (options) => ({
+	config: Config.schema(Schema.Redacted(Schema.NonEmptyString), options.runtimeKey),
+	spec: {
+		runtimeKey: options.runtimeKey,
+		type: "string",
+		required: options.required ?? true,
+		secret: true,
+		...options.description !== void 0 ? { description: options.description } : {}
+	}
+});
+/** Makes an existing field optional and represents a missing value as `Option.none()`. */
+const optional = (field) => ({
+	config: Config.option(field.config),
+	spec: {
+		...field.spec,
+		required: false
+	}
+});
+/**
+* Defines user-facing connector config once for both Effect runtime loading and
+* frontend form generation. The object keys become logical form/API field names;
+* `runtimeKey` remains the key read by Effect Config.
+*
+* @example
+* ```ts
+* const config = define({
+*   apiUrl: string({ runtimeKey: "API_URL" }),
+*   token: secret({ runtimeKey: "API_TOKEN" }),
+* });
+*
+* const RuntimeConfig = config.config;
+* const formFields = config.spec;
+* ```
+*/
+const define = (fields) => {
+	const spec = Object.entries(fields).map(([name, field]) => ({
+		name,
+		...field.spec
+	}));
+	const platformField = spec.find((field) => isPlatformRuntimeKey(field.runtimeKey));
+	if (platformField !== void 0) throw new Error(`Connector runtime key is platform-owned: ${platformField.runtimeKey}`);
+	const configFields = Struct.map(fields, configFieldToConfig);
+	return {
+		fields: configFields,
+		config: Config.unwrap(configFields),
+		spec
+	};
+};
+
+//#endregion
+//#region src/manifest/manifest.ts
+/**
+* Defines browser-safe connector metadata while preserving literal names and capabilities.
+*
+* @example
+* ```ts
+* const manifest = define({
+*   name: "producer-example",
+*   title: "Example",
+*   config: ExampleConfig.spec,
+*   resources: [{ name: "orders", capabilities: ["backfill"] }],
+* });
+* ```
+*/
+const define$1 = (manifest) => manifest;
+
+//#endregion
+//#region src/manifest/schema.ts
+const numberFromForm = (field) => {
+	const numberSchema = makeNumberSchema(field);
+	return Schema.Union([numberSchema, Schema.NonEmptyString.pipe(Schema.decodeTo(numberSchema, SchemaTransformation.numberFromString))]);
+};
+const BooleanFromForm = Schema.Union([Schema.Boolean, Schema.Literals(["true", "false"]).pipe(Schema.decodeTo(Schema.Boolean, SchemaTransformation.transform({
+	decode: (value) => value === "true",
+	encode: (value) => value ? "true" : "false"
+})))]);
+const EmptyStringAsUndefined = Schema.Literal("").pipe(Schema.decodeTo(Schema.Undefined, SchemaTransformation.transform({
+	decode: () => void 0,
+	encode: () => ""
+})));
+const baseFieldSchema = (field) => {
+	switch (field.type) {
+		case "number": return numberFromForm(field);
+		case "boolean": return BooleanFromForm;
+		case "select": return Schema.Literals(field.values);
+		case "string": return Schema.NonEmptyString;
+	}
+};
+const fieldSchema = (field) => field.required ? baseFieldSchema(field) : Schema.optional(Schema.Union([EmptyStringAsUndefined, baseFieldSchema(field)]));
+/**
+* Builds an Effect Schema for logical form/API values from a connector manifest.
+*
+* @example
+* ```ts
+* const decode = Schema.decodeUnknownEffect(configSchema(manifest));
+* const values = decode({
+*   apiUrl: "https://example.com",
+*   retries: "3",
+* });
+* ```
+*/
+const configSchema = (manifest) => {
+	const fields = Object.fromEntries(manifest.config.map((field) => [field.name, fieldSchema(field)]));
+	return Schema.Struct(fields);
+};
+/** Decodes browser/API config, applies manifest defaults, and rejects undeclared fields. */
+const decodeConfig = (manifest, input) => {
+	const defaults = Object.fromEntries(manifest.config.flatMap((field) => field.default === void 0 ? [] : [[field.name, field.default]]));
+	const optionalFields = new Set(manifest.config.flatMap((field) => field.required ? [] : [field.name]));
+	const normalizedInput = Predicate.isObject(input) ? Object.fromEntries(Object.entries(input).filter(([name, value]) => value !== "" || !optionalFields.has(name))) : input;
+	const inputWithDefaults = Predicate.isObject(normalizedInput) ? Object.assign({}, defaults, normalizedInput) : normalizedInput;
+	return Schema.decodeUnknownEffect(configSchema(manifest))(inputWithDefaults, { onExcessProperty: "error" });
+};
+/**
+* Maps validated logical values to the flat runtime-key document mounted in a pod.
+* Missing optional values are omitted.
+*
+* @example
+* ```ts
+* toRuntimeDocument(manifest, { apiUrl: "https://example.com" });
+* // { API_URL: "https://example.com" }
+* ```
+*/
+const toRuntimeDocument = (manifest, values) => {
+	const document = {};
+	for (const field of manifest.config) {
+		const value = Reflect.get(values, field.name);
+		if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") document[field.runtimeKey] = value;
+	}
+	return document;
+};
+
+//#endregion
+//#region src/manifest/index.ts
+var manifest_exports = /* @__PURE__ */ __export({
+	boolean: () => boolean,
+	configSchema: () => configSchema,
+	decodeConfig: () => decodeConfig,
+	define: () => define$1,
+	defineConfig: () => define,
+	number: () => number,
+	optional: () => optional,
+	secret: () => secret,
+	select: () => select,
+	string: () => string,
+	toRuntimeDocument: () => toRuntimeDocument
+});
+
+//#endregion
+export { define$1 as a, number as c, select as d, string as f, toRuntimeDocument as i, optional as l, configSchema as n, boolean as o, decodeConfig as r, define as s, manifest_exports as t, secret as u };
+//# sourceMappingURL=manifest-ByE6YPG0.js.map