@useairfoil/connector-kit 0.5.3 → 0.6.1

package/dist/index-ygpml2kC.d.mts

@@ -0,0 +1,73 @@
+import { E as TableSource } from "./types-xZtSmrGA.mjs";
+import { Config, Effect, FileSystem } from "effect";
+import { TableIdentifier } from "iceberg-js";
+//#region src/runtime-config/error.d.ts
+type RuntimeConfigErrorCode = "CONFIG_PATH_MISSING" | "CONFIG_FILE_UNREADABLE" | "CONFIG_JSON_INVALID" | "TABLE_BINDINGS_MISSING" | "TABLE_BINDINGS_INVALID" | "TABLE_BINDINGS_MISMATCH";
+declare const RuntimeConfigError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "RuntimeConfigError";
+} & Readonly<A>;
+/** Expected hosted-config failure with a stable code and diagnostic cause. */
+declare class RuntimeConfigError extends RuntimeConfigError_base<{
+  readonly code: RuntimeConfigErrorCode;
+  readonly message: string;
+  readonly cause: Error;
+}> {}
+//#endregion
+//#region src/runtime-config/config.d.ts
+/** Shared HTTP port used by hosted and sandbox connector entrypoints. */
+declare const httpPort: Config.Config<number>;
+/** Wings URL, including the protocol. */
+declare const wingsUri: Config.Config<URL>;
+/** Wings catalog ID. */
+declare const catalog: Config.Config<string>;
+//#endregion
+//#region src/runtime-config/constants.d.ts
+/** Shared runtime keys owned by Airfoil rather than an individual connector. */
+declare const PlatformRuntimeKey: {
+  readonly configPath: "AIRFOIL_CONFIG_PATH";
+  readonly httpPort: "AIRFOIL_HTTP_PORT";
+  readonly tableBindings: "AIRFOIL_TABLE_BINDINGS";
+  readonly connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID";
+  readonly stateTable: "AIRFOIL_STATE_TABLE";
+  readonly postgresConnectionString: "POSTGRES_CONNECTION_STRING";
+  readonly wingsUri: "WINGS_URI";
+  readonly catalog: "AIRFOIL_CATALOG";
+  readonly otelEnabled: "OTEL_ENABLED";
+  readonly otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT";
+  readonly otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS";
+  readonly otelServiceName: "OTEL_SERVICE_NAME";
+  readonly otelServiceVersion: "OTEL_SERVICE_VERSION";
+  readonly otelResourceAttributes: "OTEL_RESOURCE_ATTRIBUTES";
+};
+/** Defaults for optional platform-owned runtime configuration. */
+declare const PlatformRuntimeDefault: {
+  readonly httpPort: 8080;
+  readonly stateTable: "_airfoil_connectors_state";
+};
+//#endregion
+//#region src/runtime-config/layer.d.ts
+/**
+ * Loads the required JSON file and lets environment values override it per key.
+ * Connector field validation remains in the connector's Effect Config.
+ *
+ * @example
+ * ```ts
+ * program.pipe(Effect.provide(layerHosted()));
+ * ```
+ */
+declare const layerHosted: () => import("effect/Layer").Layer<never, RuntimeConfigError, FileSystem.FileSystem>;
+//#endregion
+//#region src/runtime-config/table-bindings.d.ts
+/** An Iceberg table and its storage location for each connector resource. */
+type TableBinding = TableIdentifier & {
+  readonly location?: string | undefined;
+};
+type ResolvedTableBindings<Source extends TableSource> = Readonly<Partial<Record<Source["resources"][number]["name"], TableBinding>>>;
+/** Reads hosted table bindings and checks they name resources of the connector. */
+declare const loadTableBindings: <const Source extends TableSource>(connector: Source) => Effect.Effect<ResolvedTableBindings<Source>, RuntimeConfigError>;
+declare namespace index_d_exports {
+  export { PlatformRuntimeDefault, PlatformRuntimeKey, ResolvedTableBindings, RuntimeConfigError, RuntimeConfigErrorCode, TableBinding, catalog, httpPort, layerHosted, loadTableBindings, wingsUri };
+}
+//#endregion
+export { layerHosted as a, catalog as c, RuntimeConfigError as d, RuntimeConfigErrorCode as f, loadTableBindings as i, httpPort as l, ResolvedTableBindings as n, PlatformRuntimeDefault as o, TableBinding as r, PlatformRuntimeKey as s, index_d_exports as t, wingsUri as u };
+//# sourceMappingURL=index-ygpml2kC.d.mts.map

package/dist/index.d.mts

@@ -0,0 +1,16 @@
+import { t as index_d_exports } from "./index-BX8G7d41.mjs";
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
+import { t as Identified } from "./annotations-BE74YmAO.mjs";
+import { A as WebhookRouteContext, C as ResourceUpdateOf, D as WebhookAckMode, E as TableSource, O as WebhookHandler, S as ResourceUpdate, T as TableResource, _ as ResourceRequiredField, a as FeedRows, b as ResourceSchema, c as FetchPageResult, d as ResourceDefinition, f as ResourceDelete, g as ResourcePayload, i as Cursor$1, k as WebhookRoute, l as PageFetch, m as ResourceKey, n as ChangesFetch, o as FetchChangesResult, p as ResourceField, r as ConnectorDefinition, s as FetchFeedResult, t as ChangesFeed, u as ResourceBatch, v as ResourceRow, x as ResourceState, y as ResourceRows } from "./types-xZtSmrGA.mjs";
+import { c as telemetry_d_exports, d as Fetch, f as Resource, l as Connector, t as index_d_exports$1, u as Cursor } from "./index-NnCutIqM.mjs";
+import { t as index_d_exports$2 } from "./index-Dk7urvpF.mjs";
+import { t as index_d_exports$3 } from "./index-DJhDp4Xc.mjs";
+import { t as index_d_exports$8 } from "./index-CgvFomO6.mjs";
+import { t as index_d_exports$4 } from "./index-BfXpgxk9.mjs";
+import { t as index_d_exports$5 } from "./index-CUPP2A_m.mjs";
+import { t as index_d_exports$7 } from "./index-ygpml2kC.mjs";
+import { t as index_d_exports$6 } from "./index-eFuoMZZ3.mjs";
+import { h as metrics_d_exports } from "./metrics-CVXFL7YT.mjs";
+import { t as index_d_exports$9 } from "./index-BUHSGm9Q.mjs";
+import { t as index_d_exports$10 } from "./index-CRVAwJ5-.mjs";
+export { index_d_exports as Auth, type ChangesFeed, type ChangesFetch, Connector, index_d_exports$1 as ConnectorApp, type ConnectorDefinition, ConnectorError, Cursor, type Cursor$1 as CursorTypes, index_d_exports$2 as Demo, type FeedRows, Fetch, type FetchChangesResult, type FetchFeedResult, type FetchPageResult, index_d_exports$3 as Iceberg, type Identified, index_d_exports$4 as Ingestion, index_d_exports$5 as Ingestor, index_d_exports$6 as Manifest, metrics_d_exports as Metrics, type PageFetch, Resource, type ResourceBatch, type ResourceDefinition, type ResourceDelete, type ResourceField, type ResourceKey, type ResourcePayload, type ResourceRequiredField, type ResourceRow, type ResourceRows, type ResourceSchema, type ResourceState, type ResourceUpdate, type ResourceUpdateOf, index_d_exports$7 as RuntimeConfig, index_d_exports$8 as StateStore, index_d_exports$9 as Status, type TableResource, type TableSource, telemetry_d_exports as Telemetry, index_d_exports$10 as Webhook, type WebhookAckMode, type WebhookHandler, type WebhookRoute, type WebhookRouteContext };

package/dist/index.mjs

@@ -0,0 +1,45 @@
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { t as auth_exports } from "./auth/index.mjs";
+import { t as connector_app_exports } from "./connector-app-C-L4QqG6.mjs";
+import { t as runtime_config_exports } from "./runtime-config-Oa55gAew.mjs";
+import { t as demo_exports } from "./demo-B30Vlkk2.mjs";
+import { o as metrics_exports, w as telemetry_exports } from "./metrics-BaEX__MN.mjs";
+import { t as state_store_exports } from "./state-store-uFvHY7Ax.mjs";
+import { t as status_exports } from "./status-CKb9IhQ9.mjs";
+import { t as ingestion_exports } from "./ingestion-q0WwqUSt.mjs";
+import { t as iceberg_exports } from "./iceberg-D6L8I6vA.mjs";
+import { t as ingestor_exports } from "./ingestor-D4L-HPkg.mjs";
+import { t as manifest_exports } from "./manifest-BGKxmKLy.mjs";
+import { t as webhook_exports } from "./webhook/index.mjs";
+import { DateTime, Effect, Option } from "effect";
+//#region src/core/builder.ts
+const Connector = { define: (definition) => definition };
+const Resource = { entity: (definition) => definition };
+const Fetch = {
+	page: (definition) => definition,
+	changes: (definition) => definition,
+	feed: (definition) => definition
+};
+const decodeFailure = (kind, value) => new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });
+const Cursor = {
+	string: () => ({
+		kind: "string",
+		decode: (value) => typeof value === "string" ? Effect.succeed(value) : Effect.fail(decodeFailure("string", value))
+	}),
+	number: () => ({
+		kind: "number",
+		decode: (value) => typeof value === "number" && Number.isFinite(value) ? Effect.succeed(value) : Effect.fail(decodeFailure("number", value))
+	}),
+	isoDateTime: () => ({
+		kind: "isoDateTime",
+		decode: (value) => {
+			if (typeof value !== "string" || Option.isNone(DateTime.make(value))) return Effect.fail(decodeFailure("ISO datetime", value));
+			return Effect.succeed(value);
+		}
+	}),
+	nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso)
+};
+//#endregion
+export { auth_exports as Auth, Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, demo_exports as Demo, Fetch, iceberg_exports as Iceberg, ingestion_exports as Ingestion, ingestor_exports as Ingestor, manifest_exports as Manifest, metrics_exports as Metrics, Resource, runtime_config_exports as RuntimeConfig, state_store_exports as StateStore, status_exports as Status, telemetry_exports as Telemetry, webhook_exports as Webhook };
+
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect, Option } from \"effect\";\n\nimport type {\n  ChangesFeed,\n  ChangesFetch,\n  ConnectorDefinition,\n  Cursor as CursorType,\n  PageFetch,\n  ResourceDefinition,\n  ResourceRequiredField,\n  ResourceSchema,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n  define: <\n    const Resources extends ReadonlyArray<ResourceDefinition>,\n    const FeedResources extends ReadonlyArray<Resources[number]> = Resources,\n  >(\n    definition: ConnectorDefinition<Resources, FeedResources>,\n  ) => definition,\n};\n\nexport const Resource = {\n  entity: <\n    S extends ResourceSchema,\n    Payload = never,\n    R = never,\n    const Name extends string = string,\n    const Version extends ResourceRequiredField<S> = ResourceRequiredField<S>,\n  >(\n    definition: ResourceDefinition<S, Payload, R, Name, Version>,\n  ): ResourceDefinition<S, Payload, R, Name, Version> => definition,\n};\n\nexport const Fetch = {\n  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n  feed: <const Resources extends ReadonlyArray<ResourceDefinition>, R = never>(\n    definition: ChangesFeed<Resources, R>,\n  ) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n  string: (): CursorType.Definition<string> => ({\n    kind: \"string\",\n    decode: (value) =>\n      typeof value === \"string\"\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"string\", value)),\n  }),\n\n  number: (): CursorType.Definition<number> => ({\n    kind: \"number\",\n    decode: (value) =>\n      typeof value === \"number\" && Number.isFinite(value)\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"number\", value)),\n  }),\n\n  isoDateTime: (): CursorType.Definition<string> => ({\n    kind: \"isoDateTime\",\n    decode: (value) => {\n      if (typeof value !== \"string\" || Option.isNone(DateTime.make(value))) {\n        return Effect.fail(decodeFailure(\"ISO datetime\", value));\n      }\n      return Effect.succeed(value);\n    },\n  }),\n\n  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;;;;;AAeA,MAAa,YAAY,EACvB,SAIE,eACG,WACP;AAEA,MAAa,WAAW,EACtB,SAOE,eACqD,WACzD;AAEA,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACtF,OACE,eACG;AACP;AAEA,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,KAAK,IAAI,CAAC;AAElF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,KAAK,IACpB,OAAO,KAAK,cAAc,UAAU,KAAK,CAAC;CAClD;CAEA,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAC9C,OAAO,QAAQ,KAAK,IACpB,OAAO,KAAK,cAAc,UAAU,KAAK,CAAC;CAClD;CAEA,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;GACjB,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,SAAS,KAAK,KAAK,CAAC,GACjE,OAAO,OAAO,KAAK,cAAc,gBAAgB,KAAK,CAAC;GAEzD,OAAO,OAAO,QAAQ,KAAK;EAC7B;CACF;CAEA,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,SAAS;AAC7D"}

package/dist/ingestion/index.d.mts

@@ -0,0 +1,2 @@
+import { n as RunOptions, r as run } from "../index-BfXpgxk9.mjs";
+export { type RunOptions, run };

package/dist/ingestion/index.mjs

@@ -0,0 +1,2 @@
+import { n as run } from "../ingestion-q0WwqUSt.mjs";
+export { run };

package/dist/ingestion-q0WwqUSt.mjs

@@ -0,0 +1,275 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { C as annotateError, d as setLastSuccessTimestamp, f as setSyncState, p as setWebhookQueueDepth, v as Attr, x as SpanName } from "./metrics-BaEX__MN.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { t as ingestBatch } from "./instrumented-Bhp7fyvh.mjs";
+import { o as StateStore } from "./state-store-uFvHY7Ax.mjs";
+import { c as deriveSyncState, l as hasChanges, s as statusRoute, u as normalizeCursor } from "./status-CKb9IhQ9.mjs";
+import { n as router, t as WebhookQueue } from "./server-CMnxz3h0.mjs";
+import { Array, DateTime, Effect, Layer, Order, Queue } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/http";
+import * as Observability from "effect/observability";
+//#region src/ingestion/engine.ts
+function run(definition, options) {
+	return Effect.gen(function* () {
+		if (definition.changes) yield* validateFeed(definition, definition.changes);
+		const ingestor = yield* Ingestor;
+		const connector = selectResources(definition, ingestor.resourceNames);
+		const queueLayer = makeWebhookQueueLayer(connector.name);
+		const runtimeLayer = options?.webhook ? Layer.mergeAll(queueLayer, makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer))) : Layer.empty;
+		return yield* Effect.gen(function* () {
+			const initialCutoff = options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+			yield* Effect.logInfo("Connector started").pipe(Effect.annotateLogs({
+				[Attr.connectorName]: connector.name,
+				resources: connector.resources.length
+			}));
+			const sourceRun = runIngestion(connector, initialCutoff);
+			const webhookRun = options?.webhook ? runWebhookRuntime(connector, options.webhook.routes) : Effect.void;
+			return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+		}).pipe(Effect.provide(runtimeLayer));
+	});
+}
+const selectResources = (connector, names) => {
+	if (names === void 0) return connector;
+	const isSelected = (resource) => names.includes(resource.name);
+	const { changes, ...rest } = connector;
+	const feedResources = changes?.resources.filter(isSelected) ?? [];
+	return {
+		...rest,
+		resources: connector.resources.filter(isSelected),
+		...changes && feedResources.length > 0 ? { changes: {
+			...changes,
+			resources: feedResources
+		} } : {}
+	};
+};
+const makeWebhookQueueLayer = (connector) => Layer.effect(WebhookQueue)(Effect.gen(function* () {
+	const queue = yield* Queue.bounded(1024);
+	yield* setWebhookQueueDepth(connector, 0);
+	return WebhookQueue.of({ queue });
+}));
+const runWebhookRuntime = (connector, routes) => {
+	const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
+	return Effect.all([hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+};
+const runWebhookQueueConsumer = (connector) => Effect.gen(function* () {
+	const queue = yield* WebhookQueue;
+	while (true) {
+		const batch = yield* Queue.take(queue.queue);
+		const depth = yield* Queue.size(queue.queue);
+		yield* setWebhookQueueDepth(connector.name, depth);
+		yield* ingestBatch({
+			connector: connector.name,
+			resource: batch.resource,
+			source: "webhook",
+			batch: batch.batch
+		});
+	}
+});
+const makeWebhookServerLayer = (connector, options) => {
+	const healthPath = options.healthPath ?? "/health";
+	const metricsPath = options.metricsPath === void 0 ? "/metrics" : options.metricsPath;
+	const statusPath = options.statusPath === void 0 ? "/status" : options.statusPath;
+	const app = Layer.mergeAll(router(options.routes, {
+		connectorName: connector.name,
+		resourceNames: connector.resources.map((resource) => resource.name)
+	}), HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))), ...metricsPath === false ? [] : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })], ...statusPath === false ? [] : [statusRoute(connector, statusPath)]);
+	return HttpRouter.serve(app, { disableLogger: options.disableHttpLogger ?? true });
+};
+const runIngestion = (connector, initialCutoff) => Effect.gen(function* () {
+	const feed = connector.changes;
+	yield* Effect.all([Effect.forEach(connector.resources, (resource) => runResourceSources(connector, resource, initialCutoff), { concurrency: "unbounded" }), feed ? runChangesFeed(connector, feed, initialCutoff) : Effect.void], { concurrency: "unbounded" });
+});
+const bySavedAt = Order.mapInput(Order.String, (state) => state.lastSuccessAt ?? "");
+const validateFeed = (connector, feed) => Effect.forEach(feed.resources, (member, index) => {
+	if (feed.resources.findIndex((item) => item.name === member.name) !== index) return Effect.fail(new ConnectorError({ message: `Resource ${member.name} is in the changes feed twice` }));
+	const resource = connector.resources.find((item) => item.name === member.name);
+	if (!resource) return Effect.fail(new ConnectorError({ message: `Changes feed resource ${member.name} is not a resource of connector ${connector.name}` }));
+	if (resource.changes) return Effect.fail(new ConnectorError({ message: `Resource ${member.name} has its own changes and is also in the changes feed` }));
+	return Effect.void;
+}).pipe(Effect.andThen(feed.resources.length === 0 ? Effect.fail(new ConnectorError({ message: "The changes feed has no resources" })) : Effect.void));
+const initializeResourceState = (existing, initialCutoff) => ({
+	...existing,
+	changes: existing?.changes ?? { cursor: initialCutoff },
+	backfill: existing?.backfill ?? {
+		cutoff: initialCutoff,
+		completed: false
+	}
+});
+const getInitializedState = Effect.fnUntraced(function* (resource, initialCutoff) {
+	const existing = yield* (yield* StateStore).getResourceState(resource);
+	return initializeResourceState(existing, initialCutoff);
+});
+const runResourceSources = (connector, resource, initialCutoff) => {
+	const runs = [resource.backfill ? isolateSourceFailure(runBackfill(connector, resource, initialCutoff), connector, resource, "backfill") : Effect.void, resource.changes ? isolateSourceFailure(runChanges(connector, resource, initialCutoff), connector, resource, "changes") : Effect.void];
+	return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })), Effect.asVoid, Effect.catch((error) => setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, "error").pipe(Effect.andThen(Effect.logError("Connector resource initialization failed", error)), Effect.annotateLogs({
+		[Attr.connectorName]: connector.name,
+		resource: resource.name
+	}), Effect.andThen(Effect.never))));
+};
+const recordResourceError = (connector, resource, source, operation) => (effect) => effect.pipe(Effect.tapError(() => setSyncState({
+	connector: connector.name,
+	resource: resource.name
+}, "error").pipe(Effect.andThen(StateStore.pipe(Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)))))));
+const isolateSourceFailure = (effect, connector, resource, source) => effect.pipe(Effect.catch((error) => Effect.logError("Connector source parked", error).pipe(Effect.annotateLogs({
+	[Attr.connectorName]: connector.name,
+	resource: resource.name,
+	source
+}), Effect.andThen(Effect.never))));
+const refreshRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(connector, resource, state));
+	return state;
+});
+const initializeRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(connector, resource, state));
+	if (resource.backfill) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "backfill"
+	}, state.backfill?.lastSuccessAt);
+	if (hasChanges(connector, resource)) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "changes"
+	}, state.changes?.lastSuccessAt);
+});
+const runBackfill = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.backfill) return;
+	const store = yield* StateStore;
+	let state = yield* getInitializedState(resource.name, initialCutoff);
+	if (state.backfill?.completed === true) {
+		yield* store.clearResourceError(resource.name, "backfill").pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		return;
+	}
+	while (state.backfill?.completed !== true) {
+		const backfill = state.backfill ?? {
+			cutoff: initialCutoff,
+			completed: false
+		};
+		const clearsVisibleError = state.lastError?.source === "backfill";
+		const page = yield* resource.backfill.fetch({
+			pageCursor: backfill.pageCursor,
+			cutoff: backfill.cutoff
+		}).pipe(recordResourceError(connector, resource, "backfill", "fetch"));
+		yield* ingestBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill",
+			batch: {
+				cursor: page.nextPageCursor ?? backfill.cutoff,
+				rows: page.rows
+			}
+		}).pipe(recordResourceError(connector, resource, "backfill", "ingest"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		state = {
+			...state,
+			backfill: {
+				cutoff: backfill.cutoff,
+				pageCursor: page.nextPageCursor,
+				completed: !page.hasMore,
+				lastSuccessAt
+			}
+		};
+		yield* Effect.withSpan(store.setBackfillState(resource.name, {
+			cutoff: normalizeCursor(backfill.cutoff),
+			pageCursor: page.nextPageCursor === void 0 ? void 0 : normalizeCursor(page.nextPageCursor),
+			completed: !page.hasMore,
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "backfill")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		if (clearsVisibleError || !page.hasMore) state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+	}
+});
+const runChanges = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.changes) return;
+	const store = yield* StateStore;
+	while (true) {
+		const state = yield* getInitializedState(resource.name, initialCutoff);
+		const cursor = state.changes?.cursor ?? initialCutoff;
+		const clearsVisibleError = state.lastError?.source === "changes";
+		const page = yield* resource.changes.fetch({ cursor }).pipe(recordResourceError(connector, resource, "changes", "fetch"));
+		yield* ingestBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes",
+			batch: {
+				cursor: page.cursor,
+				rows: page.rows
+			}
+		}).pipe(recordResourceError(connector, resource, "changes", "ingest"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		yield* Effect.withSpan(store.setChangesState(resource.name, {
+			cursor: normalizeCursor(page.cursor),
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
+		if (clearsVisibleError) yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		if (!page.hasMore) yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+	}
+});
+const runChangesFeed = (connector, feed, initialCutoff) => Effect.gen(function* () {
+	const store = yield* StateStore;
+	const members = feed.resources;
+	const selected = new Set(members.map((resource) => resource.name));
+	const recordFeedError = (operation) => (effect) => effect.pipe(Effect.tapError(() => Effect.forEach(members, (resource) => setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, "error").pipe(Effect.andThen(store.setResourceError(resource.name, "changes", operation))), { discard: true })));
+	while (true) {
+		const states = yield* Effect.forEach(members, (resource) => store.getResourceState(resource.name));
+		const saved = states.flatMap((state) => state?.changes ? [state.changes] : []);
+		const cursor = Array.isReadonlyArrayNonEmpty(saved) ? Array.min(saved, bySavedAt).cursor : initialCutoff;
+		const result = yield* feed.fetch({
+			cursor,
+			selected
+		}).pipe(recordFeedError("fetch"));
+		yield* Effect.forEach(members, (resource) => ingestBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes",
+			batch: {
+				cursor: result.cursor,
+				rows: result.rows[resource.name] ?? []
+			}
+		}), { discard: true }).pipe(recordFeedError("ingest"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		yield* Effect.forEach(members, (resource) => Effect.withSpan(store.setChangesState(resource.name, {
+			cursor: normalizeCursor(result.cursor),
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }), { discard: true }).pipe(recordFeedError("checkpoint"));
+		yield* Effect.forEach(members, (resource, index) => states[index]?.lastError?.source === "changes" ? refreshRuntimeStatus(connector, resource, initialCutoff) : Effect.void);
+		if (!result.hasMore) yield* Effect.sleep(feed.interval ?? "1 minute");
+	}
+}).pipe(Effect.catch((error) => Effect.logError("Connector changes feed parked", error).pipe(Effect.annotateLogs({
+	[Attr.connectorName]: connector.name,
+	resources: feed.resources.map((resource) => resource.name).join(",")
+}), Effect.andThen(Effect.never))));
+//#endregion
+//#region src/ingestion/index.ts
+var ingestion_exports = /* @__PURE__ */ __exportAll({ run: () => run });
+//#endregion
+export { run as n, ingestion_exports as t };
+
+//# sourceMappingURL=ingestion-q0WwqUSt.mjs.map

package/dist/ingestion-q0WwqUSt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestion-q0WwqUSt.mjs","names":["Metrics.setWebhookQueueDepth","Metrics.setSyncState","Metrics.setLastSuccessTimestamp","Arr"],"sources":["../src/ingestion/engine.ts","../src/ingestion/index.ts"],"sourcesContent":["import { Array as Arr, DateTime, Effect, Layer, Order, Queue } from \"effect\";\nimport { HttpRouter, type HttpServer, HttpServerResponse } from \"effect/http\";\nimport * as Observability from \"effect/observability\";\n\nimport type {\n  ChangesFeed,\n  ConnectorDefinition,\n  Cursor,\n  ResourceDefinition,\n  ResourceState,\n  WebhookRoute,\n} from \"../core/types\";\nimport type { StateOperation, StateSource } from \"../state-store/schema\";\n\nimport { ConnectorError } from \"../errors\";\nimport { ingestBatch } from \"../ingestor/instrumented\";\nimport { Ingestor } from \"../ingestor/service\";\nimport * as Metrics from \"../metrics\";\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, hasChanges, normalizeCursor } from \"../state-store/state\";\nimport { statusRoute } from \"../status\";\nimport { Attr, SpanName, annotateError } from \"../telemetry\";\nimport { router, WebhookQueue, type QueuedWebhookBatch } from \"../webhook/server\";\n\ntype RunBaseOptions = {\n  readonly initialCutoff?: Cursor.Value;\n};\n\nexport type RunOptions = RunBaseOptions & {\n  readonly webhook?: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  };\n};\n\ntype RunNoWebhookOptions = RunBaseOptions & {\n  readonly webhook?: undefined;\n};\n\ntype RunWebhookOptions = RunOptions & {\n  readonly webhook: NonNullable<RunOptions[\"webhook\"]>;\n};\n\n/**\n * Runs all connector ingestion loops. Callers provide the Ingestor and\n * StateStore layers; webhook mode additionally requires an HttpServer layer.\n */\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options?: RunNoWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Ingestor>;\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options: RunWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Ingestor | HttpServer.HttpServer>;\nexport function run(definition: ConnectorDefinition, options?: RunOptions) {\n  return Effect.gen(function* () {\n    // Check the whole feed, even the resources that don't run.\n    if (definition.changes) yield* validateFeed(definition, definition.changes);\n    const ingestor = yield* Ingestor;\n    const connector = selectResources(definition, ingestor.resourceNames);\n    const queueLayer = makeWebhookQueueLayer(connector.name);\n    const runtimeLayer = options?.webhook\n      ? Layer.mergeAll(\n          queueLayer,\n          makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),\n        )\n      : Layer.empty;\n\n    return yield* Effect.gen(function* () {\n      const initialCutoff =\n        options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n      yield* Effect.logInfo(\"Connector started\").pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resources: connector.resources.length,\n        }),\n      );\n\n      const sourceRun = runIngestion(connector, initialCutoff);\n      const webhookRun = options?.webhook\n        ? runWebhookRuntime(connector, options.webhook.routes)\n        : Effect.void;\n\n      return yield* Effect.all([sourceRun, webhookRun], { concurrency: \"unbounded\" }).pipe(\n        Effect.asVoid,\n      );\n    }).pipe(Effect.provide(runtimeLayer));\n  });\n}\n\nconst selectResources = (\n  connector: ConnectorDefinition,\n  names: ReadonlyArray<string> | undefined,\n): ConnectorDefinition => {\n  if (names === undefined) return connector;\n  const isSelected = (resource: ResourceDefinition) => names.includes(resource.name);\n  const { changes, ...rest } = connector;\n  const feedResources = changes?.resources.filter(isSelected) ?? [];\n  return {\n    ...rest,\n    resources: connector.resources.filter(isSelected),\n    ...(changes && feedResources.length > 0\n      ? { changes: { ...changes, resources: feedResources } }\n      : {}),\n  };\n};\n\nconst makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>\n  Layer.effect(WebhookQueue)(\n    Effect.gen(function* () {\n      const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);\n      yield* Metrics.setWebhookQueueDepth(connector, 0);\n      return WebhookQueue.of({ queue });\n    }),\n  );\n\nconst runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {\n  const hasAfterEnqueue = routes.some((route) => route.ackMode === \"after-enqueue\");\n  return Effect.all(\n    [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],\n    {\n      concurrency: \"unbounded\",\n    },\n  ).pipe(Effect.asVoid);\n};\n\nconst runWebhookQueueConsumer = (connector: ConnectorDefinition) =>\n  Effect.gen(function* () {\n    const queue = yield* WebhookQueue;\n    while (true) {\n      const batch = yield* Queue.take(queue.queue);\n      const depth = yield* Queue.size(queue.queue);\n      yield* Metrics.setWebhookQueueDepth(connector.name, depth);\n      yield* ingestBatch({\n        connector: connector.name,\n        resource: batch.resource,\n        source: \"webhook\",\n        batch: batch.batch,\n      });\n    }\n  });\n\nconst makeWebhookServerLayer = (\n  connector: ConnectorDefinition,\n  options: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  },\n): Layer.Layer<never, never, HttpServer.HttpServer> => {\n  const healthPath: HttpRouter.PathInput = options.healthPath ?? \"/health\";\n  const metricsPath = options.metricsPath === undefined ? \"/metrics\" : options.metricsPath;\n  const statusPath = options.statusPath === undefined ? \"/status\" : options.statusPath;\n  const app = Layer.mergeAll(\n    router(options.routes, {\n      connectorName: connector.name,\n      resourceNames: connector.resources.map((resource) => resource.name),\n    }),\n    HttpRouter.add(\"GET\", healthPath, Effect.succeed(HttpServerResponse.text(\"ok\"))),\n    ...(metricsPath === false\n      ? []\n      : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),\n    ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),\n  );\n\n  return HttpRouter.serve(app, {\n    disableLogger: options.disableHttpLogger ?? true,\n  });\n};\n\nconst runIngestion = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor.Value,\n): Effect.Effect<void, ConnectorError, StateStore | Ingestor> =>\n  Effect.gen(function* () {\n    const feed = connector.changes;\n    yield* Effect.all(\n      [\n        Effect.forEach(\n          connector.resources,\n          (resource) => runResourceSources(connector, resource, initialCutoff),\n          { concurrency: \"unbounded\" },\n        ),\n        feed ? runChangesFeed(connector, feed, initialCutoff) : Effect.void,\n      ],\n      { concurrency: \"unbounded\" },\n    );\n  });\n\nconst bySavedAt = Order.mapInput(\n  Order.String,\n  (state: NonNullable<ResourceState[\"changes\"]>) => state.lastSuccessAt ?? \"\",\n);\n\nconst validateFeed = (connector: ConnectorDefinition, feed: ChangesFeed) =>\n  Effect.forEach(feed.resources, (member, index) => {\n    if (feed.resources.findIndex((item) => item.name === member.name) !== index) {\n      return Effect.fail(\n        new ConnectorError({ message: `Resource ${member.name} is in the changes feed twice` }),\n      );\n    }\n    const resource = connector.resources.find((item) => item.name === member.name);\n    if (!resource) {\n      return Effect.fail(\n        new ConnectorError({\n          message: `Changes feed resource ${member.name} is not a resource of connector ${connector.name}`,\n        }),\n      );\n    }\n    if (resource.changes) {\n      return Effect.fail(\n        new ConnectorError({\n          message: `Resource ${member.name} has its own changes and is also in the changes feed`,\n        }),\n      );\n    }\n    return Effect.void;\n  }).pipe(\n    Effect.andThen(\n      feed.resources.length === 0\n        ? Effect.fail(new ConnectorError({ message: \"The changes feed has no resources\" }))\n        : Effect.void,\n    ),\n  );\n\nconst initializeResourceState = (\n  existing: ResourceState | undefined,\n  initialCutoff: Cursor.Value,\n): ResourceState => ({\n  ...existing,\n  changes: existing?.changes ?? { cursor: initialCutoff },\n  backfill: existing?.backfill ?? {\n    cutoff: initialCutoff,\n    completed: false,\n  },\n});\n\nconst getInitializedState = Effect.fnUntraced(function* (\n  resource: string,\n  initialCutoff: Cursor.Value,\n) {\n  const store = yield* StateStore;\n  const existing = yield* store.getResourceState(resource);\n  return initializeResourceState(existing, initialCutoff);\n});\n\nconst runResourceSources = (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) => {\n  const runs = [\n    resource.backfill\n      ? isolateSourceFailure(\n          runBackfill(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"backfill\",\n        )\n      : Effect.void,\n    resource.changes\n      ? isolateSourceFailure(\n          runChanges(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"changes\",\n        )\n      : Effect.void,\n  ];\n\n  return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(\n    Effect.andThen(Effect.all(runs, { concurrency: \"unbounded\" })),\n    Effect.asVoid,\n    Effect.catch((error) =>\n      Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n        Effect.andThen(Effect.logError(\"Connector resource initialization failed\", error)),\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n};\n\nconst recordResourceError =\n  (\n    connector: ConnectorDefinition,\n    resource: ResourceDefinition,\n    source: StateSource,\n    operation: StateOperation,\n  ) =>\n  <A, R>(\n    effect: Effect.Effect<A, ConnectorError, R>,\n  ): Effect.Effect<A, ConnectorError, R | StateStore> =>\n    effect.pipe(\n      Effect.tapError(() =>\n        Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n          Effect.andThen(\n            StateStore.pipe(\n              Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),\n            ),\n          ),\n        ),\n      ),\n    );\n\nconst isolateSourceFailure = <A, R>(\n  effect: Effect.Effect<A, ConnectorError, R>,\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  source: StateSource,\n): Effect.Effect<A, never, R> =>\n  // Only typed source errors are isolated. Defects and interruption still stop the runtime.\n  effect.pipe(\n    Effect.catch((error) =>\n      Effect.logError(\"Connector source parked\", error).pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n          source,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n\nconst refreshRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(connector, resource, state),\n  );\n\n  return state;\n});\n\nconst initializeRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(connector, resource, state),\n  );\n\n  if (resource.backfill) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"backfill\" },\n      state.backfill?.lastSuccessAt,\n    );\n  }\n  if (hasChanges(connector, resource)) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"changes\" },\n      state.changes?.lastSuccessAt,\n    );\n  }\n});\n\nconst runBackfill = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.backfill) return;\n  const store = yield* StateStore;\n  let state = yield* getInitializedState(resource.name, initialCutoff);\n\n  if (state.backfill?.completed === true) {\n    // The checkpoint may have committed before the previous process cleared\n    // its error marker. It may also be hidden by a newer changes error.\n    yield* store\n      .clearResourceError(resource.name, \"backfill\")\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n    yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    return;\n  }\n\n  while (state.backfill?.completed !== true) {\n    const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };\n    const clearsVisibleError = state.lastError?.source === \"backfill\";\n    const page = yield* resource.backfill\n      .fetch({\n        pageCursor: backfill.pageCursor,\n        cutoff: backfill.cutoff,\n      })\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"fetch\"));\n\n    yield* ingestBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"backfill\",\n      batch: {\n        cursor: page.nextPageCursor ?? backfill.cutoff,\n        rows: page.rows,\n      },\n    }).pipe(recordResourceError(connector, resource, \"backfill\", \"ingest\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    state = {\n      ...state,\n      backfill: {\n        cutoff: backfill.cutoff,\n        pageCursor: page.nextPageCursor,\n        completed: !page.hasMore,\n        lastSuccessAt,\n      },\n    };\n\n    // Checkpoint only after ingestion succeeds so a crash cannot skip an\n    // unacknowledged page; replay therefore remains at-least-once.\n    yield* Effect.withSpan(\n      store\n        .setBackfillState(resource.name, {\n          cutoff: normalizeCursor(backfill.cutoff),\n          pageCursor:\n            page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),\n          completed: !page.hasMore,\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"backfill\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"backfill\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n\n    // Completion or clearing an error can change the resource-wide sync state.\n    if (clearsVisibleError || !page.hasMore) {\n      state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n  }\n});\n\nconst runChanges = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.changes) return;\n  const store = yield* StateStore;\n\n  while (true) {\n    const state = yield* getInitializedState(resource.name, initialCutoff);\n    const cursor = state.changes?.cursor ?? initialCutoff;\n    const clearsVisibleError = state.lastError?.source === \"changes\";\n    const page = yield* resource.changes\n      .fetch({ cursor })\n      .pipe(recordResourceError(connector, resource, \"changes\", \"fetch\"));\n\n    yield* ingestBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"changes\",\n      batch: {\n        cursor: page.cursor,\n        rows: page.rows,\n      },\n    }).pipe(recordResourceError(connector, resource, \"changes\", \"ingest\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    // Advance the durable cursor only after Wings acknowledges the page.\n    yield* Effect.withSpan(\n      store\n        .setChangesState(resource.name, {\n          cursor: normalizeCursor(page.cursor),\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"changes\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"changes\", \"checkpoint\"));\n\n    if (clearsVisibleError) {\n      yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n\n    if (!page.hasMore) {\n      yield* Effect.sleep(resource.changes.interval ?? \"1 minute\");\n    }\n  }\n});\n\n// Every member resource stores the feed cursor. All rows are ingested before any\n// cursor is written. If a crash stops the writes partway, the next run starts\n// from the oldest cursor.\nconst runChangesFeed = (\n  connector: ConnectorDefinition,\n  feed: ChangesFeed,\n  initialCutoff: Cursor.Value,\n): Effect.Effect<void, never, StateStore | Ingestor> =>\n  Effect.gen(function* () {\n    const store = yield* StateStore;\n    const members = feed.resources;\n    const selected = new Set(members.map((resource) => resource.name));\n    // Any failure stops the whole feed, so it is an error for every resource in it.\n    const recordFeedError =\n      (operation: StateOperation) =>\n      <A, R>(effect: Effect.Effect<A, ConnectorError, R>) =>\n        effect.pipe(\n          Effect.tapError(() =>\n            Effect.forEach(\n              members,\n              (resource) =>\n                Metrics.setSyncState(\n                  { connector: connector.name, resource: resource.name },\n                  \"error\",\n                ).pipe(Effect.andThen(store.setResourceError(resource.name, \"changes\", operation))),\n              { discard: true },\n            ),\n          ),\n        );\n\n    while (true) {\n      const states = yield* Effect.forEach(members, (resource) =>\n        store.getResourceState(resource.name),\n      );\n      // A resource that was off has an older cursor. Resume from the oldest, so it\n      // gets the changes it missed. The others read some changes again.\n      const saved = states.flatMap((state) => (state?.changes ? [state.changes] : []));\n      const cursor = Arr.isReadonlyArrayNonEmpty(saved)\n        ? Arr.min(saved, bySavedAt).cursor\n        : initialCutoff;\n      const result = yield* feed.fetch({ cursor, selected }).pipe(recordFeedError(\"fetch\"));\n\n      yield* Effect.forEach(\n        members,\n        (resource) =>\n          ingestBatch({\n            connector: connector.name,\n            resource: resource.name,\n            source: \"changes\",\n            batch: {\n              cursor: result.cursor,\n              rows: result.rows[resource.name] ?? [],\n            },\n          }),\n        { discard: true },\n      ).pipe(recordFeedError(\"ingest\"));\n\n      const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n      yield* Effect.forEach(\n        members,\n        (resource) =>\n          Effect.withSpan(\n            store\n              .setChangesState(resource.name, {\n                cursor: normalizeCursor(result.cursor),\n                lastSuccessAt,\n              })\n              .pipe(\n                Effect.andThen(\n                  Metrics.setLastSuccessTimestamp(\n                    { connector: connector.name, resource: resource.name, source: \"changes\" },\n                    lastSuccessAt,\n                  ),\n                ),\n                Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n                Effect.tapError((error) => annotateError(\"state_set\", error)),\n              ),\n            SpanName.stateSet,\n            { attributes: { [Attr.stateKey]: resource.name } },\n          ),\n        { discard: true },\n      ).pipe(recordFeedError(\"checkpoint\"));\n\n      yield* Effect.forEach(members, (resource, index) =>\n        states[index]?.lastError?.source === \"changes\"\n          ? refreshRuntimeStatus(connector, resource, initialCutoff)\n          : Effect.void,\n      );\n\n      if (!result.hasMore) {\n        yield* Effect.sleep(feed.interval ?? \"1 minute\");\n      }\n    }\n  }).pipe(\n    // Only typed errors park the feed. Defects and interruption still stop the runtime.\n    Effect.catch((error) =>\n      Effect.logError(\"Connector changes feed parked\", error).pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resources: feed.resources.map((resource) => resource.name).join(\",\"),\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n","export { run, type RunOptions } from \"./engine\";\n"],"mappings":";;;;;;;;;;;;AA0DA,SAAgB,IAAI,YAAiC,SAAsB;CACzE,OAAO,OAAO,IAAI,aAAa;EAE7B,IAAI,WAAW,SAAS,OAAO,aAAa,YAAY,WAAW,OAAO;EAC1E,MAAM,WAAW,OAAO;EACxB,MAAM,YAAY,gBAAgB,YAAY,SAAS,aAAa;EACpE,MAAM,aAAa,sBAAsB,UAAU,IAAI;EACvD,MAAM,eAAe,SAAS,UAC1B,MAAM,SACJ,YACA,uBAAuB,WAAW,QAAQ,OAAO,CAAC,CAAC,KAAK,MAAM,QAAQ,UAAU,CAAC,CACnF,IACA,MAAM;EAEV,OAAO,OAAO,OAAO,IAAI,aAAa;GACpC,MAAM,gBACJ,SAAS,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;GACpF,OAAO,OAAO,QAAQ,mBAAmB,CAAC,CAAC,KACzC,OAAO,aAAa;KACjB,KAAK,gBAAgB,UAAU;IAChC,WAAW,UAAU,UAAU;GACjC,CAAC,CACH;GAEA,MAAM,YAAY,aAAa,WAAW,aAAa;GACvD,MAAM,aAAa,SAAS,UACxB,kBAAkB,WAAW,QAAQ,QAAQ,MAAM,IACnD,OAAO;GAEX,OAAO,OAAO,OAAO,IAAI,CAAC,WAAW,UAAU,GAAG,EAAE,aAAa,YAAY,CAAC,CAAC,CAAC,KAC9E,OAAO,MACT;EACF,CAAC,CAAC,CAAC,KAAK,OAAO,QAAQ,YAAY,CAAC;CACtC,CAAC;AACH;AAEA,MAAM,mBACJ,WACA,UACwB;CACxB,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,cAAc,aAAiC,MAAM,SAAS,SAAS,IAAI;CACjF,MAAM,EAAE,SAAS,GAAG,SAAS;CAC7B,MAAM,gBAAgB,SAAS,UAAU,OAAO,UAAU,KAAK,CAAC;CAChE,OAAO;EACL,GAAG;EACH,WAAW,UAAU,UAAU,OAAO,UAAU;EAChD,GAAI,WAAW,cAAc,SAAS,IAClC,EAAE,SAAS;GAAE,GAAG;GAAS,WAAW;EAAc,EAAE,IACpD,CAAC;CACP;AACF;AAEA,MAAM,yBAAyB,cAC7B,MAAM,OAAO,YAAY,CAAC,CACxB,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,MAAM,QAA4B,IAAI;CAC3D,OAAOA,qBAA6B,WAAW,CAAC;CAChD,OAAO,aAAa,GAAG,EAAE,MAAM,CAAC;AAClC,CAAC,CACH;AAEF,MAAM,qBAAqB,WAAgC,WAAwC;CACjG,MAAM,kBAAkB,OAAO,MAAM,UAAU,MAAM,YAAY,eAAe;CAChF,OAAO,OAAO,IACZ,CAAC,kBAAkB,wBAAwB,SAAS,IAAI,OAAO,MAAM,OAAO,KAAK,GACjF,EACE,aAAa,YACf,CACF,CAAC,CAAC,KAAK,OAAO,MAAM;AACtB;AAEA,MAAM,2BAA2B,cAC/B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CACrB,OAAO,MAAM;EACX,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,KAAK;EAC3C,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,KAAK;EAC3C,OAAOA,qBAA6B,UAAU,MAAM,KAAK;EACzD,OAAO,YAAY;GACjB,WAAW,UAAU;GACrB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,MAAM;EACf,CAAC;CACH;AACF,CAAC;AAEH,MAAM,0BACJ,WACA,YAOqD;CACrD,MAAM,aAAmC,QAAQ,cAAc;CAC/D,MAAM,cAAc,QAAQ,gBAAgB,KAAA,IAAY,aAAa,QAAQ;CAC7E,MAAM,aAAa,QAAQ,eAAe,KAAA,IAAY,YAAY,QAAQ;CAC1E,MAAM,MAAM,MAAM,SAChB,OAAO,QAAQ,QAAQ;EACrB,eAAe,UAAU;EACzB,eAAe,UAAU,UAAU,KAAK,aAAa,SAAS,IAAI;CACpE,CAAC,GACD,WAAW,IAAI,OAAO,YAAY,OAAO,QAAQ,mBAAmB,KAAK,IAAI,CAAC,CAAC,GAC/E,GAAI,gBAAgB,QAChB,CAAC,IACD,CAAC,cAAc,kBAAkB,UAAU,EAAE,MAAM,YAAY,CAAC,CAAC,GACrE,GAAI,eAAe,QAAQ,CAAC,IAAI,CAAC,YAAY,WAAW,UAAU,CAAC,CACrE;CAEA,OAAO,WAAW,MAAM,KAAK,EAC3B,eAAe,QAAQ,qBAAqB,KAC9C,CAAC;AACH;AAEA,MAAM,gBACJ,WACA,kBAEA,OAAO,IAAI,aAAa;CACtB,MAAM,OAAO,UAAU;CACvB,OAAO,OAAO,IACZ,CACE,OAAO,QACL,UAAU,YACT,aAAa,mBAAmB,WAAW,UAAU,aAAa,GACnE,EAAE,aAAa,YAAY,CAC7B,GACA,OAAO,eAAe,WAAW,MAAM,aAAa,IAAI,OAAO,IACjE,GACA,EAAE,aAAa,YAAY,CAC7B;AACF,CAAC;AAEH,MAAM,YAAY,MAAM,SACtB,MAAM,SACL,UAAiD,MAAM,iBAAiB,EAC3E;AAEA,MAAM,gBAAgB,WAAgC,SACpD,OAAO,QAAQ,KAAK,YAAY,QAAQ,UAAU;CAChD,IAAI,KAAK,UAAU,WAAW,SAAS,KAAK,SAAS,OAAO,IAAI,MAAM,OACpE,OAAO,OAAO,KACZ,IAAI,eAAe,EAAE,SAAS,YAAY,OAAO,KAAK,+BAA+B,CAAC,CACxF;CAEF,MAAM,WAAW,UAAU,UAAU,MAAM,SAAS,KAAK,SAAS,OAAO,IAAI;CAC7E,IAAI,CAAC,UACH,OAAO,OAAO,KACZ,IAAI,eAAe,EACjB,SAAS,yBAAyB,OAAO,KAAK,kCAAkC,UAAU,OAC5F,CAAC,CACH;CAEF,IAAI,SAAS,SACX,OAAO,OAAO,KACZ,IAAI,eAAe,EACjB,SAAS,YAAY,OAAO,KAAK,sDACnC,CAAC,CACH;CAEF,OAAO,OAAO;AAChB,CAAC,CAAC,CAAC,KACD,OAAO,QACL,KAAK,UAAU,WAAW,IACtB,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,oCAAoC,CAAC,CAAC,IAChF,OAAO,IACb,CACF;AAEF,MAAM,2BACJ,UACA,mBACmB;CACnB,GAAG;CACH,SAAS,UAAU,WAAW,EAAE,QAAQ,cAAc;CACtD,UAAU,UAAU,YAAY;EAC9B,QAAQ;EACR,WAAW;CACb;AACF;AAEA,MAAM,sBAAsB,OAAO,WAAW,WAC5C,UACA,eACA;CAEA,MAAM,WAAW,QAAO,OADH,WAAA,CACS,iBAAiB,QAAQ;CACvD,OAAO,wBAAwB,UAAU,aAAa;AACxD,CAAC;AAED,MAAM,sBACJ,WACA,UACA,kBACG;CACH,MAAM,OAAO,CACX,SAAS,WACL,qBACE,YAAY,WAAW,UAAU,aAAa,GAC9C,WACA,UACA,UACF,IACA,OAAO,MACX,SAAS,UACL,qBACE,WAAW,WAAW,UAAU,aAAa,GAC7C,WACA,UACA,SACF,IACA,OAAO,IACb;CAEA,OAAO,wBAAwB,WAAW,UAAU,aAAa,CAAC,CAAC,KACjE,OAAO,QAAQ,OAAO,IAAI,MAAM,EAAE,aAAa,YAAY,CAAC,CAAC,GAC7D,OAAO,QACP,OAAO,OAAO,UACZC,aAAqB;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GAAG,OAAO,CAAC,CAAC,KACpF,OAAO,QAAQ,OAAO,SAAS,4CAA4C,KAAK,CAAC,GACjF,OAAO,aAAa;GACjB,KAAK,gBAAgB,UAAU;EAChC,UAAU,SAAS;CACrB,CAAC,GACD,OAAO,QAAQ,OAAO,KAAK,CAC7B,CACF,CACF;AACF;AAEA,MAAM,uBAEF,WACA,UACA,QACA,eAGA,WAEA,OAAO,KACL,OAAO,eACLA,aAAqB;CAAE,WAAW,UAAU;CAAM,UAAU,SAAS;AAAK,GAAG,OAAO,CAAC,CAAC,KACpF,OAAO,QACL,WAAW,KACT,OAAO,SAAS,UAAU,MAAM,iBAAiB,SAAS,MAAM,QAAQ,SAAS,CAAC,CACpF,CACF,CACF,CACF,CACF;AAEJ,MAAM,wBACJ,QACA,WACA,UACA,WAGA,OAAO,KACL,OAAO,OAAO,UACZ,OAAO,SAAS,2BAA2B,KAAK,CAAC,CAAC,KAChD,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,UAAU,SAAS;CACnB;AACF,CAAC,GACD,OAAO,QAAQ,OAAO,KAAK,CAC7B,CACF,CACF;AAEF,MAAM,uBAAuB,OAAO,WAAW,WAC7C,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;CACrE,OAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GACrD,gBAAgB,WAAW,UAAU,KAAK,CAC5C;CAEA,OAAO;AACT,CAAC;AAED,MAAM,0BAA0B,OAAO,WAAW,WAChD,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;CACrE,OAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GACrD,gBAAgB,WAAW,UAAU,KAAK,CAC5C;CAEA,IAAI,SAAS,UACX,OAAOC,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;CAAW,GACzE,MAAM,UAAU,aAClB;CAEF,IAAI,WAAW,WAAW,QAAQ,GAChC,OAAOA,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;CAAU,GACxE,MAAM,SAAS,aACjB;AAEJ,CAAC;AAED,MAAM,cAAc,OAAO,WAAW,WACpC,WACA,UACA,eACA;CACA,IAAI,CAAC,SAAS,UAAU;CACxB,MAAM,QAAQ,OAAO;CACrB,IAAI,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;CAEnE,IAAI,MAAM,UAAU,cAAc,MAAM;EAGtC,OAAO,MACJ,mBAAmB,SAAS,MAAM,UAAU,CAAC,CAC7C,KAAK,oBAAoB,WAAW,UAAU,YAAY,YAAY,CAAC;EAC1E,OAAO,qBAAqB,WAAW,UAAU,aAAa;EAC9D;CACF;CAEA,OAAO,MAAM,UAAU,cAAc,MAAM;EACzC,MAAM,WAAW,MAAM,YAAY;GAAE,QAAQ;GAAe,WAAW;EAAM;EAC7E,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,SAC1B,MAAM;GACL,YAAY,SAAS;GACrB,QAAQ,SAAS;EACnB,CAAC,CAAC,CACD,KAAK,oBAAoB,WAAW,UAAU,YAAY,OAAO,CAAC;EAErE,OAAO,YAAY;GACjB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK,kBAAkB,SAAS;IACxC,MAAM,KAAK;GACb;EACF,CAAC,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,QAAQ,CAAC;EAEtE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;EAC7E,QAAQ;GACN,GAAG;GACH,UAAU;IACR,QAAQ,SAAS;IACjB,YAAY,KAAK;IACjB,WAAW,CAAC,KAAK;IACjB;GACF;EACF;EAIA,OAAO,OAAO,SACZ,MACG,iBAAiB,SAAS,MAAM;GAC/B,QAAQ,gBAAgB,SAAS,MAAM;GACvC,YACE,KAAK,mBAAmB,KAAA,IAAY,KAAA,IAAY,gBAAgB,KAAK,cAAc;GACrF,WAAW,CAAC,KAAK;GACjB;EACF,CAAC,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;EAAW,GACzE,aACF,CACF,GACA,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,UAAU,CAAC,GAClE,OAAO,UAAU,UAAU,cAAc,aAAa,KAAK,CAAC,CAC9D,GACF,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,KAAK,EAAE,CACnD,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,YAAY,CAAC;EAGzE,IAAI,sBAAsB,CAAC,KAAK,SAC9B,QAAQ,OAAO,qBAAqB,WAAW,UAAU,aAAa;CAE1E;AACF,CAAC;AAED,MAAM,aAAa,OAAO,WAAW,WACnC,WACA,UACA,eACA;CACA,IAAI,CAAC,SAAS,SAAS;CACvB,MAAM,QAAQ,OAAO;CAErB,OAAO,MAAM;EACX,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;EACrE,MAAM,SAAS,MAAM,SAAS,UAAU;EACxC,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,QAC1B,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,KAAK,oBAAoB,WAAW,UAAU,WAAW,OAAO,CAAC;EAEpE,OAAO,YAAY;GACjB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK;IACb,MAAM,KAAK;GACb;EACF,CAAC,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,QAAQ,CAAC;EAErE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;EAE7E,OAAO,OAAO,SACZ,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,KAAK,MAAM;GACnC;EACF,CAAC,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;EAAU,GACxE,aACF,CACF,GACA,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,SAAS,CAAC,GACjE,OAAO,UAAU,UAAU,cAAc,aAAa,KAAK,CAAC,CAC9D,GACF,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,KAAK,EAAE,CACnD,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,YAAY,CAAC;EAExE,IAAI,oBACF,OAAO,qBAAqB,WAAW,UAAU,aAAa;EAGhE,IAAI,CAAC,KAAK,SACR,OAAO,OAAO,MAAM,SAAS,QAAQ,YAAY,UAAU;CAE/D;AACF,CAAC;AAKD,MAAM,kBACJ,WACA,MACA,kBAEA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CACrB,MAAM,UAAU,KAAK;CACrB,MAAM,WAAW,IAAI,IAAI,QAAQ,KAAK,aAAa,SAAS,IAAI,CAAC;CAEjE,MAAM,mBACH,eACM,WACL,OAAO,KACL,OAAO,eACL,OAAO,QACL,UACC,aACCD,aACE;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GACrD,OACF,CAAC,CAAC,KAAK,OAAO,QAAQ,MAAM,iBAAiB,SAAS,MAAM,WAAW,SAAS,CAAC,CAAC,GACpF,EAAE,SAAS,KAAK,CAClB,CACF,CACF;CAEJ,OAAO,MAAM;EACX,MAAM,SAAS,OAAO,OAAO,QAAQ,UAAU,aAC7C,MAAM,iBAAiB,SAAS,IAAI,CACtC;EAGA,MAAM,QAAQ,OAAO,SAAS,UAAW,OAAO,UAAU,CAAC,MAAM,OAAO,IAAI,CAAC,CAAE;EAC/E,MAAM,SAASE,MAAI,wBAAwB,KAAK,IAC5CA,MAAI,IAAI,OAAO,SAAS,CAAC,CAAC,SAC1B;EACJ,MAAM,SAAS,OAAO,KAAK,MAAM;GAAE;GAAQ;EAAS,CAAC,CAAC,CAAC,KAAK,gBAAgB,OAAO,CAAC;EAEpF,OAAO,OAAO,QACZ,UACC,aACC,YAAY;GACV,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,OAAO;IACf,MAAM,OAAO,KAAK,SAAS,SAAS,CAAC;GACvC;EACF,CAAC,GACH,EAAE,SAAS,KAAK,CAClB,CAAC,CAAC,KAAK,gBAAgB,QAAQ,CAAC;EAEhC,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;EAC7E,OAAO,OAAO,QACZ,UACC,aACC,OAAO,SACL,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,OAAO,MAAM;GACrC;EACF,CAAC,CAAC,CACD,KACC,OAAO,QACLD,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;EAAU,GACxE,aACF,CACF,GACA,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,SAAS,CAAC,GACjE,OAAO,UAAU,UAAU,cAAc,aAAa,KAAK,CAAC,CAC9D,GACF,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,KAAK,EAAE,CACnD,GACF,EAAE,SAAS,KAAK,CAClB,CAAC,CAAC,KAAK,gBAAgB,YAAY,CAAC;EAEpC,OAAO,OAAO,QAAQ,UAAU,UAAU,UACxC,OAAO,MAAM,EAAE,WAAW,WAAW,YACjC,qBAAqB,WAAW,UAAU,aAAa,IACvD,OAAO,IACb;EAEA,IAAI,CAAC,OAAO,SACV,OAAO,OAAO,MAAM,KAAK,YAAY,UAAU;CAEnD;AACF,CAAC,CAAC,CAAC,KAED,OAAO,OAAO,UACZ,OAAO,SAAS,iCAAiC,KAAK,CAAC,CAAC,KACtD,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,WAAW,KAAK,UAAU,KAAK,aAAa,SAAS,IAAI,CAAC,CAAC,KAAK,GAAG;AACrE,CAAC,GACD,OAAO,QAAQ,OAAO,KAAK,CAC7B,CACF,CACF"}

package/dist/ingestor/index.d.mts

@@ -0,0 +1,3 @@
+import { a as IngestorService, i as Ingestor, n as IngestResult, r as IngestSource, t as IngestOptions } from "../service-DLkhuxYl.mjs";
+import { a as layerConsole, i as layerWingsConfig, n as WingsIngestorConfig, r as layerWings } from "../index-CUPP2A_m.mjs";
+export { type IngestOptions, type IngestResult, type IngestSource, Ingestor, type IngestorService, type WingsIngestorConfig, layerConsole, layerWings, layerWingsConfig };

package/dist/ingestor/index.mjs

@@ -0,0 +1,4 @@
+import { n as layerWingsConfig, t as layerWings } from "../wings-I56hGtNv.mjs";
+import { t as Ingestor } from "../service-jWxxfEht.mjs";
+import { n as layerConsole } from "../ingestor-D4L-HPkg.mjs";
+export { Ingestor, layerConsole, layerWings, layerWingsConfig };

package/dist/ingestor-D4L-HPkg.mjs

@@ -0,0 +1,26 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { n as layerWingsConfig, t as layerWings } from "./wings-I56hGtNv.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { Effect, Layer } from "effect";
+//#region src/ingestor/console.ts
+/** Logs batches without sending them to Wings. */
+const layerConsole = Layer.succeed(Ingestor)({ ingest: ({ resource, source, batch }) => Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(Effect.annotateLogs({
+	rows: batch.rows.length,
+	cursor: batch.cursor,
+	source
+}), Effect.as({
+	rows: batch.rows.length,
+	bytes: 0
+})) });
+//#endregion
+//#region src/ingestor/index.ts
+var ingestor_exports = /* @__PURE__ */ __exportAll({
+	Ingestor: () => Ingestor,
+	layerConsole: () => layerConsole,
+	layerWings: () => layerWings,
+	layerWingsConfig: () => layerWingsConfig
+});
+//#endregion
+export { layerConsole as n, ingestor_exports as t };
+
+//# sourceMappingURL=ingestor-D4L-HPkg.mjs.map

package/dist/ingestor-D4L-HPkg.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestor-D4L-HPkg.mjs","names":[],"sources":["../src/ingestor/console.ts","../src/ingestor/index.ts"],"sourcesContent":["import { Effect, Layer } from \"effect\";\n\nimport { Ingestor } from \"./service\";\n\n/** Logs batches without sending them to Wings. */\nexport const layerConsole = Layer.succeed(Ingestor)({\n  ingest: ({ resource, source, batch }) =>\n    Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(\n      Effect.annotateLogs({ rows: batch.rows.length, cursor: batch.cursor, source }),\n      // Nothing is encoded, so there are no bytes to count.\n      Effect.as({ rows: batch.rows.length, bytes: 0 }),\n    ),\n});\n","export { layerConsole } from \"./console\";\nexport {\n  Ingestor,\n  type IngestOptions,\n  type IngestorService,\n  type IngestResult,\n  type IngestSource,\n} from \"./service\";\nexport { layerWings, layerWingsConfig, type WingsIngestorConfig } from \"./wings\";\n"],"mappings":";;;;;;AAKA,MAAa,eAAe,MAAM,QAAQ,QAAQ,CAAC,CAAC,EAClD,SAAS,EAAE,UAAU,QAAQ,YAC3B,OAAO,QAAQ,yBAAyB,OAAO,eAAe,UAAU,CAAC,CAAC,KACxE,OAAO,aAAa;CAAE,MAAM,MAAM,KAAK;CAAQ,QAAQ,MAAM;CAAQ;AAAO,CAAC,GAE7E,OAAO,GAAG;CAAE,MAAM,MAAM,KAAK;CAAQ,OAAO;AAAE,CAAC,CACjD,EACJ,CAAC"}

package/dist/instrumented-Bhp7fyvh.mjs

@@ -0,0 +1,29 @@
+import { S as addCurrentSpanEvent, _ as withResourceAttributes, b as EventName, i as ingestDuration, n as batchSize, r as batches, u as rowsIngested, v as Attr, x as SpanName, y as EventAttr } from "./metrics-BaEX__MN.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { Effect, Metric } from "effect";
+//#region src/ingestor/instrumented.ts
+const ingestBatch = Effect.fnUntraced(function* (options) {
+	const metricAttributes = withResourceAttributes({
+		connector: options.connector ?? "unknown",
+		resource: options.resource,
+		source: options.source
+	});
+	const batchOutcome = (outcome) => Metric.withAttributes(batches, {
+		...metricAttributes,
+		[Attr.batchOutcome]: outcome
+	});
+	const ingest = (yield* Ingestor).ingest(options).pipe(Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })), Effect.tapError(() => Effect.annotateCurrentSpan({
+		[Attr.ingestionSuccess]: false,
+		[Attr.errorPhase]: "ingest"
+	})));
+	const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: "producer" }).pipe(Effect.trackDuration(Metric.withAttributes(ingestDuration, metricAttributes)), Effect.onError(() => Metric.update(batchOutcome("error"), 1)));
+	yield* Metric.update(batchOutcome("success"), 1);
+	yield* Metric.update(Metric.withAttributes(rowsIngested, metricAttributes), options.batch.rows.length);
+	yield* Metric.update(Metric.withAttributes(batchSize, metricAttributes), options.batch.rows.length);
+	if (options.batch.cursor !== void 0) yield* addCurrentSpanEvent(EventName.batchCheckpoint, { [EventAttr.batchCursor]: options.batch.cursor });
+	return result;
+});
+//#endregion
+export { ingestBatch as t };
+
+//# sourceMappingURL=instrumented-Bhp7fyvh.mjs.map

package/dist/instrumented-Bhp7fyvh.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"instrumented-Bhp7fyvh.mjs","names":["Metrics.withResourceAttributes","Metrics.batches","Metrics.ingestDuration","Metrics.rowsIngested","Metrics.batchSize"],"sources":["../src/ingestor/instrumented.ts"],"sourcesContent":["import { Effect, Metric } from \"effect\";\n\nimport type { ResourceBatch } from \"../core/types\";\n\nimport * as Metrics from \"../metrics\";\nimport { Attr, EventAttr, EventName, SpanName, addCurrentSpanEvent } from \"../telemetry\";\nimport { Ingestor, type IngestSource } from \"./service\";\n\nexport const ingestBatch = Effect.fnUntraced(function* (options: {\n  readonly connector?: string;\n  readonly resource: string;\n  readonly source: IngestSource;\n  readonly batch: ResourceBatch;\n}) {\n  const metricAttributes = Metrics.withResourceAttributes({\n    connector: options.connector ?? \"unknown\",\n    resource: options.resource,\n    source: options.source,\n  });\n  const batchOutcome = (outcome: \"success\" | \"error\") =>\n    Metric.withAttributes(Metrics.batches, {\n      ...metricAttributes,\n      [Attr.batchOutcome]: outcome,\n    });\n\n  const ingestor = yield* Ingestor;\n\n  const ingest = ingestor.ingest(options).pipe(\n    Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })),\n    Effect.tapError(() =>\n      Effect.annotateCurrentSpan({\n        [Attr.ingestionSuccess]: false,\n        [Attr.errorPhase]: \"ingest\",\n      }),\n    ),\n  );\n\n  // onError keeps the failure, so the engine still sees it and skips the checkpoint.\n  const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: \"producer\" }).pipe(\n    Effect.trackDuration(Metric.withAttributes(Metrics.ingestDuration, metricAttributes)),\n    Effect.onError(() => Metric.update(batchOutcome(\"error\"), 1)),\n  );\n\n  yield* Metric.update(batchOutcome(\"success\"), 1);\n  yield* Metric.update(\n    Metric.withAttributes(Metrics.rowsIngested, metricAttributes),\n    options.batch.rows.length,\n  );\n  yield* Metric.update(\n    Metric.withAttributes(Metrics.batchSize, metricAttributes),\n    options.batch.rows.length,\n  );\n\n  if (options.batch.cursor !== undefined) {\n    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {\n      [EventAttr.batchCursor]: options.batch.cursor,\n    });\n  }\n\n  return result;\n});\n"],"mappings":";;;;AAQA,MAAa,cAAc,OAAO,WAAW,WAAW,SAKrD;CACD,MAAM,mBAAmBA,uBAA+B;EACtD,WAAW,QAAQ,aAAa;EAChC,UAAU,QAAQ;EAClB,QAAQ,QAAQ;CAClB,CAAC;CACD,MAAM,gBAAgB,YACpB,OAAO,eAAeC,SAAiB;EACrC,GAAG;GACF,KAAK,eAAe;CACvB,CAAC;CAIH,MAAM,UAAS,OAFS,SAAA,CAEA,OAAO,OAAO,CAAC,CAAC,KACtC,OAAO,UAAU,OAAO,oBAAoB,GAAG,KAAK,mBAAmB,KAAK,CAAC,CAAC,GAC9E,OAAO,eACL,OAAO,oBAAoB;GACxB,KAAK,mBAAmB;GACxB,KAAK,aAAa;CACrB,CAAC,CACH,CACF;CAGA,MAAM,SAAS,OAAO,OAAO,SAAS,QAAQ,SAAS,QAAQ,EAAE,MAAM,WAAW,CAAC,CAAC,CAAC,KACnF,OAAO,cAAc,OAAO,eAAeC,gBAAwB,gBAAgB,CAAC,GACpF,OAAO,cAAc,OAAO,OAAO,aAAa,OAAO,GAAG,CAAC,CAAC,CAC9D;CAEA,OAAO,OAAO,OAAO,aAAa,SAAS,GAAG,CAAC;CAC/C,OAAO,OAAO,OACZ,OAAO,eAAeC,cAAsB,gBAAgB,GAC5D,QAAQ,MAAM,KAAK,MACrB;CACA,OAAO,OAAO,OACZ,OAAO,eAAeC,WAAmB,gBAAgB,GACzD,QAAQ,MAAM,KAAK,MACrB;CAEA,IAAI,QAAQ,MAAM,WAAW,KAAA,GAC3B,OAAO,oBAAoB,UAAU,iBAAiB,GACnD,UAAU,cAAc,QAAQ,MAAM,OACzC,CAAC;CAGH,OAAO;AACT,CAAC"}

package/dist/manifest/index.d.mts

@@ -0,0 +1,2 @@
+import { _ as number, a as configSchema, b as select, c as ConnectorManifest, d as ConfigField, f as ConfigFieldSpec, g as define$1, h as boolean, i as RuntimeConfigDocument, l as ResourceCapability, m as ConnectorConfigDef, n as ConnectorConfigSchema, o as decodeConfig, p as ConfigValuesOf, r as ConnectorConfigValues, s as toRuntimeDocument, u as define, v as optional, x as string, y as secret } from "../index-eFuoMZZ3.mjs";
+export { type ConfigField, type ConfigFieldSpec, type ConfigValuesOf, type ConnectorConfigDef, type ConnectorConfigSchema, type ConnectorConfigValues, type ConnectorManifest, type ResourceCapability, type RuntimeConfigDocument, boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };

package/dist/manifest/{index.js → index.mjs}

@@ -1,4 +1,2 @@
-import "../constants-BRhIX_rz.js";
-import { a as define, c as number, d as select, f as string, i as toRuntimeDocument, l as optional, n as configSchema, o as boolean, r as decodeConfig, s as define$1, u as secret } from "../manifest-ByE6YPG0.js";
-
-export { boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };
+import { a as define, c as number, d as select, f as string, i as toRuntimeDocument, l as optional, n as configSchema, o as boolean, r as decodeConfig, s as define$1, u as secret } from "../manifest-BGKxmKLy.mjs";
+export { boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };