@useairfoil/connector-kit 0.6.0 → 0.7.0

package/dist/index.d.mts

@@ -0,0 +1,16 @@
+import { t as index_d_exports } from "./index-BX8G7d41.mjs";
+import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
+import { t as Identified } from "./annotations-vzSzXtr-.mjs";
+import { A as WebhookRouteContext, C as ResourceUpdateOf, D as WebhookAckMode, E as TableSource, O as WebhookHandler, S as ResourceUpdate, T as TableResource, _ as ResourceRequiredField, a as FeedRows, b as ResourceSchema, c as FetchPageResult, d as ResourceDefinition, f as ResourceDelete, g as ResourcePayload, i as Cursor$1, k as WebhookRoute, l as PageFetch, m as ResourceKey, n as ChangesFetch, o as FetchChangesResult, p as ResourceField, r as ConnectorDefinition, s as FetchFeedResult, t as ChangesFeed, u as ResourceBatch, v as ResourceRow, x as ResourceState, y as ResourceRows } from "./types-NtSv6gwl.mjs";
+import { d as Cursor, f as Fetch, l as telemetry_d_exports, p as Resource, t as index_d_exports$1, u as Connector } from "./index-CZKx2Yce.mjs";
+import { t as index_d_exports$2 } from "./index-CV6V6fyx.mjs";
+import { t as index_d_exports$3 } from "./index-BTAbexBp.mjs";
+import { t as index_d_exports$8 } from "./index-CgvFomO6.mjs";
+import { t as index_d_exports$4 } from "./index-DeTwIUKl.mjs";
+import { t as index_d_exports$5 } from "./index-uFa2wwYo.mjs";
+import { t as index_d_exports$7 } from "./index-fHIJJoRe.mjs";
+import { t as index_d_exports$6 } from "./index-eFuoMZZ3.mjs";
+import { h as metrics_d_exports } from "./metrics-DoJ0WeKu.mjs";
+import { t as index_d_exports$9 } from "./index-Byr8NTdG.mjs";
+import { t as index_d_exports$10 } from "./index-dHS59EZl.mjs";
+export { index_d_exports as Auth, type ChangesFeed, type ChangesFetch, Connector, index_d_exports$1 as ConnectorApp, type ConnectorDefinition, ConnectorError, Cursor, type Cursor$1 as CursorTypes, index_d_exports$2 as Demo, type FeedRows, Fetch, type FetchChangesResult, type FetchFeedResult, type FetchPageResult, index_d_exports$3 as Iceberg, type Identified, index_d_exports$4 as Ingestion, index_d_exports$5 as Ingestor, index_d_exports$6 as Manifest, metrics_d_exports as Metrics, type PageFetch, Resource, type ResourceBatch, type ResourceDefinition, type ResourceDelete, type ResourceField, type ResourceKey, type ResourcePayload, type ResourceRequiredField, type ResourceRow, type ResourceRows, type ResourceSchema, type ResourceState, type ResourceUpdate, type ResourceUpdateOf, index_d_exports$7 as RuntimeConfig, index_d_exports$8 as StateStore, index_d_exports$9 as Status, type TableResource, type TableSource, telemetry_d_exports as Telemetry, index_d_exports$10 as Webhook, type WebhookAckMode, type WebhookHandler, type WebhookRoute, type WebhookRouteContext };

package/dist/index.mjs

@@ -0,0 +1,45 @@
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { t as auth_exports } from "./auth/index.mjs";
+import { t as connector_app_exports } from "./connector-app-C6_dZnhE.mjs";
+import { t as runtime_config_exports } from "./runtime-config-Bn8Fivt1.mjs";
+import { t as demo_exports } from "./demo-DqTJ7TWI.mjs";
+import { o as metrics_exports, w as telemetry_exports } from "./metrics-C901IIYZ.mjs";
+import { t as state_store_exports } from "./state-store-DhSrKVEH.mjs";
+import { t as status_exports } from "./status-4h6Oxh-O.mjs";
+import { t as ingestion_exports } from "./ingestion-DoCUwm5N.mjs";
+import { t as iceberg_exports } from "./iceberg-Cj2PwlNm.mjs";
+import { t as ingestor_exports } from "./ingestor-DjCEzBzZ.mjs";
+import { t as manifest_exports } from "./manifest-CABeIhAG.mjs";
+import { t as webhook_exports } from "./webhook/index.mjs";
+import { DateTime, Effect, Option } from "effect";
+//#region src/core/builder.ts
+const Connector = { define: (definition) => definition };
+const Resource = { entity: (definition) => definition };
+const Fetch = {
+	page: (definition) => definition,
+	changes: (definition) => definition,
+	feed: (definition) => definition
+};
+const decodeFailure = (kind, value) => new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });
+const Cursor = {
+	string: () => ({
+		kind: "string",
+		decode: (value) => typeof value === "string" ? Effect.succeed(value) : Effect.fail(decodeFailure("string", value))
+	}),
+	number: () => ({
+		kind: "number",
+		decode: (value) => typeof value === "number" && Number.isFinite(value) ? Effect.succeed(value) : Effect.fail(decodeFailure("number", value))
+	}),
+	isoDateTime: () => ({
+		kind: "isoDateTime",
+		decode: (value) => {
+			if (typeof value !== "string" || Option.isNone(DateTime.make(value))) return Effect.fail(decodeFailure("ISO datetime", value));
+			return Effect.succeed(value);
+		}
+	}),
+	nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso)
+};
+//#endregion
+export { auth_exports as Auth, Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, demo_exports as Demo, Fetch, iceberg_exports as Iceberg, ingestion_exports as Ingestion, ingestor_exports as Ingestor, manifest_exports as Manifest, metrics_exports as Metrics, Resource, runtime_config_exports as RuntimeConfig, state_store_exports as StateStore, status_exports as Status, telemetry_exports as Telemetry, webhook_exports as Webhook };
+
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect, Option } from \"effect\";\n\nimport type {\n  ChangesFeed,\n  ChangesFetch,\n  ConnectorDefinition,\n  Cursor as CursorType,\n  PageFetch,\n  ResourceDefinition,\n  ResourceRequiredField,\n  ResourceSchema,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n  define: <\n    const Resources extends ReadonlyArray<ResourceDefinition>,\n    const FeedResources extends ReadonlyArray<Resources[number]> = Resources,\n  >(\n    definition: ConnectorDefinition<Resources, FeedResources>,\n  ) => definition,\n};\n\nexport const Resource = {\n  entity: <\n    S extends ResourceSchema,\n    Payload = never,\n    R = never,\n    const Name extends string = string,\n    const Version extends ResourceRequiredField<S> = ResourceRequiredField<S>,\n  >(\n    definition: ResourceDefinition<S, Payload, R, Name, Version>,\n  ): ResourceDefinition<S, Payload, R, Name, Version> => definition,\n};\n\nexport const Fetch = {\n  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n  feed: <const Resources extends ReadonlyArray<ResourceDefinition>, R = never>(\n    definition: ChangesFeed<Resources, R>,\n  ) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n  string: (): CursorType.Definition<string> => ({\n    kind: \"string\",\n    decode: (value) =>\n      typeof value === \"string\"\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"string\", value)),\n  }),\n\n  number: (): CursorType.Definition<number> => ({\n    kind: \"number\",\n    decode: (value) =>\n      typeof value === \"number\" && Number.isFinite(value)\n        ? Effect.succeed(value)\n        : Effect.fail(decodeFailure(\"number\", value)),\n  }),\n\n  isoDateTime: (): CursorType.Definition<string> => ({\n    kind: \"isoDateTime\",\n    decode: (value) => {\n      if (typeof value !== \"string\" || Option.isNone(DateTime.make(value))) {\n        return Effect.fail(decodeFailure(\"ISO datetime\", value));\n      }\n      return Effect.succeed(value);\n    },\n  }),\n\n  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;;;;;AAeA,MAAa,YAAY,EACvB,SAIE,eACG,WACP;AAEA,MAAa,WAAW,EACtB,SAOE,eACqD,WACzD;AAEA,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACtF,OACE,eACG;AACP;AAEA,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,KAAK,IAAI,CAAC;AAElF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,KAAK,IACpB,OAAO,KAAK,cAAc,UAAU,KAAK,CAAC;CAClD;CAEA,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAC9C,OAAO,QAAQ,KAAK,IACpB,OAAO,KAAK,cAAc,UAAU,KAAK,CAAC;CAClD;CAEA,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;GACjB,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,SAAS,KAAK,KAAK,CAAC,GACjE,OAAO,OAAO,KAAK,cAAc,gBAAgB,KAAK,CAAC;GAEzD,OAAO,OAAO,QAAQ,KAAK;EAC7B;CACF;CAEA,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,SAAS;AAC7D"}

package/dist/ingestion/index.d.mts

@@ -0,0 +1,2 @@
+import { n as RunOptions, r as run } from "../index-DeTwIUKl.mjs";
+export { type RunOptions, run };

package/dist/ingestion/index.mjs

@@ -0,0 +1,2 @@
+import { n as run } from "../ingestion-DoCUwm5N.mjs";
+export { run };

package/dist/ingestion-DoCUwm5N.mjs

@@ -0,0 +1,275 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { C as annotateError, d as setLastSuccessTimestamp, f as setSyncState, p as setWebhookQueueDepth, v as Attr, x as SpanName } from "./metrics-C901IIYZ.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { t as ingestBatch } from "./instrumented-Bq1WX2L5.mjs";
+import { o as StateStore } from "./state-store-DhSrKVEH.mjs";
+import { c as deriveSyncState, l as hasChanges, s as statusRoute, u as normalizeCursor } from "./status-4h6Oxh-O.mjs";
+import { n as router, t as WebhookQueue } from "./server-D1TTN0_D.mjs";
+import { Array, DateTime, Effect, Layer, Order, Queue } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/http";
+import * as Observability from "effect/observability";
+//#region src/ingestion/engine.ts
+function run(definition, options) {
+	return Effect.gen(function* () {
+		if (definition.changes) yield* validateFeed(definition, definition.changes);
+		const ingestor = yield* Ingestor;
+		const connector = selectResources(definition, ingestor.resourceNames);
+		const queueLayer = makeWebhookQueueLayer(connector.name);
+		const runtimeLayer = options?.webhook ? Layer.mergeAll(queueLayer, makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer))) : Layer.empty;
+		return yield* Effect.gen(function* () {
+			const initialCutoff = options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+			yield* Effect.logInfo("Connector started").pipe(Effect.annotateLogs({
+				[Attr.connectorName]: connector.name,
+				resources: connector.resources.length
+			}));
+			const sourceRun = runIngestion(connector, initialCutoff);
+			const webhookRun = options?.webhook ? runWebhookRuntime(connector, options.webhook.routes) : Effect.void;
+			return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+		}).pipe(Effect.provide(runtimeLayer));
+	});
+}
+const selectResources = (connector, names) => {
+	if (names === void 0) return connector;
+	const isSelected = (resource) => names.includes(resource.name);
+	const { changes, ...rest } = connector;
+	const feedResources = changes?.resources.filter(isSelected) ?? [];
+	return {
+		...rest,
+		resources: connector.resources.filter(isSelected),
+		...changes && feedResources.length > 0 ? { changes: {
+			...changes,
+			resources: feedResources
+		} } : {}
+	};
+};
+const makeWebhookQueueLayer = (connector) => Layer.effect(WebhookQueue)(Effect.gen(function* () {
+	const queue = yield* Queue.bounded(1024);
+	yield* setWebhookQueueDepth(connector, 0);
+	return WebhookQueue.of({ queue });
+}));
+const runWebhookRuntime = (connector, routes) => {
+	const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
+	return Effect.all([hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never], { concurrency: "unbounded" }).pipe(Effect.asVoid);
+};
+const runWebhookQueueConsumer = (connector) => Effect.gen(function* () {
+	const queue = yield* WebhookQueue;
+	while (true) {
+		const batch = yield* Queue.take(queue.queue);
+		const depth = yield* Queue.size(queue.queue);
+		yield* setWebhookQueueDepth(connector.name, depth);
+		yield* ingestBatch({
+			connector: connector.name,
+			resource: batch.resource,
+			source: "webhook",
+			batch: batch.batch
+		});
+	}
+});
+const makeWebhookServerLayer = (connector, options) => {
+	const healthPath = options.healthPath ?? "/health";
+	const metricsPath = options.metricsPath === void 0 ? "/metrics" : options.metricsPath;
+	const statusPath = options.statusPath === void 0 ? "/status" : options.statusPath;
+	const app = Layer.mergeAll(router(options.routes, {
+		connectorName: connector.name,
+		resourceNames: connector.resources.map((resource) => resource.name)
+	}), HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))), ...metricsPath === false ? [] : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })], ...statusPath === false ? [] : [statusRoute(connector, statusPath)]);
+	return HttpRouter.serve(app, { disableLogger: options.disableHttpLogger ?? true });
+};
+const runIngestion = (connector, initialCutoff) => Effect.gen(function* () {
+	const feed = connector.changes;
+	yield* Effect.all([Effect.forEach(connector.resources, (resource) => runResourceSources(connector, resource, initialCutoff), { concurrency: "unbounded" }), feed ? runChangesFeed(connector, feed, initialCutoff) : Effect.void], { concurrency: "unbounded" });
+});
+const bySavedAt = Order.mapInput(Order.String, (state) => state.lastSuccessAt ?? "");
+const validateFeed = (connector, feed) => Effect.forEach(feed.resources, (member, index) => {
+	if (feed.resources.findIndex((item) => item.name === member.name) !== index) return Effect.fail(new ConnectorError({ message: `Resource ${member.name} is in the changes feed twice` }));
+	const resource = connector.resources.find((item) => item.name === member.name);
+	if (!resource) return Effect.fail(new ConnectorError({ message: `Changes feed resource ${member.name} is not a resource of connector ${connector.name}` }));
+	if (resource.changes) return Effect.fail(new ConnectorError({ message: `Resource ${member.name} has its own changes and is also in the changes feed` }));
+	return Effect.void;
+}).pipe(Effect.andThen(feed.resources.length === 0 ? Effect.fail(new ConnectorError({ message: "The changes feed has no resources" })) : Effect.void));
+const initializeResourceState = (existing, initialCutoff) => ({
+	...existing,
+	changes: existing?.changes ?? { cursor: initialCutoff },
+	backfill: existing?.backfill ?? {
+		cutoff: initialCutoff,
+		completed: false
+	}
+});
+const getInitializedState = Effect.fnUntraced(function* (resource, initialCutoff) {
+	const existing = yield* (yield* StateStore).getResourceState(resource);
+	return initializeResourceState(existing, initialCutoff);
+});
+const runResourceSources = (connector, resource, initialCutoff) => {
+	const runs = [resource.backfill ? isolateSourceFailure(runBackfill(connector, resource, initialCutoff), connector, resource, "backfill") : Effect.void, resource.changes ? isolateSourceFailure(runChanges(connector, resource, initialCutoff), connector, resource, "changes") : Effect.void];
+	return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })), Effect.asVoid, Effect.catch((error) => setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, "error").pipe(Effect.andThen(Effect.logError("Connector resource initialization failed", error)), Effect.annotateLogs({
+		[Attr.connectorName]: connector.name,
+		resource: resource.name
+	}), Effect.andThen(Effect.never))));
+};
+const recordResourceError = (connector, resource, source, operation) => (effect) => effect.pipe(Effect.tapError(() => setSyncState({
+	connector: connector.name,
+	resource: resource.name
+}, "error").pipe(Effect.andThen(StateStore.pipe(Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)))))));
+const isolateSourceFailure = (effect, connector, resource, source) => effect.pipe(Effect.catch((error) => Effect.logError("Connector source parked", error).pipe(Effect.annotateLogs({
+	[Attr.connectorName]: connector.name,
+	resource: resource.name,
+	source
+}), Effect.andThen(Effect.never))));
+const refreshRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(connector, resource, state));
+	return state;
+});
+const initializeRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	const state = yield* getInitializedState(resource.name, initialCutoff);
+	yield* setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, deriveSyncState(connector, resource, state));
+	if (resource.backfill) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "backfill"
+	}, state.backfill?.lastSuccessAt);
+	if (hasChanges(connector, resource)) yield* setLastSuccessTimestamp({
+		connector: connector.name,
+		resource: resource.name,
+		source: "changes"
+	}, state.changes?.lastSuccessAt);
+});
+const runBackfill = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.backfill) return;
+	const store = yield* StateStore;
+	let state = yield* getInitializedState(resource.name, initialCutoff);
+	if (state.backfill?.completed === true) {
+		yield* store.clearResourceError(resource.name, "backfill").pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		return;
+	}
+	while (state.backfill?.completed !== true) {
+		const backfill = state.backfill ?? {
+			cutoff: initialCutoff,
+			completed: false
+		};
+		const clearsVisibleError = state.lastError?.source === "backfill";
+		const page = yield* resource.backfill.fetch({
+			pageCursor: backfill.pageCursor,
+			cutoff: backfill.cutoff
+		}).pipe(recordResourceError(connector, resource, "backfill", "fetch"));
+		yield* ingestBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill",
+			batch: {
+				cursor: page.nextPageCursor ?? backfill.cutoff,
+				rows: page.rows
+			}
+		}).pipe(recordResourceError(connector, resource, "backfill", "ingest"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		state = {
+			...state,
+			backfill: {
+				cutoff: backfill.cutoff,
+				pageCursor: page.nextPageCursor,
+				completed: !page.hasMore,
+				lastSuccessAt
+			}
+		};
+		yield* Effect.withSpan(store.setBackfillState(resource.name, {
+			cutoff: normalizeCursor(backfill.cutoff),
+			pageCursor: page.nextPageCursor === void 0 ? void 0 : normalizeCursor(page.nextPageCursor),
+			completed: !page.hasMore,
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "backfill"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "backfill")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
+		if (clearsVisibleError || !page.hasMore) state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+	}
+});
+const runChanges = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
+	if (!resource.changes) return;
+	const store = yield* StateStore;
+	while (true) {
+		const state = yield* getInitializedState(resource.name, initialCutoff);
+		const cursor = state.changes?.cursor ?? initialCutoff;
+		const clearsVisibleError = state.lastError?.source === "changes";
+		const page = yield* resource.changes.fetch({ cursor }).pipe(recordResourceError(connector, resource, "changes", "fetch"));
+		yield* ingestBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes",
+			batch: {
+				cursor: page.cursor,
+				rows: page.rows
+			}
+		}).pipe(recordResourceError(connector, resource, "changes", "ingest"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		yield* Effect.withSpan(store.setChangesState(resource.name, {
+			cursor: normalizeCursor(page.cursor),
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
+		if (clearsVisibleError) yield* refreshRuntimeStatus(connector, resource, initialCutoff);
+		if (!page.hasMore) yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+	}
+});
+const runChangesFeed = (connector, feed, initialCutoff) => Effect.gen(function* () {
+	const store = yield* StateStore;
+	const members = feed.resources;
+	const selected = new Set(members.map((resource) => resource.name));
+	const recordFeedError = (operation) => (effect) => effect.pipe(Effect.tapError(() => Effect.forEach(members, (resource) => setSyncState({
+		connector: connector.name,
+		resource: resource.name
+	}, "error").pipe(Effect.andThen(store.setResourceError(resource.name, "changes", operation))), { discard: true })));
+	while (true) {
+		const states = yield* Effect.forEach(members, (resource) => store.getResourceState(resource.name));
+		const saved = states.flatMap((state) => state?.changes ? [state.changes] : []);
+		const cursor = Array.isReadonlyArrayNonEmpty(saved) ? Array.min(saved, bySavedAt).cursor : initialCutoff;
+		const result = yield* feed.fetch({
+			cursor,
+			selected
+		}).pipe(recordFeedError("fetch"));
+		yield* Effect.forEach(members, (resource) => ingestBatch({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes",
+			batch: {
+				cursor: result.cursor,
+				rows: result.rows[resource.name] ?? []
+			}
+		}), { discard: true }).pipe(recordFeedError("ingest"));
+		const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+		yield* Effect.forEach(members, (resource) => Effect.withSpan(store.setChangesState(resource.name, {
+			cursor: normalizeCursor(result.cursor),
+			lastSuccessAt
+		}).pipe(Effect.andThen(setLastSuccessTimestamp({
+			connector: connector.name,
+			resource: resource.name,
+			source: "changes"
+		}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }), { discard: true }).pipe(recordFeedError("checkpoint"));
+		yield* Effect.forEach(members, (resource, index) => states[index]?.lastError?.source === "changes" ? refreshRuntimeStatus(connector, resource, initialCutoff) : Effect.void);
+		if (!result.hasMore) yield* Effect.sleep(feed.interval ?? "1 minute");
+	}
+}).pipe(Effect.catch((error) => Effect.logError("Connector changes feed parked", error).pipe(Effect.annotateLogs({
+	[Attr.connectorName]: connector.name,
+	resources: feed.resources.map((resource) => resource.name).join(",")
+}), Effect.andThen(Effect.never))));
+//#endregion
+//#region src/ingestion/index.ts
+var ingestion_exports = /* @__PURE__ */ __exportAll({ run: () => run });
+//#endregion
+export { run as n, ingestion_exports as t };
+
+//# sourceMappingURL=ingestion-DoCUwm5N.mjs.map

package/dist/ingestion-DoCUwm5N.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestion-DoCUwm5N.mjs","names":["Metrics.setWebhookQueueDepth","Metrics.setSyncState","Metrics.setLastSuccessTimestamp","Arr"],"sources":["../src/ingestion/engine.ts","../src/ingestion/index.ts"],"sourcesContent":["import { Array as Arr, DateTime, Effect, Layer, Order, Queue } from \"effect\";\nimport { HttpRouter, type HttpServer, HttpServerResponse } from \"effect/http\";\nimport * as Observability from \"effect/observability\";\n\nimport type {\n  ChangesFeed,\n  ConnectorDefinition,\n  Cursor,\n  ResourceDefinition,\n  ResourceState,\n  WebhookRoute,\n} from \"../core/types\";\nimport type { StateOperation, StateSource } from \"../state-store/schema\";\n\nimport { ConnectorError } from \"../errors\";\nimport { ingestBatch } from \"../ingestor/instrumented\";\nimport { Ingestor } from \"../ingestor/service\";\nimport * as Metrics from \"../metrics\";\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, hasChanges, normalizeCursor } from \"../state-store/state\";\nimport { statusRoute } from \"../status\";\nimport { Attr, SpanName, annotateError } from \"../telemetry\";\nimport { router, WebhookQueue, type QueuedWebhookBatch } from \"../webhook/server\";\n\ntype RunBaseOptions = {\n  readonly initialCutoff?: Cursor.Value;\n};\n\nexport type RunOptions = RunBaseOptions & {\n  readonly webhook?: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  };\n};\n\ntype RunNoWebhookOptions = RunBaseOptions & {\n  readonly webhook?: undefined;\n};\n\ntype RunWebhookOptions = RunOptions & {\n  readonly webhook: NonNullable<RunOptions[\"webhook\"]>;\n};\n\n/**\n * Runs all connector ingestion loops. Callers provide the Ingestor and\n * StateStore layers; webhook mode additionally requires an HttpServer layer.\n */\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options?: RunNoWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Ingestor>;\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n  connector: ConnectorDefinition<Resources>,\n  options: RunWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Ingestor | HttpServer.HttpServer>;\nexport function run(definition: ConnectorDefinition, options?: RunOptions) {\n  return Effect.gen(function* () {\n    // Check the whole feed, even the resources that don't run.\n    if (definition.changes) yield* validateFeed(definition, definition.changes);\n    const ingestor = yield* Ingestor;\n    const connector = selectResources(definition, ingestor.resourceNames);\n    const queueLayer = makeWebhookQueueLayer(connector.name);\n    const runtimeLayer = options?.webhook\n      ? Layer.mergeAll(\n          queueLayer,\n          makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),\n        )\n      : Layer.empty;\n\n    return yield* Effect.gen(function* () {\n      const initialCutoff =\n        options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n      yield* Effect.logInfo(\"Connector started\").pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resources: connector.resources.length,\n        }),\n      );\n\n      const sourceRun = runIngestion(connector, initialCutoff);\n      const webhookRun = options?.webhook\n        ? runWebhookRuntime(connector, options.webhook.routes)\n        : Effect.void;\n\n      return yield* Effect.all([sourceRun, webhookRun], { concurrency: \"unbounded\" }).pipe(\n        Effect.asVoid,\n      );\n    }).pipe(Effect.provide(runtimeLayer));\n  });\n}\n\nconst selectResources = (\n  connector: ConnectorDefinition,\n  names: ReadonlyArray<string> | undefined,\n): ConnectorDefinition => {\n  if (names === undefined) return connector;\n  const isSelected = (resource: ResourceDefinition) => names.includes(resource.name);\n  const { changes, ...rest } = connector;\n  const feedResources = changes?.resources.filter(isSelected) ?? [];\n  return {\n    ...rest,\n    resources: connector.resources.filter(isSelected),\n    ...(changes && feedResources.length > 0\n      ? { changes: { ...changes, resources: feedResources } }\n      : {}),\n  };\n};\n\nconst makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>\n  Layer.effect(WebhookQueue)(\n    Effect.gen(function* () {\n      const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);\n      yield* Metrics.setWebhookQueueDepth(connector, 0);\n      return WebhookQueue.of({ queue });\n    }),\n  );\n\nconst runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {\n  const hasAfterEnqueue = routes.some((route) => route.ackMode === \"after-enqueue\");\n  return Effect.all(\n    [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],\n    {\n      concurrency: \"unbounded\",\n    },\n  ).pipe(Effect.asVoid);\n};\n\nconst runWebhookQueueConsumer = (connector: ConnectorDefinition) =>\n  Effect.gen(function* () {\n    const queue = yield* WebhookQueue;\n    while (true) {\n      const batch = yield* Queue.take(queue.queue);\n      const depth = yield* Queue.size(queue.queue);\n      yield* Metrics.setWebhookQueueDepth(connector.name, depth);\n      yield* ingestBatch({\n        connector: connector.name,\n        resource: batch.resource,\n        source: \"webhook\",\n        batch: batch.batch,\n      });\n    }\n  });\n\nconst makeWebhookServerLayer = (\n  connector: ConnectorDefinition,\n  options: {\n    readonly routes: ReadonlyArray<WebhookRoute>;\n    readonly healthPath?: HttpRouter.PathInput;\n    readonly metricsPath?: HttpRouter.PathInput | false;\n    readonly statusPath?: HttpRouter.PathInput | false;\n    readonly disableHttpLogger?: boolean;\n  },\n): Layer.Layer<never, never, HttpServer.HttpServer> => {\n  const healthPath: HttpRouter.PathInput = options.healthPath ?? \"/health\";\n  const metricsPath = options.metricsPath === undefined ? \"/metrics\" : options.metricsPath;\n  const statusPath = options.statusPath === undefined ? \"/status\" : options.statusPath;\n  const app = Layer.mergeAll(\n    router(options.routes, {\n      connectorName: connector.name,\n      resourceNames: connector.resources.map((resource) => resource.name),\n    }),\n    HttpRouter.add(\"GET\", healthPath, Effect.succeed(HttpServerResponse.text(\"ok\"))),\n    ...(metricsPath === false\n      ? []\n      : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),\n    ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),\n  );\n\n  return HttpRouter.serve(app, {\n    disableLogger: options.disableHttpLogger ?? true,\n  });\n};\n\nconst runIngestion = (\n  connector: ConnectorDefinition,\n  initialCutoff: Cursor.Value,\n): Effect.Effect<void, ConnectorError, StateStore | Ingestor> =>\n  Effect.gen(function* () {\n    const feed = connector.changes;\n    yield* Effect.all(\n      [\n        Effect.forEach(\n          connector.resources,\n          (resource) => runResourceSources(connector, resource, initialCutoff),\n          { concurrency: \"unbounded\" },\n        ),\n        feed ? runChangesFeed(connector, feed, initialCutoff) : Effect.void,\n      ],\n      { concurrency: \"unbounded\" },\n    );\n  });\n\nconst bySavedAt = Order.mapInput(\n  Order.String,\n  (state: NonNullable<ResourceState[\"changes\"]>) => state.lastSuccessAt ?? \"\",\n);\n\nconst validateFeed = (connector: ConnectorDefinition, feed: ChangesFeed) =>\n  Effect.forEach(feed.resources, (member, index) => {\n    if (feed.resources.findIndex((item) => item.name === member.name) !== index) {\n      return Effect.fail(\n        new ConnectorError({ message: `Resource ${member.name} is in the changes feed twice` }),\n      );\n    }\n    const resource = connector.resources.find((item) => item.name === member.name);\n    if (!resource) {\n      return Effect.fail(\n        new ConnectorError({\n          message: `Changes feed resource ${member.name} is not a resource of connector ${connector.name}`,\n        }),\n      );\n    }\n    if (resource.changes) {\n      return Effect.fail(\n        new ConnectorError({\n          message: `Resource ${member.name} has its own changes and is also in the changes feed`,\n        }),\n      );\n    }\n    return Effect.void;\n  }).pipe(\n    Effect.andThen(\n      feed.resources.length === 0\n        ? Effect.fail(new ConnectorError({ message: \"The changes feed has no resources\" }))\n        : Effect.void,\n    ),\n  );\n\nconst initializeResourceState = (\n  existing: ResourceState | undefined,\n  initialCutoff: Cursor.Value,\n): ResourceState => ({\n  ...existing,\n  changes: existing?.changes ?? { cursor: initialCutoff },\n  backfill: existing?.backfill ?? {\n    cutoff: initialCutoff,\n    completed: false,\n  },\n});\n\nconst getInitializedState = Effect.fnUntraced(function* (\n  resource: string,\n  initialCutoff: Cursor.Value,\n) {\n  const store = yield* StateStore;\n  const existing = yield* store.getResourceState(resource);\n  return initializeResourceState(existing, initialCutoff);\n});\n\nconst runResourceSources = (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) => {\n  const runs = [\n    resource.backfill\n      ? isolateSourceFailure(\n          runBackfill(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"backfill\",\n        )\n      : Effect.void,\n    resource.changes\n      ? isolateSourceFailure(\n          runChanges(connector, resource, initialCutoff),\n          connector,\n          resource,\n          \"changes\",\n        )\n      : Effect.void,\n  ];\n\n  return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(\n    Effect.andThen(Effect.all(runs, { concurrency: \"unbounded\" })),\n    Effect.asVoid,\n    Effect.catch((error) =>\n      Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n        Effect.andThen(Effect.logError(\"Connector resource initialization failed\", error)),\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n};\n\nconst recordResourceError =\n  (\n    connector: ConnectorDefinition,\n    resource: ResourceDefinition,\n    source: StateSource,\n    operation: StateOperation,\n  ) =>\n  <A, R>(\n    effect: Effect.Effect<A, ConnectorError, R>,\n  ): Effect.Effect<A, ConnectorError, R | StateStore> =>\n    effect.pipe(\n      Effect.tapError(() =>\n        Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n          Effect.andThen(\n            StateStore.pipe(\n              Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),\n            ),\n          ),\n        ),\n      ),\n    );\n\nconst isolateSourceFailure = <A, R>(\n  effect: Effect.Effect<A, ConnectorError, R>,\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  source: StateSource,\n): Effect.Effect<A, never, R> =>\n  // Only typed source errors are isolated. Defects and interruption still stop the runtime.\n  effect.pipe(\n    Effect.catch((error) =>\n      Effect.logError(\"Connector source parked\", error).pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resource: resource.name,\n          source,\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n\nconst refreshRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(connector, resource, state),\n  );\n\n  return state;\n});\n\nconst initializeRuntimeStatus = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  const state = yield* getInitializedState(resource.name, initialCutoff);\n  yield* Metrics.setSyncState(\n    { connector: connector.name, resource: resource.name },\n    deriveSyncState(connector, resource, state),\n  );\n\n  if (resource.backfill) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"backfill\" },\n      state.backfill?.lastSuccessAt,\n    );\n  }\n  if (hasChanges(connector, resource)) {\n    yield* Metrics.setLastSuccessTimestamp(\n      { connector: connector.name, resource: resource.name, source: \"changes\" },\n      state.changes?.lastSuccessAt,\n    );\n  }\n});\n\nconst runBackfill = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.backfill) return;\n  const store = yield* StateStore;\n  let state = yield* getInitializedState(resource.name, initialCutoff);\n\n  if (state.backfill?.completed === true) {\n    // The checkpoint may have committed before the previous process cleared\n    // its error marker. It may also be hidden by a newer changes error.\n    yield* store\n      .clearResourceError(resource.name, \"backfill\")\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n    yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    return;\n  }\n\n  while (state.backfill?.completed !== true) {\n    const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };\n    const clearsVisibleError = state.lastError?.source === \"backfill\";\n    const page = yield* resource.backfill\n      .fetch({\n        pageCursor: backfill.pageCursor,\n        cutoff: backfill.cutoff,\n      })\n      .pipe(recordResourceError(connector, resource, \"backfill\", \"fetch\"));\n\n    yield* ingestBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"backfill\",\n      batch: {\n        cursor: page.nextPageCursor ?? backfill.cutoff,\n        rows: page.rows,\n      },\n    }).pipe(recordResourceError(connector, resource, \"backfill\", \"ingest\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    state = {\n      ...state,\n      backfill: {\n        cutoff: backfill.cutoff,\n        pageCursor: page.nextPageCursor,\n        completed: !page.hasMore,\n        lastSuccessAt,\n      },\n    };\n\n    // Checkpoint only after ingestion succeeds so a crash cannot skip an\n    // unacknowledged page; replay therefore remains at-least-once.\n    yield* Effect.withSpan(\n      store\n        .setBackfillState(resource.name, {\n          cutoff: normalizeCursor(backfill.cutoff),\n          pageCursor:\n            page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),\n          completed: !page.hasMore,\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"backfill\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"backfill\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n\n    // Completion or clearing an error can change the resource-wide sync state.\n    if (clearsVisibleError || !page.hasMore) {\n      state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n  }\n});\n\nconst runChanges = Effect.fnUntraced(function* (\n  connector: ConnectorDefinition,\n  resource: ResourceDefinition,\n  initialCutoff: Cursor.Value,\n) {\n  if (!resource.changes) return;\n  const store = yield* StateStore;\n\n  while (true) {\n    const state = yield* getInitializedState(resource.name, initialCutoff);\n    const cursor = state.changes?.cursor ?? initialCutoff;\n    const clearsVisibleError = state.lastError?.source === \"changes\";\n    const page = yield* resource.changes\n      .fetch({ cursor })\n      .pipe(recordResourceError(connector, resource, \"changes\", \"fetch\"));\n\n    yield* ingestBatch({\n      connector: connector.name,\n      resource: resource.name,\n      source: \"changes\",\n      batch: {\n        cursor: page.cursor,\n        rows: page.rows,\n      },\n    }).pipe(recordResourceError(connector, resource, \"changes\", \"ingest\"));\n\n    const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n    // Advance the durable cursor only after the engine acknowledges the page.\n    yield* Effect.withSpan(\n      store\n        .setChangesState(resource.name, {\n          cursor: normalizeCursor(page.cursor),\n          lastSuccessAt,\n        })\n        .pipe(\n          Effect.andThen(\n            Metrics.setLastSuccessTimestamp(\n              { connector: connector.name, resource: resource.name, source: \"changes\" },\n              lastSuccessAt,\n            ),\n          ),\n          Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n          Effect.tapError((error) => annotateError(\"state_set\", error)),\n        ),\n      SpanName.stateSet,\n      { attributes: { [Attr.stateKey]: resource.name } },\n    ).pipe(recordResourceError(connector, resource, \"changes\", \"checkpoint\"));\n\n    if (clearsVisibleError) {\n      yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n    }\n\n    if (!page.hasMore) {\n      yield* Effect.sleep(resource.changes.interval ?? \"1 minute\");\n    }\n  }\n});\n\n// Every member resource stores the feed cursor. All rows are ingested before any\n// cursor is written. If a crash stops the writes partway, the next run starts\n// from the oldest cursor.\nconst runChangesFeed = (\n  connector: ConnectorDefinition,\n  feed: ChangesFeed,\n  initialCutoff: Cursor.Value,\n): Effect.Effect<void, never, StateStore | Ingestor> =>\n  Effect.gen(function* () {\n    const store = yield* StateStore;\n    const members = feed.resources;\n    const selected = new Set(members.map((resource) => resource.name));\n    // Any failure stops the whole feed, so it is an error for every resource in it.\n    const recordFeedError =\n      (operation: StateOperation) =>\n      <A, R>(effect: Effect.Effect<A, ConnectorError, R>) =>\n        effect.pipe(\n          Effect.tapError(() =>\n            Effect.forEach(\n              members,\n              (resource) =>\n                Metrics.setSyncState(\n                  { connector: connector.name, resource: resource.name },\n                  \"error\",\n                ).pipe(Effect.andThen(store.setResourceError(resource.name, \"changes\", operation))),\n              { discard: true },\n            ),\n          ),\n        );\n\n    while (true) {\n      const states = yield* Effect.forEach(members, (resource) =>\n        store.getResourceState(resource.name),\n      );\n      // A resource that was off has an older cursor. Resume from the oldest, so it\n      // gets the changes it missed. The others read some changes again.\n      const saved = states.flatMap((state) => (state?.changes ? [state.changes] : []));\n      const cursor = Arr.isReadonlyArrayNonEmpty(saved)\n        ? Arr.min(saved, bySavedAt).cursor\n        : initialCutoff;\n      const result = yield* feed.fetch({ cursor, selected }).pipe(recordFeedError(\"fetch\"));\n\n      yield* Effect.forEach(\n        members,\n        (resource) =>\n          ingestBatch({\n            connector: connector.name,\n            resource: resource.name,\n            source: \"changes\",\n            batch: {\n              cursor: result.cursor,\n              rows: result.rows[resource.name] ?? [],\n            },\n          }),\n        { discard: true },\n      ).pipe(recordFeedError(\"ingest\"));\n\n      const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n      yield* Effect.forEach(\n        members,\n        (resource) =>\n          Effect.withSpan(\n            store\n              .setChangesState(resource.name, {\n                cursor: normalizeCursor(result.cursor),\n                lastSuccessAt,\n              })\n              .pipe(\n                Effect.andThen(\n                  Metrics.setLastSuccessTimestamp(\n                    { connector: connector.name, resource: resource.name, source: \"changes\" },\n                    lastSuccessAt,\n                  ),\n                ),\n                Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n                Effect.tapError((error) => annotateError(\"state_set\", error)),\n              ),\n            SpanName.stateSet,\n            { attributes: { [Attr.stateKey]: resource.name } },\n          ),\n        { discard: true },\n      ).pipe(recordFeedError(\"checkpoint\"));\n\n      yield* Effect.forEach(members, (resource, index) =>\n        states[index]?.lastError?.source === \"changes\"\n          ? refreshRuntimeStatus(connector, resource, initialCutoff)\n          : Effect.void,\n      );\n\n      if (!result.hasMore) {\n        yield* Effect.sleep(feed.interval ?? \"1 minute\");\n      }\n    }\n  }).pipe(\n    // Only typed errors park the feed. Defects and interruption still stop the runtime.\n    Effect.catch((error) =>\n      Effect.logError(\"Connector changes feed parked\", error).pipe(\n        Effect.annotateLogs({\n          [Attr.connectorName]: connector.name,\n          resources: feed.resources.map((resource) => resource.name).join(\",\"),\n        }),\n        Effect.andThen(Effect.never),\n      ),\n    ),\n  );\n","export { run, type RunOptions } from \"./engine\";\n"],"mappings":";;;;;;;;;;;;AA0DA,SAAgB,IAAI,YAAiC,SAAsB;CACzE,OAAO,OAAO,IAAI,aAAa;EAE7B,IAAI,WAAW,SAAS,OAAO,aAAa,YAAY,WAAW,OAAO;EAC1E,MAAM,WAAW,OAAO;EACxB,MAAM,YAAY,gBAAgB,YAAY,SAAS,aAAa;EACpE,MAAM,aAAa,sBAAsB,UAAU,IAAI;EACvD,MAAM,eAAe,SAAS,UAC1B,MAAM,SACJ,YACA,uBAAuB,WAAW,QAAQ,OAAO,CAAC,CAAC,KAAK,MAAM,QAAQ,UAAU,CAAC,CACnF,IACA,MAAM;EAEV,OAAO,OAAO,OAAO,IAAI,aAAa;GACpC,MAAM,gBACJ,SAAS,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;GACpF,OAAO,OAAO,QAAQ,mBAAmB,CAAC,CAAC,KACzC,OAAO,aAAa;KACjB,KAAK,gBAAgB,UAAU;IAChC,WAAW,UAAU,UAAU;GACjC,CAAC,CACH;GAEA,MAAM,YAAY,aAAa,WAAW,aAAa;GACvD,MAAM,aAAa,SAAS,UACxB,kBAAkB,WAAW,QAAQ,QAAQ,MAAM,IACnD,OAAO;GAEX,OAAO,OAAO,OAAO,IAAI,CAAC,WAAW,UAAU,GAAG,EAAE,aAAa,YAAY,CAAC,CAAC,CAAC,KAC9E,OAAO,MACT;EACF,CAAC,CAAC,CAAC,KAAK,OAAO,QAAQ,YAAY,CAAC;CACtC,CAAC;AACH;AAEA,MAAM,mBACJ,WACA,UACwB;CACxB,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,cAAc,aAAiC,MAAM,SAAS,SAAS,IAAI;CACjF,MAAM,EAAE,SAAS,GAAG,SAAS;CAC7B,MAAM,gBAAgB,SAAS,UAAU,OAAO,UAAU,KAAK,CAAC;CAChE,OAAO;EACL,GAAG;EACH,WAAW,UAAU,UAAU,OAAO,UAAU;EAChD,GAAI,WAAW,cAAc,SAAS,IAClC,EAAE,SAAS;GAAE,GAAG;GAAS,WAAW;EAAc,EAAE,IACpD,CAAC;CACP;AACF;AAEA,MAAM,yBAAyB,cAC7B,MAAM,OAAO,YAAY,CAAC,CACxB,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,MAAM,QAA4B,IAAI;CAC3D,OAAOA,qBAA6B,WAAW,CAAC;CAChD,OAAO,aAAa,GAAG,EAAE,MAAM,CAAC;AAClC,CAAC,CACH;AAEF,MAAM,qBAAqB,WAAgC,WAAwC;CACjG,MAAM,kBAAkB,OAAO,MAAM,UAAU,MAAM,YAAY,eAAe;CAChF,OAAO,OAAO,IACZ,CAAC,kBAAkB,wBAAwB,SAAS,IAAI,OAAO,MAAM,OAAO,KAAK,GACjF,EACE,aAAa,YACf,CACF,CAAC,CAAC,KAAK,OAAO,MAAM;AACtB;AAEA,MAAM,2BAA2B,cAC/B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CACrB,OAAO,MAAM;EACX,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,KAAK;EAC3C,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,KAAK;EAC3C,OAAOA,qBAA6B,UAAU,MAAM,KAAK;EACzD,OAAO,YAAY;GACjB,WAAW,UAAU;GACrB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,MAAM;EACf,CAAC;CACH;AACF,CAAC;AAEH,MAAM,0BACJ,WACA,YAOqD;CACrD,MAAM,aAAmC,QAAQ,cAAc;CAC/D,MAAM,cAAc,QAAQ,gBAAgB,KAAA,IAAY,aAAa,QAAQ;CAC7E,MAAM,aAAa,QAAQ,eAAe,KAAA,IAAY,YAAY,QAAQ;CAC1E,MAAM,MAAM,MAAM,SAChB,OAAO,QAAQ,QAAQ;EACrB,eAAe,UAAU;EACzB,eAAe,UAAU,UAAU,KAAK,aAAa,SAAS,IAAI;CACpE,CAAC,GACD,WAAW,IAAI,OAAO,YAAY,OAAO,QAAQ,mBAAmB,KAAK,IAAI,CAAC,CAAC,GAC/E,GAAI,gBAAgB,QAChB,CAAC,IACD,CAAC,cAAc,kBAAkB,UAAU,EAAE,MAAM,YAAY,CAAC,CAAC,GACrE,GAAI,eAAe,QAAQ,CAAC,IAAI,CAAC,YAAY,WAAW,UAAU,CAAC,CACrE;CAEA,OAAO,WAAW,MAAM,KAAK,EAC3B,eAAe,QAAQ,qBAAqB,KAC9C,CAAC;AACH;AAEA,MAAM,gBACJ,WACA,kBAEA,OAAO,IAAI,aAAa;CACtB,MAAM,OAAO,UAAU;CACvB,OAAO,OAAO,IACZ,CACE,OAAO,QACL,UAAU,YACT,aAAa,mBAAmB,WAAW,UAAU,aAAa,GACnE,EAAE,aAAa,YAAY,CAC7B,GACA,OAAO,eAAe,WAAW,MAAM,aAAa,IAAI,OAAO,IACjE,GACA,EAAE,aAAa,YAAY,CAC7B;AACF,CAAC;AAEH,MAAM,YAAY,MAAM,SACtB,MAAM,SACL,UAAiD,MAAM,iBAAiB,EAC3E;AAEA,MAAM,gBAAgB,WAAgC,SACpD,OAAO,QAAQ,KAAK,YAAY,QAAQ,UAAU;CAChD,IAAI,KAAK,UAAU,WAAW,SAAS,KAAK,SAAS,OAAO,IAAI,MAAM,OACpE,OAAO,OAAO,KACZ,IAAI,eAAe,EAAE,SAAS,YAAY,OAAO,KAAK,+BAA+B,CAAC,CACxF;CAEF,MAAM,WAAW,UAAU,UAAU,MAAM,SAAS,KAAK,SAAS,OAAO,IAAI;CAC7E,IAAI,CAAC,UACH,OAAO,OAAO,KACZ,IAAI,eAAe,EACjB,SAAS,yBAAyB,OAAO,KAAK,kCAAkC,UAAU,OAC5F,CAAC,CACH;CAEF,IAAI,SAAS,SACX,OAAO,OAAO,KACZ,IAAI,eAAe,EACjB,SAAS,YAAY,OAAO,KAAK,sDACnC,CAAC,CACH;CAEF,OAAO,OAAO;AAChB,CAAC,CAAC,CAAC,KACD,OAAO,QACL,KAAK,UAAU,WAAW,IACtB,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,oCAAoC,CAAC,CAAC,IAChF,OAAO,IACb,CACF;AAEF,MAAM,2BACJ,UACA,mBACmB;CACnB,GAAG;CACH,SAAS,UAAU,WAAW,EAAE,QAAQ,cAAc;CACtD,UAAU,UAAU,YAAY;EAC9B,QAAQ;EACR,WAAW;CACb;AACF;AAEA,MAAM,sBAAsB,OAAO,WAAW,WAC5C,UACA,eACA;CAEA,MAAM,WAAW,QAAO,OADH,WAAA,CACS,iBAAiB,QAAQ;CACvD,OAAO,wBAAwB,UAAU,aAAa;AACxD,CAAC;AAED,MAAM,sBACJ,WACA,UACA,kBACG;CACH,MAAM,OAAO,CACX,SAAS,WACL,qBACE,YAAY,WAAW,UAAU,aAAa,GAC9C,WACA,UACA,UACF,IACA,OAAO,MACX,SAAS,UACL,qBACE,WAAW,WAAW,UAAU,aAAa,GAC7C,WACA,UACA,SACF,IACA,OAAO,IACb;CAEA,OAAO,wBAAwB,WAAW,UAAU,aAAa,CAAC,CAAC,KACjE,OAAO,QAAQ,OAAO,IAAI,MAAM,EAAE,aAAa,YAAY,CAAC,CAAC,GAC7D,OAAO,QACP,OAAO,OAAO,UACZC,aAAqB;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GAAG,OAAO,CAAC,CAAC,KACpF,OAAO,QAAQ,OAAO,SAAS,4CAA4C,KAAK,CAAC,GACjF,OAAO,aAAa;GACjB,KAAK,gBAAgB,UAAU;EAChC,UAAU,SAAS;CACrB,CAAC,GACD,OAAO,QAAQ,OAAO,KAAK,CAC7B,CACF,CACF;AACF;AAEA,MAAM,uBAEF,WACA,UACA,QACA,eAGA,WAEA,OAAO,KACL,OAAO,eACLA,aAAqB;CAAE,WAAW,UAAU;CAAM,UAAU,SAAS;AAAK,GAAG,OAAO,CAAC,CAAC,KACpF,OAAO,QACL,WAAW,KACT,OAAO,SAAS,UAAU,MAAM,iBAAiB,SAAS,MAAM,QAAQ,SAAS,CAAC,CACpF,CACF,CACF,CACF,CACF;AAEJ,MAAM,wBACJ,QACA,WACA,UACA,WAGA,OAAO,KACL,OAAO,OAAO,UACZ,OAAO,SAAS,2BAA2B,KAAK,CAAC,CAAC,KAChD,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,UAAU,SAAS;CACnB;AACF,CAAC,GACD,OAAO,QAAQ,OAAO,KAAK,CAC7B,CACF,CACF;AAEF,MAAM,uBAAuB,OAAO,WAAW,WAC7C,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;CACrE,OAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GACrD,gBAAgB,WAAW,UAAU,KAAK,CAC5C;CAEA,OAAO;AACT,CAAC;AAED,MAAM,0BAA0B,OAAO,WAAW,WAChD,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;CACrE,OAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GACrD,gBAAgB,WAAW,UAAU,KAAK,CAC5C;CAEA,IAAI,SAAS,UACX,OAAOC,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;CAAW,GACzE,MAAM,UAAU,aAClB;CAEF,IAAI,WAAW,WAAW,QAAQ,GAChC,OAAOA,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;CAAU,GACxE,MAAM,SAAS,aACjB;AAEJ,CAAC;AAED,MAAM,cAAc,OAAO,WAAW,WACpC,WACA,UACA,eACA;CACA,IAAI,CAAC,SAAS,UAAU;CACxB,MAAM,QAAQ,OAAO;CACrB,IAAI,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;CAEnE,IAAI,MAAM,UAAU,cAAc,MAAM;EAGtC,OAAO,MACJ,mBAAmB,SAAS,MAAM,UAAU,CAAC,CAC7C,KAAK,oBAAoB,WAAW,UAAU,YAAY,YAAY,CAAC;EAC1E,OAAO,qBAAqB,WAAW,UAAU,aAAa;EAC9D;CACF;CAEA,OAAO,MAAM,UAAU,cAAc,MAAM;EACzC,MAAM,WAAW,MAAM,YAAY;GAAE,QAAQ;GAAe,WAAW;EAAM;EAC7E,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,SAC1B,MAAM;GACL,YAAY,SAAS;GACrB,QAAQ,SAAS;EACnB,CAAC,CAAC,CACD,KAAK,oBAAoB,WAAW,UAAU,YAAY,OAAO,CAAC;EAErE,OAAO,YAAY;GACjB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK,kBAAkB,SAAS;IACxC,MAAM,KAAK;GACb;EACF,CAAC,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,QAAQ,CAAC;EAEtE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;EAC7E,QAAQ;GACN,GAAG;GACH,UAAU;IACR,QAAQ,SAAS;IACjB,YAAY,KAAK;IACjB,WAAW,CAAC,KAAK;IACjB;GACF;EACF;EAIA,OAAO,OAAO,SACZ,MACG,iBAAiB,SAAS,MAAM;GAC/B,QAAQ,gBAAgB,SAAS,MAAM;GACvC,YACE,KAAK,mBAAmB,KAAA,IAAY,KAAA,IAAY,gBAAgB,KAAK,cAAc;GACrF,WAAW,CAAC,KAAK;GACjB;EACF,CAAC,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;EAAW,GACzE,aACF,CACF,GACA,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,UAAU,CAAC,GAClE,OAAO,UAAU,UAAU,cAAc,aAAa,KAAK,CAAC,CAC9D,GACF,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,KAAK,EAAE,CACnD,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,YAAY,CAAC;EAGzE,IAAI,sBAAsB,CAAC,KAAK,SAC9B,QAAQ,OAAO,qBAAqB,WAAW,UAAU,aAAa;CAE1E;AACF,CAAC;AAED,MAAM,aAAa,OAAO,WAAW,WACnC,WACA,UACA,eACA;CACA,IAAI,CAAC,SAAS,SAAS;CACvB,MAAM,QAAQ,OAAO;CAErB,OAAO,MAAM;EACX,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,aAAa;EACrE,MAAM,SAAS,MAAM,SAAS,UAAU;EACxC,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,QAC1B,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,KAAK,oBAAoB,WAAW,UAAU,WAAW,OAAO,CAAC;EAEpE,OAAO,YAAY;GACjB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK;IACb,MAAM,KAAK;GACb;EACF,CAAC,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,QAAQ,CAAC;EAErE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;EAE7E,OAAO,OAAO,SACZ,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,KAAK,MAAM;GACnC;EACF,CAAC,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;EAAU,GACxE,aACF,CACF,GACA,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,SAAS,CAAC,GACjE,OAAO,UAAU,UAAU,cAAc,aAAa,KAAK,CAAC,CAC9D,GACF,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,KAAK,EAAE,CACnD,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,YAAY,CAAC;EAExE,IAAI,oBACF,OAAO,qBAAqB,WAAW,UAAU,aAAa;EAGhE,IAAI,CAAC,KAAK,SACR,OAAO,OAAO,MAAM,SAAS,QAAQ,YAAY,UAAU;CAE/D;AACF,CAAC;AAKD,MAAM,kBACJ,WACA,MACA,kBAEA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CACrB,MAAM,UAAU,KAAK;CACrB,MAAM,WAAW,IAAI,IAAI,QAAQ,KAAK,aAAa,SAAS,IAAI,CAAC;CAEjE,MAAM,mBACH,eACM,WACL,OAAO,KACL,OAAO,eACL,OAAO,QACL,UACC,aACCD,aACE;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;CAAK,GACrD,OACF,CAAC,CAAC,KAAK,OAAO,QAAQ,MAAM,iBAAiB,SAAS,MAAM,WAAW,SAAS,CAAC,CAAC,GACpF,EAAE,SAAS,KAAK,CAClB,CACF,CACF;CAEJ,OAAO,MAAM;EACX,MAAM,SAAS,OAAO,OAAO,QAAQ,UAAU,aAC7C,MAAM,iBAAiB,SAAS,IAAI,CACtC;EAGA,MAAM,QAAQ,OAAO,SAAS,UAAW,OAAO,UAAU,CAAC,MAAM,OAAO,IAAI,CAAC,CAAE;EAC/E,MAAM,SAASE,MAAI,wBAAwB,KAAK,IAC5CA,MAAI,IAAI,OAAO,SAAS,CAAC,CAAC,SAC1B;EACJ,MAAM,SAAS,OAAO,KAAK,MAAM;GAAE;GAAQ;EAAS,CAAC,CAAC,CAAC,KAAK,gBAAgB,OAAO,CAAC;EAEpF,OAAO,OAAO,QACZ,UACC,aACC,YAAY;GACV,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,OAAO;IACf,MAAM,OAAO,KAAK,SAAS,SAAS,CAAC;GACvC;EACF,CAAC,GACH,EAAE,SAAS,KAAK,CAClB,CAAC,CAAC,KAAK,gBAAgB,QAAQ,CAAC;EAEhC,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;EAC7E,OAAO,OAAO,QACZ,UACC,aACC,OAAO,SACL,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,OAAO,MAAM;GACrC;EACF,CAAC,CAAC,CACD,KACC,OAAO,QACLD,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;EAAU,GACxE,aACF,CACF,GACA,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,SAAS,CAAC,GACjE,OAAO,UAAU,UAAU,cAAc,aAAa,KAAK,CAAC,CAC9D,GACF,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,KAAK,EAAE,CACnD,GACF,EAAE,SAAS,KAAK,CAClB,CAAC,CAAC,KAAK,gBAAgB,YAAY,CAAC;EAEpC,OAAO,OAAO,QAAQ,UAAU,UAAU,UACxC,OAAO,MAAM,EAAE,WAAW,WAAW,YACjC,qBAAqB,WAAW,UAAU,aAAa,IACvD,OAAO,IACb;EAEA,IAAI,CAAC,OAAO,SACV,OAAO,OAAO,MAAM,KAAK,YAAY,UAAU;CAEnD;AACF,CAAC,CAAC,CAAC,KAED,OAAO,OAAO,UACZ,OAAO,SAAS,iCAAiC,KAAK,CAAC,CAAC,KACtD,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,WAAW,KAAK,UAAU,KAAK,aAAa,SAAS,IAAI,CAAC,CAAC,KAAK,GAAG;AACrE,CAAC,GACD,OAAO,QAAQ,OAAO,KAAK,CAC7B,CACF,CACF"}

package/dist/ingestor/index.d.mts

@@ -0,0 +1,3 @@
+import { a as IngestorService, i as Ingestor, n as IngestResult, r as IngestSource, t as IngestOptions } from "../service-BRoy_EId.mjs";
+import { a as layerConsole, i as layerArrowFlightConfig, n as ArrowFlightIngestorConfig, r as layerArrowFlight } from "../index-uFa2wwYo.mjs";
+export { type ArrowFlightIngestorConfig, type IngestOptions, type IngestResult, type IngestSource, Ingestor, type IngestorService, layerArrowFlight, layerArrowFlightConfig, layerConsole };

package/dist/ingestor/index.mjs

@@ -0,0 +1,4 @@
+import { t as Ingestor } from "../service-jWxxfEht.mjs";
+import { n as layerArrowFlightConfig, t as layerArrowFlight } from "../arrow-flight-DDQ0ktx9.mjs";
+import { n as layerConsole } from "../ingestor-DjCEzBzZ.mjs";
+export { Ingestor, layerArrowFlight, layerArrowFlightConfig, layerConsole };

package/dist/ingestor-DjCEzBzZ.mjs

@@ -0,0 +1,26 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { n as layerArrowFlightConfig, t as layerArrowFlight } from "./arrow-flight-DDQ0ktx9.mjs";
+import { Effect, Layer } from "effect";
+//#region src/ingestor/console.ts
+/** Logs batches without sending them over Arrow Flight. */
+const layerConsole = Layer.succeed(Ingestor)({ ingest: ({ resource, source, batch }) => Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(Effect.annotateLogs({
+	rows: batch.rows.length,
+	cursor: batch.cursor,
+	source
+}), Effect.as({
+	rows: batch.rows.length,
+	bytes: 0
+})) });
+//#endregion
+//#region src/ingestor/index.ts
+var ingestor_exports = /* @__PURE__ */ __exportAll({
+	Ingestor: () => Ingestor,
+	layerArrowFlight: () => layerArrowFlight,
+	layerArrowFlightConfig: () => layerArrowFlightConfig,
+	layerConsole: () => layerConsole
+});
+//#endregion
+export { layerConsole as n, ingestor_exports as t };
+
+//# sourceMappingURL=ingestor-DjCEzBzZ.mjs.map

package/dist/ingestor-DjCEzBzZ.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestor-DjCEzBzZ.mjs","names":[],"sources":["../src/ingestor/console.ts","../src/ingestor/index.ts"],"sourcesContent":["import { Effect, Layer } from \"effect\";\n\nimport { Ingestor } from \"./service\";\n\n/** Logs batches without sending them over Arrow Flight. */\nexport const layerConsole = Layer.succeed(Ingestor)({\n  ingest: ({ resource, source, batch }) =>\n    Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(\n      Effect.annotateLogs({ rows: batch.rows.length, cursor: batch.cursor, source }),\n      // Nothing is encoded, so there are no bytes to count.\n      Effect.as({ rows: batch.rows.length, bytes: 0 }),\n    ),\n});\n","export { layerConsole } from \"./console\";\nexport {\n  Ingestor,\n  type IngestOptions,\n  type IngestorService,\n  type IngestResult,\n  type IngestSource,\n} from \"./service\";\nexport {\n  layerArrowFlight,\n  layerArrowFlightConfig,\n  type ArrowFlightIngestorConfig,\n} from \"./arrow-flight\";\n"],"mappings":";;;;;;AAKA,MAAa,eAAe,MAAM,QAAQ,QAAQ,CAAC,CAAC,EAClD,SAAS,EAAE,UAAU,QAAQ,YAC3B,OAAO,QAAQ,yBAAyB,OAAO,eAAe,UAAU,CAAC,CAAC,KACxE,OAAO,aAAa;CAAE,MAAM,MAAM,KAAK;CAAQ,QAAQ,MAAM;CAAQ;AAAO,CAAC,GAE7E,OAAO,GAAG;CAAE,MAAM,MAAM,KAAK;CAAQ,OAAO;AAAE,CAAC,CACjD,EACJ,CAAC"}

package/dist/instrumented-Bq1WX2L5.mjs

@@ -0,0 +1,29 @@
+import { S as addCurrentSpanEvent, _ as withResourceAttributes, b as EventName, i as ingestDuration, n as batchSize, r as batches, u as rowsIngested, v as Attr, x as SpanName, y as EventAttr } from "./metrics-C901IIYZ.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { Effect, Metric } from "effect";
+//#region src/ingestor/instrumented.ts
+const ingestBatch = Effect.fnUntraced(function* (options) {
+	const metricAttributes = withResourceAttributes({
+		connector: options.connector ?? "unknown",
+		resource: options.resource,
+		source: options.source
+	});
+	const batchOutcome = (outcome) => Metric.withAttributes(batches, {
+		...metricAttributes,
+		[Attr.batchOutcome]: outcome
+	});
+	const ingest = (yield* Ingestor).ingest(options).pipe(Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })), Effect.tapError(() => Effect.annotateCurrentSpan({
+		[Attr.ingestionSuccess]: false,
+		[Attr.errorPhase]: "ingest"
+	})));
+	const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: "producer" }).pipe(Effect.trackDuration(Metric.withAttributes(ingestDuration, metricAttributes)), Effect.onError(() => Metric.update(batchOutcome("error"), 1)));
+	yield* Metric.update(batchOutcome("success"), 1);
+	yield* Metric.update(Metric.withAttributes(rowsIngested, metricAttributes), options.batch.rows.length);
+	yield* Metric.update(Metric.withAttributes(batchSize, metricAttributes), options.batch.rows.length);
+	if (options.batch.cursor !== void 0) yield* addCurrentSpanEvent(EventName.batchCheckpoint, { [EventAttr.batchCursor]: options.batch.cursor });
+	return result;
+});
+//#endregion
+export { ingestBatch as t };
+
+//# sourceMappingURL=instrumented-Bq1WX2L5.mjs.map

package/dist/instrumented-Bq1WX2L5.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"instrumented-Bq1WX2L5.mjs","names":["Metrics.withResourceAttributes","Metrics.batches","Metrics.ingestDuration","Metrics.rowsIngested","Metrics.batchSize"],"sources":["../src/ingestor/instrumented.ts"],"sourcesContent":["import { Effect, Metric } from \"effect\";\n\nimport type { ResourceBatch } from \"../core/types\";\n\nimport * as Metrics from \"../metrics\";\nimport { Attr, EventAttr, EventName, SpanName, addCurrentSpanEvent } from \"../telemetry\";\nimport { Ingestor, type IngestSource } from \"./service\";\n\nexport const ingestBatch = Effect.fnUntraced(function* (options: {\n  readonly connector?: string;\n  readonly resource: string;\n  readonly source: IngestSource;\n  readonly batch: ResourceBatch;\n}) {\n  const metricAttributes = Metrics.withResourceAttributes({\n    connector: options.connector ?? \"unknown\",\n    resource: options.resource,\n    source: options.source,\n  });\n  const batchOutcome = (outcome: \"success\" | \"error\") =>\n    Metric.withAttributes(Metrics.batches, {\n      ...metricAttributes,\n      [Attr.batchOutcome]: outcome,\n    });\n\n  const ingestor = yield* Ingestor;\n\n  const ingest = ingestor.ingest(options).pipe(\n    Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })),\n    Effect.tapError(() =>\n      Effect.annotateCurrentSpan({\n        [Attr.ingestionSuccess]: false,\n        [Attr.errorPhase]: \"ingest\",\n      }),\n    ),\n  );\n\n  // onError keeps the failure, so the engine still sees it and skips the checkpoint.\n  const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: \"producer\" }).pipe(\n    Effect.trackDuration(Metric.withAttributes(Metrics.ingestDuration, metricAttributes)),\n    Effect.onError(() => Metric.update(batchOutcome(\"error\"), 1)),\n  );\n\n  yield* Metric.update(batchOutcome(\"success\"), 1);\n  yield* Metric.update(\n    Metric.withAttributes(Metrics.rowsIngested, metricAttributes),\n    options.batch.rows.length,\n  );\n  yield* Metric.update(\n    Metric.withAttributes(Metrics.batchSize, metricAttributes),\n    options.batch.rows.length,\n  );\n\n  if (options.batch.cursor !== undefined) {\n    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {\n      [EventAttr.batchCursor]: options.batch.cursor,\n    });\n  }\n\n  return result;\n});\n"],"mappings":";;;;AAQA,MAAa,cAAc,OAAO,WAAW,WAAW,SAKrD;CACD,MAAM,mBAAmBA,uBAA+B;EACtD,WAAW,QAAQ,aAAa;EAChC,UAAU,QAAQ;EAClB,QAAQ,QAAQ;CAClB,CAAC;CACD,MAAM,gBAAgB,YACpB,OAAO,eAAeC,SAAiB;EACrC,GAAG;GACF,KAAK,eAAe;CACvB,CAAC;CAIH,MAAM,UAAS,OAFS,SAAA,CAEA,OAAO,OAAO,CAAC,CAAC,KACtC,OAAO,UAAU,OAAO,oBAAoB,GAAG,KAAK,mBAAmB,KAAK,CAAC,CAAC,GAC9E,OAAO,eACL,OAAO,oBAAoB;GACxB,KAAK,mBAAmB;GACxB,KAAK,aAAa;CACrB,CAAC,CACH,CACF;CAGA,MAAM,SAAS,OAAO,OAAO,SAAS,QAAQ,SAAS,QAAQ,EAAE,MAAM,WAAW,CAAC,CAAC,CAAC,KACnF,OAAO,cAAc,OAAO,eAAeC,gBAAwB,gBAAgB,CAAC,GACpF,OAAO,cAAc,OAAO,OAAO,aAAa,OAAO,GAAG,CAAC,CAAC,CAC9D;CAEA,OAAO,OAAO,OAAO,aAAa,SAAS,GAAG,CAAC;CAC/C,OAAO,OAAO,OACZ,OAAO,eAAeC,cAAsB,gBAAgB,GAC5D,QAAQ,MAAM,KAAK,MACrB;CACA,OAAO,OAAO,OACZ,OAAO,eAAeC,WAAmB,gBAAgB,GACzD,QAAQ,MAAM,KAAK,MACrB;CAEA,IAAI,QAAQ,MAAM,WAAW,KAAA,GAC3B,OAAO,oBAAoB,UAAU,iBAAiB,GACnD,UAAU,cAAc,QAAQ,MAAM,OACzC,CAAC;CAGH,OAAO;AACT,CAAC"}

package/dist/manifest/index.d.mts

@@ -0,0 +1,2 @@
+import { _ as number, a as configSchema, b as select, c as ConnectorManifest, d as ConfigField, f as ConfigFieldSpec, g as define$1, h as boolean, i as RuntimeConfigDocument, l as ResourceCapability, m as ConnectorConfigDef, n as ConnectorConfigSchema, o as decodeConfig, p as ConfigValuesOf, r as ConnectorConfigValues, s as toRuntimeDocument, u as define, v as optional, x as string, y as secret } from "../index-eFuoMZZ3.mjs";
+export { type ConfigField, type ConfigFieldSpec, type ConfigValuesOf, type ConnectorConfigDef, type ConnectorConfigSchema, type ConnectorConfigValues, type ConnectorManifest, type ResourceCapability, type RuntimeConfigDocument, boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };

package/dist/manifest/{index.js → index.mjs}

@@ -1,4 +1,2 @@
-import "../constants-D113Ut1x.js";
-import { a as define, c as number, d as select, f as string, i as toRuntimeDocument, l as optional, n as configSchema, o as boolean, r as decodeConfig, s as define$1, u as secret } from "../manifest-P7kBnch9.js";
-
-export { boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };
+import { a as define, c as number, d as select, f as string, i as toRuntimeDocument, l as optional, n as configSchema, o as boolean, r as decodeConfig, s as define$1, u as secret } from "../manifest-CABeIhAG.mjs";
+export { boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };

package/dist/{manifest-P7kBnch9.js → manifest-CABeIhAG.mjs}

@@ -1,20 +1,18 @@
-import { t as __export } from "./chunk-Bp6m_JJh.js";
-import { r as isPlatformRuntimeKey } from "./constants-D113Ut1x.js";
-import { Config, Option, Predicate, Redacted, Schema, SchemaTransformation, Struct } from "effect";
-
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { r as isPlatformRuntimeKey } from "./constants-v3UX-hf6.mjs";
+import { Config, Predicate, Schema, SchemaTransformation, Struct } from "effect";
 //#region src/manifest/number-schema.ts
 const makeNumberSchema = (constraints) => {
 	if (constraints.integer) return constraints.minimum === void 0 ? Schema.Int : Schema.Int.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
 	return constraints.minimum === void 0 ? Schema.Finite : Schema.Finite.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
 };
-
 //#endregion
 //#region src/manifest/config.ts
 const configFieldToConfig = Struct.lambda((field) => field.config);
 const withDefault = (config, defaultValue) => defaultValue === void 0 ? config : Config.withDefault(config, defaultValue);
 /** Defines a required non-empty string, optionally with a default. */
 const string = (options) => ({
-	config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
+	config: withDefault(Config.NonEmptyString(options.runtimeKey), options.default),
 	spec: {
 		runtimeKey: options.runtimeKey,
 		type: "string",
@@ -44,7 +42,7 @@ const number = (options) => {
 };
 /** Defines a boolean, optionally with a default. */
 const boolean = (options) => ({
-	config: withDefault(Config.boolean(options.runtimeKey), options.default),
+	config: withDefault(Config.Boolean(options.runtimeKey), options.default),
 	spec: {
 		runtimeKey: options.runtimeKey,
 		type: "boolean",
@@ -56,7 +54,7 @@ const boolean = (options) => ({
 });
 /** Defines a string literal choice whose values are preserved in the inferred type. */
 const select = (options) => ({
-	config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
+	config: withDefault(Config.Literals(options.values, options.runtimeKey), options.default),
 	spec: {
 		runtimeKey: options.runtimeKey,
 		type: "select",
@@ -102,7 +100,7 @@ const optional = (field) => ({
 * const formFields = config.spec;
 * ```
 */
-const define = (fields) => {
+const define$1 = (fields) => {
 	const spec = Object.entries(fields).map(([name, field]) => ({
 		name,
 		...field.spec
@@ -116,7 +114,6 @@ const define = (fields) => {
 		spec
 	};
 };
-
 //#endregion
 //#region src/manifest/manifest.ts
 /**
@@ -132,8 +129,7 @@ const define = (fields) => {
 * });
 * ```
 */
-const define$1 = (manifest) => manifest;
-
+const define = (manifest) => manifest;
 //#endregion
 //#region src/manifest/schema.ts
 const numberFromForm = (field) => {
@@ -199,15 +195,14 @@ const toRuntimeDocument = (manifest, values) => {
 	}
 	return document;
 };
-
 //#endregion
 //#region src/manifest/index.ts
-var manifest_exports = /* @__PURE__ */ __export({
+var manifest_exports = /* @__PURE__ */ __exportAll({
 	boolean: () => boolean,
 	configSchema: () => configSchema,
 	decodeConfig: () => decodeConfig,
-	define: () => define$1,
-	defineConfig: () => define,
+	define: () => define,
+	defineConfig: () => define$1,
 	number: () => number,
 	optional: () => optional,
 	secret: () => secret,
@@ -215,7 +210,7 @@ var manifest_exports = /* @__PURE__ */ __export({
 	string: () => string,
 	toRuntimeDocument: () => toRuntimeDocument
 });
-
 //#endregion
-export { define$1 as a, number as c, select as d, string as f, toRuntimeDocument as i, optional as l, configSchema as n, boolean as o, decodeConfig as r, define as s, manifest_exports as t, secret as u };
-//# sourceMappingURL=manifest-P7kBnch9.js.map
+export { define as a, number as c, select as d, string as f, toRuntimeDocument as i, optional as l, configSchema as n, boolean as o, decodeConfig as r, define$1 as s, manifest_exports as t, secret as u };
+
+//# sourceMappingURL=manifest-CABeIhAG.mjs.map