@chkit/plugin-ingest 0.2.0-beta.8

package/README.md

@@ -0,0 +1,30 @@
+# @chkit/plugin-ingest
+
+Write TypeScript readers to load application API data into ClickHouse with [chkit](https://www.npmjs.com/package/chkit). Use an external scheduler for repeated runs.
+
+The executor records progress after writes succeed. A retry can reread rows from the last committed checkpoint.
+
+```ts
+import { defineConfig } from '@chkit/core'
+import { ingest } from '@chkit/plugin-ingest'
+
+export default defineConfig({
+  entry: './src/chkit.ts',
+  plugins: [ingest()],
+  clickhouse: { url: process.env.CLICKHOUSE_URL ?? '' },
+})
+```
+
+```sh
+chkit ingest run --tag schedule:1h
+```
+
+Documentation: https://chkit.obsessiondb.com/api-sync/
+
+For raw storage, typed rows, and where to map fields, see [Destinations and transformations](https://chkit.obsessiondb.com/api-sync/destinations/).
+
+Install the source-authoring skill for a coding agent:
+
+```sh
+npx skills add obsessiondb/chkit --skill chkit-ingestion
+```

package/dist/destination.d.ts

@@ -0,0 +1,40 @@
+import type { ClickHouseExecutor } from '@chkit/clickhouse';
+import { type ColumnDefinition, type TableDefinition } from '@chkit/core';
+import type { DestinationAdapter, Row } from './types.js';
+export declare const BATCH_ID_COLUMN = "_chkit_batch_id";
+export declare const RUN_ID_COLUMN = "_chkit_run_id";
+export declare const INGESTED_AT_COLUMN = "_chkit_ingested_at";
+/**
+ * Runtime-owned metadata every ingestion destination table carries. Spread it
+ * into the table's `columns`. `_chkit_ingested_at` is destination-owned: it is
+ * never sent by the loader, so it records the physical publication time even
+ * when a retry recomputes it under the same deduplication token.
+ */
+export declare const ingestionColumns: readonly ColumnDefinition[];
+export type RawRow = {
+    id: string;
+    raw: unknown;
+};
+/**
+ * Landing table for provider objects exactly as received: a stable id plus the
+ * untouched object in a native JSON column. Typed shapes are derived from it
+ * inside ClickHouse (views or materialized views), so changing a transform
+ * never requires re-fetching the source. Replays and overlapping windows
+ * collapse to the latest ingested version of each id.
+ */
+export declare function rawTable(input: {
+    database: string;
+    name: string;
+    comment?: string;
+}): TableDefinition;
+/** Shape provider objects for a {@link rawTable} without mapping their fields. */
+export declare function rawRows<T>(items: readonly T[], id: (item: T) => string): RawRow[];
+/**
+ * A successful synchronous insert response (or an awaited async insert) is the
+ * sink evidence ChKit trusts. A missing response stays ambiguous and is retried
+ * with the same token.
+ */
+export declare function createClickHouseDestination(executor: ClickHouseExecutor): DestinationAdapter;
+/** JSONEachRow encodes big integers as exact decimal strings, including nested values. */
+export declare function toJsonRows(rows: readonly Row[]): Row[];
+//# sourceMappingURL=destination.d.ts.map

package/dist/destination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"destination.d.ts","sourceRoot":"","sources":["../src/destination.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AAC3D,OAAO,EAAS,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AAEhF,OAAO,KAAK,EAAE,kBAAkB,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAEzD,eAAO,MAAM,eAAe,oBAAoB,CAAA;AAChD,eAAO,MAAM,aAAa,kBAAkB,CAAA;AAC5C,eAAO,MAAM,kBAAkB,uBAAuB,CAAA;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,gBAAgB,EAIvD,CAAA;AAGD,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAA;IACV,GAAG,EAAE,OAAO,CAAA;CACb,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,eAAe,CAUrG;AAED,kFAAkF;AAClF,wBAAgB,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAAG,MAAM,EAAE,CAEjF;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,kBAAkB,GAAG,kBAAkB,CAc5F;AAED,0FAA0F;AAC1F,wBAAgB,UAAU,CAAC,IAAI,EAAE,SAAS,GAAG,EAAE,GAAG,GAAG,EAAE,CAEtD"}

package/dist/destination.js

@@ -0,0 +1,63 @@
+import { table } from '@chkit/core';
+export const BATCH_ID_COLUMN = '_chkit_batch_id';
+export const RUN_ID_COLUMN = '_chkit_run_id';
+export const INGESTED_AT_COLUMN = '_chkit_ingested_at';
+/**
+ * Runtime-owned metadata every ingestion destination table carries. Spread it
+ * into the table's `columns`. `_chkit_ingested_at` is destination-owned: it is
+ * never sent by the loader, so it records the physical publication time even
+ * when a retry recomputes it under the same deduplication token.
+ */
+export const ingestionColumns = [
+    { name: BATCH_ID_COLUMN, type: 'String' },
+    { name: RUN_ID_COLUMN, type: 'String' },
+    { name: INGESTED_AT_COLUMN, type: "DateTime64(6, 'UTC')", default: 'fn:now64(6)' },
+];
+/**
+ * Landing table for provider objects exactly as received: a stable id plus the
+ * untouched object in a native JSON column. Typed shapes are derived from it
+ * inside ClickHouse (views or materialized views), so changing a transform
+ * never requires re-fetching the source. Replays and overlapping windows
+ * collapse to the latest ingested version of each id.
+ */
+export function rawTable(input) {
+    return table({
+        database: input.database,
+        name: input.name,
+        comment: input.comment,
+        columns: [{ name: 'id', type: 'String' }, { name: 'raw', type: 'JSON' }, ...ingestionColumns],
+        engine: `ReplacingMergeTree(${INGESTED_AT_COLUMN})`,
+        primaryKey: ['id'],
+        orderBy: ['id'],
+    });
+}
+/** Shape provider objects for a {@link rawTable} without mapping their fields. */
+export function rawRows(items, id) {
+    return items.map((item) => ({ id: id(item), raw: item }));
+}
+/**
+ * A successful synchronous insert response (or an awaited async insert) is the
+ * sink evidence ChKit trusts. A missing response stays ambiguous and is retried
+ * with the same token.
+ */
+export function createClickHouseDestination(executor) {
+    return {
+        async insert({ table, rows, token }) {
+            if (rows.length === 0)
+                return;
+            await executor.insert({
+                table: `${table.database}.${table.name}`,
+                values: toJsonRows(rows),
+                settings: {
+                    insert_deduplication_token: token,
+                    wait_for_async_insert: 1,
+                },
+            });
+        },
+    };
+}
+/** JSONEachRow encodes big integers as exact decimal strings, including nested values. */
+export function toJsonRows(rows) {
+    return JSON.parse(JSON.stringify(rows, (_key, value) => typeof value === 'bigint' ? value.toString() : value));
+}
+//# sourceMappingURL=destination.js.map

package/dist/destination.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"destination.js","sourceRoot":"","sources":["../src/destination.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAA+C,MAAM,aAAa,CAAA;AAIhF,MAAM,CAAC,MAAM,eAAe,GAAG,iBAAiB,CAAA;AAChD,MAAM,CAAC,MAAM,aAAa,GAAG,eAAe,CAAA;AAC5C,MAAM,CAAC,MAAM,kBAAkB,GAAG,oBAAoB,CAAA;AAEtD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAgC;IAC3D,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IACzC,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvC,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,aAAa,EAAE;CACnF,CAAA;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,KAA2D;IAClF,OAAO,KAAK,CAAC;QACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,gBAAgB,CAAC;QAC7F,MAAM,EAAE,sBAAsB,kBAAkB,GAAG;QACnD,UAAU,EAAE,CAAC,IAAI,CAAC;QAClB,OAAO,EAAE,CAAC,IAAI,CAAC;KAChB,CAAC,CAAA;AACJ,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,OAAO,CAAI,KAAmB,EAAE,EAAuB;IACrE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;AAC3D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CAAC,QAA4B;IACtE,OAAO;QACL,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAC7B,MAAM,QAAQ,CAAC,MAAM,CAAC;gBACpB,KAAK,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;gBACxC,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC;gBACxB,QAAQ,EAAE;oBACR,0BAA0B,EAAE,KAAK;oBACjC,qBAAqB,EAAE,CAAC;iBACzB;aACF,CAAC,CAAA;QACJ,CAAC;KACF,CAAA;AACH,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,UAAU,CAAC,IAAoB;IAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAc,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAU,CAAA;AAClI,CAAC"}

package/dist/errors.d.ts

@@ -0,0 +1,33 @@
+import type { ErrorClassifier, FailureClass } from './types.js';
+export declare class IngestConfigError extends Error {
+    constructor(message: string);
+}
+/**
+ * Canonical boundary for fetch-based readers when `response.ok` is false.
+ * Keeps normalized HTTP facts plus the original Response for raw inspection.
+ */
+export declare class HttpError extends Error {
+    readonly status: number;
+    readonly statusText: string;
+    readonly url: string;
+    readonly retryAfterMs: number | undefined;
+    readonly body: string;
+    readonly response: Response;
+    private constructor();
+    static fromResponse(response: Response): Promise<HttpError>;
+}
+/** Executor-owned failure that always preserves the exact thrown value. */
+export declare class FetchFailure extends Error {
+    readonly classification: FailureClass;
+    readonly cause: unknown;
+    constructor(cause: unknown, classification: FailureClass);
+}
+export declare class BudgetExhausted extends Error {
+    constructor(message: string);
+}
+/**
+ * Cancellation is authoritative, then the common HTTP/network fallback is
+ * derived, then an optional provider classifier may enrich or override it.
+ */
+export declare function classifyFailure(cause: unknown, signal: AbortSignal, classifier: ErrorClassifier | undefined): FailureClass;
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAiB/D,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAI5B;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,KAAK;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAA;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAE3B,OAAO;WAWM,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;CAIlE;AAED,2EAA2E;AAC3E,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,cAAc,EAAE,YAAY,CAAA;IACrC,SAAkB,KAAK,EAAE,OAAO,CAAA;gBAEpB,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,YAAY;CAMzD;AAED,qBAAa,eAAgB,SAAQ,KAAK;gBAC5B,OAAO,EAAE,MAAM;CAI5B;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,eAAe,GAAG,SAAS,GACtC,YAAY,CAId"}

package/dist/errors.js

@@ -0,0 +1,133 @@
+const DIAGNOSTIC_BODY_LIMIT = 2048;
+const TRANSIENT_NETWORK_CODES = new Set([
+    'ECONNRESET',
+    'ECONNREFUSED',
+    'ECONNABORTED',
+    'ETIMEDOUT',
+    'EPIPE',
+    'EAI_AGAIN',
+    'ENOTFOUND',
+    'UND_ERR_SOCKET',
+    'UND_ERR_CONNECT_TIMEOUT',
+    'UND_ERR_HEADERS_TIMEOUT',
+    'UND_ERR_BODY_TIMEOUT',
+]);
+export class IngestConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'IngestConfigError';
+    }
+}
+/**
+ * Canonical boundary for fetch-based readers when `response.ok` is false.
+ * Keeps normalized HTTP facts plus the original Response for raw inspection.
+ */
+export class HttpError extends Error {
+    status;
+    statusText;
+    url;
+    retryAfterMs;
+    body;
+    response;
+    constructor(response, body) {
+        super(`HTTP ${response.status} ${response.statusText} for ${redactUrl(response.url)}${body ? `: ${body}` : ''}`);
+        this.name = 'HttpError';
+        this.status = response.status;
+        this.statusText = response.statusText;
+        this.url = redactUrl(response.url);
+        this.retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
+        this.body = body;
+        this.response = response;
+    }
+    static async fromResponse(response) {
+        const body = await readDiagnosticBody(response);
+        return new HttpError(response, body);
+    }
+}
+/** Executor-owned failure that always preserves the exact thrown value. */
+export class FetchFailure extends Error {
+    classification;
+    cause;
+    constructor(cause, classification) {
+        super(cause instanceof Error ? cause.message : String(cause));
+        this.name = 'FetchFailure';
+        this.cause = cause;
+        this.classification = classification;
+    }
+}
+export class BudgetExhausted extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'BudgetExhausted';
+    }
+}
+/**
+ * Cancellation is authoritative, then the common HTTP/network fallback is
+ * derived, then an optional provider classifier may enrich or override it.
+ */
+export function classifyFailure(cause, signal, classifier) {
+    if (signal.aborted)
+        return { kind: 'cancelled' };
+    const fallback = classifyCommon(cause);
+    return classifier?.(cause, fallback) ?? fallback;
+}
+function classifyCommon(cause) {
+    if (cause instanceof HttpError) {
+        if (cause.status === 429)
+            return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs };
+        if (cause.status === 408 || cause.status === 425 || cause.status >= 500)
+            return { kind: 'transient' };
+        if (cause.status >= 400)
+            return { kind: 'permanent' };
+        return { kind: 'unknown' };
+    }
+    const code = errorCode(cause);
+    if (code !== undefined && TRANSIENT_NETWORK_CODES.has(code))
+        return { kind: 'transient' };
+    if (cause instanceof TypeError && /fetch failed|network|socket/i.test(cause.message))
+        return { kind: 'transient' };
+    // A request's own timeout/abort does not cancel the ingestion execution.
+    if (cause instanceof Error && (cause.name === 'TimeoutError' || cause.name === 'AbortError'))
+        return { kind: 'transient' };
+    return { kind: 'unknown' };
+}
+function errorCode(value) {
+    if (typeof value !== 'object' || value === null)
+        return undefined;
+    if ('code' in value && typeof value.code === 'string')
+        return value.code;
+    if ('cause' in value)
+        return errorCode(value.cause);
+    return undefined;
+}
+function parseRetryAfter(header) {
+    if (header === null)
+        return undefined;
+    const seconds = Number(header);
+    if (Number.isFinite(seconds))
+        return Math.max(0, seconds * 1000);
+    const date = Date.parse(header);
+    if (Number.isNaN(date))
+        return undefined;
+    return Math.max(0, date - Date.now());
+}
+async function readDiagnosticBody(response) {
+    try {
+        const text = await response.clone().text();
+        return text.length > DIAGNOSTIC_BODY_LIMIT ? `${text.slice(0, DIAGNOSTIC_BODY_LIMIT)}…` : text;
+    }
+    catch {
+        return '';
+    }
+}
+// Query strings may carry API keys; keep only origin and path in diagnostics.
+function redactUrl(url) {
+    try {
+        const parsed = new URL(url);
+        return `${parsed.origin}${parsed.pathname}`;
+    }
+    catch {
+        return url;
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAEA,MAAM,qBAAqB,GAAG,IAAI,CAAA;AAClC,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IACtC,YAAY;IACZ,cAAc;IACd,cAAc;IACd,WAAW;IACX,OAAO;IACP,WAAW;IACX,WAAW;IACX,gBAAgB;IAChB,yBAAyB;IACzB,yBAAyB;IACzB,sBAAsB;CACvB,CAAC,CAAA;AAEF,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAA;IACjC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,SAAU,SAAQ,KAAK;IACzB,MAAM,CAAQ;IACd,UAAU,CAAQ;IAClB,GAAG,CAAQ;IACX,YAAY,CAAoB;IAChC,IAAI,CAAQ;IACZ,QAAQ,CAAU;IAE3B,YAAoB,QAAkB,EAAE,IAAY;QAClD,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,QAAQ,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAChH,IAAI,CAAC,IAAI,GAAG,WAAW,CAAA;QACvB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC7B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;QACrC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;QACxE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,QAAkB;QAC1C,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QAC/C,OAAO,IAAI,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;CACF;AAED,2EAA2E;AAC3E,MAAM,OAAO,YAAa,SAAQ,KAAK;IAC5B,cAAc,CAAc;IACnB,KAAK,CAAS;IAEhC,YAAY,KAAc,EAAE,cAA4B;QACtD,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;IACtC,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;IAC/B,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAc,EACd,MAAmB,EACnB,UAAuC;IAEvC,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAA;IACtC,OAAO,UAAU,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAA;AAClD,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAA;QAC3F,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACrG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACrD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;IAC7B,IAAI,IAAI,KAAK,SAAS,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACzF,IAAI,KAAK,YAAY,SAAS,IAAI,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IAClH,yEAAyE;IACzE,IAAI,KAAK,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IAC1H,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAA;IACjE,IAAI,MAAM,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IACxE,IAAI,OAAO,IAAI,KAAK;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACnD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,MAAqB;IAC5C,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,SAAS,CAAA;IACrC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9B,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAA;IAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC/B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IACxC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;AACvC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,QAAkB;IAClD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAA;QAC1C,OAAO,IAAI,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;IAChG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAA;IACX,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAA;IACZ,CAAC;AACH,CAAC"}

package/dist/executor.d.ts

@@ -0,0 +1,30 @@
+import type { SelectedStream } from './registry.js';
+import type { DestinationAdapter, ExecutionResult, Journal } from './types.js';
+export interface BackfillRequest {
+    /** Stable identity: resuming the same backfill reuses its isolated checkpoint namespace. */
+    id: string;
+    from: Date | undefined;
+    to: Date | undefined;
+}
+export interface ExecutionRequest {
+    selected: readonly SelectedStream[];
+    backfill: BackfillRequest | undefined;
+}
+export interface ExecutionEnv {
+    journal: Journal;
+    destination: DestinationAdapter;
+    signal?: AbortSignal;
+    /** Host-default execution budget. Exhaustion preserves committed progress. */
+    maxDurationMs?: number;
+    /** Finite number of mapped batches buffered between fetch and load. */
+    prefetchBatches?: number;
+    now?: () => Date;
+    log?: (message: string) => void;
+}
+/**
+ * One transient execution context for one exact stream selection. The run id
+ * correlates journal and telemetry evidence only: every stream plans and
+ * recovers independently from its own journal-backed checkpoint.
+ */
+export declare function runIngestion(request: ExecutionRequest, input: ExecutionEnv): Promise<ExecutionResult>;
+//# sourceMappingURL=executor.d.ts.map

package/dist/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../src/executor.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAEnD,OAAO,KAAK,EAGV,kBAAkB,EAClB,eAAe,EACf,OAAO,EAOR,MAAM,YAAY,CAAA;AAiBnB,MAAM,WAAW,eAAe;IAC9B,4FAA4F;IAC5F,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,IAAI,GAAG,SAAS,CAAA;IACtB,EAAE,EAAE,IAAI,GAAG,SAAS,CAAA;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,QAAQ,EAAE,eAAe,GAAG,SAAS,CAAA;CACtC;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAA;IAChB,WAAW,EAAE,kBAAkB,CAAA;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;IAChB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CAChC;AAoCD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAsC3G"}