@chkit/plugin-ingest 0.2.0-beta.11

package/README.md

@@ -0,0 +1,43 @@
+# @chkit/plugin-ingest
+
+`@chkit/plugin-ingest` implements **chkit Sync (beta)**, the optional TypeScript layer for syncing business application data into ClickHouse. Define and migrate destination tables with [chkit](https://www.npmjs.com/package/chkit), then add readers to load them. Use an external scheduler for repeated runs.
+
+The executor records progress after writes succeed. Run-scoped journal identities keep overlapping executions independent. A retry resumes from a validated checkpoint and may reread rows.
+
+`paginate()` yields complete `{ items, next, metadata? }` pages, including empty terminal pages. Read rows from `page.items`; use `next` for request continuation and explicitly map checkpoint metadata into a chunk's `state` with `cursorState`. The executor commits that state after the rows are saved. This replaces the previous item-array return type; existing readers must use `page.items`.
+
+```ts
+import { defineConfig } from '@chkit/core'
+import { ingest } from '@chkit/plugin-ingest'
+
+export default defineConfig({
+  entry: './src/chkit.ts',
+  plugins: [ingest()],
+  clickhouse: { url: process.env.CLICKHOUSE_URL ?? '' },
+})
+```
+
+```sh
+chkit ingest run --tag schedule:1h
+```
+
+The append-only ClickHouse journal records each run's starting checkpoint and local event sequence. Two processes may start from the same checkpoint without reusing event identities. Complete snapshot validation excludes malformed run tails while retaining acknowledged progress. A stale but valid read can cause replay; destination writes remain at-least-once.
+
+Diagnose the journal or preview evidence-preserving recovery:
+
+```sh
+chkit ingest doctor --tag stream:helpdesk.tickets
+chkit ingest repair --tag stream:helpdesk.tickets
+```
+
+Overlapping valid runs need no repair. For damaged evidence, stop all ingestion writers and review the plan. `chkit ingest repair --tag stream:helpdesk.tickets --apply <fingerprint>` materializes a verified archive and replacement journal while preserving the original. Keep writers stopped, set `ingest({ journalTable: '<replacementTable>' })` to the returned table, then restart them. The command leaves activation to this explicit configuration change. Other streams' evidence is copied unchanged.
+
+Documentation: [chkit Sync (beta)](https://chkit.obsessiondb.com/sync/)
+
+For raw storage, typed rows, and where to map fields, see [Destinations and transformations](https://chkit.obsessiondb.com/api-sync/destinations/).
+
+Install the source-authoring skill for a coding agent:
+
+```sh
+npx skills add obsessiondb/chkit --skill chkit-ingestion
+```

package/dist/destination.d.ts

@@ -0,0 +1,40 @@
+import type { ClickHouseExecutor } from '@chkit/clickhouse';
+import { type ColumnDefinition, type TableDefinition } from '@chkit/core';
+import type { DestinationAdapter, Row } from './types.js';
+export declare const BATCH_ID_COLUMN = "_chkit_batch_id";
+export declare const RUN_ID_COLUMN = "_chkit_run_id";
+export declare const INGESTED_AT_COLUMN = "_chkit_ingested_at";
+/**
+ * Runtime-owned metadata every ingestion destination table carries. Spread it
+ * into the table's `columns`. `_chkit_ingested_at` is destination-owned: it is
+ * never sent by the loader, so it records the physical publication time even
+ * when a retry recomputes it under the same deduplication token.
+ */
+export declare const ingestionColumns: readonly ColumnDefinition[];
+export type RawRow = {
+    id: string;
+    raw: unknown;
+};
+/**
+ * Landing table for provider objects exactly as received: a stable id plus the
+ * untouched object in a native JSON column. Typed shapes are derived from it
+ * inside ClickHouse (views or materialized views), so changing a transform
+ * never requires re-fetching the source. Replays and overlapping windows
+ * collapse to the latest ingested version of each id.
+ */
+export declare function rawTable(input: {
+    database: string;
+    name: string;
+    comment?: string;
+}): TableDefinition;
+/** Shape provider objects for a {@link rawTable} without mapping their fields. */
+export declare function rawRows<T>(items: readonly T[], id: (item: T) => string): RawRow[];
+/**
+ * A successful synchronous insert response (or an awaited async insert) is the
+ * sink evidence ChKit trusts. A missing response stays ambiguous and is retried
+ * with the same token.
+ */
+export declare function createClickHouseDestination(executor: ClickHouseExecutor): DestinationAdapter;
+/** JSONEachRow encodes big integers as exact decimal strings, including nested values. */
+export declare function toJsonRows(rows: readonly Row[]): Row[];
+//# sourceMappingURL=destination.d.ts.map

package/dist/destination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"destination.d.ts","sourceRoot":"","sources":["../src/destination.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AAC3D,OAAO,EAA2B,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AAElG,OAAO,KAAK,EAAE,kBAAkB,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAEzD,eAAO,MAAM,eAAe,oBAAoB,CAAA;AAChD,eAAO,MAAM,aAAa,kBAAkB,CAAA;AAC5C,eAAO,MAAM,kBAAkB,uBAAuB,CAAA;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,gBAAgB,EAIvD,CAAA;AAGD,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAA;IACV,GAAG,EAAE,OAAO,CAAA;CACb,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,eAAe,CAUrG;AAED,kFAAkF;AAClF,wBAAgB,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAAG,MAAM,EAAE,CAEjF;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,kBAAkB,GAAG,kBAAkB,CAgB5F;AAED,0FAA0F;AAC1F,wBAAgB,UAAU,CAAC,IAAI,EAAE,SAAS,GAAG,EAAE,GAAG,GAAG,EAAE,CAEtD"}

package/dist/destination.js

@@ -0,0 +1,65 @@
+import { insertColumnList, table } from '@chkit/core';
+export const BATCH_ID_COLUMN = '_chkit_batch_id';
+export const RUN_ID_COLUMN = '_chkit_run_id';
+export const INGESTED_AT_COLUMN = '_chkit_ingested_at';
+/**
+ * Runtime-owned metadata every ingestion destination table carries. Spread it
+ * into the table's `columns`. `_chkit_ingested_at` is destination-owned: it is
+ * never sent by the loader, so it records the physical publication time even
+ * when a retry recomputes it under the same deduplication token.
+ */
+export const ingestionColumns = [
+    { name: BATCH_ID_COLUMN, type: 'String' },
+    { name: RUN_ID_COLUMN, type: 'String' },
+    { name: INGESTED_AT_COLUMN, type: "DateTime64(6, 'UTC')", default: 'fn:now64(6)' },
+];
+/**
+ * Landing table for provider objects exactly as received: a stable id plus the
+ * untouched object in a native JSON column. Typed shapes are derived from it
+ * inside ClickHouse (views or materialized views), so changing a transform
+ * never requires re-fetching the source. Replays and overlapping windows
+ * collapse to the latest ingested version of each id.
+ */
+export function rawTable(input) {
+    return table({
+        database: input.database,
+        name: input.name,
+        comment: input.comment,
+        columns: [{ name: 'id', type: 'String' }, { name: 'raw', type: 'JSON' }, ...ingestionColumns],
+        engine: `ReplacingMergeTree(${INGESTED_AT_COLUMN})`,
+        primaryKey: ['id'],
+        orderBy: ['id'],
+    });
+}
+/** Shape provider objects for a {@link rawTable} without mapping their fields. */
+export function rawRows(items, id) {
+    return items.map((item) => ({ id: id(item), raw: item }));
+}
+/**
+ * A successful synchronous insert response (or an awaited async insert) is the
+ * sink evidence ChKit trusts. A missing response stays ambiguous and is retried
+ * with the same token.
+ */
+export function createClickHouseDestination(executor) {
+    return {
+        async insert({ table, rows, token }) {
+            if (rows.length === 0)
+                return;
+            const columns = insertColumnList(table);
+            await executor.insert({
+                table: `${table.database}.${table.name}`,
+                values: toJsonRows(rows),
+                ...(columns ? { columns } : {}),
+                settings: {
+                    insert_deduplication_token: token,
+                    wait_for_async_insert: 1,
+                },
+            });
+        },
+    };
+}
+/** JSONEachRow encodes big integers as exact decimal strings, including nested values. */
+export function toJsonRows(rows) {
+    return JSON.parse(JSON.stringify(rows, (_key, value) => typeof value === 'bigint' ? value.toString() : value));
+}
+//# sourceMappingURL=destination.js.map

package/dist/destination.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"destination.js","sourceRoot":"","sources":["../src/destination.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAA+C,MAAM,aAAa,CAAA;AAIlG,MAAM,CAAC,MAAM,eAAe,GAAG,iBAAiB,CAAA;AAChD,MAAM,CAAC,MAAM,aAAa,GAAG,eAAe,CAAA;AAC5C,MAAM,CAAC,MAAM,kBAAkB,GAAG,oBAAoB,CAAA;AAEtD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAgC;IAC3D,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IACzC,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvC,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,aAAa,EAAE;CACnF,CAAA;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,KAA2D;IAClF,OAAO,KAAK,CAAC;QACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,gBAAgB,CAAC;QAC7F,MAAM,EAAE,sBAAsB,kBAAkB,GAAG;QACnD,UAAU,EAAE,CAAC,IAAI,CAAC;QAClB,OAAO,EAAE,CAAC,IAAI,CAAC;KAChB,CAAC,CAAA;AACJ,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,OAAO,CAAI,KAAmB,EAAE,EAAuB;IACrE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;AAC3D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CAAC,QAA4B;IACtE,OAAO;QACL,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAC7B,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;YACvC,MAAM,QAAQ,CAAC,MAAM,CAAC;gBACpB,KAAK,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;gBACxC,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC;gBACxB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/B,QAAQ,EAAE;oBACR,0BAA0B,EAAE,KAAK;oBACjC,qBAAqB,EAAE,CAAC;iBACzB;aACF,CAAC,CAAA;QACJ,CAAC;KACF,CAAA;AACH,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,UAAU,CAAC,IAAoB;IAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAc,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAU,CAAA;AAClI,CAAC"}

package/dist/errors.d.ts

@@ -0,0 +1,33 @@
+import type { ErrorClassifier, FailureClass } from './types.js';
+export declare class IngestConfigError extends Error {
+    constructor(message: string);
+}
+/**
+ * Canonical boundary for fetch-based readers when `response.ok` is false.
+ * Keeps normalized HTTP facts plus the original Response for raw inspection.
+ */
+export declare class HttpError extends Error {
+    readonly status: number;
+    readonly statusText: string;
+    readonly url: string;
+    readonly retryAfterMs: number | undefined;
+    readonly body: string;
+    readonly response: Response;
+    private constructor();
+    static fromResponse(response: Response): Promise<HttpError>;
+}
+/** Executor-owned failure that always preserves the exact thrown value. */
+export declare class FetchFailure extends Error {
+    readonly classification: FailureClass;
+    readonly cause: unknown;
+    constructor(cause: unknown, classification: FailureClass);
+}
+export declare class BudgetExhausted extends Error {
+    constructor(message: string);
+}
+/**
+ * Cancellation is authoritative, then the common HTTP/network fallback is
+ * derived, then an optional provider classifier may enrich or override it.
+ */
+export declare function classifyFailure(cause: unknown, signal: AbortSignal, classifier: ErrorClassifier | undefined): FailureClass;
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAiB/D,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAI5B;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,KAAK;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAA;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAE3B,OAAO;WAWM,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;CAIlE;AAED,2EAA2E;AAC3E,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,cAAc,EAAE,YAAY,CAAA;IACrC,SAAkB,KAAK,EAAE,OAAO,CAAA;gBAEpB,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,YAAY;CAMzD;AAED,qBAAa,eAAgB,SAAQ,KAAK;gBAC5B,OAAO,EAAE,MAAM;CAI5B;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,eAAe,GAAG,SAAS,GACtC,YAAY,CAId"}

package/dist/errors.js

@@ -0,0 +1,133 @@
+const DIAGNOSTIC_BODY_LIMIT = 2048;
+const TRANSIENT_NETWORK_CODES = new Set([
+    'ECONNRESET',
+    'ECONNREFUSED',
+    'ECONNABORTED',
+    'ETIMEDOUT',
+    'EPIPE',
+    'EAI_AGAIN',
+    'ENOTFOUND',
+    'UND_ERR_SOCKET',
+    'UND_ERR_CONNECT_TIMEOUT',
+    'UND_ERR_HEADERS_TIMEOUT',
+    'UND_ERR_BODY_TIMEOUT',
+]);
+export class IngestConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'IngestConfigError';
+    }
+}
+/**
+ * Canonical boundary for fetch-based readers when `response.ok` is false.
+ * Keeps normalized HTTP facts plus the original Response for raw inspection.
+ */
+export class HttpError extends Error {
+    status;
+    statusText;
+    url;
+    retryAfterMs;
+    body;
+    response;
+    constructor(response, body) {
+        super(`HTTP ${response.status} ${response.statusText} for ${redactUrl(response.url)}${body ? `: ${body}` : ''}`);
+        this.name = 'HttpError';
+        this.status = response.status;
+        this.statusText = response.statusText;
+        this.url = redactUrl(response.url);
+        this.retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
+        this.body = body;
+        this.response = response;
+    }
+    static async fromResponse(response) {
+        const body = await readDiagnosticBody(response);
+        return new HttpError(response, body);
+    }
+}
+/** Executor-owned failure that always preserves the exact thrown value. */
+export class FetchFailure extends Error {
+    classification;
+    cause;
+    constructor(cause, classification) {
+        super(cause instanceof Error ? cause.message : String(cause));
+        this.name = 'FetchFailure';
+        this.cause = cause;
+        this.classification = classification;
+    }
+}
+export class BudgetExhausted extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'BudgetExhausted';
+    }
+}
+/**
+ * Cancellation is authoritative, then the common HTTP/network fallback is
+ * derived, then an optional provider classifier may enrich or override it.
+ */
+export function classifyFailure(cause, signal, classifier) {
+    if (signal.aborted)
+        return { kind: 'cancelled' };
+    const fallback = classifyCommon(cause);
+    return classifier?.(cause, fallback) ?? fallback;
+}
+function classifyCommon(cause) {
+    if (cause instanceof HttpError) {
+        if (cause.status === 429)
+            return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs };
+        if (cause.status === 408 || cause.status === 425 || cause.status >= 500)
+            return { kind: 'transient' };
+        if (cause.status >= 400)
+            return { kind: 'permanent' };
+        return { kind: 'unknown' };
+    }
+    const code = errorCode(cause);
+    if (code !== undefined && TRANSIENT_NETWORK_CODES.has(code))
+        return { kind: 'transient' };
+    if (cause instanceof TypeError && /fetch failed|network|socket/i.test(cause.message))
+        return { kind: 'transient' };
+    // A request's own timeout/abort does not cancel the ingestion execution.
+    if (cause instanceof Error && (cause.name === 'TimeoutError' || cause.name === 'AbortError'))
+        return { kind: 'transient' };
+    return { kind: 'unknown' };
+}
+function errorCode(value) {
+    if (typeof value !== 'object' || value === null)
+        return undefined;
+    if ('code' in value && typeof value.code === 'string')
+        return value.code;
+    if ('cause' in value)
+        return errorCode(value.cause);
+    return undefined;
+}
+function parseRetryAfter(header) {
+    if (header === null)
+        return undefined;
+    const seconds = Number(header);
+    if (Number.isFinite(seconds))
+        return Math.max(0, seconds * 1000);
+    const date = Date.parse(header);
+    if (Number.isNaN(date))
+        return undefined;
+    return Math.max(0, date - Date.now());
+}
+async function readDiagnosticBody(response) {
+    try {
+        const text = await response.clone().text();
+        return text.length > DIAGNOSTIC_BODY_LIMIT ? `${text.slice(0, DIAGNOSTIC_BODY_LIMIT)}…` : text;
+    }
+    catch {
+        return '';
+    }
+}
+// Query strings may carry API keys; keep only origin and path in diagnostics.
+function redactUrl(url) {
+    try {
+        const parsed = new URL(url);
+        return `${parsed.origin}${parsed.pathname}`;
+    }
+    catch {
+        return url;
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAEA,MAAM,qBAAqB,GAAG,IAAI,CAAA;AAClC,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IACtC,YAAY;IACZ,cAAc;IACd,cAAc;IACd,WAAW;IACX,OAAO;IACP,WAAW;IACX,WAAW;IACX,gBAAgB;IAChB,yBAAyB;IACzB,yBAAyB;IACzB,sBAAsB;CACvB,CAAC,CAAA;AAEF,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAA;IACjC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,SAAU,SAAQ,KAAK;IACzB,MAAM,CAAQ;IACd,UAAU,CAAQ;IAClB,GAAG,CAAQ;IACX,YAAY,CAAoB;IAChC,IAAI,CAAQ;IACZ,QAAQ,CAAU;IAE3B,YAAoB,QAAkB,EAAE,IAAY;QAClD,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,QAAQ,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAChH,IAAI,CAAC,IAAI,GAAG,WAAW,CAAA;QACvB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC7B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;QACrC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;QACxE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,QAAkB;QAC1C,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QAC/C,OAAO,IAAI,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;CACF;AAED,2EAA2E;AAC3E,MAAM,OAAO,YAAa,SAAQ,KAAK;IAC5B,cAAc,CAAc;IACnB,KAAK,CAAS;IAEhC,YAAY,KAAc,EAAE,cAA4B;QACtD,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;IACtC,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;IAC/B,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAc,EACd,MAAmB,EACnB,UAAuC;IAEvC,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAA;IACtC,OAAO,UAAU,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAA;AAClD,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAA;QAC3F,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACrG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACrD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;IAC7B,IAAI,IAAI,KAAK,SAAS,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACzF,IAAI,KAAK,YAAY,SAAS,IAAI,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IAClH,yEAAyE;IACzE,IAAI,KAAK,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IAC1H,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAA;IACjE,IAAI,MAAM,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IACxE,IAAI,OAAO,IAAI,KAAK;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACnD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,MAAqB;IAC5C,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,SAAS,CAAA;IACrC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9B,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAA;IAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC/B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IACxC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;AACvC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,QAAkB;IAClD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAA;QAC1C,OAAO,IAAI,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;IAChG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAA;IACX,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAA;IACZ,CAAC;AACH,CAAC"}

package/dist/executor.d.ts

@@ -0,0 +1,30 @@
+import type { SelectedStream } from './registry.js';
+import type { DestinationAdapter, ExecutionResult, Journal } from './types.js';
+export interface BackfillRequest {
+    /** Stable identity: resuming the same backfill reuses its isolated checkpoint namespace. */
+    id: string;
+    from: Date | undefined;
+    to: Date | undefined;
+}
+export interface ExecutionRequest {
+    selected: readonly SelectedStream[];
+    backfill: BackfillRequest | undefined;
+}
+export interface ExecutionEnv {
+    journal: Journal;
+    destination: DestinationAdapter;
+    signal?: AbortSignal;
+    /** Host-default execution budget. Exhaustion preserves committed progress. */
+    maxDurationMs?: number;
+    /** Finite number of mapped batches buffered between fetch and load. */
+    prefetchBatches?: number;
+    now?: () => Date;
+    log?: (message: string) => void;
+}
+/**
+ * One transient execution context for one exact stream selection. The run id
+ * correlates journal and telemetry evidence only: every stream plans and
+ * recovers independently from its own journal-backed checkpoint.
+ */
+export declare function runIngestion(request: ExecutionRequest, input: ExecutionEnv): Promise<ExecutionResult>;
+//# sourceMappingURL=executor.d.ts.map

package/dist/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../src/executor.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAEnD,OAAO,KAAK,EAIV,kBAAkB,EAClB,eAAe,EACf,OAAO,EAOR,MAAM,YAAY,CAAA;AAiBnB,MAAM,WAAW,eAAe;IAC9B,4FAA4F;IAC5F,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,IAAI,GAAG,SAAS,CAAA;IACtB,EAAE,EAAE,IAAI,GAAG,SAAS,CAAA;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,QAAQ,EAAE,eAAe,GAAG,SAAS,CAAA;CACtC;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAA;IAChB,WAAW,EAAE,kBAAkB,CAAA;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;IAChB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CAChC;AA0CD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAsC3G"}