@chkit/plugin-ingest 0.2.0-beta.11

package/dist/journal-records.js

@@ -0,0 +1,76 @@
+import { createHash } from 'node:crypto';
+export function toJournalRow(event, targetId, at) {
+    const checkpointJson = event.checkpoint ? canonicalJson(event.checkpoint) : '';
+    const detailJson = canonicalJson(event.detail);
+    // Identity covers what makes the fact unique; the payload hash covers every
+    // authoritative field a retry of that same fact must reproduce. Only the
+    // physical append time (event_at) is excluded.
+    const eventId = digest([targetId, event.namespaceId, event.runId, String(event.eventSeq), event.eventKind, event.workId, event.batchId, String(event.attemptNo)]);
+    const payload = digest([
+        eventId,
+        String(event.expectedCheckpointVersion),
+        String(event.checkpointVersion),
+        checkpointJson,
+        event.workState,
+        event.sinkEvidence,
+        event.errorClass,
+        event.runId,
+        event.retryAt ? event.retryAt.toISOString() : '',
+        detailJson,
+    ]);
+    return {
+        target_id: targetId,
+        namespace_id: event.namespaceId,
+        event_seq: String(event.eventSeq),
+        event_id: eventId,
+        payload_hash: BigInt(`0x${payload.slice(0, 16)}`).toString(),
+        event_at: toClickHouseDateTime(at),
+        event_kind: event.eventKind,
+        run_id: event.runId,
+        work_id: event.workId,
+        attempt_no: event.attemptNo,
+        batch_id: event.batchId,
+        expected_checkpoint_version: String(event.expectedCheckpointVersion),
+        checkpoint_version: String(event.checkpointVersion),
+        checkpoint_json: checkpointJson,
+        work_state: event.workState,
+        sink_evidence: event.sinkEvidence,
+        retry_at: event.retryAt ? toClickHouseDateTime(event.retryAt) : null,
+        error_class: event.errorClass,
+        detail_json: detailJson,
+    };
+}
+/** Stable key order so equal values always serialize identically. */
+export function canonicalJson(value) {
+    // JSON.stringify(undefined) is undefined, not a string.
+    return JSON.stringify(sortKeys(value)) ?? 'null';
+}
+export function digest(parts) {
+    const hash = createHash('sha256');
+    for (const part of parts) {
+        hash.update(String(part.length));
+        hash.update(':');
+        hash.update(part);
+    }
+    return hash.digest('hex');
+}
+export function emptyCheckpoint() {
+    return { version: 0, envelope: undefined, checkpointId: '', successId: '', headSeq: 0, lastSuccessSeq: 0 };
+}
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (value instanceof Date)
+        return value.toISOString();
+    if (typeof value === 'object' && value !== null) {
+        return Object.fromEntries(Object.entries(value)
+            .filter(([, entry]) => entry !== undefined)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([key, entry]) => [key, sortKeys(entry)]));
+    }
+    return value;
+}
+function toClickHouseDateTime(date) {
+    return date.toISOString().replace('T', ' ').replace('Z', '');
+}
+//# sourceMappingURL=journal-records.js.map

package/dist/journal-records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"journal-records.js","sourceRoot":"","sources":["../src/journal-records.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AA2BxC,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,QAAgB,EAAE,EAAQ;IAC1E,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9E,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC9C,4EAA4E;IAC5E,yEAAyE;IACzE,+CAA+C;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IACjK,MAAM,OAAO,GAAG,MAAM,CAAC;QACrB,OAAO;QACP,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC/B,cAAc;QACd,KAAK,CAAC,SAAS;QACf,KAAK,CAAC,YAAY;QAClB,KAAK,CAAC,UAAU;QAChB,KAAK,CAAC,KAAK;QACX,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE;QAChD,UAAU;KACX,CAAC,CAAA;IACF,OAAO;QACL,SAAS,EAAE,QAAQ;QACnB,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjC,QAAQ,EAAE,OAAO;QACjB,YAAY,EAAE,MAAM,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;QAC5D,QAAQ,EAAE,oBAAoB,CAAC,EAAE,CAAC;QAClC,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,MAAM,EAAE,KAAK,CAAC,KAAK;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,2BAA2B,EAAE,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;QACpE,kBAAkB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;QACnD,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;QACpE,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,WAAW,EAAE,UAAU;KACxB,CAAA;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,wDAAwD;IACxD,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAAwB;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACnB,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAA;AAC5G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IACpD,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC,WAAW,EAAE,CAAA;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;aAC1C,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAChD,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CACjD,CAAA;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAU;IACtC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;AAC9D,CAAC"}

package/dist/journal.d.ts

@@ -0,0 +1,16 @@
+import { type ClickHouseExecutor } from '@chkit/clickhouse';
+import type { Journal } from './types.js';
+export { canonicalJson, digest, emptyCheckpoint, toJournalRow, type JournalRow } from './journal-records.js';
+export declare const DEFAULT_JOURNAL_TABLE = "_chkit_ingestion_journal";
+export interface ClickHouseJournalOptions {
+    /** Must allow concurrent queries: use a stateless executor, not a session-bound one. */
+    executor: ClickHouseExecutor;
+    database: string;
+    targetId: string;
+    table?: string;
+    now?: () => Date;
+}
+/** Immutable run-scoped facts are projected from one ClickHouse snapshot. */
+export declare function createClickHouseJournal(options: ClickHouseJournalOptions): Journal;
+export declare function journalTableSql(qualified: string): string;
+//# sourceMappingURL=journal.d.ts.map

package/dist/journal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"journal.d.ts","sourceRoot":"","sources":["../src/journal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AAKzE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAE5G,eAAO,MAAM,qBAAqB,6BAA6B,CAAA;AAG/D,MAAM,WAAW,wBAAwB;IACvC,wFAAwF;IACxF,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;CACjB;AAED,6EAA6E;AAC7E,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CA8BlF;AAMD,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA0BzD"}

package/dist/journal.js

@@ -0,0 +1,72 @@
+import { waitForTable } from '@chkit/clickhouse';
+import { IngestConfigError } from './errors.js';
+import { validateJournalHistory } from './journal-history.js';
+import { digest, toJournalRow } from './journal-records.js';
+export { canonicalJson, digest, emptyCheckpoint, toJournalRow } from './journal-records.js';
+export const DEFAULT_JOURNAL_TABLE = '_chkit_ingestion_journal';
+const TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** Immutable run-scoped facts are projected from one ClickHouse snapshot. */
+export function createClickHouseJournal(options) {
+    const table = options.table ?? DEFAULT_JOURNAL_TABLE;
+    validateTableName(table);
+    validateTableName(options.database);
+    const now = options.now ?? (() => new Date());
+    const qualified = `\`${options.database}\`.\`${table}\``;
+    return {
+        async ensure() {
+            await options.executor.command(journalTableSql(qualified));
+            await waitForTable(options.executor, options.database, table);
+        },
+        async append(events) {
+            if (events.length === 0)
+                return;
+            const at = now();
+            const rows = events.map((event) => toJournalRow(event, options.targetId, at));
+            await options.executor.insert({
+                table: `${options.database}.${table}`, values: rows,
+                settings: { insert_deduplication_token: digest(rows.flatMap((row) => [row.event_id, row.payload_hash])), async_insert: 0 },
+            });
+        },
+        async readCheckpoint(namespaceId) {
+            const rows = await options.executor.query(`SELECT * FROM ${qualified} WHERE target_id = ${sqlString(options.targetId)} AND namespace_id = ${sqlString(namespaceId)} ORDER BY run_id, event_seq, event_id, payload_hash`, { select_sequential_consistency: '1', use_query_cache: 0, output_format_json_quote_64bit_integers: 1 });
+            // A stale snapshot can cause replay, but its runs cannot overwrite the
+            // identities or checkpoint lineage of another run.
+            return validateJournalHistory(rows, namespaceId).checkpoint;
+        },
+    };
+}
+function validateTableName(name) {
+    if (!TABLE_NAME_PATTERN.test(name))
+        throw new IngestConfigError(`Invalid journal table or database name "${name}".`);
+}
+export function journalTableSql(qualified) {
+    return `CREATE TABLE IF NOT EXISTS ${qualified}
+(
+    target_id LowCardinality(String),
+    namespace_id String,
+    event_seq UInt64,
+    event_id String,
+    payload_hash UInt64,
+    event_at DateTime64(6, 'UTC'),
+    event_kind LowCardinality(String),
+    run_id String,
+    work_id String,
+    attempt_no UInt32,
+    batch_id String,
+    expected_checkpoint_version UInt64,
+    checkpoint_version UInt64,
+    checkpoint_json String,
+    work_state LowCardinality(String),
+    sink_evidence LowCardinality(String),
+    retry_at Nullable(DateTime64(6, 'UTC')),
+    error_class LowCardinality(String),
+    detail_json String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(event_at)
+ORDER BY (target_id, namespace_id, event_seq, event_id)`;
+}
+function sqlString(value) {
+    return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+}
+//# sourceMappingURL=journal.js.map

package/dist/journal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"journal.js","sourceRoot":"","sources":["../src/journal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAA2B,MAAM,mBAAmB,CAAA;AAEzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAmB,MAAM,sBAAsB,CAAA;AAG5E,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,YAAY,EAAmB,MAAM,sBAAsB,CAAA;AAE5G,MAAM,CAAC,MAAM,qBAAqB,GAAG,0BAA0B,CAAA;AAC/D,MAAM,kBAAkB,GAAG,0BAA0B,CAAA;AAWrD,6EAA6E;AAC7E,MAAM,UAAU,uBAAuB,CAAC,OAAiC;IACvE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,qBAAqB,CAAA;IACpD,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACxB,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;IAC7C,MAAM,SAAS,GAAG,KAAK,OAAO,CAAC,QAAQ,QAAQ,KAAK,IAAI,CAAA;IAExD,OAAO;QACL,KAAK,CAAC,MAAM;YACV,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAA;YAC1D,MAAM,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QAC/D,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,MAAM;YACjB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAC/B,MAAM,EAAE,GAAG,GAAG,EAAE,CAAA;YAChB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAA;YAC7E,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC5B,KAAK,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI;gBACnD,QAAQ,EAAE,EAAE,0BAA0B,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;aAC3H,CAAC,CAAA;QACJ,CAAC;QACD,KAAK,CAAC,cAAc,CAAC,WAAW;YAC9B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,KAAK,CACvC,iBAAiB,SAAS,sBAAsB,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,SAAS,CAAC,WAAW,CAAC,qDAAqD,EAC7K,EAAE,6BAA6B,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,uCAAuC,EAAE,CAAC,EAAE,CAAC,CAAA;YACzG,uEAAuE;YACvE,mDAAmD;YACnD,OAAO,sBAAsB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,UAAU,CAAA;QAC7D,CAAC;KACF,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,2CAA2C,IAAI,IAAI,CAAC,CAAA;AACtH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAiB;IAC/C,OAAO,8BAA8B,SAAS;;;;;;;;;;;;;;;;;;;;;;;;wDAwBQ,CAAA;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAA;AACjE,CAAC"}

package/dist/loader.d.ts

@@ -0,0 +1,12 @@
+import type { LoaderFactory } from './types.js';
+export interface SimpleLoaderOptions {
+    /** Rows per physical INSERT. A larger batch is split into deterministic chunks. */
+    maxRowsPerInsert?: number;
+}
+/**
+ * Direct at-least-once destination writes. Every write unit reuses one stable
+ * token across retries: a batch written by one INSERT has one token, a
+ * deterministically chunked batch has one distinct token per chunk.
+ */
+export declare function simpleLoader(options?: SimpleLoaderOptions): LoaderFactory;
+//# sourceMappingURL=loader.d.ts.map

package/dist/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,MAAM,WAAW,mBAAmB;IAClC,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAID;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,aAAa,CA8B7E"}

package/dist/loader.js

@@ -0,0 +1,37 @@
+import { BATCH_ID_COLUMN, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js';
+const DEFAULT_MAX_ROWS_PER_INSERT = 50_000;
+/**
+ * Direct at-least-once destination writes. Every write unit reuses one stable
+ * token across retries: a batch written by one INSERT has one token, a
+ * deterministically chunked batch has one distinct token per chunk.
+ */
+export function simpleLoader(options = {}) {
+    const maxRows = options.maxRowsPerInsert ?? DEFAULT_MAX_ROWS_PER_INSERT;
+    return (ctx) => {
+        let rows = 0;
+        let writeUnits = 0;
+        return {
+            ctx,
+            async write(batch) {
+                for (let offset = 0, unit = 0; offset < batch.rows.length; offset += maxRows, unit += 1) {
+                    ctx.signal.throwIfAborted();
+                    const slice = batch.rows.slice(offset, offset + maxRows).map((row) => {
+                        // Publication time is destination-owned: never let a mapped row supply it.
+                        const { [INGESTED_AT_COLUMN]: _ignored, ...authored } = row;
+                        return { ...authored, [BATCH_ID_COLUMN]: batch.batchId, [RUN_ID_COLUMN]: ctx.runId };
+                    });
+                    await ctx.destination.insert({ table: ctx.table, rows: slice, token: `${batch.batchId}:${unit}` });
+                    rows += slice.length;
+                    writeUnits += 1;
+                }
+            },
+            async finalize() {
+                return { evidence: writeUnits === 0 ? 'none_required' : 'clickhouse_ack', rows, writeUnits };
+            },
+            async abort() {
+                // Direct writes cannot be withdrawn; replay reuses the same tokens.
+            },
+        };
+    };
+}
+//# sourceMappingURL=loader.js.map

package/dist/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAQrF,MAAM,2BAA2B,GAAG,MAAM,CAAA;AAE1C;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,UAA+B,EAAE;IAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,gBAAgB,IAAI,2BAA2B,CAAA;IAEvE,OAAO,CAAC,GAAG,EAAE,EAAE;QACb,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,OAAO;YACL,GAAG;YACH,KAAK,CAAC,KAAK,CAAC,KAAK;gBACf,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;oBACxF,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;oBAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBACnE,2EAA2E;wBAC3E,MAAM,EAAE,CAAC,kBAAkB,CAAC,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,GAAG,CAAA;wBAC3D,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,EAAE,GAAG,CAAC,KAAK,EAAE,CAAA;oBACtF,CAAC,CAAC,CAAA;oBACF,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,OAAO,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;oBAClG,IAAI,IAAI,KAAK,CAAC,MAAM,CAAA;oBACpB,UAAU,IAAI,CAAC,CAAA;gBACjB,CAAC;YACH,CAAC;YACD,KAAK,CAAC,QAAQ;gBACZ,OAAO,EAAE,QAAQ,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAA;YAC9F,CAAC;YACD,KAAK,CAAC,KAAK;gBACT,oEAAoE;YACtE,CAAC;SACF,CAAA;IACH,CAAC,CAAA;AACH,CAAC"}

package/dist/paginate.d.ts

@@ -0,0 +1,24 @@
+import type { AttemptOptions, FetchContext } from './types.js';
+export interface Page<TItem, TCursor, TMetadata = undefined> {
+    items: readonly TItem[];
+    /** Continuation for the next request; `undefined` ends the sequence. */
+    next: TCursor | undefined;
+    /** Provider metadata passed through unchanged; readers explicitly map it to candidate state. */
+    metadata?: TMetadata;
+}
+export interface PaginateOptions<TItem, TCursor, TMetadata = undefined> {
+    context: FetchContext;
+    fetchPage(cursor: TCursor | undefined, signal: AbortSignal): Promise<Page<TItem, TCursor, TMetadata>>;
+    initial?: TCursor;
+    label?: AttemptOptions['label'];
+}
+/**
+ * Pull-based pagination over one retryable Promise step per page. Every request
+ * runs through the executor attempt capability. A continuation that was already
+ * seen (compared by canonical serialization) is rejected, so a cyclic provider
+ * cursor cannot loop forever; other non-progress is bounded by execution budgets.
+ * Every complete page is yielded, including empty and terminal pages. Metadata
+ * does not advance checkpoints unless the reader includes it in SourceChunk.state.
+ */
+export declare function paginate<TItem, TCursor, TMetadata = undefined>(options: PaginateOptions<TItem, TCursor, TMetadata>): AsyncGenerator<Page<TItem, TCursor, TMetadata>, void, void>;
+//# sourceMappingURL=paginate.d.ts.map

package/dist/paginate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"paginate.d.ts","sourceRoot":"","sources":["../src/paginate.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9D,MAAM,WAAW,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,SAAS;IACzD,KAAK,EAAE,SAAS,KAAK,EAAE,CAAA;IACvB,wEAAwE;IACxE,IAAI,EAAE,OAAO,GAAG,SAAS,CAAA;IACzB,gGAAgG;IAChG,QAAQ,CAAC,EAAE,SAAS,CAAA;CACrB;AAED,MAAM,WAAW,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,SAAS;IACpE,OAAO,EAAE,YAAY,CAAA;IACrB,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAA;IACrG,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,KAAK,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAA;CAChC;AAED;;;;;;;GAOG;AACH,wBAAuB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,SAAS,EACnE,OAAO,EAAE,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,GAClD,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAwB7D"}

package/dist/paginate.js

@@ -0,0 +1,37 @@
+import { IngestConfigError } from './errors.js';
+import { canonicalJson } from './journal.js';
+/**
+ * Pull-based pagination over one retryable Promise step per page. Every request
+ * runs through the executor attempt capability. A continuation that was already
+ * seen (compared by canonical serialization) is rejected, so a cyclic provider
+ * cursor cannot loop forever; other non-progress is bounded by execution budgets.
+ * Every complete page is yielded, including empty and terminal pages. Metadata
+ * does not advance checkpoints unless the reader includes it in SourceChunk.state.
+ */
+export async function* paginate(options) {
+    const seen = new Set();
+    let cursor = options.initial;
+    if (cursor !== undefined)
+        seen.add(canonicalJson(cursor));
+    while (true) {
+        options.context.signal.throwIfAborted();
+        const current = cursor;
+        const page = await options.context.attempt((signal) => options.fetchPage(current, signal), {
+            label: options.label,
+        });
+        options.context.signal.throwIfAborted();
+        const next = page.next;
+        if (next !== undefined && next !== null) {
+            const key = canonicalJson(next);
+            if (seen.has(key)) {
+                throw new IngestConfigError(`paginate: provider returned a repeated continuation ${key}; refusing to loop.`);
+            }
+            seen.add(key);
+        }
+        yield page;
+        if (next === undefined || next === null)
+            return;
+        cursor = next;
+    }
+}
+//# sourceMappingURL=paginate.js.map

package/dist/paginate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paginate.js","sourceRoot":"","sources":["../src/paginate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAkB5C;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC7B,OAAmD;IAEnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,CAAA;IAC5B,IAAI,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;IAEzD,OAAO,IAAI,EAAE,CAAC;QACZ,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QACvC,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;YACzF,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAA;QACF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACtB,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;YAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,iBAAiB,CAAC,uDAAuD,GAAG,qBAAqB,CAAC,CAAA;YAC9G,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACf,CAAC;QACD,MAAM,IAAI,CAAA;QACV,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI;YAAE,OAAM;QAC/C,MAAM,GAAG,IAAI,CAAA;IACf,CAAC;AACH,CAAC"}

package/dist/plugin.d.ts

@@ -0,0 +1,155 @@
+import { type ClickHouseExecutor } from '@chkit/clickhouse';
+import { type ChxInlinePluginRegistration, type ResolvedChxConfig } from '@chkit/core';
+import { z } from 'zod';
+import type { PipelineDefinition } from './types.js';
+declare const IngestOptionsSchema: z.ZodObject<{
+    journalTable: z.ZodDefault<z.ZodString>;
+    maxDurationSeconds: z.ZodDefault<z.ZodNumber>;
+    prefetchBatches: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+type IngestOptions = z.infer<typeof IngestOptionsSchema>;
+export type IngestPluginOptions = Partial<IngestOptions>;
+interface IngestPluginCommandContext {
+    args: string[];
+    flags: Record<string, string | string[] | boolean | undefined>;
+    jsonMode: boolean;
+    options: IngestOptions;
+    config: ResolvedChxConfig;
+    configPath: string;
+    print: (value: unknown) => void;
+    pluginContext?: {
+        executor: ClickHouseExecutor;
+        hasExecutor: boolean;
+    };
+}
+export declare function createIngestPlugin(options?: IngestPluginOptions): {
+    manifest: {
+        name: "ingest";
+        apiVersion: 1;
+    };
+    optionsSchema: import("@chkit/core").SafeParseable<{
+        journalTable: string;
+        maxDurationSeconds: number;
+        prefetchBatches: number;
+    }>;
+    commands: ({
+        name: string;
+        description: string;
+        flags: readonly [{
+            readonly name: "--tag";
+            readonly type: "string[]";
+            readonly description: "Exact tag every selected stream must carry (repeatable, AND)";
+            readonly placeholder: "<tag>";
+        }, {
+            readonly name: "--backfill";
+            readonly type: "string";
+            readonly description: "Stable backfill id; uses an isolated checkpoint namespace";
+            readonly placeholder: "<id>";
+        }, {
+            readonly name: "--from";
+            readonly type: "string";
+            readonly description: "Backfill range lower bound (ISO timestamp)";
+            readonly placeholder: "<timestamp>";
+        }, {
+            readonly name: "--to";
+            readonly type: "string";
+            readonly description: "Backfill range upper bound (ISO timestamp)";
+            readonly placeholder: "<timestamp>";
+        }, {
+            readonly name: "--max-duration";
+            readonly type: "string";
+            readonly description: "Execution budget in seconds";
+            readonly placeholder: "<seconds>";
+        }];
+        optionsSchema: import("@chkit/core").SafeParseable<{
+            journalTable: string;
+            maxDurationSeconds: number;
+            prefetchBatches: number;
+        }>;
+        run: (context: IngestPluginCommandContext) => Promise<number | undefined>;
+    } | {
+        name: string;
+        description: string;
+        flags: readonly [{
+            readonly name: "--tag";
+            readonly type: "string[]";
+            readonly description: "Exact tag every selected stream must carry (repeatable, AND)";
+            readonly placeholder: "<tag>";
+        }];
+        optionsSchema: import("@chkit/core").SafeParseable<{
+            journalTable: string;
+            maxDurationSeconds: number;
+            prefetchBatches: number;
+        }>;
+        run: (context: IngestPluginCommandContext) => Promise<number | undefined>;
+    } | {
+        name: string;
+        description: string;
+        flags: readonly [{
+            readonly name: "--tag";
+            readonly type: "string[]";
+            readonly description: "Exact tag every selected stream must carry (repeatable, AND)";
+            readonly placeholder: "<tag>";
+        }, {
+            readonly name: "--backfill";
+            readonly type: "string";
+            readonly description: "Inspect an isolated backfill checkpoint namespace";
+            readonly placeholder: "<id>";
+        }];
+        optionsSchema: import("@chkit/core").SafeParseable<{
+            journalTable: string;
+            maxDurationSeconds: number;
+            prefetchBatches: number;
+        }>;
+        run: (context: IngestPluginCommandContext) => Promise<number | undefined>;
+    } | {
+        name: string;
+        description: string;
+        flags: readonly [{
+            readonly name: "--tag";
+            readonly type: "string[]";
+            readonly description: "Exact tag every selected stream must carry (repeatable, AND)";
+            readonly placeholder: "<tag>";
+        }, {
+            readonly name: "--backfill";
+            readonly type: "string";
+            readonly description: "Inspect an isolated backfill checkpoint namespace";
+            readonly placeholder: "<id>";
+        }, {
+            readonly name: "--apply";
+            readonly type: "string";
+            readonly description: "Materialize the reviewed fingerprint with writers stopped (select one stream)";
+            readonly placeholder: "<fingerprint>";
+        }];
+        optionsSchema: import("@chkit/core").SafeParseable<{
+            journalTable: string;
+            maxDurationSeconds: number;
+            prefetchBatches: number;
+        }>;
+        run: (context: IngestPluginCommandContext) => Promise<number | undefined>;
+    })[];
+    hooks: {
+        onCheck(context: {
+            config: ResolvedChxConfig;
+        }): Promise<{
+            plugin: string;
+            evaluated: boolean;
+            ok: boolean;
+            findings: {
+                code: string;
+                message: string;
+                severity: "info" | "warn" | "error";
+            }[];
+        }>;
+    };
+};
+export type IngestPlugin = ReturnType<typeof createIngestPlugin>;
+export declare function ingest(options?: IngestPluginOptions): ChxInlinePluginRegistration<IngestPlugin, IngestPluginOptions>;
+/** Local, zero-network graph checks: they hold in `--offline` mode too. */
+export declare function checkGraph(pipelines: readonly PipelineDefinition[]): {
+    code: string;
+    message: string;
+    severity: "info" | "warn" | "error";
+}[];
+export {};
+//# sourceMappingURL=plugin.d.ts.map

package/dist/plugin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAEA,OAAO,EAAqC,KAAK,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AAC9F,OAAO,EAIL,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACvB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAQvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAIpD,QAAA,MAAM,mBAAmB;;;;iBAKvB,CAAA;AACF,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAA;AAExD,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;AAwBxD,UAAU,0BAA0B;IAClC,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;IAC9D,QAAQ,EAAE,OAAO,CAAA;IACjB,OAAO,EAAE,aAAa,CAAA;IACtB,MAAM,EAAE,iBAAiB,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;IAC/B,aAAa,CAAC,EAAE;QAAE,QAAQ,EAAE,kBAAkB,CAAC;QAAC,WAAW,EAAE,OAAO,CAAA;KAAE,CAAA;CACvE;AAID,wBAAgB,kBAAkB,CAAC,OAAO,GAAE,mBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA2MvC;YAAE,MAAM,EAAE,iBAAiB,CAAA;SAAE;;;;;sBAqB1B,MAAM;yBAAW,MAAM;0BAAY,MAAM,GAAG,MAAM,GAAG,OAAO;;;;EAV3F;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAA;AAEhE,wBAAgB,MAAM,CAAC,OAAO,GAAE,mBAAwB,GAAG,2BAA2B,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAExH;AAED,2EAA2E;AAC3E,wBAAgB,UAAU,CAAC,SAAS,EAAE,SAAS,kBAAkB,EAAE;UACnC,MAAM;aAAW,MAAM;cAAY,MAAM,GAAG,MAAM,GAAG,OAAO;IAkB3F"}