@chkit/plugin-backfill 0.1.0-beta.2 → 0.1.0-beta.20

package/dist/async-backfill.js

@@ -0,0 +1,251 @@
+import pMap from 'p-map';
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** Build the deterministic query ID for a chunk. */
+function chunkQueryId(planId, chunkId) {
+    return `backfill-${planId}-${chunkId}`;
+}
+function applyQueryStatus(state, qs) {
+    if (qs.status === 'running') {
+        const next = {
+            ...state,
+            status: 'running',
+            readRows: qs.readRows,
+            readBytes: qs.readBytes,
+            writtenRows: qs.writtenRows,
+            writtenBytes: qs.writtenBytes,
+            elapsedMs: qs.elapsedMs,
+        };
+        const metricsChanged = state.status !== 'running' ||
+            state.readRows !== qs.readRows ||
+            state.writtenRows !== qs.writtenRows ||
+            state.elapsedMs !== qs.elapsedMs;
+        return { state: next, changed: metricsChanged };
+    }
+    if (qs.status === 'finished') {
+        return {
+            state: {
+                ...state,
+                status: 'done',
+                finishedAt: new Date().toISOString(),
+                durationMs: qs.durationMs,
+                writtenRows: qs.writtenRows,
+                writtenBytes: qs.writtenBytes,
+            },
+            changed: true,
+        };
+    }
+    if (qs.status === 'failed') {
+        return {
+            state: {
+                ...state,
+                status: 'failed',
+                finishedAt: new Date().toISOString(),
+                durationMs: qs.durationMs,
+                error: qs.error,
+            },
+            changed: true,
+        };
+    }
+    // 'unknown' — leave status as-is (query_log may not have flushed yet)
+    return { state, changed: false };
+}
+function getChunk(progress, id) {
+    const state = progress[id];
+    if (!state)
+        throw new Error(`No progress entry for chunk ${id}`);
+    return state;
+}
+function updateChunk(progress, id, next) {
+    return { ...progress, [id]: next };
+}
+/**
+ * Reconcile local progress with server-side state.
+ *
+ * Queries system.processes and system.query_log for all chunk query IDs
+ * to discover queries that were submitted but whose status was never
+ * persisted locally (e.g. client crash between submit and state write).
+ */
+export async function syncProgress(executor, planId, chunks, progress) {
+    const prefix = `backfill-${planId}-`;
+    // Collect query IDs for non-terminal chunks that need reconciliation
+    const chunkIdsToSync = [];
+    for (const chunk of chunks) {
+        const state = progress[chunk.id];
+        if (!state || state.status === 'done')
+            continue;
+        chunkIdsToSync.push(chunk.id);
+    }
+    if (chunkIdsToSync.length === 0)
+        return progress;
+    // Escape single-quotes in the prefix for safe SQL embedding
+    const safePrefix = prefix.replace(/'/g, "''").replace(/%/g, '\\%').replace(/_/g, '\\_');
+    const runningRows = await executor.query(`SELECT query_id FROM clusterAllReplicas('parallel_replicas', system.processes) WHERE user = currentUser() AND query_id LIKE '${safePrefix}%' SETTINGS skip_unavailable_shards = 1`);
+    const runningSet = new Set(runningRows.map((r) => r.query_id));
+    const logRows = await executor.query(`SELECT query_id, type, written_rows, written_bytes, query_duration_ms, exception
+FROM clusterAllReplicas('parallel_replicas', system.query_log)
+WHERE user = currentUser()
+  AND query_id LIKE '${safePrefix}%'
+  AND type IN ('QueryFinish', 'ExceptionWhileProcessing')
+  AND is_initial_query = 1
+ORDER BY event_time DESC
+SETTINGS skip_unavailable_shards = 1`);
+    // Deduplicate: take the latest log entry per query_id (results are ordered by event_time DESC)
+    const latestLogByQueryId = new Map();
+    for (const row of logRows) {
+        if (!latestLogByQueryId.has(row.query_id)) {
+            latestLogByQueryId.set(row.query_id, row);
+        }
+    }
+    let updated = { ...progress };
+    for (const chunkId of chunkIdsToSync) {
+        const queryId = chunkQueryId(planId, chunkId);
+        const current = updated[chunkId];
+        if (!current)
+            continue;
+        if (runningSet.has(queryId)) {
+            updated = updateChunk(updated, chunkId, { ...current, status: 'running', queryId });
+        }
+        else {
+            const logEntry = latestLogByQueryId.get(queryId);
+            if (logEntry) {
+                if (logEntry.type === 'QueryFinish') {
+                    updated = updateChunk(updated, chunkId, {
+                        ...current,
+                        status: 'done',
+                        queryId,
+                        finishedAt: new Date().toISOString(),
+                        writtenRows: Number(logEntry.written_rows),
+                        writtenBytes: Number(logEntry.written_bytes),
+                        durationMs: Number(logEntry.query_duration_ms),
+                    });
+                }
+                else {
+                    updated = updateChunk(updated, chunkId, {
+                        ...current,
+                        status: 'failed',
+                        queryId,
+                        finishedAt: new Date().toISOString(),
+                        durationMs: Number(logEntry.query_duration_ms),
+                        error: logEntry.exception,
+                    });
+                }
+            }
+        }
+    }
+    return updated;
+}
+async function pollChunk(executor, initial, pollIntervalMs, maxPollErrors, onChanged) {
+    let state = initial;
+    let consecutiveErrors = 0;
+    while (state.status === 'submitted' || state.status === 'running') {
+        await sleep(pollIntervalMs);
+        if (!state.queryId)
+            break;
+        let qs;
+        try {
+            qs = await executor.queryStatus(state.queryId, {
+                afterTime: state.submittedAt,
+            });
+        }
+        catch {
+            consecutiveErrors++;
+            if (consecutiveErrors >= maxPollErrors) {
+                state = {
+                    ...state,
+                    status: 'failed',
+                    finishedAt: new Date().toISOString(),
+                    error: `Lost contact with query after ${consecutiveErrors} consecutive poll errors`,
+                };
+                await onChanged(state);
+                break;
+            }
+            continue;
+        }
+        consecutiveErrors = 0;
+        const result = applyQueryStatus(state, qs);
+        if (result.changed) {
+            state = result.state;
+            await onChanged(state);
+        }
+    }
+    return state;
+}
+export async function executeBackfill(options) {
+    const { executor, planId, chunks, buildQuery, concurrency = 3, pollIntervalMs = 5000, maxPollErrors = 10, onProgress, resumeFrom, replayFailed, } = options;
+    let progress = Object.fromEntries(chunks.map((chunk) => {
+        const resumed = resumeFrom?.[chunk.id];
+        return [chunk.id, resumed ? { ...resumed } : { status: 'pending' }];
+    }));
+    // When resuming, reconcile local state with the server before processing.
+    // This catches queries that were submitted but whose status was never
+    // persisted (e.g. client crash between submit() and state file write).
+    if (resumeFrom) {
+        progress = await syncProgress(executor, planId, chunks, progress);
+    }
+    // Reset confirmed-failed chunks to pending AFTER sync so we operate on
+    // ground truth. The deterministic query_id is reused; the afterTime filter
+    // in queryStatus ensures we ignore stale query_log entries from prior attempts.
+    if (replayFailed) {
+        // Stamp submittedAt with a 60s buffer so the afterTime filter in
+        // queryStatus ignores stale query_log entries from the prior failed
+        // attempt while tolerating clock skew between client and server.
+        const replayAfterTime = new Date(Date.now() - 60_000).toISOString();
+        for (const chunk of chunks) {
+            const state = progress[chunk.id];
+            if (state?.status === 'failed') {
+                progress = updateChunk(progress, chunk.id, { status: 'pending', submittedAt: replayAfterTime });
+            }
+        }
+    }
+    // Persist the reconciled state so the caller's checkpoint is up to date
+    if (resumeFrom || replayFailed) {
+        await onProgress?.(progress);
+    }
+    const setChunk = (id, next) => {
+        progress = updateChunk(progress, id, next);
+        return onProgress?.(progress);
+    };
+    await pMap(chunks, async (chunk) => {
+        const state = getChunk(progress, chunk.id);
+        // Already terminal from a previous run
+        if (state.status === 'done' || state.status === 'failed')
+            return;
+        // Resumed in-flight: poll to completion
+        if (state.status === 'submitted' || state.status === 'running') {
+            if (!state.queryId) {
+                await setChunk(chunk.id, { ...state, status: 'pending' });
+            }
+            else {
+                await pollChunk(executor, state, pollIntervalMs, maxPollErrors, (s) => setChunk(chunk.id, s));
+                return;
+            }
+        }
+        // Submit and poll
+        // submittedAt is intentionally omitted on first submission — it's only
+        // used as an afterTime filter to ignore stale query_log entries when
+        // replaying a previously failed chunk with the same deterministic query_id.
+        // Setting it to local time here would cause clock-skew issues with the
+        // ClickHouse server, making the filter exclude valid entries.
+        const queryId = chunkQueryId(planId, chunk.id);
+        const sql = buildQuery(chunk);
+        await executor.submit(sql, queryId);
+        const submitted = {
+            ...getChunk(progress, chunk.id),
+            status: 'submitted',
+            queryId,
+        };
+        await setChunk(chunk.id, submitted);
+        await pollChunk(executor, submitted, pollIntervalMs, maxPollErrors, (s) => setChunk(chunk.id, s));
+    }, { concurrency });
+    const completed = chunks.filter((c) => getChunk(progress, c.id).status === 'done').length;
+    const failed = chunks.filter((c) => getChunk(progress, c.id).status === 'failed').length;
+    return {
+        total: chunks.length,
+        completed,
+        failed,
+        progress,
+    };
+}
+//# sourceMappingURL=async-backfill.js.map

package/dist/async-backfill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,OAAO,CAAA;AAgDxB,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,oDAAoD;AACpD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe;IACnD,OAAO,YAAY,MAAM,IAAI,OAAO,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAyB,EACzB,EAAe;IAEf,IAAI,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAuB;YAC/B,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAA;QACD,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,KAAK,SAAS;YAC1B,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,QAAQ;YAC9B,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,WAAW;YACpC,KAAK,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAA;QAClC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;IACjD,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;aAC9B;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,KAAK,EAAE,EAAE,CAAC,KAAK;aAChB;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,sEAAsE;IACtE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,QAA0B,EAAE,EAAU;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAA;IAChE,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,EAAU,EACV,IAAwB;IAExB,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA4B,EAC5B,MAAc,EACd,MAA6B,EAC7B,QAA0B;IAE1B,MAAM,MAAM,GAAG,YAAY,MAAM,GAAG,CAAA;IAEpC,qEAAqE;IACrE,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;YAAE,SAAQ;QAC/C,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhD,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEvF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CACtC,gIAAgI,UAAU,yCAAyC,CACpL,CAAA;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAQlC;;;uBAGmB,UAAU;;;;qCAII,CAClC,CAAA;IAED,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAA;IACjE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;IAE7B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAChC,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;QACrF,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAChD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACpC,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,MAAM;wBACd,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAC1C,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;wBAC5C,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;qBAC/C,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,QAAQ;wBAChB,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;wBAC9C,KAAK,EAAE,QAAQ,CAAC,SAAS;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAA4B,EAC5B,OAA2B,EAC3B,cAAsB,EACtB,aAAqB,EACrB,SAA8D;IAE9D,IAAI,KAAK,GAAG,OAAO,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,MAAK;QACzB,IAAI,EAAe,CAAA;QACnB,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;gBAC7C,SAAS,EAAE,KAAK,CAAC,WAAW;aAC7B,CAAC,CAAA;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAA;YACnB,IAAI,iBAAiB,IAAI,aAAa,EAAE,CAAC;gBACvC,KAAK,GAAG;oBACN,GAAG,KAAK;oBACR,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,KAAK,EAAE,iCAAiC,iBAAiB,0BAA0B;iBACpF,CAAA;gBACD,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;gBACtB,MAAK;YACP,CAAC;YACD,SAAQ;QACV,CAAC;QACD,iBAAiB,GAAG,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;YACpB,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAwB;IAC5D,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,MAAM,EACN,UAAU,EACV,WAAW,GAAG,CAAC,EACf,cAAc,GAAG,IAAI,EACrB,aAAa,GAAG,EAAE,EAClB,UAAU,EACV,UAAU,EACV,YAAY,GACb,GAAG,OAAO,CAAA;IAEX,IAAI,QAAQ,GAAqB,MAAM,CAAC,WAAW,CACjD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAkB,EAAE,CAAC,CAAA;IAC9E,CAAC,CAAC,CACH,CAAA;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,uEAAuE;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;IACnE,CAAC;IAED,uEAAuE;IACvE,2EAA2E;IAC3E,gFAAgF;IAChF,IAAI,YAAY,EAAE,CAAC;QACjB,iEAAiE;QACjE,oEAAoE;QACpE,iEAAiE;QACjE,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACnE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAChC,IAAI,KAAK,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAA;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,IAAwB,EAAE,EAAE;QACxD,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QAC1C,OAAO,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC,CAAA;IAED,MAAM,IAAI,CACR,MAAM,EACN,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1C,uCAAuC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAM;QAEhE,wCAAwC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;gBAC7F,OAAM;YACR,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACnC,MAAM,SAAS,GAAuB;YACpC,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,WAAW;YACnB,OAAO;SACR,CAAA;QACD,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAEnC,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACnG,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAA;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;IAExF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,SAAS;QACT,MAAM;QACN,QAAQ;KACT,CAAA;AACH,CAAC"}

package/dist/check.d.ts

@@ -0,0 +1,9 @@
+import type { ResolvedChxConfig } from '@chkit/core';
+import type { BackfillPluginCheckResult } from './types.js';
+export declare function evaluateBackfillCheck(input: {
+    configPath: string;
+    config: Pick<ResolvedChxConfig, 'metaDir'>;
+    stateDir?: string;
+    failCheckOnRequiredPendingBackfill: boolean;
+}): Promise<BackfillPluginCheckResult>;
+//# sourceMappingURL=check.d.ts.map

package/dist/check.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"check.d.ts","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAOpD,OAAO,KAAK,EACV,yBAAyB,EAC1B,MAAM,YAAY,CAAA;AAEnB,wBAAsB,qBAAqB,CAAC,KAAK,EAAE;IACjD,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAA;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kCAAkC,EAAE,OAAO,CAAA;CAC5C,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAgFrC"}

package/dist/check.js

@@ -0,0 +1,79 @@
+import { join } from 'node:path';
+import { computeBackfillStateDir, listPlanIds, readRun, } from './state.js';
+export async function evaluateBackfillCheck(input) {
+    const stateDir = computeBackfillStateDir(input.config, input.configPath, input.stateDir);
+    const plansDir = join(stateDir, 'plans');
+    const runsDir = join(stateDir, 'runs');
+    const planIds = await listPlanIds(plansDir);
+    if (planIds.length === 0) {
+        return {
+            plugin: 'backfill',
+            evaluated: true,
+            ok: true,
+            findings: [],
+            metadata: {
+                requiredCount: 0,
+                activeRuns: 0,
+                failedRuns: 0,
+            },
+        };
+    }
+    let requiredCount = 0;
+    let activeRuns = 0;
+    let failedRuns = 0;
+    for (const planId of planIds) {
+        const runPath = join(runsDir, `${planId}.json`);
+        const run = await readRun(runPath);
+        if (!run) {
+            requiredCount += 1;
+            continue;
+        }
+        if (run.status === 'running')
+            activeRuns += 1;
+        if (run.status === 'failed')
+            failedRuns += 1;
+        if (run.status !== 'completed')
+            requiredCount += 1;
+    }
+    const findings = [];
+    if (requiredCount > 0) {
+        findings.push({
+            code: 'backfill_required_pending',
+            message: `Required backfills pending completion: ${requiredCount}`,
+            severity: input.failCheckOnRequiredPendingBackfill ? 'error' : 'warn',
+            metadata: {
+                requiredCount,
+            },
+        });
+    }
+    if (failedRuns > 0) {
+        findings.push({
+            code: 'backfill_chunk_failed_retry_exhausted',
+            message: `Backfill runs failed after retry budget: ${failedRuns}`,
+            severity: 'error',
+            metadata: {
+                failedRuns,
+            },
+        });
+    }
+    if (!input.failCheckOnRequiredPendingBackfill) {
+        findings.push({
+            code: 'backfill_policy_relaxed',
+            message: 'Backfill check policy is relaxed: failCheckOnRequiredPendingBackfill=false.',
+            severity: 'warn',
+        });
+    }
+    const ok = findings.every((finding) => finding.severity !== 'error');
+    return {
+        plugin: 'backfill',
+        evaluated: true,
+        ok,
+        findings,
+        metadata: {
+            requiredCount,
+            activeRuns,
+            failedRuns,
+        },
+    };
+}
+//# sourceMappingURL=check.js.map

package/dist/check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"check.js","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAIhC,OAAO,EACL,uBAAuB,EACvB,WAAW,EACX,OAAO,GACR,MAAM,YAAY,CAAA;AAKnB,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAK3C;IACC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAEtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,IAAI;YACf,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE;gBACR,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;aACd;SACF,CAAA;IACH,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,CAAA;QAC/C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;QAClC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,aAAa,IAAI,CAAC,CAAA;YAClB,SAAQ;QACV,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,UAAU,IAAI,CAAC,CAAA;QAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,UAAU,IAAI,CAAC,CAAA;QAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,aAAa,IAAI,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,QAAQ,GAA0C,EAAE,CAAA;IAC1D,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,2BAA2B;YACjC,OAAO,EAAE,0CAA0C,aAAa,EAAE;YAClE,QAAQ,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YACrE,QAAQ,EAAE;gBACR,aAAa;aACd;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,uCAAuC;YAC7C,OAAO,EAAE,4CAA4C,UAAU,EAAE;YACjE,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE;gBACR,UAAU;aACX;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,kCAAkC,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,yBAAyB;YAC/B,OAAO,EAAE,6EAA6E;YACtF,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IACpE,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,SAAS,EAAE,IAAI;QACf,EAAE;QACF,QAAQ;QACR,QAAQ,EAAE;YACR,aAAa;YACb,UAAU;YACV,UAAU;SACX;KACF,CAAA;AACH,CAAC"}

package/dist/chunking/analyze.d.ts

@@ -0,0 +1,38 @@
+import type { ChunkBoundary, PartitionInfo, PlannedChunk, SortKeyInfo } from './types.js';
+export interface AnalyzeAndChunkInput {
+    database: string;
+    table: string;
+    from?: string;
+    to?: string;
+    maxChunkBytes: number;
+    requireIdempotencyToken: boolean;
+    query: <T>(sql: string) => Promise<T[]>;
+}
+export interface AnalyzeAndChunkResult {
+    planId: string;
+    partitions: PartitionInfo[];
+    sortKey?: SortKeyInfo;
+    chunks: PlannedChunk[];
+}
+export declare function analyzeAndChunk(input: AnalyzeAndChunkInput): Promise<AnalyzeAndChunkResult>;
+export interface AnalyzeTableInput {
+    database: string;
+    table: string;
+    from?: string;
+    to?: string;
+    maxChunkBytes: number;
+    query: <T>(sql: string) => Promise<T[]>;
+}
+export interface AnalyzeTableResult {
+    partitions: PartitionInfo[];
+    sortKey?: SortKeyInfo;
+    boundaries: ChunkBoundary[];
+}
+export declare function analyzeTable(input: AnalyzeTableInput): Promise<AnalyzeTableResult>;
+export declare function buildPlannedChunks(input: {
+    planId: string;
+    partitions: PartitionInfo[];
+    boundaries: ChunkBoundary[];
+    requireIdempotencyToken: boolean;
+}): PlannedChunk[];
+//# sourceMappingURL=analyze.d.ts.map

package/dist/chunking/analyze.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAEzF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,aAAa,EAAE,MAAM,CAAA;IACrB,uBAAuB,EAAE,OAAO,CAAA;IAChC,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,MAAM,EAAE,YAAY,EAAE,CAAA;CACvB;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAoBjG;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,UAAU,EAAE,aAAa,EAAE,CAAA;CAC5B;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgCxF;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACxC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,uBAAuB,EAAE,OAAO,CAAA;CACjC,GAAG,YAAY,EAAE,CA6BjB"}

package/dist/chunking/analyze.js

@@ -0,0 +1,76 @@
+import { hashId, randomPlanId } from '../state.js';
+import { buildChunkBoundaries } from './build.js';
+import { introspectTable, querySortKeyRanges } from './introspect.js';
+export async function analyzeAndChunk(input) {
+    const { partitions, sortKey, boundaries } = await analyzeTable({
+        database: input.database,
+        table: input.table,
+        from: input.from,
+        to: input.to,
+        maxChunkBytes: input.maxChunkBytes,
+        query: input.query,
+    });
+    const planId = randomPlanId();
+    const chunks = buildPlannedChunks({
+        planId,
+        partitions,
+        boundaries,
+        requireIdempotencyToken: input.requireIdempotencyToken,
+    });
+    return { planId, partitions, sortKey, chunks };
+}
+export async function analyzeTable(input) {
+    const { partitions, sortKey } = await introspectTable({
+        database: input.database,
+        table: input.table,
+        from: input.from,
+        to: input.to,
+        query: input.query,
+    });
+    const oversizedPartitionIds = partitions
+        .filter(p => p.bytesOnDisk > input.maxChunkBytes)
+        .map(p => p.partitionId);
+    let sortKeyRanges;
+    if (sortKey && oversizedPartitionIds.length > 0) {
+        sortKeyRanges = await querySortKeyRanges({
+            database: input.database,
+            table: input.table,
+            sortKeyColumn: sortKey.column,
+            partitionIds: oversizedPartitionIds,
+            query: input.query,
+        });
+    }
+    const boundaries = buildChunkBoundaries({
+        partitions,
+        maxChunkBytes: input.maxChunkBytes,
+        sortKey,
+        sortKeyRanges,
+    });
+    return { partitions, sortKey, boundaries };
+}
+export function buildPlannedChunks(input) {
+    const chunks = [];
+    const partitionIndex = new Map();
+    for (const boundary of input.boundaries) {
+        const idx = partitionIndex.get(boundary.partitionId) ?? 0;
+        partitionIndex.set(boundary.partitionId, idx + 1);
+        const idSeed = `${input.planId}:${boundary.partitionId}:${idx}`;
+        const chunkId = hashId(`chunk:${idSeed}`).slice(0, 16);
+        const token = input.requireIdempotencyToken ? hashId(`token:${idSeed}`) : '';
+        const partition = input.partitions.find(p => p.partitionId === boundary.partitionId);
+        const from = boundary.sortKeyFrom ?? partition?.minTime ?? '';
+        const to = boundary.sortKeyTo ?? partition?.maxTime ?? '';
+        chunks.push({
+            id: chunkId,
+            partitionId: boundary.partitionId,
+            sortKeyFrom: boundary.sortKeyFrom,
+            sortKeyTo: boundary.sortKeyTo,
+            estimatedBytes: boundary.estimatedBytes,
+            idempotencyToken: token,
+            from,
+            to,
+        });
+    }
+    return chunks;
+}
+//# sourceMappingURL=analyze.js.map

package/dist/chunking/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAElD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAoBrE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAA2B;IAC/D,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,YAAY,CAAC;QAC7D,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;IAE7B,MAAM,MAAM,GAAG,kBAAkB,CAAC;QAChC,MAAM;QACN,UAAU;QACV,UAAU;QACV,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;KACvD,CAAC,CAAA;IAEF,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC;AAiBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAwB;IACzD,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,MAAM,eAAe,CAAC;QACpD,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAA;IAEF,MAAM,qBAAqB,GAAG,UAAU;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAA;IAE1B,IAAI,aAAoE,CAAA;IACxE,IAAI,OAAO,IAAI,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,aAAa,GAAG,MAAM,kBAAkB,CAAC;YACvC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,YAAY,EAAE,qBAAqB;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC;QACtC,UAAU;QACV,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO;QACP,aAAa;KACd,CAAC,CAAA;IAEF,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAKlC;IACC,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEhD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QACzD,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA;QAEjD,MAAM,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,WAAW,IAAI,GAAG,EAAE,CAAA;QAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAE5E,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,CAAC,CAAA;QACpF,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,IAAI,SAAS,EAAE,OAAO,IAAI,EAAE,CAAA;QAC7D,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,IAAI,SAAS,EAAE,OAAO,IAAI,EAAE,CAAA;QAEzD,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,OAAO;YACX,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,EAAE;SACH,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/chunking/build.d.ts

@@ -0,0 +1,11 @@
+import type { ChunkBoundary, PartitionInfo, SortKeyInfo } from './types.js';
+export declare function buildChunkBoundaries(input: {
+    partitions: PartitionInfo[];
+    maxChunkBytes: number;
+    sortKey?: SortKeyInfo;
+    sortKeyRanges?: Map<string, {
+        min: string;
+        max: string;
+    }>;
+}): ChunkBoundary[];
+//# sourceMappingURL=build.d.ts.map

package/dist/chunking/build.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/chunking/build.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE3E,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAC1D,GAAG,aAAa,EAAE,CAmDlB"}

package/dist/chunking/build.js

@@ -0,0 +1,51 @@
+import { splitSortKeyRange } from './splitter.js';
+export function buildChunkBoundaries(input) {
+    const boundaries = [];
+    for (const partition of input.partitions) {
+        if (partition.bytesOnDisk <= input.maxChunkBytes) {
+            boundaries.push({
+                partitionId: partition.partitionId,
+                estimatedBytes: partition.bytesOnDisk,
+            });
+        }
+        else if (input.sortKey && input.sortKeyRanges) {
+            const range = input.sortKeyRanges.get(partition.partitionId);
+            if (!range) {
+                // No range data — emit as single chunk
+                boundaries.push({
+                    partitionId: partition.partitionId,
+                    estimatedBytes: partition.bytesOnDisk,
+                });
+                continue;
+            }
+            // If min === max, splitting would produce empty sub-ranges; emit as single chunk
+            if (range.min === range.max) {
+                boundaries.push({
+                    partitionId: partition.partitionId,
+                    estimatedBytes: partition.bytesOnDisk,
+                });
+                continue;
+            }
+            const subCount = Math.ceil(partition.bytesOnDisk / input.maxChunkBytes);
+            const subRanges = splitSortKeyRange(input.sortKey.category, range.min, range.max, subCount);
+            const estimatedBytesPerSub = Math.ceil(partition.bytesOnDisk / subCount);
+            for (const sub of subRanges) {
+                boundaries.push({
+                    partitionId: partition.partitionId,
+                    sortKeyFrom: sub.from,
+                    sortKeyTo: sub.to,
+                    estimatedBytes: estimatedBytesPerSub,
+                });
+            }
+        }
+        else {
+            // No sort key info — emit as single chunk despite being oversized
+            boundaries.push({
+                partitionId: partition.partitionId,
+                estimatedBytes: partition.bytesOnDisk,
+            });
+        }
+    }
+    return boundaries;
+}
+//# sourceMappingURL=build.js.map

package/dist/chunking/build.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/chunking/build.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAGjD,MAAM,UAAU,oBAAoB,CAAC,KAKpC;IACC,MAAM,UAAU,GAAoB,EAAE,CAAA;IAEtC,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,SAAS,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACjD,UAAU,CAAC,IAAI,CAAC;gBACd,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;aACtC,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAA;YAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,uCAAuC;gBACvC,UAAU,CAAC,IAAI,CAAC;oBACd,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;iBACtC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,iFAAiF;YACjF,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,EAAE,CAAC;gBAC5B,UAAU,CAAC,IAAI,CAAC;oBACd,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;iBACtC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;YACvE,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YAC3F,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,GAAG,QAAQ,CAAC,CAAA;YAExE,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,UAAU,CAAC,IAAI,CAAC;oBACd,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,WAAW,EAAE,GAAG,CAAC,IAAI;oBACrB,SAAS,EAAE,GAAG,CAAC,EAAE;oBACjB,cAAc,EAAE,oBAAoB;iBACrC,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,kEAAkE;YAClE,UAAU,CAAC,IAAI,CAAC;gBACd,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;aACtC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAA;AACnB,CAAC"}

package/dist/chunking/introspect.d.ts

@@ -0,0 +1,34 @@
+import type { PartitionInfo, SortKeyInfo } from './types.js';
+export declare function queryPartitionInfo(input: {
+    database: string;
+    table: string;
+    from?: string;
+    to?: string;
+    query: <T>(sql: string) => Promise<T[]>;
+}): Promise<PartitionInfo[]>;
+export declare function querySortKeyInfo(input: {
+    database: string;
+    table: string;
+    query: <T>(sql: string) => Promise<T[]>;
+}): Promise<SortKeyInfo | undefined>;
+export declare function querySortKeyRanges(input: {
+    database: string;
+    table: string;
+    sortKeyColumn: string;
+    partitionIds: string[];
+    query: <T>(sql: string) => Promise<T[]>;
+}): Promise<Map<string, {
+    min: string;
+    max: string;
+}>>;
+export declare function introspectTable(input: {
+    database: string;
+    table: string;
+    from?: string;
+    to?: string;
+    query: <T>(sql: string) => Promise<T[]>;
+}): Promise<{
+    partitions: PartitionInfo[];
+    sortKey?: SortKeyInfo;
+}>;
+//# sourceMappingURL=introspect.d.ts.map

package/dist/chunking/introspect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"introspect.d.ts","sourceRoot":"","sources":["../../src/chunking/introspect.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAkB5D,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CA4C3B;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE;IAC5C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CA6BnC;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAiBrD;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,aAAa,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,WAAW,CAAA;CAAE,CAAC,CASlE"}