@chkit/plugin-backfill 0.1.0-beta.2 → 0.1.0-beta.21

package/dist/plugin.js

@@ -1,12 +1,134 @@
-import { parseCancelArgs, parseDoctorArgs, parsePlanArgs, parseResumeArgs, parseRunArgs, parseStatusArgs } from './args.js';
+import { createClickHouseExecutor } from '@chkit/clickhouse';
+import { wrapPluginRun } from '@chkit/core';
+import { executeBackfill } from './async-backfill.js';
+import { buildChunkExecutionSql } from './chunking/sql.js';
+import { generateIdempotencyToken } from './chunking/utils/ids.js';
 import { BackfillConfigError } from './errors.js';
-import { normalizeBackfillOptions, mergeOptions, validateBaseOptions } from './options.js';
-import { planPayload, runPayload, statusPayload, cancelPayload, doctorPayload } from './payload.js';
+import { PLAN_FLAGS, PLAN_ID_FLAGS, RESUME_FLAGS, RUN_FLAGS, PluginConfigSchema, resolveCheckOptions, resolvePlanOptions, resolveResumeOptions, resolveRunOptions, resolveStatusOptions, } from './options.js';
+import { planPayload, statusPayload, cancelPayload, doctorPayload } from './payload.js';
 import { buildBackfillPlan } from './planner.js';
-import { cancelBackfillRun, evaluateBackfillCheck, executeBackfillRun, getBackfillDoctorReport, getBackfillStatus, resumeBackfillRun, } from './runtime.js';
+import { evaluateBackfillCheck } from './check.js';
+import { cancelBackfillRun, getBackfillDoctorReport, getBackfillStatus } from './queries.js';
+import { backfillPaths, ensureEnvironmentMatch, nowIso, readPlan, readRun, summarizeRunStatus, writeJson, } from './state.js';
+function formatBytes(bytes) {
+    if (bytes >= 1024 ** 4)
+        return `${(bytes / 1024 ** 4).toFixed(1)} TiB`;
+    if (bytes >= 1024 ** 3)
+        return `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
+    if (bytes >= 1024 ** 2)
+        return `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
+    if (bytes >= 1024)
+        return `${(bytes / 1024).toFixed(1)} KiB`;
+    return `${bytes} B`;
+}
+async function runBackfill(input) {
+    const { plan, stateDir } = await readPlan({
+        planId: input.planId,
+        configPath: input.configPath,
+        config: input.config,
+        stateDir: input.stateDir,
+    });
+    ensureEnvironmentMatch({
+        plan,
+        clickhouse: input.clickhouse,
+        forceEnvironment: input.forceEnvironment,
+    });
+    const paths = backfillPaths(stateDir, plan.planId);
+    // Check for existing run state
+    const existingRun = await readRun(paths.runPath);
+    const resumeFrom = input.resumeFrom;
+    if (existingRun && !resumeFrom) {
+        // `run` command (no resumeFrom) must not silently continue an existing run.
+        // Users should use `backfill resume` instead.
+        const status = existingRun.status;
+        if (status === 'completed') {
+            throw new BackfillConfigError(`Run already completed for plan ${plan.planId}. Nothing to do.`);
+        }
+        if (status === 'cancelled') {
+            throw new BackfillConfigError(`Run is cancelled for plan ${plan.planId}. Create a new plan or inspect with backfill doctor.`);
+        }
+        throw new BackfillConfigError(`A run already exists for plan ${plan.planId} (status: ${status}). Use backfill resume to continue.`);
+    }
+    const db = createClickHouseExecutor(input.clickhouse);
+    try {
+        const runState = {
+            planId: plan.planId,
+            target: plan.target,
+            status: 'running',
+            startedAt: existingRun?.startedAt ?? nowIso(),
+            updatedAt: nowIso(),
+            progress: resumeFrom ?? {},
+        };
+        await writeJson(paths.runPath, runState);
+        const result = await executeBackfill({
+            executor: db,
+            planId: plan.planId,
+            chunks: plan.chunkPlan.chunks.map((chunk) => ({ id: chunk.id })),
+            buildQuery: (chunk) => {
+                const planChunk = plan.chunkPlan.chunks.find((candidate) => candidate.id === chunk.id);
+                if (!planChunk)
+                    throw new Error(`Chunk ${chunk.id} not found in plan`);
+                return buildChunkExecutionSql({
+                    planId: plan.planId,
+                    chunk: planChunk,
+                    target: plan.target,
+                    sourceTarget: plan.execution.sourceTarget,
+                    table: plan.chunkPlan.table,
+                    mvAsQuery: plan.execution.mvAsQuery,
+                    targetColumns: plan.execution.targetColumns,
+                    idempotencyToken: plan.execution.requireIdempotencyToken
+                        ? generateIdempotencyToken(plan.planId, planChunk.id)
+                        : '',
+                });
+            },
+            concurrency: input.concurrency,
+            pollIntervalMs: input.pollIntervalMs,
+            resumeFrom,
+            replayFailed: input.replayFailed,
+            onProgress: async (progress) => {
+                runState.progress = progress;
+                runState.updatedAt = nowIso();
+                await writeJson(paths.runPath, runState);
+            },
+        });
+        runState.status = result.failed > 0 ? 'failed' : 'completed';
+        runState.completedAt = nowIso();
+        runState.updatedAt = nowIso();
+        runState.progress = result.progress;
+        if (result.failed > 0) {
+            const failedEntry = Object.values(result.progress).find((c) => c.status === 'failed');
+            runState.lastError = failedEntry?.error ?? 'One or more chunks failed';
+        }
+        await writeJson(paths.runPath, runState);
+        const summary = summarizeRunStatus(runState, paths.runPath, plan);
+        if (input.jsonMode) {
+            input.print({
+                ok: result.failed === 0,
+                planId: plan.planId,
+                status: runState.status,
+                chunkCounts: summary.totals,
+                rowsWritten: summary.rowsWritten,
+                runPath: paths.runPath,
+                lastError: runState.lastError,
+            });
+        }
+        else {
+            let line = `Backfill ${plan.planId}: ${runState.status} (done=${summary.totals.done}/${summary.totals.total}, ${summary.rowsWritten} rows written)`;
+            if (runState.lastError)
+                line += ` \u2014 ${runState.lastError}`;
+            input.print(line);
+            if (runState.status === 'completed' && summary.rowsWritten === 0) {
+                input.print('Warning: 0 rows written across all chunks. Verify that source data exists in the time range and passes the query\'s WHERE filters.');
+            }
+        }
+        return result.failed > 0 ? 1 : 0;
+    }
+    finally {
+        await db.close();
+    }
+}
 export function createBackfillPlugin(options = {}) {
-    const base = normalizeBackfillOptions(options);
-    validateBaseOptions(base);
+    const config = PluginConfigSchema.parse(options);
     return {
         manifest: {
             name: 'backfill',
@@ -16,271 +138,245 @@ export function createBackfillPlugin(options = {}) {
             {
                 name: 'plan',
                 description: 'Build a deterministic backfill plan and persist immutable plan state',
-                async run({ args, jsonMode, print, options: runtimeOptions, config, configPath }) {
-                    try {
-                        const parsed = parsePlanArgs(args);
-                        const effectiveOptions = mergeOptions(base, runtimeOptions);
-                        validateBaseOptions(effectiveOptions);
-                        const output = await buildBackfillPlan({
-                            target: parsed.target,
-                            from: parsed.from,
-                            to: parsed.to,
-                            config,
-                            configPath,
-                            options: effectiveOptions,
-                            chunkHours: parsed.chunkHours,
-                            forceLargeWindow: parsed.forceLargeWindow,
-                        });
-                        const payload = planPayload(output);
-                        if (jsonMode) {
-                            print(payload);
-                        }
-                        else {
-                            print(`Backfill plan ${payload.planId} for ${payload.target} (${payload.chunkCount} chunks at ${payload.chunkHours}h) -> ${payload.planPath}${payload.existed ? ' [existing]' : ''}`);
+                flags: PLAN_FLAGS,
+                run: async (context) => wrapPluginRun({
+                    command: 'plan',
+                    label: 'Backfill plan',
+                    jsonMode: context.jsonMode,
+                    print: context.print,
+                    configErrorClass: BackfillConfigError,
+                    fn: async () => {
+                        const opts = resolvePlanOptions(config, context.options, context.flags);
+                        if (!context.config.clickhouse) {
+                            throw new BackfillConfigError('ClickHouse connection is required for backfill planning. Configure clickhouse in your clickhouse.config.ts.');
                         }
-                        return 0;
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        if (jsonMode) {
-                            print({
-                                ok: false,
-                                command: 'plan',
-                                error: message,
+                        const db = createClickHouseExecutor(context.config.clickhouse);
+                        try {
+                            const output = await buildBackfillPlan({
+                                opts,
+                                configPath: context.configPath,
+                                config: context.config,
+                                clickhouse: context.config.clickhouse,
+                                clickhouseQuery: async (sql, settings) => {
+                                    const result = await db.query(sql, settings);
+                                    return result;
+                                },
+                                // ObsessionDB (ClickHouse Cloud) enables parallel replicas by default,
+                                // which inflates aggregate results (count, GROUP BY). Disable for planning
+                                // queries until ObsessionDB handles it at the profile level.
+                                querySettings: { enable_parallel_replicas: 0 },
                             });
+                            const payload = planPayload(output);
+                            if (context.jsonMode) {
+                                context.print(payload);
+                            }
+                            else {
+                                const partitionCount = output.plan.chunkPlan.partitions.length;
+                                const totalBytes = formatBytes(output.plan.chunkPlan.totalBytesCompressed);
+                                const primarySortKey = output.plan.chunkPlan.table.sortKeys[0];
+                                const sortKeyLabel = primarySortKey
+                                    ? `, sort key: ${primarySortKey.name} (${primarySortKey.category})`
+                                    : '';
+                                context.print(`Backfill plan ${payload.planId} for ${payload.target} (${payload.chunkCount} chunks across ${partitionCount} partitions, ~${totalBytes}${sortKeyLabel}) -> ${payload.planPath}`);
+                            }
+                            return 0;
                         }
-                        else {
-                            print(`Backfill plan failed: ${message}`);
+                        finally {
+                            await db.close();
                         }
-                        if (error instanceof BackfillConfigError)
-                            return 2;
-                        return 1;
-                    }
-                },
+                    },
+                }),
             },
             {
                 name: 'run',
-                description: 'Execute a planned backfill with checkpointed chunk progress',
-                async run({ args, jsonMode, print, options: runtimeOptions, config, configPath }) {
-                    try {
-                        const parsed = parseRunArgs(args);
-                        const effectiveOptions = mergeOptions(base, runtimeOptions);
-                        validateBaseOptions(effectiveOptions);
-                        const output = await executeBackfillRun({
-                            planId: parsed.planId,
-                            config,
-                            configPath,
-                            options: effectiveOptions,
-                            execution: {
-                                replayDone: parsed.replayDone,
-                                replayFailed: parsed.replayFailed,
-                                forceOverlap: parsed.forceOverlap,
-                                forceCompatibility: parsed.forceCompatibility,
-                                simulation: {
-                                    failChunkId: parsed.simulateFailChunk,
-                                    failCount: parsed.simulateFailCount,
-                                },
-                            },
-                        });
-                        const payload = {
-                            ...runPayload(output),
-                            command: 'run',
-                        };
-                        if (jsonMode) {
-                            print(payload);
-                        }
-                        else {
-                            print(`Backfill run ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
-                        }
-                        return payload.ok ? 0 : 1;
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        if (jsonMode) {
-                            print({ ok: false, command: 'run', error: message });
-                        }
-                        else {
-                            print(`Backfill run failed: ${message}`);
+                description: 'Execute a planned backfill with async query submission and polling',
+                flags: RUN_FLAGS,
+                run: async (context) => wrapPluginRun({
+                    command: 'run',
+                    label: 'Backfill run',
+                    jsonMode: context.jsonMode,
+                    print: context.print,
+                    configErrorClass: BackfillConfigError,
+                    fn: async () => {
+                        const opts = resolveRunOptions(config, context.options, context.flags);
+                        if (!context.config.clickhouse) {
+                            throw new BackfillConfigError('ClickHouse connection is required for backfill execution. Configure clickhouse in your clickhouse.config.ts.');
                         }
-                        if (error instanceof BackfillConfigError)
-                            return 2;
-                        return 1;
-                    }
-                },
+                        return runBackfill({
+                            planId: opts.planId,
+                            forceEnvironment: opts.forceEnvironment,
+                            concurrency: opts.concurrency,
+                            pollIntervalMs: opts.pollIntervalMs,
+                            stateDir: opts.stateDir,
+                            configPath: context.configPath,
+                            config: context.config,
+                            clickhouse: context.config.clickhouse,
+                            print: context.print,
+                            jsonMode: context.jsonMode,
+                        });
+                    },
+                }),
             },
             {
                 name: 'resume',
                 description: 'Resume a backfill run from last checkpoint',
-                async run({ args, jsonMode, print, options: runtimeOptions, config, configPath }) {
-                    try {
-                        const parsed = parseResumeArgs(args);
-                        const effectiveOptions = mergeOptions(base, runtimeOptions);
-                        validateBaseOptions(effectiveOptions);
-                        const output = await resumeBackfillRun({
-                            planId: parsed.planId,
-                            config,
-                            configPath,
-                            options: effectiveOptions,
-                            execution: {
-                                replayDone: parsed.replayDone,
-                                replayFailed: parsed.replayFailed,
-                                forceOverlap: parsed.forceOverlap,
-                                forceCompatibility: parsed.forceCompatibility,
-                            },
-                        });
-                        const payload = {
-                            ...runPayload(output),
-                            command: 'resume',
-                        };
-                        if (jsonMode) {
-                            print(payload);
+                flags: RESUME_FLAGS,
+                run: async (context) => wrapPluginRun({
+                    command: 'resume',
+                    label: 'Backfill resume',
+                    jsonMode: context.jsonMode,
+                    print: context.print,
+                    configErrorClass: BackfillConfigError,
+                    fn: async () => {
+                        const opts = resolveResumeOptions(config, context.options, context.flags);
+                        if (!context.config.clickhouse) {
+                            throw new BackfillConfigError('ClickHouse connection is required for backfill execution. Configure clickhouse in your clickhouse.config.ts.');
                         }
-                        else {
-                            print(`Backfill resume ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
-                        }
-                        return payload.ok ? 0 : 1;
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        if (jsonMode) {
-                            print({ ok: false, command: 'resume', error: message });
+                        const { stateDir } = await readPlan({
+                            planId: opts.planId,
+                            configPath: context.configPath,
+                            config: context.config,
+                            stateDir: opts.stateDir,
+                        });
+                        const paths = backfillPaths(stateDir, opts.planId);
+                        const existingRun = await readRun(paths.runPath);
+                        if (!existingRun) {
+                            throw new BackfillConfigError(`Run state not found for plan ${opts.planId}. Start with backfill run before resume.`);
                         }
-                        else {
-                            print(`Backfill resume failed: ${message}`);
+                        if (existingRun.status === 'completed') {
+                            if (context.jsonMode) {
+                                context.print({ ok: true, noop: true, planId: opts.planId, status: 'completed', message: 'Run already completed. Nothing to resume.' });
+                            }
+                            else {
+                                context.print(`Backfill ${opts.planId}: already completed. Nothing to resume.`);
+                            }
+                            return 0;
                         }
-                        if (error instanceof BackfillConfigError)
-                            return 2;
-                        return 1;
-                    }
-                },
+                        return runBackfill({
+                            planId: opts.planId,
+                            forceEnvironment: opts.forceEnvironment,
+                            concurrency: opts.concurrency,
+                            pollIntervalMs: opts.pollIntervalMs,
+                            stateDir: opts.stateDir,
+                            resumeFrom: existingRun.progress,
+                            replayFailed: opts.replayFailed,
+                            configPath: context.configPath,
+                            config: context.config,
+                            clickhouse: context.config.clickhouse,
+                            print: context.print,
+                            jsonMode: context.jsonMode,
+                        });
+                    },
+                }),
             },
             {
                 name: 'status',
                 description: 'Show checkpoint and chunk progress for a backfill run',
-                async run({ args, jsonMode, print, options: runtimeOptions, config, configPath }) {
-                    try {
-                        const parsed = parseStatusArgs(args);
-                        const effectiveOptions = mergeOptions(base, runtimeOptions);
-                        validateBaseOptions(effectiveOptions);
+                flags: PLAN_ID_FLAGS,
+                run: async (context) => wrapPluginRun({
+                    command: 'status',
+                    label: 'Backfill status',
+                    jsonMode: context.jsonMode,
+                    print: context.print,
+                    configErrorClass: BackfillConfigError,
+                    fn: async () => {
+                        const opts = resolveStatusOptions(config, context.options, context.flags);
                         const summary = await getBackfillStatus({
-                            planId: parsed.planId,
-                            config,
-                            configPath,
-                            options: effectiveOptions,
+                            planId: opts.planId,
+                            config: context.config,
+                            configPath: context.configPath,
+                            stateDir: opts.stateDir,
                         });
                         const payload = statusPayload(summary);
-                        if (jsonMode) {
-                            print(payload);
+                        if (context.jsonMode) {
+                            context.print(payload);
                         }
                         else {
-                            print(`Backfill status ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total}, failed=${payload.chunkCounts.failed})`);
+                            let line = `Backfill status ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total}, failed=${payload.chunkCounts.failed}, ${payload.rowsWritten} rows written)`;
+                            if (payload.lastError)
+                                line += ` \u2014 ${payload.lastError}`;
+                            context.print(line);
+                            if (payload.status === 'completed' && payload.rowsWritten === 0) {
+                                context.print('Warning: 0 rows written across all chunks. Verify that source data exists in the time range and passes the query\'s WHERE filters.');
+                            }
                         }
                         return payload.ok ? 0 : 1;
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        if (jsonMode) {
-                            print({ ok: false, command: 'status', error: message });
-                        }
-                        else {
-                            print(`Backfill status failed: ${message}`);
-                        }
-                        if (error instanceof BackfillConfigError)
-                            return 2;
-                        return 1;
-                    }
-                },
+                    },
+                }),
             },
             {
                 name: 'cancel',
                 description: 'Cancel an in-progress backfill run and prevent further chunk execution',
-                async run({ args, jsonMode, print, options: runtimeOptions, config, configPath }) {
-                    try {
-                        const parsed = parseCancelArgs(args);
-                        const effectiveOptions = mergeOptions(base, runtimeOptions);
-                        validateBaseOptions(effectiveOptions);
+                flags: PLAN_ID_FLAGS,
+                run: async (context) => wrapPluginRun({
+                    command: 'cancel',
+                    label: 'Backfill cancel',
+                    jsonMode: context.jsonMode,
+                    print: context.print,
+                    configErrorClass: BackfillConfigError,
+                    fn: async () => {
+                        const opts = resolveStatusOptions(config, context.options, context.flags);
                         const summary = await cancelBackfillRun({
-                            planId: parsed.planId,
-                            config,
-                            configPath,
-                            options: effectiveOptions,
+                            planId: opts.planId,
+                            config: context.config,
+                            configPath: context.configPath,
+                            stateDir: opts.stateDir,
                         });
                         const payload = cancelPayload(summary);
-                        if (jsonMode) {
-                            print(payload);
+                        if (context.jsonMode) {
+                            context.print(payload);
                         }
                         else {
-                            print(`Backfill cancel ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
+                            context.print(`Backfill cancel ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
                         }
                         return payload.ok ? 0 : 1;
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        if (jsonMode) {
-                            print({ ok: false, command: 'cancel', error: message });
-                        }
-                        else {
-                            print(`Backfill cancel failed: ${message}`);
-                        }
-                        if (error instanceof BackfillConfigError)
-                            return 2;
-                        return 1;
-                    }
-                },
+                    },
+                }),
             },
             {
                 name: 'doctor',
                 description: 'Provide actionable remediation steps for failed or pending backfill runs',
-                async run({ args, jsonMode, print, options: runtimeOptions, config, configPath }) {
-                    try {
-                        const parsed = parseDoctorArgs(args);
-                        const effectiveOptions = mergeOptions(base, runtimeOptions);
-                        validateBaseOptions(effectiveOptions);
+                flags: PLAN_ID_FLAGS,
+                run: async (context) => wrapPluginRun({
+                    command: 'doctor',
+                    label: 'Backfill doctor',
+                    jsonMode: context.jsonMode,
+                    print: context.print,
+                    configErrorClass: BackfillConfigError,
+                    fn: async () => {
+                        const opts = resolveStatusOptions(config, context.options, context.flags);
                         const report = await getBackfillDoctorReport({
-                            planId: parsed.planId,
-                            config,
-                            configPath,
-                            options: effectiveOptions,
+                            planId: opts.planId,
+                            config: context.config,
+                            configPath: context.configPath,
+                            stateDir: opts.stateDir,
                         });
                         const payload = doctorPayload(report);
-                        if (jsonMode) {
-                            print(payload);
+                        if (context.jsonMode) {
+                            context.print(payload);
                         }
                         else {
-                            print(`Backfill doctor ${payload.planId}: ${payload.issueCodes.length === 0 ? 'ok' : payload.issueCodes.join(', ')}`);
+                            context.print(`Backfill doctor ${payload.planId}: ${payload.issueCodes.length === 0 ? 'ok' : payload.issueCodes.join(', ')}`);
                             for (const recommendation of payload.recommendations) {
-                                print(`- ${recommendation}`);
+                                context.print(`- ${recommendation}`);
                             }
                         }
                         return payload.ok ? 0 : 1;
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        if (jsonMode) {
-                            print({ ok: false, command: 'doctor', error: message });
-                        }
-                        else {
-                            print(`Backfill doctor failed: ${message}`);
-                        }
-                        if (error instanceof BackfillConfigError)
-                            return 2;
-                        return 1;
-                    }
-                },
+                    },
+                }),
             },
         ],
         hooks: {
             onConfigLoaded({ options: runtimeOptions }) {
-                const merged = mergeOptions(base, runtimeOptions);
-                validateBaseOptions(merged);
+                resolveCheckOptions(config, runtimeOptions);
             },
-            async onCheck({ config, configPath, options: runtimeOptions }) {
-                const effectiveOptions = mergeOptions(base, runtimeOptions);
-                validateBaseOptions(effectiveOptions);
+            async onCheck({ config: appConfig, configPath, options: runtimeOptions }) {
+                const opts = resolveCheckOptions(config, runtimeOptions);
                 return evaluateBackfillCheck({
                     configPath,
-                    config,
-                    options: effectiveOptions,
+                    config: appConfig,
+                    stateDir: opts.stateDir,
+                    failCheckOnRequiredPendingBackfill: opts.failCheckOnRequiredPendingBackfill,
                 });
             },
             onCheckReport({ result, print }) {
@@ -296,7 +392,7 @@ export function createBackfillPlugin(options = {}) {
 }
 export function backfill(options = {}) {
     return {
-        plugin: createBackfillPlugin(),
+        plugin: createBackfillPlugin(options),
         name: 'backfill',
         enabled: true,
         options,