@chkit/plugin-backfill 0.1.0-beta.8 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -0
- package/dist/args.d.ts +17 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +11 -0
- package/dist/args.js.map +1 -1
- package/dist/async-backfill.d.ts +64 -0
- package/dist/async-backfill.d.ts.map +1 -0
- package/dist/async-backfill.js +251 -0
- package/dist/async-backfill.js.map +1 -0
- package/dist/check.d.ts +9 -0
- package/dist/check.d.ts.map +1 -0
- package/dist/check.js +79 -0
- package/dist/check.js.map +1 -0
- package/dist/chunking/analyze.d.ts +8 -0
- package/dist/chunking/analyze.d.ts.map +1 -0
- package/dist/chunking/analyze.js +8 -0
- package/dist/chunking/analyze.js.map +1 -0
- package/dist/chunking/boundary-codec.d.ts +4 -0
- package/dist/chunking/boundary-codec.d.ts.map +1 -0
- package/dist/chunking/boundary-codec.js +79 -0
- package/dist/chunking/boundary-codec.js.map +1 -0
- package/dist/chunking/build.d.ts +11 -0
- package/dist/chunking/build.d.ts.map +1 -0
- package/dist/chunking/build.js +51 -0
- package/dist/chunking/build.js.map +1 -0
- package/dist/chunking/e2e/constants.d.ts +2 -0
- package/dist/chunking/e2e/constants.d.ts.map +1 -0
- package/dist/chunking/e2e/constants.js +2 -0
- package/dist/chunking/e2e/constants.js.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.js +134 -0
- package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
- package/dist/chunking/introspect.d.ts +40 -0
- package/dist/chunking/introspect.d.ts.map +1 -0
- package/dist/chunking/introspect.js +187 -0
- package/dist/chunking/introspect.js.map +1 -0
- package/dist/chunking/partition-slices.d.ts +14 -0
- package/dist/chunking/partition-slices.d.ts.map +1 -0
- package/dist/chunking/partition-slices.js +111 -0
- package/dist/chunking/partition-slices.js.map +1 -0
- package/dist/chunking/planner.d.ts +3 -0
- package/dist/chunking/planner.d.ts.map +1 -0
- package/dist/chunking/planner.js +343 -0
- package/dist/chunking/planner.js.map +1 -0
- package/dist/chunking/services/distribution-source.d.ts +11 -0
- package/dist/chunking/services/distribution-source.d.ts.map +1 -0
- package/dist/chunking/services/distribution-source.js +60 -0
- package/dist/chunking/services/distribution-source.js.map +1 -0
- package/dist/chunking/services/metadata-source.d.ts +4 -0
- package/dist/chunking/services/metadata-source.d.ts.map +1 -0
- package/dist/chunking/services/metadata-source.js +138 -0
- package/dist/chunking/services/metadata-source.js.map +1 -0
- package/dist/chunking/services/row-probe.d.ts +12 -0
- package/dist/chunking/services/row-probe.d.ts.map +1 -0
- package/dist/chunking/services/row-probe.js +49 -0
- package/dist/chunking/services/row-probe.js.map +1 -0
- package/dist/chunking/splitter.d.ts +20 -0
- package/dist/chunking/splitter.d.ts.map +1 -0
- package/dist/chunking/splitter.js +76 -0
- package/dist/chunking/splitter.js.map +1 -0
- package/dist/chunking/sql.d.ts +18 -0
- package/dist/chunking/sql.d.ts.map +1 -0
- package/dist/chunking/sql.js +304 -0
- package/dist/chunking/sql.js.map +1 -0
- package/dist/chunking/strategies/equal-width-split.d.ts +3 -0
- package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
- package/dist/chunking/strategies/equal-width-split.js +46 -0
- package/dist/chunking/strategies/equal-width-split.js.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.js +54 -0
- package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
- package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.js +132 -0
- package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
- package/dist/chunking/strategies/refinement.d.ts +3 -0
- package/dist/chunking/strategies/refinement.d.ts.map +1 -0
- package/dist/chunking/strategies/refinement.js +61 -0
- package/dist/chunking/strategies/refinement.js.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.js +73 -0
- package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts +3 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
- package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
- package/dist/chunking/strategy-policy.d.ts +3 -0
- package/dist/chunking/strategy-policy.d.ts.map +1 -0
- package/dist/chunking/strategy-policy.js +4 -0
- package/dist/chunking/strategy-policy.js.map +1 -0
- package/dist/chunking/types.d.ts +137 -0
- package/dist/chunking/types.d.ts.map +1 -0
- package/dist/chunking/types.js +2 -0
- package/dist/chunking/types.js.map +1 -0
- package/dist/chunking/utils/binary-string.d.ts +8 -0
- package/dist/chunking/utils/binary-string.d.ts.map +1 -0
- package/dist/chunking/utils/binary-string.js +52 -0
- package/dist/chunking/utils/binary-string.js.map +1 -0
- package/dist/chunking/utils/ids.d.ts +4 -0
- package/dist/chunking/utils/ids.d.ts.map +1 -0
- package/dist/chunking/utils/ids.js +11 -0
- package/dist/chunking/utils/ids.js.map +1 -0
- package/dist/chunking/utils/ranges.d.ts +5 -0
- package/dist/chunking/utils/ranges.d.ts.map +1 -0
- package/dist/chunking/utils/ranges.js +19 -0
- package/dist/chunking/utils/ranges.js.map +1 -0
- package/dist/detect.d.ts +8 -0
- package/dist/detect.d.ts.map +1 -0
- package/dist/detect.js +72 -0
- package/dist/detect.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/logging.d.ts +12 -0
- package/dist/logging.d.ts.map +1 -0
- package/dist/logging.js +61 -0
- package/dist/logging.js.map +1 -0
- package/dist/options.d.ts +140 -4
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +145 -111
- package/dist/options.js.map +1 -1
- package/dist/payload.d.ts +7 -17
- package/dist/payload.d.ts.map +1 -1
- package/dist/payload.js +7 -19
- package/dist/payload.js.map +1 -1
- package/dist/planner.d.ts +10 -8
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +76 -97
- package/dist/planner.js.map +1 -1
- package/dist/plugin.d.ts +4 -3
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +351 -201
- package/dist/plugin.js.map +1 -1
- package/dist/queries.d.ts +21 -0
- package/dist/queries.d.ts.map +1 -0
- package/dist/queries.js +113 -0
- package/dist/queries.js.map +1 -0
- package/dist/runtime.d.ts +14 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +162 -83
- package/dist/runtime.js.map +1 -1
- package/dist/sdk.d.ts +12 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +9 -0
- package/dist/sdk.js.map +1 -0
- package/dist/state.d.ts +16 -28
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +73 -127
- package/dist/state.js.map +1 -1
- package/dist/table-config.d.ts +9 -0
- package/dist/table-config.d.ts.map +1 -0
- package/dist/table-config.js +2 -0
- package/dist/table-config.js.map +1 -0
- package/dist/types.d.ts +48 -118
- package/dist/types.d.ts.map +1 -1
- package/package.json +25 -2
package/dist/plugin.js
CHANGED
|
@@ -1,259 +1,409 @@
|
|
|
1
|
+
import { createClickHouseExecutor } from '@chkit/clickhouse';
|
|
1
2
|
import { wrapPluginRun } from '@chkit/core';
|
|
2
|
-
import {
|
|
3
|
+
import { executeBackfill } from './async-backfill.js';
|
|
4
|
+
import { buildChunkExecutionSql } from './chunking/sql.js';
|
|
5
|
+
import { generateIdempotencyToken } from './chunking/utils/ids.js';
|
|
3
6
|
import { BackfillConfigError } from './errors.js';
|
|
4
|
-
import {
|
|
5
|
-
import { planPayload,
|
|
7
|
+
import { CheckSchema, PLAN_FLAGS, PLAN_FLAG_MAP, PLAN_ID_FLAGS, PLAN_ID_FLAG_MAP, PlanSchema, PluginConfigSchema, RESUME_FLAGS, RESUME_FLAG_MAP, ResumeSchema, RUN_FLAGS, RUN_FLAG_MAP, RunSchema, StatusSchema, } from './options.js';
|
|
8
|
+
import { planPayload, statusPayload, cancelPayload, doctorPayload } from './payload.js';
|
|
6
9
|
import { buildBackfillPlan } from './planner.js';
|
|
7
|
-
import {
|
|
10
|
+
import { evaluateBackfillCheck } from './check.js';
|
|
11
|
+
import { cancelBackfillRun, getBackfillDoctorReport, getBackfillStatus } from './queries.js';
|
|
12
|
+
import { backfillPaths, ensureEnvironmentMatch, nowIso, readPlan, readRun, summarizeRunStatus, writeJson, } from './state.js';
|
|
13
|
+
function formatBytes(bytes) {
|
|
14
|
+
if (bytes >= 1024 ** 4)
|
|
15
|
+
return `${(bytes / 1024 ** 4).toFixed(1)} TiB`;
|
|
16
|
+
if (bytes >= 1024 ** 3)
|
|
17
|
+
return `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
|
|
18
|
+
if (bytes >= 1024 ** 2)
|
|
19
|
+
return `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
|
|
20
|
+
if (bytes >= 1024)
|
|
21
|
+
return `${(bytes / 1024).toFixed(1)} KiB`;
|
|
22
|
+
return `${bytes} B`;
|
|
23
|
+
}
|
|
24
|
+
async function runBackfill(input) {
|
|
25
|
+
const { plan, stateDir } = await readPlan({
|
|
26
|
+
planId: input.planId,
|
|
27
|
+
configPath: input.configPath,
|
|
28
|
+
config: input.config,
|
|
29
|
+
stateDir: input.stateDir,
|
|
30
|
+
});
|
|
31
|
+
ensureEnvironmentMatch({
|
|
32
|
+
plan,
|
|
33
|
+
clickhouse: input.clickhouse,
|
|
34
|
+
forceEnvironment: input.forceEnvironment,
|
|
35
|
+
});
|
|
36
|
+
const paths = backfillPaths(stateDir, plan.planId);
|
|
37
|
+
// Check for existing run state
|
|
38
|
+
const existingRun = await readRun(paths.runPath);
|
|
39
|
+
const resumeFrom = input.resumeFrom;
|
|
40
|
+
if (existingRun && !resumeFrom) {
|
|
41
|
+
// `run` command (no resumeFrom) must not silently continue an existing run.
|
|
42
|
+
// Users should use `backfill resume` instead.
|
|
43
|
+
const status = existingRun.status;
|
|
44
|
+
if (status === 'completed') {
|
|
45
|
+
throw new BackfillConfigError(`Run already completed for plan ${plan.planId}. Nothing to do.`);
|
|
46
|
+
}
|
|
47
|
+
if (status === 'cancelled') {
|
|
48
|
+
throw new BackfillConfigError(`Run is cancelled for plan ${plan.planId}. Create a new plan or inspect with backfill doctor.`);
|
|
49
|
+
}
|
|
50
|
+
throw new BackfillConfigError(`A run already exists for plan ${plan.planId} (status: ${status}). Use backfill resume to continue.`);
|
|
51
|
+
}
|
|
52
|
+
const db = createClickHouseExecutor(input.clickhouse);
|
|
53
|
+
try {
|
|
54
|
+
const runState = {
|
|
55
|
+
planId: plan.planId,
|
|
56
|
+
target: plan.target,
|
|
57
|
+
status: 'running',
|
|
58
|
+
startedAt: existingRun?.startedAt ?? nowIso(),
|
|
59
|
+
updatedAt: nowIso(),
|
|
60
|
+
progress: resumeFrom ?? {},
|
|
61
|
+
};
|
|
62
|
+
await writeJson(paths.runPath, runState);
|
|
63
|
+
const result = await executeBackfill({
|
|
64
|
+
executor: db,
|
|
65
|
+
planId: plan.planId,
|
|
66
|
+
chunks: plan.chunkPlan.chunks.map((chunk) => ({ id: chunk.id })),
|
|
67
|
+
buildQuery: (chunk) => {
|
|
68
|
+
const planChunk = plan.chunkPlan.chunks.find((candidate) => candidate.id === chunk.id);
|
|
69
|
+
if (!planChunk)
|
|
70
|
+
throw new Error(`Chunk ${chunk.id} not found in plan`);
|
|
71
|
+
return buildChunkExecutionSql({
|
|
72
|
+
planId: plan.planId,
|
|
73
|
+
chunk: planChunk,
|
|
74
|
+
target: plan.target,
|
|
75
|
+
sourceTarget: plan.execution.sourceTarget,
|
|
76
|
+
table: plan.chunkPlan.table,
|
|
77
|
+
mvAsQuery: plan.execution.mvAsQuery,
|
|
78
|
+
targetColumns: plan.execution.targetColumns,
|
|
79
|
+
idempotencyToken: plan.execution.requireIdempotencyToken
|
|
80
|
+
? generateIdempotencyToken(plan.planId, planChunk.id)
|
|
81
|
+
: '',
|
|
82
|
+
});
|
|
83
|
+
},
|
|
84
|
+
concurrency: input.concurrency,
|
|
85
|
+
pollIntervalMs: input.pollIntervalMs,
|
|
86
|
+
resumeFrom,
|
|
87
|
+
replayFailed: input.replayFailed,
|
|
88
|
+
onProgress: async (progress) => {
|
|
89
|
+
runState.progress = progress;
|
|
90
|
+
runState.updatedAt = nowIso();
|
|
91
|
+
await writeJson(paths.runPath, runState);
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
runState.status = result.failed > 0 ? 'failed' : 'completed';
|
|
95
|
+
runState.completedAt = nowIso();
|
|
96
|
+
runState.updatedAt = nowIso();
|
|
97
|
+
runState.progress = result.progress;
|
|
98
|
+
if (result.failed > 0) {
|
|
99
|
+
const failedEntry = Object.values(result.progress).find((c) => c.status === 'failed');
|
|
100
|
+
runState.lastError = failedEntry?.error ?? 'One or more chunks failed';
|
|
101
|
+
}
|
|
102
|
+
await writeJson(paths.runPath, runState);
|
|
103
|
+
const summary = summarizeRunStatus(runState, paths.runPath, plan);
|
|
104
|
+
if (input.jsonMode) {
|
|
105
|
+
input.print({
|
|
106
|
+
ok: result.failed === 0,
|
|
107
|
+
planId: plan.planId,
|
|
108
|
+
status: runState.status,
|
|
109
|
+
chunkCounts: summary.totals,
|
|
110
|
+
rowsWritten: summary.rowsWritten,
|
|
111
|
+
runPath: paths.runPath,
|
|
112
|
+
lastError: runState.lastError,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
let line = `Backfill ${plan.planId}: ${runState.status} (done=${summary.totals.done}/${summary.totals.total}, ${summary.rowsWritten} rows written)`;
|
|
117
|
+
if (runState.lastError)
|
|
118
|
+
line += ` \u2014 ${runState.lastError}`;
|
|
119
|
+
input.print(line);
|
|
120
|
+
if (runState.status === 'completed' && summary.rowsWritten === 0) {
|
|
121
|
+
input.print('Warning: 0 rows written across all chunks. Verify that source data exists in the time range and passes the query\'s WHERE filters.');
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return result.failed > 0 ? 1 : 0;
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
await db.close();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
function isRecord(value) {
|
|
131
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
132
|
+
}
|
|
133
|
+
function withFactoryDefaults(schema, defaults) {
|
|
134
|
+
return {
|
|
135
|
+
safeParse(data) {
|
|
136
|
+
return schema.safeParse({ ...defaults, ...(isRecord(data) ? data : {}) });
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
8
140
|
export function createBackfillPlugin(options = {}) {
|
|
9
|
-
const
|
|
10
|
-
|
|
141
|
+
const factoryOptions = options;
|
|
142
|
+
const planOptionsSchema = withFactoryDefaults(PlanSchema, factoryOptions);
|
|
143
|
+
const runOptionsSchema = withFactoryDefaults(RunSchema, factoryOptions);
|
|
144
|
+
const resumeOptionsSchema = withFactoryDefaults(ResumeSchema, factoryOptions);
|
|
145
|
+
const statusOptionsSchema = withFactoryDefaults(StatusSchema, factoryOptions);
|
|
11
146
|
return {
|
|
12
147
|
manifest: {
|
|
13
148
|
name: 'backfill',
|
|
14
149
|
apiVersion: 1,
|
|
15
150
|
},
|
|
151
|
+
optionsSchema: withFactoryDefaults(PluginConfigSchema, factoryOptions),
|
|
16
152
|
commands: [
|
|
17
153
|
{
|
|
18
154
|
name: 'plan',
|
|
19
155
|
description: 'Build a deterministic backfill plan and persist immutable plan state',
|
|
20
156
|
flags: PLAN_FLAGS,
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
157
|
+
optionsSchema: planOptionsSchema,
|
|
158
|
+
flagMapping: PLAN_FLAG_MAP,
|
|
159
|
+
run: async (context) => wrapPluginRun({
|
|
160
|
+
command: 'plan',
|
|
161
|
+
label: 'Backfill plan',
|
|
162
|
+
jsonMode: context.jsonMode,
|
|
163
|
+
print: context.print,
|
|
164
|
+
configErrorClass: BackfillConfigError,
|
|
165
|
+
fn: async () => {
|
|
166
|
+
const opts = context.options;
|
|
167
|
+
if (!context.config.clickhouse) {
|
|
168
|
+
throw new BackfillConfigError('ClickHouse connection is required for backfill planning. Configure clickhouse in your clickhouse.config.ts.');
|
|
169
|
+
}
|
|
170
|
+
const db = createClickHouseExecutor(context.config.clickhouse);
|
|
171
|
+
try {
|
|
32
172
|
const output = await buildBackfillPlan({
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
config,
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
173
|
+
opts,
|
|
174
|
+
configPath: context.configPath,
|
|
175
|
+
config: context.config,
|
|
176
|
+
clickhouse: context.config.clickhouse,
|
|
177
|
+
clickhouseQuery: async (sql, settings) => {
|
|
178
|
+
const result = await db.query(sql, settings);
|
|
179
|
+
return result;
|
|
180
|
+
},
|
|
181
|
+
// ObsessionDB enables parallel replicas by default, which inflates
|
|
182
|
+
// aggregate results (count, GROUP BY). Disable for planning queries
|
|
183
|
+
// until ObsessionDB handles it at the profile level.
|
|
184
|
+
querySettings: { enable_parallel_replicas: 0 },
|
|
41
185
|
});
|
|
42
186
|
const payload = planPayload(output);
|
|
43
|
-
if (jsonMode) {
|
|
44
|
-
print(payload);
|
|
187
|
+
if (context.jsonMode) {
|
|
188
|
+
context.print(payload);
|
|
45
189
|
}
|
|
46
190
|
else {
|
|
47
|
-
|
|
191
|
+
const partitionCount = output.plan.chunkPlan.partitions.length;
|
|
192
|
+
const totalBytes = formatBytes(output.plan.chunkPlan.totalBytesCompressed);
|
|
193
|
+
const primarySortKey = output.plan.chunkPlan.table.sortKeys[0];
|
|
194
|
+
const sortKeyLabel = primarySortKey
|
|
195
|
+
? `, sort key: ${primarySortKey.name} (${primarySortKey.category})`
|
|
196
|
+
: '';
|
|
197
|
+
context.print(`Backfill plan ${payload.planId} for ${payload.target} (${payload.chunkCount} chunks across ${partitionCount} partitions, ~${totalBytes}${sortKeyLabel}) -> ${payload.planPath}`);
|
|
48
198
|
}
|
|
49
199
|
return 0;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
await db.close();
|
|
203
|
+
}
|
|
204
|
+
},
|
|
205
|
+
}),
|
|
53
206
|
},
|
|
54
207
|
{
|
|
55
208
|
name: 'run',
|
|
56
|
-
description: 'Execute a planned backfill with
|
|
209
|
+
description: 'Execute a planned backfill with async query submission and polling',
|
|
57
210
|
flags: RUN_FLAGS,
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
const payload = {
|
|
86
|
-
...runPayload(output),
|
|
87
|
-
command: 'run',
|
|
88
|
-
};
|
|
89
|
-
if (jsonMode) {
|
|
90
|
-
print(payload);
|
|
91
|
-
}
|
|
92
|
-
else {
|
|
93
|
-
print(`Backfill run ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
|
|
94
|
-
}
|
|
95
|
-
return payload.ok ? 0 : 1;
|
|
96
|
-
},
|
|
97
|
-
});
|
|
98
|
-
},
|
|
211
|
+
optionsSchema: runOptionsSchema,
|
|
212
|
+
flagMapping: RUN_FLAG_MAP,
|
|
213
|
+
run: async (context) => wrapPluginRun({
|
|
214
|
+
command: 'run',
|
|
215
|
+
label: 'Backfill run',
|
|
216
|
+
jsonMode: context.jsonMode,
|
|
217
|
+
print: context.print,
|
|
218
|
+
configErrorClass: BackfillConfigError,
|
|
219
|
+
fn: async () => {
|
|
220
|
+
const opts = context.options;
|
|
221
|
+
if (!context.config.clickhouse) {
|
|
222
|
+
throw new BackfillConfigError('ClickHouse connection is required for backfill execution. Configure clickhouse in your clickhouse.config.ts.');
|
|
223
|
+
}
|
|
224
|
+
return runBackfill({
|
|
225
|
+
planId: opts.planId,
|
|
226
|
+
forceEnvironment: opts.forceEnvironment,
|
|
227
|
+
concurrency: opts.concurrency,
|
|
228
|
+
pollIntervalMs: opts.pollIntervalMs,
|
|
229
|
+
stateDir: opts.stateDir,
|
|
230
|
+
configPath: context.configPath,
|
|
231
|
+
config: context.config,
|
|
232
|
+
clickhouse: context.config.clickhouse,
|
|
233
|
+
print: context.print,
|
|
234
|
+
jsonMode: context.jsonMode,
|
|
235
|
+
});
|
|
236
|
+
},
|
|
237
|
+
}),
|
|
99
238
|
},
|
|
100
239
|
{
|
|
101
240
|
name: 'resume',
|
|
102
241
|
description: 'Resume a backfill run from last checkpoint',
|
|
103
242
|
flags: RESUME_FLAGS,
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
if (jsonMode) {
|
|
132
|
-
print(payload);
|
|
243
|
+
optionsSchema: resumeOptionsSchema,
|
|
244
|
+
flagMapping: RESUME_FLAG_MAP,
|
|
245
|
+
run: async (context) => wrapPluginRun({
|
|
246
|
+
command: 'resume',
|
|
247
|
+
label: 'Backfill resume',
|
|
248
|
+
jsonMode: context.jsonMode,
|
|
249
|
+
print: context.print,
|
|
250
|
+
configErrorClass: BackfillConfigError,
|
|
251
|
+
fn: async () => {
|
|
252
|
+
const opts = context.options;
|
|
253
|
+
if (!context.config.clickhouse) {
|
|
254
|
+
throw new BackfillConfigError('ClickHouse connection is required for backfill execution. Configure clickhouse in your clickhouse.config.ts.');
|
|
255
|
+
}
|
|
256
|
+
const { stateDir } = await readPlan({
|
|
257
|
+
planId: opts.planId,
|
|
258
|
+
configPath: context.configPath,
|
|
259
|
+
config: context.config,
|
|
260
|
+
stateDir: opts.stateDir,
|
|
261
|
+
});
|
|
262
|
+
const paths = backfillPaths(stateDir, opts.planId);
|
|
263
|
+
const existingRun = await readRun(paths.runPath);
|
|
264
|
+
if (!existingRun) {
|
|
265
|
+
throw new BackfillConfigError(`Run state not found for plan ${opts.planId}. Start with backfill run before resume.`);
|
|
266
|
+
}
|
|
267
|
+
if (existingRun.status === 'completed') {
|
|
268
|
+
if (context.jsonMode) {
|
|
269
|
+
context.print({ ok: true, noop: true, planId: opts.planId, status: 'completed', message: 'Run already completed. Nothing to resume.' });
|
|
133
270
|
}
|
|
134
271
|
else {
|
|
135
|
-
print(`Backfill
|
|
272
|
+
context.print(`Backfill ${opts.planId}: already completed. Nothing to resume.`);
|
|
136
273
|
}
|
|
137
|
-
return
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
|
|
274
|
+
return 0;
|
|
275
|
+
}
|
|
276
|
+
return runBackfill({
|
|
277
|
+
planId: opts.planId,
|
|
278
|
+
forceEnvironment: opts.forceEnvironment,
|
|
279
|
+
concurrency: opts.concurrency,
|
|
280
|
+
pollIntervalMs: opts.pollIntervalMs,
|
|
281
|
+
stateDir: opts.stateDir,
|
|
282
|
+
resumeFrom: existingRun.progress,
|
|
283
|
+
replayFailed: opts.replayFailed,
|
|
284
|
+
configPath: context.configPath,
|
|
285
|
+
config: context.config,
|
|
286
|
+
clickhouse: context.config.clickhouse,
|
|
287
|
+
print: context.print,
|
|
288
|
+
jsonMode: context.jsonMode,
|
|
289
|
+
});
|
|
290
|
+
},
|
|
291
|
+
}),
|
|
141
292
|
},
|
|
142
293
|
{
|
|
143
294
|
name: 'status',
|
|
144
295
|
description: 'Show checkpoint and chunk progress for a backfill run',
|
|
145
296
|
flags: PLAN_ID_FLAGS,
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
297
|
+
optionsSchema: statusOptionsSchema,
|
|
298
|
+
flagMapping: PLAN_ID_FLAG_MAP,
|
|
299
|
+
run: async (context) => wrapPluginRun({
|
|
300
|
+
command: 'status',
|
|
301
|
+
label: 'Backfill status',
|
|
302
|
+
jsonMode: context.jsonMode,
|
|
303
|
+
print: context.print,
|
|
304
|
+
configErrorClass: BackfillConfigError,
|
|
305
|
+
fn: async () => {
|
|
306
|
+
const opts = context.options;
|
|
307
|
+
const summary = await getBackfillStatus({
|
|
308
|
+
planId: opts.planId,
|
|
309
|
+
config: context.config,
|
|
310
|
+
configPath: context.configPath,
|
|
311
|
+
stateDir: opts.stateDir,
|
|
312
|
+
});
|
|
313
|
+
const payload = statusPayload(summary);
|
|
314
|
+
if (context.jsonMode) {
|
|
315
|
+
context.print(payload);
|
|
316
|
+
}
|
|
317
|
+
else {
|
|
318
|
+
let line = `Backfill status ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total}, failed=${payload.chunkCounts.failed}, ${payload.rowsWritten} rows written)`;
|
|
319
|
+
if (payload.lastError)
|
|
320
|
+
line += ` \u2014 ${payload.lastError}`;
|
|
321
|
+
context.print(line);
|
|
322
|
+
if (payload.status === 'completed' && payload.rowsWritten === 0) {
|
|
323
|
+
context.print('Warning: 0 rows written across all chunks. Verify that source data exists in the time range and passes the query\'s WHERE filters.');
|
|
169
324
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
}
|
|
173
|
-
},
|
|
325
|
+
}
|
|
326
|
+
return payload.ok ? 0 : 1;
|
|
327
|
+
},
|
|
328
|
+
}),
|
|
174
329
|
},
|
|
175
330
|
{
|
|
176
331
|
name: 'cancel',
|
|
177
332
|
description: 'Cancel an in-progress backfill run and prevent further chunk execution',
|
|
178
333
|
flags: PLAN_ID_FLAGS,
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
});
|
|
206
|
-
},
|
|
334
|
+
optionsSchema: statusOptionsSchema,
|
|
335
|
+
flagMapping: PLAN_ID_FLAG_MAP,
|
|
336
|
+
run: async (context) => wrapPluginRun({
|
|
337
|
+
command: 'cancel',
|
|
338
|
+
label: 'Backfill cancel',
|
|
339
|
+
jsonMode: context.jsonMode,
|
|
340
|
+
print: context.print,
|
|
341
|
+
configErrorClass: BackfillConfigError,
|
|
342
|
+
fn: async () => {
|
|
343
|
+
const opts = context.options;
|
|
344
|
+
const summary = await cancelBackfillRun({
|
|
345
|
+
planId: opts.planId,
|
|
346
|
+
config: context.config,
|
|
347
|
+
configPath: context.configPath,
|
|
348
|
+
stateDir: opts.stateDir,
|
|
349
|
+
});
|
|
350
|
+
const payload = cancelPayload(summary);
|
|
351
|
+
if (context.jsonMode) {
|
|
352
|
+
context.print(payload);
|
|
353
|
+
}
|
|
354
|
+
else {
|
|
355
|
+
context.print(`Backfill cancel ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
|
|
356
|
+
}
|
|
357
|
+
return payload.ok ? 0 : 1;
|
|
358
|
+
},
|
|
359
|
+
}),
|
|
207
360
|
},
|
|
208
361
|
{
|
|
209
362
|
name: 'doctor',
|
|
210
363
|
description: 'Provide actionable remediation steps for failed or pending backfill runs',
|
|
211
364
|
flags: PLAN_ID_FLAGS,
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
print(`- ${recommendation}`);
|
|
237
|
-
}
|
|
365
|
+
optionsSchema: statusOptionsSchema,
|
|
366
|
+
flagMapping: PLAN_ID_FLAG_MAP,
|
|
367
|
+
run: async (context) => wrapPluginRun({
|
|
368
|
+
command: 'doctor',
|
|
369
|
+
label: 'Backfill doctor',
|
|
370
|
+
jsonMode: context.jsonMode,
|
|
371
|
+
print: context.print,
|
|
372
|
+
configErrorClass: BackfillConfigError,
|
|
373
|
+
fn: async () => {
|
|
374
|
+
const opts = context.options;
|
|
375
|
+
const report = await getBackfillDoctorReport({
|
|
376
|
+
planId: opts.planId,
|
|
377
|
+
config: context.config,
|
|
378
|
+
configPath: context.configPath,
|
|
379
|
+
stateDir: opts.stateDir,
|
|
380
|
+
});
|
|
381
|
+
const payload = doctorPayload(report);
|
|
382
|
+
if (context.jsonMode) {
|
|
383
|
+
context.print(payload);
|
|
384
|
+
}
|
|
385
|
+
else {
|
|
386
|
+
context.print(`Backfill doctor ${payload.planId}: ${payload.issueCodes.length === 0 ? 'ok' : payload.issueCodes.join(', ')}`);
|
|
387
|
+
for (const recommendation of payload.recommendations) {
|
|
388
|
+
context.print(`- ${recommendation}`);
|
|
238
389
|
}
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
}
|
|
242
|
-
},
|
|
390
|
+
}
|
|
391
|
+
return payload.ok ? 0 : 1;
|
|
392
|
+
},
|
|
393
|
+
}),
|
|
243
394
|
},
|
|
244
395
|
],
|
|
245
396
|
hooks: {
|
|
246
|
-
onConfigLoaded(
|
|
247
|
-
|
|
248
|
-
validateBaseOptions(merged);
|
|
397
|
+
onConfigLoaded() {
|
|
398
|
+
// Plugin-level options validated softly at load time via PluginConfigSchema.
|
|
249
399
|
},
|
|
250
|
-
async onCheck({ config, configPath, options
|
|
251
|
-
const
|
|
252
|
-
validateBaseOptions(effectiveOptions);
|
|
400
|
+
async onCheck({ config: appConfig, configPath, options }) {
|
|
401
|
+
const opts = CheckSchema.parse(options);
|
|
253
402
|
return evaluateBackfillCheck({
|
|
254
403
|
configPath,
|
|
255
|
-
config,
|
|
256
|
-
|
|
404
|
+
config: appConfig,
|
|
405
|
+
stateDir: opts.stateDir,
|
|
406
|
+
failCheckOnRequiredPendingBackfill: opts.failCheckOnRequiredPendingBackfill,
|
|
257
407
|
});
|
|
258
408
|
},
|
|
259
409
|
onCheckReport({ result, print }) {
|
|
@@ -269,7 +419,7 @@ export function createBackfillPlugin(options = {}) {
|
|
|
269
419
|
}
|
|
270
420
|
export function backfill(options = {}) {
|
|
271
421
|
return {
|
|
272
|
-
plugin: createBackfillPlugin(),
|
|
422
|
+
plugin: createBackfillPlugin(options),
|
|
273
423
|
name: 'backfill',
|
|
274
424
|
enabled: true,
|
|
275
425
|
options,
|