@chkit/plugin-backfill 0.1.0-beta.9 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/README.md +130 -0
  2. package/dist/args.d.ts +17 -0
  3. package/dist/args.d.ts.map +1 -1
  4. package/dist/args.js +11 -0
  5. package/dist/args.js.map +1 -1
  6. package/dist/async-backfill.d.ts +64 -0
  7. package/dist/async-backfill.d.ts.map +1 -0
  8. package/dist/async-backfill.js +251 -0
  9. package/dist/async-backfill.js.map +1 -0
  10. package/dist/check.d.ts +9 -0
  11. package/dist/check.d.ts.map +1 -0
  12. package/dist/check.js +79 -0
  13. package/dist/check.js.map +1 -0
  14. package/dist/chunking/analyze.d.ts +8 -0
  15. package/dist/chunking/analyze.d.ts.map +1 -0
  16. package/dist/chunking/analyze.js +8 -0
  17. package/dist/chunking/analyze.js.map +1 -0
  18. package/dist/chunking/boundary-codec.d.ts +4 -0
  19. package/dist/chunking/boundary-codec.d.ts.map +1 -0
  20. package/dist/chunking/boundary-codec.js +79 -0
  21. package/dist/chunking/boundary-codec.js.map +1 -0
  22. package/dist/chunking/build.d.ts +11 -0
  23. package/dist/chunking/build.d.ts.map +1 -0
  24. package/dist/chunking/build.js +51 -0
  25. package/dist/chunking/build.js.map +1 -0
  26. package/dist/chunking/e2e/constants.d.ts +2 -0
  27. package/dist/chunking/e2e/constants.d.ts.map +1 -0
  28. package/dist/chunking/e2e/constants.js +2 -0
  29. package/dist/chunking/e2e/constants.js.map +1 -0
  30. package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
  31. package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
  32. package/dist/chunking/e2e/seed-datasets.script.js +134 -0
  33. package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
  34. package/dist/chunking/introspect.d.ts +40 -0
  35. package/dist/chunking/introspect.d.ts.map +1 -0
  36. package/dist/chunking/introspect.js +187 -0
  37. package/dist/chunking/introspect.js.map +1 -0
  38. package/dist/chunking/partition-slices.d.ts +14 -0
  39. package/dist/chunking/partition-slices.d.ts.map +1 -0
  40. package/dist/chunking/partition-slices.js +111 -0
  41. package/dist/chunking/partition-slices.js.map +1 -0
  42. package/dist/chunking/planner.d.ts +3 -0
  43. package/dist/chunking/planner.d.ts.map +1 -0
  44. package/dist/chunking/planner.js +343 -0
  45. package/dist/chunking/planner.js.map +1 -0
  46. package/dist/chunking/services/distribution-source.d.ts +11 -0
  47. package/dist/chunking/services/distribution-source.d.ts.map +1 -0
  48. package/dist/chunking/services/distribution-source.js +60 -0
  49. package/dist/chunking/services/distribution-source.js.map +1 -0
  50. package/dist/chunking/services/metadata-source.d.ts +4 -0
  51. package/dist/chunking/services/metadata-source.d.ts.map +1 -0
  52. package/dist/chunking/services/metadata-source.js +138 -0
  53. package/dist/chunking/services/metadata-source.js.map +1 -0
  54. package/dist/chunking/services/row-probe.d.ts +12 -0
  55. package/dist/chunking/services/row-probe.d.ts.map +1 -0
  56. package/dist/chunking/services/row-probe.js +49 -0
  57. package/dist/chunking/services/row-probe.js.map +1 -0
  58. package/dist/chunking/splitter.d.ts +20 -0
  59. package/dist/chunking/splitter.d.ts.map +1 -0
  60. package/dist/chunking/splitter.js +76 -0
  61. package/dist/chunking/splitter.js.map +1 -0
  62. package/dist/chunking/sql.d.ts +18 -0
  63. package/dist/chunking/sql.d.ts.map +1 -0
  64. package/dist/chunking/sql.js +304 -0
  65. package/dist/chunking/sql.js.map +1 -0
  66. package/dist/chunking/strategies/equal-width-split.d.ts +3 -0
  67. package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
  68. package/dist/chunking/strategies/equal-width-split.js +46 -0
  69. package/dist/chunking/strategies/equal-width-split.js.map +1 -0
  70. package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
  71. package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
  72. package/dist/chunking/strategies/group-by-key-split.js +54 -0
  73. package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
  74. package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
  75. package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
  76. package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
  77. package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
  78. package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
  79. package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
  80. package/dist/chunking/strategies/quantile-range-split.js +132 -0
  81. package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
  82. package/dist/chunking/strategies/refinement.d.ts +3 -0
  83. package/dist/chunking/strategies/refinement.d.ts.map +1 -0
  84. package/dist/chunking/strategies/refinement.js +61 -0
  85. package/dist/chunking/strategies/refinement.js.map +1 -0
  86. package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
  87. package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
  88. package/dist/chunking/strategies/string-prefix-split.js +73 -0
  89. package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
  90. package/dist/chunking/strategies/temporal-bucket-split.d.ts +3 -0
  91. package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
  92. package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
  93. package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
  94. package/dist/chunking/strategy-policy.d.ts +3 -0
  95. package/dist/chunking/strategy-policy.d.ts.map +1 -0
  96. package/dist/chunking/strategy-policy.js +4 -0
  97. package/dist/chunking/strategy-policy.js.map +1 -0
  98. package/dist/chunking/types.d.ts +137 -0
  99. package/dist/chunking/types.d.ts.map +1 -0
  100. package/dist/chunking/types.js +2 -0
  101. package/dist/chunking/types.js.map +1 -0
  102. package/dist/chunking/utils/binary-string.d.ts +8 -0
  103. package/dist/chunking/utils/binary-string.d.ts.map +1 -0
  104. package/dist/chunking/utils/binary-string.js +52 -0
  105. package/dist/chunking/utils/binary-string.js.map +1 -0
  106. package/dist/chunking/utils/ids.d.ts +4 -0
  107. package/dist/chunking/utils/ids.d.ts.map +1 -0
  108. package/dist/chunking/utils/ids.js +11 -0
  109. package/dist/chunking/utils/ids.js.map +1 -0
  110. package/dist/chunking/utils/ranges.d.ts +5 -0
  111. package/dist/chunking/utils/ranges.d.ts.map +1 -0
  112. package/dist/chunking/utils/ranges.js +19 -0
  113. package/dist/chunking/utils/ranges.js.map +1 -0
  114. package/dist/detect.d.ts +8 -0
  115. package/dist/detect.d.ts.map +1 -0
  116. package/dist/detect.js +72 -0
  117. package/dist/detect.js.map +1 -0
  118. package/dist/index.d.ts +3 -0
  119. package/dist/index.d.ts.map +1 -1
  120. package/dist/index.js +1 -0
  121. package/dist/index.js.map +1 -1
  122. package/dist/logging.d.ts +12 -0
  123. package/dist/logging.d.ts.map +1 -0
  124. package/dist/logging.js +61 -0
  125. package/dist/logging.js.map +1 -0
  126. package/dist/options.d.ts +140 -4
  127. package/dist/options.d.ts.map +1 -1
  128. package/dist/options.js +145 -111
  129. package/dist/options.js.map +1 -1
  130. package/dist/payload.d.ts +7 -17
  131. package/dist/payload.d.ts.map +1 -1
  132. package/dist/payload.js +7 -19
  133. package/dist/payload.js.map +1 -1
  134. package/dist/planner.d.ts +10 -8
  135. package/dist/planner.d.ts.map +1 -1
  136. package/dist/planner.js +76 -97
  137. package/dist/planner.js.map +1 -1
  138. package/dist/plugin.d.ts +4 -3
  139. package/dist/plugin.d.ts.map +1 -1
  140. package/dist/plugin.js +351 -201
  141. package/dist/plugin.js.map +1 -1
  142. package/dist/queries.d.ts +21 -0
  143. package/dist/queries.d.ts.map +1 -0
  144. package/dist/queries.js +113 -0
  145. package/dist/queries.js.map +1 -0
  146. package/dist/runtime.d.ts +14 -0
  147. package/dist/runtime.d.ts.map +1 -1
  148. package/dist/runtime.js +162 -83
  149. package/dist/runtime.js.map +1 -1
  150. package/dist/sdk.d.ts +12 -0
  151. package/dist/sdk.d.ts.map +1 -0
  152. package/dist/sdk.js +9 -0
  153. package/dist/sdk.js.map +1 -0
  154. package/dist/state.d.ts +16 -28
  155. package/dist/state.d.ts.map +1 -1
  156. package/dist/state.js +73 -127
  157. package/dist/state.js.map +1 -1
  158. package/dist/table-config.d.ts +9 -0
  159. package/dist/table-config.d.ts.map +1 -0
  160. package/dist/table-config.js +2 -0
  161. package/dist/table-config.js.map +1 -0
  162. package/dist/types.d.ts +48 -118
  163. package/dist/types.d.ts.map +1 -1
  164. package/package.json +25 -2
package/dist/plugin.js CHANGED
@@ -1,259 +1,409 @@
1
+ import { createClickHouseExecutor } from '@chkit/clickhouse';
1
2
  import { wrapPluginRun } from '@chkit/core';
2
- import { PLAN_FLAGS, PLAN_ID_FLAGS, RESUME_FLAGS, RUN_FLAGS, parseCancelArgs, parseDoctorArgs, parsePlanArgs, parseResumeArgs, parseRunArgs, parseStatusArgs, } from './args.js';
3
+ import { executeBackfill } from './async-backfill.js';
4
+ import { buildChunkExecutionSql } from './chunking/sql.js';
5
+ import { generateIdempotencyToken } from './chunking/utils/ids.js';
3
6
  import { BackfillConfigError } from './errors.js';
4
- import { normalizeBackfillOptions, mergeOptions, validateBaseOptions } from './options.js';
5
- import { planPayload, runPayload, statusPayload, cancelPayload, doctorPayload } from './payload.js';
7
+ import { CheckSchema, PLAN_FLAGS, PLAN_FLAG_MAP, PLAN_ID_FLAGS, PLAN_ID_FLAG_MAP, PlanSchema, PluginConfigSchema, RESUME_FLAGS, RESUME_FLAG_MAP, ResumeSchema, RUN_FLAGS, RUN_FLAG_MAP, RunSchema, StatusSchema, } from './options.js';
8
+ import { planPayload, statusPayload, cancelPayload, doctorPayload } from './payload.js';
6
9
  import { buildBackfillPlan } from './planner.js';
7
- import { cancelBackfillRun, evaluateBackfillCheck, executeBackfillRun, getBackfillDoctorReport, getBackfillStatus, resumeBackfillRun, } from './runtime.js';
10
+ import { evaluateBackfillCheck } from './check.js';
11
+ import { cancelBackfillRun, getBackfillDoctorReport, getBackfillStatus } from './queries.js';
12
+ import { backfillPaths, ensureEnvironmentMatch, nowIso, readPlan, readRun, summarizeRunStatus, writeJson, } from './state.js';
13
+ function formatBytes(bytes) {
14
+ if (bytes >= 1024 ** 4)
15
+ return `${(bytes / 1024 ** 4).toFixed(1)} TiB`;
16
+ if (bytes >= 1024 ** 3)
17
+ return `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
18
+ if (bytes >= 1024 ** 2)
19
+ return `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
20
+ if (bytes >= 1024)
21
+ return `${(bytes / 1024).toFixed(1)} KiB`;
22
+ return `${bytes} B`;
23
+ }
24
+ async function runBackfill(input) {
25
+ const { plan, stateDir } = await readPlan({
26
+ planId: input.planId,
27
+ configPath: input.configPath,
28
+ config: input.config,
29
+ stateDir: input.stateDir,
30
+ });
31
+ ensureEnvironmentMatch({
32
+ plan,
33
+ clickhouse: input.clickhouse,
34
+ forceEnvironment: input.forceEnvironment,
35
+ });
36
+ const paths = backfillPaths(stateDir, plan.planId);
37
+ // Check for existing run state
38
+ const existingRun = await readRun(paths.runPath);
39
+ const resumeFrom = input.resumeFrom;
40
+ if (existingRun && !resumeFrom) {
41
+ // `run` command (no resumeFrom) must not silently continue an existing run.
42
+ // Users should use `backfill resume` instead.
43
+ const status = existingRun.status;
44
+ if (status === 'completed') {
45
+ throw new BackfillConfigError(`Run already completed for plan ${plan.planId}. Nothing to do.`);
46
+ }
47
+ if (status === 'cancelled') {
48
+ throw new BackfillConfigError(`Run is cancelled for plan ${plan.planId}. Create a new plan or inspect with backfill doctor.`);
49
+ }
50
+ throw new BackfillConfigError(`A run already exists for plan ${plan.planId} (status: ${status}). Use backfill resume to continue.`);
51
+ }
52
+ const db = createClickHouseExecutor(input.clickhouse);
53
+ try {
54
+ const runState = {
55
+ planId: plan.planId,
56
+ target: plan.target,
57
+ status: 'running',
58
+ startedAt: existingRun?.startedAt ?? nowIso(),
59
+ updatedAt: nowIso(),
60
+ progress: resumeFrom ?? {},
61
+ };
62
+ await writeJson(paths.runPath, runState);
63
+ const result = await executeBackfill({
64
+ executor: db,
65
+ planId: plan.planId,
66
+ chunks: plan.chunkPlan.chunks.map((chunk) => ({ id: chunk.id })),
67
+ buildQuery: (chunk) => {
68
+ const planChunk = plan.chunkPlan.chunks.find((candidate) => candidate.id === chunk.id);
69
+ if (!planChunk)
70
+ throw new Error(`Chunk ${chunk.id} not found in plan`);
71
+ return buildChunkExecutionSql({
72
+ planId: plan.planId,
73
+ chunk: planChunk,
74
+ target: plan.target,
75
+ sourceTarget: plan.execution.sourceTarget,
76
+ table: plan.chunkPlan.table,
77
+ mvAsQuery: plan.execution.mvAsQuery,
78
+ targetColumns: plan.execution.targetColumns,
79
+ idempotencyToken: plan.execution.requireIdempotencyToken
80
+ ? generateIdempotencyToken(plan.planId, planChunk.id)
81
+ : '',
82
+ });
83
+ },
84
+ concurrency: input.concurrency,
85
+ pollIntervalMs: input.pollIntervalMs,
86
+ resumeFrom,
87
+ replayFailed: input.replayFailed,
88
+ onProgress: async (progress) => {
89
+ runState.progress = progress;
90
+ runState.updatedAt = nowIso();
91
+ await writeJson(paths.runPath, runState);
92
+ },
93
+ });
94
+ runState.status = result.failed > 0 ? 'failed' : 'completed';
95
+ runState.completedAt = nowIso();
96
+ runState.updatedAt = nowIso();
97
+ runState.progress = result.progress;
98
+ if (result.failed > 0) {
99
+ const failedEntry = Object.values(result.progress).find((c) => c.status === 'failed');
100
+ runState.lastError = failedEntry?.error ?? 'One or more chunks failed';
101
+ }
102
+ await writeJson(paths.runPath, runState);
103
+ const summary = summarizeRunStatus(runState, paths.runPath, plan);
104
+ if (input.jsonMode) {
105
+ input.print({
106
+ ok: result.failed === 0,
107
+ planId: plan.planId,
108
+ status: runState.status,
109
+ chunkCounts: summary.totals,
110
+ rowsWritten: summary.rowsWritten,
111
+ runPath: paths.runPath,
112
+ lastError: runState.lastError,
113
+ });
114
+ }
115
+ else {
116
+ let line = `Backfill ${plan.planId}: ${runState.status} (done=${summary.totals.done}/${summary.totals.total}, ${summary.rowsWritten} rows written)`;
117
+ if (runState.lastError)
118
+ line += ` \u2014 ${runState.lastError}`;
119
+ input.print(line);
120
+ if (runState.status === 'completed' && summary.rowsWritten === 0) {
121
+ input.print('Warning: 0 rows written across all chunks. Verify that source data exists in the time range and passes the query\'s WHERE filters.');
122
+ }
123
+ }
124
+ return result.failed > 0 ? 1 : 0;
125
+ }
126
+ finally {
127
+ await db.close();
128
+ }
129
+ }
130
+ function isRecord(value) {
131
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
132
+ }
133
+ function withFactoryDefaults(schema, defaults) {
134
+ return {
135
+ safeParse(data) {
136
+ return schema.safeParse({ ...defaults, ...(isRecord(data) ? data : {}) });
137
+ },
138
+ };
139
+ }
8
140
  export function createBackfillPlugin(options = {}) {
9
- const base = normalizeBackfillOptions(options);
10
- validateBaseOptions(base);
141
+ const factoryOptions = options;
142
+ const planOptionsSchema = withFactoryDefaults(PlanSchema, factoryOptions);
143
+ const runOptionsSchema = withFactoryDefaults(RunSchema, factoryOptions);
144
+ const resumeOptionsSchema = withFactoryDefaults(ResumeSchema, factoryOptions);
145
+ const statusOptionsSchema = withFactoryDefaults(StatusSchema, factoryOptions);
11
146
  return {
12
147
  manifest: {
13
148
  name: 'backfill',
14
149
  apiVersion: 1,
15
150
  },
151
+ optionsSchema: withFactoryDefaults(PluginConfigSchema, factoryOptions),
16
152
  commands: [
17
153
  {
18
154
  name: 'plan',
19
155
  description: 'Build a deterministic backfill plan and persist immutable plan state',
20
156
  flags: PLAN_FLAGS,
21
- async run({ flags, jsonMode, print, options: runtimeOptions, config, configPath }) {
22
- return wrapPluginRun({
23
- command: 'plan',
24
- label: 'Backfill plan',
25
- jsonMode,
26
- print,
27
- configErrorClass: BackfillConfigError,
28
- fn: async () => {
29
- const parsed = parsePlanArgs(flags);
30
- const effectiveOptions = mergeOptions(base, runtimeOptions);
31
- validateBaseOptions(effectiveOptions);
157
+ optionsSchema: planOptionsSchema,
158
+ flagMapping: PLAN_FLAG_MAP,
159
+ run: async (context) => wrapPluginRun({
160
+ command: 'plan',
161
+ label: 'Backfill plan',
162
+ jsonMode: context.jsonMode,
163
+ print: context.print,
164
+ configErrorClass: BackfillConfigError,
165
+ fn: async () => {
166
+ const opts = context.options;
167
+ if (!context.config.clickhouse) {
168
+ throw new BackfillConfigError('ClickHouse connection is required for backfill planning. Configure clickhouse in your clickhouse.config.ts.');
169
+ }
170
+ const db = createClickHouseExecutor(context.config.clickhouse);
171
+ try {
32
172
  const output = await buildBackfillPlan({
33
- target: parsed.target,
34
- from: parsed.from,
35
- to: parsed.to,
36
- config,
37
- configPath,
38
- options: effectiveOptions,
39
- chunkHours: parsed.chunkHours,
40
- forceLargeWindow: parsed.forceLargeWindow,
173
+ opts,
174
+ configPath: context.configPath,
175
+ config: context.config,
176
+ clickhouse: context.config.clickhouse,
177
+ clickhouseQuery: async (sql, settings) => {
178
+ const result = await db.query(sql, settings);
179
+ return result;
180
+ },
181
+ // ObsessionDB enables parallel replicas by default, which inflates
182
+ // aggregate results (count, GROUP BY). Disable for planning queries
183
+ // until ObsessionDB handles it at the profile level.
184
+ querySettings: { enable_parallel_replicas: 0 },
41
185
  });
42
186
  const payload = planPayload(output);
43
- if (jsonMode) {
44
- print(payload);
187
+ if (context.jsonMode) {
188
+ context.print(payload);
45
189
  }
46
190
  else {
47
- print(`Backfill plan ${payload.planId} for ${payload.target} (${payload.chunkCount} chunks at ${payload.chunkHours}h) -> ${payload.planPath}${payload.existed ? ' [existing]' : ''}`);
191
+ const partitionCount = output.plan.chunkPlan.partitions.length;
192
+ const totalBytes = formatBytes(output.plan.chunkPlan.totalBytesCompressed);
193
+ const primarySortKey = output.plan.chunkPlan.table.sortKeys[0];
194
+ const sortKeyLabel = primarySortKey
195
+ ? `, sort key: ${primarySortKey.name} (${primarySortKey.category})`
196
+ : '';
197
+ context.print(`Backfill plan ${payload.planId} for ${payload.target} (${payload.chunkCount} chunks across ${partitionCount} partitions, ~${totalBytes}${sortKeyLabel}) -> ${payload.planPath}`);
48
198
  }
49
199
  return 0;
50
- },
51
- });
52
- },
200
+ }
201
+ finally {
202
+ await db.close();
203
+ }
204
+ },
205
+ }),
53
206
  },
54
207
  {
55
208
  name: 'run',
56
- description: 'Execute a planned backfill with checkpointed chunk progress',
209
+ description: 'Execute a planned backfill with async query submission and polling',
57
210
  flags: RUN_FLAGS,
58
- async run({ flags, jsonMode, print, options: runtimeOptions, config, configPath }) {
59
- return wrapPluginRun({
60
- command: 'run',
61
- label: 'Backfill run',
62
- jsonMode,
63
- print,
64
- configErrorClass: BackfillConfigError,
65
- fn: async () => {
66
- const parsed = parseRunArgs(flags);
67
- const effectiveOptions = mergeOptions(base, runtimeOptions);
68
- validateBaseOptions(effectiveOptions);
69
- const output = await executeBackfillRun({
70
- planId: parsed.planId,
71
- config,
72
- configPath,
73
- options: effectiveOptions,
74
- execution: {
75
- replayDone: parsed.replayDone,
76
- replayFailed: parsed.replayFailed,
77
- forceOverlap: parsed.forceOverlap,
78
- forceCompatibility: parsed.forceCompatibility,
79
- simulation: {
80
- failChunkId: parsed.simulateFailChunk,
81
- failCount: parsed.simulateFailCount,
82
- },
83
- },
84
- });
85
- const payload = {
86
- ...runPayload(output),
87
- command: 'run',
88
- };
89
- if (jsonMode) {
90
- print(payload);
91
- }
92
- else {
93
- print(`Backfill run ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
94
- }
95
- return payload.ok ? 0 : 1;
96
- },
97
- });
98
- },
211
+ optionsSchema: runOptionsSchema,
212
+ flagMapping: RUN_FLAG_MAP,
213
+ run: async (context) => wrapPluginRun({
214
+ command: 'run',
215
+ label: 'Backfill run',
216
+ jsonMode: context.jsonMode,
217
+ print: context.print,
218
+ configErrorClass: BackfillConfigError,
219
+ fn: async () => {
220
+ const opts = context.options;
221
+ if (!context.config.clickhouse) {
222
+ throw new BackfillConfigError('ClickHouse connection is required for backfill execution. Configure clickhouse in your clickhouse.config.ts.');
223
+ }
224
+ return runBackfill({
225
+ planId: opts.planId,
226
+ forceEnvironment: opts.forceEnvironment,
227
+ concurrency: opts.concurrency,
228
+ pollIntervalMs: opts.pollIntervalMs,
229
+ stateDir: opts.stateDir,
230
+ configPath: context.configPath,
231
+ config: context.config,
232
+ clickhouse: context.config.clickhouse,
233
+ print: context.print,
234
+ jsonMode: context.jsonMode,
235
+ });
236
+ },
237
+ }),
99
238
  },
100
239
  {
101
240
  name: 'resume',
102
241
  description: 'Resume a backfill run from last checkpoint',
103
242
  flags: RESUME_FLAGS,
104
- async run({ flags, jsonMode, print, options: runtimeOptions, config, configPath }) {
105
- return wrapPluginRun({
106
- command: 'resume',
107
- label: 'Backfill resume',
108
- jsonMode,
109
- print,
110
- configErrorClass: BackfillConfigError,
111
- fn: async () => {
112
- const parsed = parseResumeArgs(flags);
113
- const effectiveOptions = mergeOptions(base, runtimeOptions);
114
- validateBaseOptions(effectiveOptions);
115
- const output = await resumeBackfillRun({
116
- planId: parsed.planId,
117
- config,
118
- configPath,
119
- options: effectiveOptions,
120
- execution: {
121
- replayDone: parsed.replayDone,
122
- replayFailed: parsed.replayFailed,
123
- forceOverlap: parsed.forceOverlap,
124
- forceCompatibility: parsed.forceCompatibility,
125
- },
126
- });
127
- const payload = {
128
- ...runPayload(output),
129
- command: 'resume',
130
- };
131
- if (jsonMode) {
132
- print(payload);
243
+ optionsSchema: resumeOptionsSchema,
244
+ flagMapping: RESUME_FLAG_MAP,
245
+ run: async (context) => wrapPluginRun({
246
+ command: 'resume',
247
+ label: 'Backfill resume',
248
+ jsonMode: context.jsonMode,
249
+ print: context.print,
250
+ configErrorClass: BackfillConfigError,
251
+ fn: async () => {
252
+ const opts = context.options;
253
+ if (!context.config.clickhouse) {
254
+ throw new BackfillConfigError('ClickHouse connection is required for backfill execution. Configure clickhouse in your clickhouse.config.ts.');
255
+ }
256
+ const { stateDir } = await readPlan({
257
+ planId: opts.planId,
258
+ configPath: context.configPath,
259
+ config: context.config,
260
+ stateDir: opts.stateDir,
261
+ });
262
+ const paths = backfillPaths(stateDir, opts.planId);
263
+ const existingRun = await readRun(paths.runPath);
264
+ if (!existingRun) {
265
+ throw new BackfillConfigError(`Run state not found for plan ${opts.planId}. Start with backfill run before resume.`);
266
+ }
267
+ if (existingRun.status === 'completed') {
268
+ if (context.jsonMode) {
269
+ context.print({ ok: true, noop: true, planId: opts.planId, status: 'completed', message: 'Run already completed. Nothing to resume.' });
133
270
  }
134
271
  else {
135
- print(`Backfill resume ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
272
+ context.print(`Backfill ${opts.planId}: already completed. Nothing to resume.`);
136
273
  }
137
- return payload.ok ? 0 : 1;
138
- },
139
- });
140
- },
274
+ return 0;
275
+ }
276
+ return runBackfill({
277
+ planId: opts.planId,
278
+ forceEnvironment: opts.forceEnvironment,
279
+ concurrency: opts.concurrency,
280
+ pollIntervalMs: opts.pollIntervalMs,
281
+ stateDir: opts.stateDir,
282
+ resumeFrom: existingRun.progress,
283
+ replayFailed: opts.replayFailed,
284
+ configPath: context.configPath,
285
+ config: context.config,
286
+ clickhouse: context.config.clickhouse,
287
+ print: context.print,
288
+ jsonMode: context.jsonMode,
289
+ });
290
+ },
291
+ }),
141
292
  },
142
293
  {
143
294
  name: 'status',
144
295
  description: 'Show checkpoint and chunk progress for a backfill run',
145
296
  flags: PLAN_ID_FLAGS,
146
- async run({ flags, jsonMode, print, options: runtimeOptions, config, configPath }) {
147
- return wrapPluginRun({
148
- command: 'status',
149
- label: 'Backfill status',
150
- jsonMode,
151
- print,
152
- configErrorClass: BackfillConfigError,
153
- fn: async () => {
154
- const parsed = parseStatusArgs(flags);
155
- const effectiveOptions = mergeOptions(base, runtimeOptions);
156
- validateBaseOptions(effectiveOptions);
157
- const summary = await getBackfillStatus({
158
- planId: parsed.planId,
159
- config,
160
- configPath,
161
- options: effectiveOptions,
162
- });
163
- const payload = statusPayload(summary);
164
- if (jsonMode) {
165
- print(payload);
166
- }
167
- else {
168
- print(`Backfill status ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total}, failed=${payload.chunkCounts.failed})`);
297
+ optionsSchema: statusOptionsSchema,
298
+ flagMapping: PLAN_ID_FLAG_MAP,
299
+ run: async (context) => wrapPluginRun({
300
+ command: 'status',
301
+ label: 'Backfill status',
302
+ jsonMode: context.jsonMode,
303
+ print: context.print,
304
+ configErrorClass: BackfillConfigError,
305
+ fn: async () => {
306
+ const opts = context.options;
307
+ const summary = await getBackfillStatus({
308
+ planId: opts.planId,
309
+ config: context.config,
310
+ configPath: context.configPath,
311
+ stateDir: opts.stateDir,
312
+ });
313
+ const payload = statusPayload(summary);
314
+ if (context.jsonMode) {
315
+ context.print(payload);
316
+ }
317
+ else {
318
+ let line = `Backfill status ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total}, failed=${payload.chunkCounts.failed}, ${payload.rowsWritten} rows written)`;
319
+ if (payload.lastError)
320
+ line += ` \u2014 ${payload.lastError}`;
321
+ context.print(line);
322
+ if (payload.status === 'completed' && payload.rowsWritten === 0) {
323
+ context.print('Warning: 0 rows written across all chunks. Verify that source data exists in the time range and passes the query\'s WHERE filters.');
169
324
  }
170
- return payload.ok ? 0 : 1;
171
- },
172
- });
173
- },
325
+ }
326
+ return payload.ok ? 0 : 1;
327
+ },
328
+ }),
174
329
  },
175
330
  {
176
331
  name: 'cancel',
177
332
  description: 'Cancel an in-progress backfill run and prevent further chunk execution',
178
333
  flags: PLAN_ID_FLAGS,
179
- async run({ flags, jsonMode, print, options: runtimeOptions, config, configPath }) {
180
- return wrapPluginRun({
181
- command: 'cancel',
182
- label: 'Backfill cancel',
183
- jsonMode,
184
- print,
185
- configErrorClass: BackfillConfigError,
186
- fn: async () => {
187
- const parsed = parseCancelArgs(flags);
188
- const effectiveOptions = mergeOptions(base, runtimeOptions);
189
- validateBaseOptions(effectiveOptions);
190
- const summary = await cancelBackfillRun({
191
- planId: parsed.planId,
192
- config,
193
- configPath,
194
- options: effectiveOptions,
195
- });
196
- const payload = cancelPayload(summary);
197
- if (jsonMode) {
198
- print(payload);
199
- }
200
- else {
201
- print(`Backfill cancel ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
202
- }
203
- return payload.ok ? 0 : 1;
204
- },
205
- });
206
- },
334
+ optionsSchema: statusOptionsSchema,
335
+ flagMapping: PLAN_ID_FLAG_MAP,
336
+ run: async (context) => wrapPluginRun({
337
+ command: 'cancel',
338
+ label: 'Backfill cancel',
339
+ jsonMode: context.jsonMode,
340
+ print: context.print,
341
+ configErrorClass: BackfillConfigError,
342
+ fn: async () => {
343
+ const opts = context.options;
344
+ const summary = await cancelBackfillRun({
345
+ planId: opts.planId,
346
+ config: context.config,
347
+ configPath: context.configPath,
348
+ stateDir: opts.stateDir,
349
+ });
350
+ const payload = cancelPayload(summary);
351
+ if (context.jsonMode) {
352
+ context.print(payload);
353
+ }
354
+ else {
355
+ context.print(`Backfill cancel ${payload.planId}: ${payload.status} (done=${payload.chunkCounts.done}/${payload.chunkCounts.total})`);
356
+ }
357
+ return payload.ok ? 0 : 1;
358
+ },
359
+ }),
207
360
  },
208
361
  {
209
362
  name: 'doctor',
210
363
  description: 'Provide actionable remediation steps for failed or pending backfill runs',
211
364
  flags: PLAN_ID_FLAGS,
212
- async run({ flags, jsonMode, print, options: runtimeOptions, config, configPath }) {
213
- return wrapPluginRun({
214
- command: 'doctor',
215
- label: 'Backfill doctor',
216
- jsonMode,
217
- print,
218
- configErrorClass: BackfillConfigError,
219
- fn: async () => {
220
- const parsed = parseDoctorArgs(flags);
221
- const effectiveOptions = mergeOptions(base, runtimeOptions);
222
- validateBaseOptions(effectiveOptions);
223
- const report = await getBackfillDoctorReport({
224
- planId: parsed.planId,
225
- config,
226
- configPath,
227
- options: effectiveOptions,
228
- });
229
- const payload = doctorPayload(report);
230
- if (jsonMode) {
231
- print(payload);
232
- }
233
- else {
234
- print(`Backfill doctor ${payload.planId}: ${payload.issueCodes.length === 0 ? 'ok' : payload.issueCodes.join(', ')}`);
235
- for (const recommendation of payload.recommendations) {
236
- print(`- ${recommendation}`);
237
- }
365
+ optionsSchema: statusOptionsSchema,
366
+ flagMapping: PLAN_ID_FLAG_MAP,
367
+ run: async (context) => wrapPluginRun({
368
+ command: 'doctor',
369
+ label: 'Backfill doctor',
370
+ jsonMode: context.jsonMode,
371
+ print: context.print,
372
+ configErrorClass: BackfillConfigError,
373
+ fn: async () => {
374
+ const opts = context.options;
375
+ const report = await getBackfillDoctorReport({
376
+ planId: opts.planId,
377
+ config: context.config,
378
+ configPath: context.configPath,
379
+ stateDir: opts.stateDir,
380
+ });
381
+ const payload = doctorPayload(report);
382
+ if (context.jsonMode) {
383
+ context.print(payload);
384
+ }
385
+ else {
386
+ context.print(`Backfill doctor ${payload.planId}: ${payload.issueCodes.length === 0 ? 'ok' : payload.issueCodes.join(', ')}`);
387
+ for (const recommendation of payload.recommendations) {
388
+ context.print(`- ${recommendation}`);
238
389
  }
239
- return payload.ok ? 0 : 1;
240
- },
241
- });
242
- },
390
+ }
391
+ return payload.ok ? 0 : 1;
392
+ },
393
+ }),
243
394
  },
244
395
  ],
245
396
  hooks: {
246
- onConfigLoaded({ options: runtimeOptions }) {
247
- const merged = mergeOptions(base, runtimeOptions);
248
- validateBaseOptions(merged);
397
+ onConfigLoaded() {
398
+ // Plugin-level options validated softly at load time via PluginConfigSchema.
249
399
  },
250
- async onCheck({ config, configPath, options: runtimeOptions }) {
251
- const effectiveOptions = mergeOptions(base, runtimeOptions);
252
- validateBaseOptions(effectiveOptions);
400
+ async onCheck({ config: appConfig, configPath, options }) {
401
+ const opts = CheckSchema.parse(options);
253
402
  return evaluateBackfillCheck({
254
403
  configPath,
255
- config,
256
- options: effectiveOptions,
404
+ config: appConfig,
405
+ stateDir: opts.stateDir,
406
+ failCheckOnRequiredPendingBackfill: opts.failCheckOnRequiredPendingBackfill,
257
407
  });
258
408
  },
259
409
  onCheckReport({ result, print }) {
@@ -269,7 +419,7 @@ export function createBackfillPlugin(options = {}) {
269
419
  }
270
420
  export function backfill(options = {}) {
271
421
  return {
272
- plugin: createBackfillPlugin(),
422
+ plugin: createBackfillPlugin(options),
273
423
  name: 'backfill',
274
424
  enabled: true,
275
425
  options,