@chkit/plugin-backfill 0.1.0-beta.2 → 0.1.0-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/README.md +170 -0
  2. package/dist/args.d.ts +109 -6
  3. package/dist/args.d.ts.map +1 -1
  4. package/dist/args.js +73 -97
  5. package/dist/args.js.map +1 -1
  6. package/dist/async-backfill.d.ts +64 -0
  7. package/dist/async-backfill.d.ts.map +1 -0
  8. package/dist/async-backfill.js +251 -0
  9. package/dist/async-backfill.js.map +1 -0
  10. package/dist/check.d.ts +9 -0
  11. package/dist/check.d.ts.map +1 -0
  12. package/dist/check.js +79 -0
  13. package/dist/check.js.map +1 -0
  14. package/dist/chunking/analyze.d.ts +8 -0
  15. package/dist/chunking/analyze.d.ts.map +1 -0
  16. package/dist/chunking/analyze.js +8 -0
  17. package/dist/chunking/analyze.js.map +1 -0
  18. package/dist/chunking/boundary-codec.d.ts +10 -0
  19. package/dist/chunking/boundary-codec.d.ts.map +1 -0
  20. package/dist/chunking/boundary-codec.js +79 -0
  21. package/dist/chunking/boundary-codec.js.map +1 -0
  22. package/dist/chunking/build.d.ts +11 -0
  23. package/dist/chunking/build.d.ts.map +1 -0
  24. package/dist/chunking/build.js +51 -0
  25. package/dist/chunking/build.js.map +1 -0
  26. package/dist/chunking/e2e/constants.d.ts +2 -0
  27. package/dist/chunking/e2e/constants.d.ts.map +1 -0
  28. package/dist/chunking/e2e/constants.js +2 -0
  29. package/dist/chunking/e2e/constants.js.map +1 -0
  30. package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
  31. package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
  32. package/dist/chunking/e2e/seed-datasets.script.js +134 -0
  33. package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
  34. package/dist/chunking/introspect.d.ts +40 -0
  35. package/dist/chunking/introspect.d.ts.map +1 -0
  36. package/dist/chunking/introspect.js +187 -0
  37. package/dist/chunking/introspect.js.map +1 -0
  38. package/dist/chunking/partition-slices.d.ts +14 -0
  39. package/dist/chunking/partition-slices.d.ts.map +1 -0
  40. package/dist/chunking/partition-slices.js +111 -0
  41. package/dist/chunking/partition-slices.js.map +1 -0
  42. package/dist/chunking/planner.d.ts +3 -0
  43. package/dist/chunking/planner.d.ts.map +1 -0
  44. package/dist/chunking/planner.js +343 -0
  45. package/dist/chunking/planner.js.map +1 -0
  46. package/dist/chunking/services/distribution-source.d.ts +11 -0
  47. package/dist/chunking/services/distribution-source.d.ts.map +1 -0
  48. package/dist/chunking/services/distribution-source.js +60 -0
  49. package/dist/chunking/services/distribution-source.js.map +1 -0
  50. package/dist/chunking/services/metadata-source.d.ts +4 -0
  51. package/dist/chunking/services/metadata-source.d.ts.map +1 -0
  52. package/dist/chunking/services/metadata-source.js +138 -0
  53. package/dist/chunking/services/metadata-source.js.map +1 -0
  54. package/dist/chunking/services/row-probe.d.ts +14 -0
  55. package/dist/chunking/services/row-probe.d.ts.map +1 -0
  56. package/dist/chunking/services/row-probe.js +62 -0
  57. package/dist/chunking/services/row-probe.js.map +1 -0
  58. package/dist/chunking/splitter.d.ts +20 -0
  59. package/dist/chunking/splitter.d.ts.map +1 -0
  60. package/dist/chunking/splitter.js +76 -0
  61. package/dist/chunking/splitter.js.map +1 -0
  62. package/dist/chunking/sql.d.ts +20 -0
  63. package/dist/chunking/sql.d.ts.map +1 -0
  64. package/dist/chunking/sql.js +304 -0
  65. package/dist/chunking/sql.js.map +1 -0
  66. package/dist/chunking/strategies/equal-width-split.d.ts +4 -0
  67. package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
  68. package/dist/chunking/strategies/equal-width-split.js +46 -0
  69. package/dist/chunking/strategies/equal-width-split.js.map +1 -0
  70. package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
  71. package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
  72. package/dist/chunking/strategies/group-by-key-split.js +54 -0
  73. package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
  74. package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
  75. package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
  76. package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
  77. package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
  78. package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
  79. package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
  80. package/dist/chunking/strategies/quantile-range-split.js +132 -0
  81. package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
  82. package/dist/chunking/strategies/refinement.d.ts +4 -0
  83. package/dist/chunking/strategies/refinement.d.ts.map +1 -0
  84. package/dist/chunking/strategies/refinement.js +61 -0
  85. package/dist/chunking/strategies/refinement.js.map +1 -0
  86. package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
  87. package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
  88. package/dist/chunking/strategies/string-prefix-split.js +73 -0
  89. package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
  90. package/dist/chunking/strategies/temporal-bucket-split.d.ts +4 -0
  91. package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
  92. package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
  93. package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
  94. package/dist/chunking/strategy-policy.d.ts +3 -0
  95. package/dist/chunking/strategy-policy.d.ts.map +1 -0
  96. package/dist/chunking/strategy-policy.js +4 -0
  97. package/dist/chunking/strategy-policy.js.map +1 -0
  98. package/dist/chunking/types.d.ts +139 -0
  99. package/dist/chunking/types.d.ts.map +1 -0
  100. package/dist/chunking/types.js +2 -0
  101. package/dist/chunking/types.js.map +1 -0
  102. package/dist/chunking/utils/binary-string.d.ts +8 -0
  103. package/dist/chunking/utils/binary-string.d.ts.map +1 -0
  104. package/dist/chunking/utils/binary-string.js +52 -0
  105. package/dist/chunking/utils/binary-string.js.map +1 -0
  106. package/dist/chunking/utils/ids.d.ts +4 -0
  107. package/dist/chunking/utils/ids.d.ts.map +1 -0
  108. package/dist/chunking/utils/ids.js +11 -0
  109. package/dist/chunking/utils/ids.js.map +1 -0
  110. package/dist/chunking/utils/ranges.d.ts +5 -0
  111. package/dist/chunking/utils/ranges.d.ts.map +1 -0
  112. package/dist/chunking/utils/ranges.js +19 -0
  113. package/dist/chunking/utils/ranges.js.map +1 -0
  114. package/dist/detect.d.ts +13 -0
  115. package/dist/detect.d.ts.map +1 -0
  116. package/dist/detect.js +113 -0
  117. package/dist/detect.js.map +1 -0
  118. package/dist/index.d.ts +3 -0
  119. package/dist/index.d.ts.map +1 -1
  120. package/dist/index.js +1 -0
  121. package/dist/index.js.map +1 -1
  122. package/dist/logging.d.ts +12 -0
  123. package/dist/logging.d.ts.map +1 -0
  124. package/dist/logging.js +61 -0
  125. package/dist/logging.js.map +1 -0
  126. package/dist/options.d.ts +151 -4
  127. package/dist/options.d.ts.map +1 -1
  128. package/dist/options.js +161 -109
  129. package/dist/options.js.map +1 -1
  130. package/dist/payload.d.ts +7 -17
  131. package/dist/payload.d.ts.map +1 -1
  132. package/dist/payload.js +7 -19
  133. package/dist/payload.js.map +1 -1
  134. package/dist/planner.d.ts +10 -8
  135. package/dist/planner.d.ts.map +1 -1
  136. package/dist/planner.js +76 -97
  137. package/dist/planner.js.map +1 -1
  138. package/dist/plugin.d.ts +4 -3
  139. package/dist/plugin.d.ts.map +1 -1
  140. package/dist/plugin.js +311 -215
  141. package/dist/plugin.js.map +1 -1
  142. package/dist/queries.d.ts +21 -0
  143. package/dist/queries.d.ts.map +1 -0
  144. package/dist/queries.js +113 -0
  145. package/dist/queries.js.map +1 -0
  146. package/dist/runtime.d.ts +14 -0
  147. package/dist/runtime.d.ts.map +1 -1
  148. package/dist/runtime.js +162 -83
  149. package/dist/runtime.js.map +1 -1
  150. package/dist/sdk.d.ts +12 -0
  151. package/dist/sdk.d.ts.map +1 -0
  152. package/dist/sdk.js +9 -0
  153. package/dist/sdk.js.map +1 -0
  154. package/dist/state.d.ts +16 -28
  155. package/dist/state.d.ts.map +1 -1
  156. package/dist/state.js +73 -127
  157. package/dist/state.js.map +1 -1
  158. package/dist/table-config.d.ts +9 -0
  159. package/dist/table-config.d.ts.map +1 -0
  160. package/dist/table-config.js +2 -0
  161. package/dist/table-config.js.map +1 -0
  162. package/dist/types.d.ts +49 -114
  163. package/dist/types.d.ts.map +1 -1
  164. package/package.json +31 -2
package/README.md ADDED
@@ -0,0 +1,170 @@
1
+ # @chkit/plugin-backfill
2
+
3
+ Plugin for data backfill operations in chkit.
4
+
5
+ Part of the [chkit](https://github.com/obsessiondb/chkit) monorepo. This plugin extends the [`chkit`](https://www.npmjs.com/package/chkit) CLI with data backfill commands.
6
+
7
+ ## Install
8
+
9
+ ```bash
10
+ bun add -d @chkit/plugin-backfill
11
+ ```
12
+
13
+ ## Usage
14
+
15
+ Register the plugin in your config:
16
+
17
+ ```ts
18
+ // clickhouse.config.ts
19
+ import { defineConfig } from '@chkit/core'
20
+ import { backfill } from '@chkit/plugin-backfill'
21
+
22
+ export default defineConfig({
23
+ schema: './src/db/schema/**/*.ts',
24
+ outDir: './chkit',
25
+ plugins: [
26
+ backfill(),
27
+ ],
28
+ clickhouse: {
29
+ url: process.env.CLICKHOUSE_URL ?? 'http://localhost:8123',
30
+ },
31
+ })
32
+ ```
33
+
34
+ ## Documentation
35
+
36
+ See the [chkit documentation](https://chkit.obsessiondb.com).
37
+
38
+ ## SDK
39
+
40
+ The package root is limited to the plugin registration API. Everything used by the CLI itself — the chunk planner, SQL builders, async executor, logging — is also exported from the `@chkit/plugin-backfill/sdk` subpath so you can build your own backfill scripts without going through the CLI.
41
+
42
+ ```ts
43
+ import {
44
+ generateChunkPlan,
45
+ buildChunkExecutionSql,
46
+ executeBackfill,
47
+ getBackfillLogger,
48
+ type ChunkPlan,
49
+ type PlannerQuery,
50
+ } from '@chkit/plugin-backfill/sdk'
51
+ ```
52
+
53
+ The pipeline has three stages, and you can use any subset:
54
+
55
+ 1. **Plan** — `generateChunkPlan(...)` introspects a table and returns a `ChunkPlan` describing how to partition the work into roughly equal-sized chunks.
56
+ 2. **Build SQL** — `buildChunkExecutionSql(...)` turns a single `Chunk` into an `INSERT … SELECT` statement.
57
+ 3. **Execute** — `executeBackfill(...)` submits chunks against a real `ClickHouseExecutor` with deterministic query IDs, polling, and resume support.
58
+
59
+ ### Plan a backfill
60
+
61
+ `generateChunkPlan` is decoupled from any ClickHouse client. You pass in a `query` function with the `PlannerQuery` shape and the planner uses it for every introspection / probe / split query. This makes the planner trivial to instrument or run against alternative clients.
62
+
63
+ ```ts
64
+ import { createClient } from '@clickhouse/client'
65
+ import { generateChunkPlan, type PlannerQuery } from '@chkit/plugin-backfill/sdk'
66
+
67
+ const client = createClient({ url: process.env.CLICKHOUSE_URL })
68
+
69
+ const query: PlannerQuery = async (sql, settings) => {
70
+ const result = await client.query({
71
+ query: sql,
72
+ format: 'JSONEachRow',
73
+ clickhouse_settings: settings as Record<string, string | number | boolean>,
74
+ })
75
+ return result.json()
76
+ }
77
+
78
+ const plan = await generateChunkPlan({
79
+ database: 'analytics',
80
+ table: 'events',
81
+ from: '2025-01-01T00:00:00Z',
82
+ to: '2025-02-01T00:00:00Z',
83
+ targetChunkBytes: 1_000_000_000, // ~1 GiB per chunk
84
+ query,
85
+ // 'count' is exact but slower; 'explain-estimate' is faster but approximate
86
+ rowProbeStrategy: 'count',
87
+ })
88
+
89
+ console.log(`${plan.chunks.length} chunks, ${plan.totalRows.toLocaleString()} rows`)
90
+ ```
91
+
92
+ ### Execute chunks against a target
93
+
94
+ `buildChunkExecutionSql` produces the per-chunk `INSERT … SELECT` and `executeBackfill` runs them with concurrency, polling, and progress callbacks. Persist the `progress` argument anywhere you like to support resume.
95
+
96
+ ```ts
97
+ import { createClickHouseExecutor } from '@chkit/clickhouse'
98
+ import {
99
+ buildChunkExecutionSql,
100
+ executeBackfill,
101
+ type BackfillProgress,
102
+ } from '@chkit/plugin-backfill/sdk'
103
+
104
+ const executor = createClickHouseExecutor({
105
+ url: process.env.CLICKHOUSE_URL!,
106
+ username: 'default',
107
+ password: process.env.CLICKHOUSE_PASSWORD!,
108
+ database: 'analytics',
109
+ })
110
+
111
+ const chunksById = new Map(plan.chunks.map((chunk) => [chunk.id, chunk]))
112
+ let saved: BackfillProgress | undefined // load from disk for resume
113
+
114
+ const result = await executeBackfill({
115
+ executor,
116
+ planId: plan.planId,
117
+ chunks: plan.chunks,
118
+ buildQuery: ({ id }) =>
119
+ buildChunkExecutionSql({
120
+ planId: plan.planId,
121
+ chunk: chunksById.get(id)!,
122
+ target: 'analytics.events_backfill',
123
+ table: plan.table,
124
+ }),
125
+ concurrency: 4,
126
+ pollIntervalMs: 5_000,
127
+ resumeFrom: saved,
128
+ onProgress: async (progress) => {
129
+ saved = progress
130
+ // persist to disk / state store
131
+ },
132
+ })
133
+
134
+ console.log(`done=${result.completed} failed=${result.failed}`)
135
+ ```
136
+
137
+ ### Plan persistence
138
+
139
+ Plans contain string boundaries that may include non-UTF-8 bytes (the planner uses `latin1`-encoded byte ranges for string sort keys), so JSON-serializing a `ChunkPlan` directly will lose information. Use the codec helpers when you need to round-trip a plan through storage:
140
+
141
+ ```ts
142
+ import {
143
+ encodeChunkPlanForPersistence,
144
+ decodeChunkPlanFromPersistence,
145
+ } from '@chkit/plugin-backfill/sdk'
146
+
147
+ const json = JSON.stringify(encodeChunkPlanForPersistence(plan))
148
+ // later …
149
+ const plan2 = decodeChunkPlanFromPersistence(JSON.parse(json))
150
+ ```
151
+
152
+ ### Logging
153
+
154
+ The planner emits structured logs via [`@logtape/logtape`](https://logtape.org/) under the `['chkit', 'backfill']` category. Configure a sink at process start to see them — slow-query warnings (>5 s) are emitted at `warning` level, planning progress at `info`, and per-strategy decisions at `debug`.
155
+
156
+ ```ts
157
+ import { configureSync, getConsoleSink, getTextFormatter } from '@chkit/plugin-backfill/sdk'
158
+
159
+ configureSync({
160
+ sinks: { console: getConsoleSink({ formatter: getTextFormatter({ timestamp: 'time' }) }) },
161
+ loggers: [{ category: 'chkit', sinks: ['console'], lowestLevel: 'info' }],
162
+ reset: true,
163
+ })
164
+ ```
165
+
166
+ To capture every SQL statement the planner runs (with timing, server-side stats, and per-strategy classification), wrap your `query` function instead of relying solely on logging — the wrapper sees the raw SQL and settings on every call and can record query IDs, response headers, and durations alongside the structured logs.
167
+
168
+ ## License
169
+
170
+ [MIT](../../LICENSE)
package/dist/args.d.ts CHANGED
@@ -1,8 +1,111 @@
1
+ import { type ParsedFlags } from '@chkit/core';
1
2
  import type { ParsedCancelArgs, ParsedDoctorArgs, ParsedPlanArgs, ParsedResumeArgs, ParsedRunArgs, ParsedStatusArgs } from './types.js';
2
- export declare function parsePlanArgs(args: string[]): ParsedPlanArgs;
3
- export declare function parseRunArgs(args: string[]): ParsedRunArgs;
4
- export declare function parseResumeArgs(args: string[]): ParsedResumeArgs;
5
- export declare function parseStatusArgs(args: string[]): ParsedStatusArgs;
6
- export declare function parseCancelArgs(args: string[]): ParsedCancelArgs;
7
- export declare function parseDoctorArgs(args: string[]): ParsedDoctorArgs;
3
+ export declare const PLAN_FLAGS: readonly [{
4
+ readonly name: "--target";
5
+ readonly type: "string";
6
+ readonly description: "Target table (database.table)";
7
+ readonly placeholder: "<database.table>";
8
+ }, {
9
+ readonly name: "--from";
10
+ readonly type: "string";
11
+ readonly description: "Start timestamp";
12
+ readonly placeholder: "<timestamp>";
13
+ }, {
14
+ readonly name: "--to";
15
+ readonly type: "string";
16
+ readonly description: "End timestamp";
17
+ readonly placeholder: "<timestamp>";
18
+ }, {
19
+ readonly name: "--chunk-hours";
20
+ readonly type: "string";
21
+ readonly description: "Hours per chunk";
22
+ readonly placeholder: "<hours>";
23
+ }, {
24
+ readonly name: "--time-column";
25
+ readonly type: "string";
26
+ readonly description: "Time column for WHERE clause";
27
+ readonly placeholder: "<column>";
28
+ }, {
29
+ readonly name: "--force-large-window";
30
+ readonly type: "boolean";
31
+ readonly description: "Allow large time windows without confirmation";
32
+ }, {
33
+ readonly name: "--force";
34
+ readonly type: "boolean";
35
+ readonly description: "Delete existing plan and regenerate from scratch";
36
+ }];
37
+ export declare const RUN_FLAGS: readonly [{
38
+ readonly name: "--plan-id";
39
+ readonly type: "string";
40
+ readonly description: "Plan ID to execute";
41
+ readonly placeholder: "<id>";
42
+ }, {
43
+ readonly name: "--replay-done";
44
+ readonly type: "boolean";
45
+ readonly description: "Re-execute already completed chunks";
46
+ }, {
47
+ readonly name: "--replay-failed";
48
+ readonly type: "boolean";
49
+ readonly description: "Re-execute failed chunks";
50
+ }, {
51
+ readonly name: "--force-overlap";
52
+ readonly type: "boolean";
53
+ readonly description: "Allow overlapping runs";
54
+ }, {
55
+ readonly name: "--force-compatibility";
56
+ readonly type: "boolean";
57
+ readonly description: "Skip compatibility checks";
58
+ }, {
59
+ readonly name: "--force-environment";
60
+ readonly type: "boolean";
61
+ readonly description: "Skip environment mismatch checks";
62
+ }, {
63
+ readonly name: "--simulate-fail-chunk";
64
+ readonly type: "string";
65
+ readonly description: "Simulate failure on chunk";
66
+ readonly placeholder: "<chunk-id>";
67
+ }, {
68
+ readonly name: "--simulate-fail-count";
69
+ readonly type: "string";
70
+ readonly description: "Number of simulated failures";
71
+ readonly placeholder: "<count>";
72
+ }];
73
+ export declare const RESUME_FLAGS: readonly [{
74
+ readonly name: "--plan-id";
75
+ readonly type: "string";
76
+ readonly description: "Plan ID to resume";
77
+ readonly placeholder: "<id>";
78
+ }, {
79
+ readonly name: "--replay-done";
80
+ readonly type: "boolean";
81
+ readonly description: "Re-execute already completed chunks";
82
+ }, {
83
+ readonly name: "--replay-failed";
84
+ readonly type: "boolean";
85
+ readonly description: "Re-execute failed chunks";
86
+ }, {
87
+ readonly name: "--force-overlap";
88
+ readonly type: "boolean";
89
+ readonly description: "Allow overlapping runs";
90
+ }, {
91
+ readonly name: "--force-compatibility";
92
+ readonly type: "boolean";
93
+ readonly description: "Skip compatibility checks";
94
+ }, {
95
+ readonly name: "--force-environment";
96
+ readonly type: "boolean";
97
+ readonly description: "Skip environment mismatch checks";
98
+ }];
99
+ export declare const PLAN_ID_FLAGS: readonly [{
100
+ readonly name: "--plan-id";
101
+ readonly type: "string";
102
+ readonly description: "Plan ID";
103
+ readonly placeholder: "<id>";
104
+ }];
105
+ export declare function parsePlanArgs(flags: ParsedFlags): ParsedPlanArgs;
106
+ export declare function parseRunArgs(flags: ParsedFlags): ParsedRunArgs;
107
+ export declare function parseResumeArgs(flags: ParsedFlags): ParsedResumeArgs;
108
+ export declare function parseStatusArgs(flags: ParsedFlags): ParsedStatusArgs;
109
+ export declare function parseCancelArgs(flags: ParsedFlags): ParsedCancelArgs;
110
+ export declare function parseDoctorArgs(flags: ParsedFlags): ParsedDoctorArgs;
8
111
  //# sourceMappingURL=args.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EACjB,MAAM,YAAY,CAAA;AAgCnB,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,cAAc,CAgD5D;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,aAAa,CAoE1D;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAShE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAkBhE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAEhE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAEhE"}
1
+ {"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,aAAa,CAAA;AAGvE,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EACjB,MAAM,YAAY,CAAA;AAEnB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQZ,CAAA;AAEX,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASX,CAAA;AAEX,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;EAOd,CAAA;AAEX,eAAO,MAAM,aAAa;;;;;EAEf,CAAA;AAgCX,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,cAAc,CAgChE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,aAAa,CAgC9D;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAUpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAKpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAEpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAEpE"}
package/dist/args.js CHANGED
@@ -1,4 +1,35 @@
1
+ import { defineFlags, typedFlags } from '@chkit/core';
1
2
  import { BackfillConfigError } from './errors.js';
3
+ export const PLAN_FLAGS = defineFlags([
4
+ { name: '--target', type: 'string', description: 'Target table (database.table)', placeholder: '<database.table>' },
5
+ { name: '--from', type: 'string', description: 'Start timestamp', placeholder: '<timestamp>' },
6
+ { name: '--to', type: 'string', description: 'End timestamp', placeholder: '<timestamp>' },
7
+ { name: '--chunk-hours', type: 'string', description: 'Hours per chunk', placeholder: '<hours>' },
8
+ { name: '--time-column', type: 'string', description: 'Time column for WHERE clause', placeholder: '<column>' },
9
+ { name: '--force-large-window', type: 'boolean', description: 'Allow large time windows without confirmation' },
10
+ { name: '--force', type: 'boolean', description: 'Delete existing plan and regenerate from scratch' },
11
+ ]);
12
+ export const RUN_FLAGS = defineFlags([
13
+ { name: '--plan-id', type: 'string', description: 'Plan ID to execute', placeholder: '<id>' },
14
+ { name: '--replay-done', type: 'boolean', description: 'Re-execute already completed chunks' },
15
+ { name: '--replay-failed', type: 'boolean', description: 'Re-execute failed chunks' },
16
+ { name: '--force-overlap', type: 'boolean', description: 'Allow overlapping runs' },
17
+ { name: '--force-compatibility', type: 'boolean', description: 'Skip compatibility checks' },
18
+ { name: '--force-environment', type: 'boolean', description: 'Skip environment mismatch checks' },
19
+ { name: '--simulate-fail-chunk', type: 'string', description: 'Simulate failure on chunk', placeholder: '<chunk-id>' },
20
+ { name: '--simulate-fail-count', type: 'string', description: 'Number of simulated failures', placeholder: '<count>' },
21
+ ]);
22
+ export const RESUME_FLAGS = defineFlags([
23
+ { name: '--plan-id', type: 'string', description: 'Plan ID to resume', placeholder: '<id>' },
24
+ { name: '--replay-done', type: 'boolean', description: 'Re-execute already completed chunks' },
25
+ { name: '--replay-failed', type: 'boolean', description: 'Re-execute failed chunks' },
26
+ { name: '--force-overlap', type: 'boolean', description: 'Allow overlapping runs' },
27
+ { name: '--force-compatibility', type: 'boolean', description: 'Skip compatibility checks' },
28
+ { name: '--force-environment', type: 'boolean', description: 'Skip environment mismatch checks' },
29
+ ]);
30
+ export const PLAN_ID_FLAGS = defineFlags([
31
+ { name: '--plan-id', type: 'string', description: 'Plan ID', placeholder: '<id>' },
32
+ ]);
2
33
  function normalizeTimestamp(raw, flagName) {
3
34
  const value = raw.trim();
4
35
  if (value.length === 0) {
@@ -24,40 +55,22 @@ function normalizePlanId(raw) {
24
55
  }
25
56
  return value;
26
57
  }
27
- export function parsePlanArgs(args) {
28
- let target;
29
- let from;
30
- let to;
58
+ export function parsePlanArgs(flags) {
59
+ const f = typedFlags(flags, PLAN_FLAGS);
60
+ const target = f['--target'];
61
+ const from = f['--from'];
62
+ const to = f['--to'];
63
+ const rawChunkHours = f['--chunk-hours'];
64
+ const timeColumn = f['--time-column'];
65
+ const forceLargeWindow = f['--force-large-window'] === true;
66
+ const force = f['--force'] === true;
31
67
  let chunkHours;
32
- let forceLargeWindow = false;
33
- for (let i = 0; i < args.length; i += 1) {
34
- const token = args[i];
35
- if (!token)
36
- continue;
37
- if (token === '--force-large-window') {
38
- forceLargeWindow = true;
39
- continue;
40
- }
41
- if (token === '--target' || token === '--from' || token === '--to' || token === '--chunk-hours') {
42
- const nextValue = args[i + 1];
43
- if (!nextValue || nextValue.startsWith('--')) {
44
- throw new BackfillConfigError(`Missing value for ${token}`);
45
- }
46
- if (token === '--target')
47
- target = nextValue;
48
- if (token === '--from')
49
- from = nextValue;
50
- if (token === '--to')
51
- to = nextValue;
52
- if (token === '--chunk-hours') {
53
- const parsed = Number(nextValue);
54
- if (!Number.isFinite(parsed) || parsed <= 0) {
55
- throw new BackfillConfigError('Invalid value for --chunk-hours. Expected a positive number.');
56
- }
57
- chunkHours = parsed;
58
- }
59
- i += 1;
68
+ if (rawChunkHours !== undefined) {
69
+ const parsed = Number(rawChunkHours);
70
+ if (!Number.isFinite(parsed) || parsed <= 0) {
71
+ throw new BackfillConfigError('Invalid value for --chunk-hours. Expected a positive number.');
60
72
  }
73
+ chunkHours = parsed;
61
74
  }
62
75
  if (!target)
63
76
  throw new BackfillConfigError('Missing required --target <database.table>');
@@ -70,57 +83,28 @@ export function parsePlanArgs(args) {
70
83
  from: normalizeTimestamp(from, '--from'),
71
84
  to: normalizeTimestamp(to, '--to'),
72
85
  chunkHours,
86
+ timeColumn: timeColumn?.trim() || undefined,
73
87
  forceLargeWindow,
88
+ force,
74
89
  };
75
90
  }
76
- export function parseRunArgs(args) {
77
- let planId;
78
- let replayDone = false;
79
- let replayFailed = false;
80
- let forceOverlap = false;
81
- let forceCompatibility = false;
82
- let simulateFailChunk;
91
+ export function parseRunArgs(flags) {
92
+ const f = typedFlags(flags, RUN_FLAGS);
93
+ const planId = f['--plan-id'];
94
+ const replayDone = f['--replay-done'] === true;
95
+ const replayFailed = f['--replay-failed'] === true;
96
+ const forceOverlap = f['--force-overlap'] === true;
97
+ const forceCompatibility = f['--force-compatibility'] === true;
98
+ const forceEnvironment = f['--force-environment'] === true;
99
+ const simulateFailChunk = f['--simulate-fail-chunk'];
83
100
  let simulateFailCount = 1;
84
- for (let i = 0; i < args.length; i += 1) {
85
- const token = args[i];
86
- if (!token)
87
- continue;
88
- if (token === '--replay-done') {
89
- replayDone = true;
90
- continue;
91
- }
92
- if (token === '--replay-failed') {
93
- replayFailed = true;
94
- continue;
95
- }
96
- if (token === '--force-overlap') {
97
- forceOverlap = true;
98
- continue;
99
- }
100
- if (token === '--force-compatibility') {
101
- forceCompatibility = true;
102
- continue;
103
- }
104
- if (token === '--plan-id' ||
105
- token === '--simulate-fail-chunk' ||
106
- token === '--simulate-fail-count') {
107
- const nextValue = args[i + 1];
108
- if (!nextValue || nextValue.startsWith('--')) {
109
- throw new BackfillConfigError(`Missing value for ${token}`);
110
- }
111
- if (token === '--plan-id')
112
- planId = nextValue;
113
- if (token === '--simulate-fail-chunk')
114
- simulateFailChunk = nextValue;
115
- if (token === '--simulate-fail-count') {
116
- const parsed = Number(nextValue);
117
- if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
118
- throw new BackfillConfigError('Invalid value for --simulate-fail-count. Expected integer > 0.');
119
- }
120
- simulateFailCount = parsed;
121
- }
122
- i += 1;
101
+ const rawSimulateFailCount = f['--simulate-fail-count'];
102
+ if (rawSimulateFailCount !== undefined) {
103
+ const parsed = Number(rawSimulateFailCount);
104
+ if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
105
+ throw new BackfillConfigError('Invalid value for --simulate-fail-count. Expected integer > 0.');
123
106
  }
107
+ simulateFailCount = parsed;
124
108
  }
125
109
  if (!planId)
126
110
  throw new BackfillConfigError('Missing required --plan-id <id>');
@@ -130,41 +114,33 @@ export function parseRunArgs(args) {
130
114
  replayFailed,
131
115
  forceOverlap,
132
116
  forceCompatibility,
117
+ forceEnvironment,
133
118
  simulateFailChunk,
134
119
  simulateFailCount,
135
120
  };
136
121
  }
137
- export function parseResumeArgs(args) {
138
- const parsed = parseRunArgs(args);
122
+ export function parseResumeArgs(flags) {
123
+ const parsed = parseRunArgs(flags);
139
124
  return {
140
125
  planId: parsed.planId,
141
126
  replayDone: parsed.replayDone,
142
127
  replayFailed: parsed.replayFailed,
143
128
  forceOverlap: parsed.forceOverlap,
144
129
  forceCompatibility: parsed.forceCompatibility,
130
+ forceEnvironment: parsed.forceEnvironment,
145
131
  };
146
132
  }
147
- export function parseStatusArgs(args) {
148
- let planId;
149
- for (let i = 0; i < args.length; i += 1) {
150
- const token = args[i];
151
- if (token !== '--plan-id')
152
- continue;
153
- const nextValue = args[i + 1];
154
- if (!nextValue || nextValue.startsWith('--')) {
155
- throw new BackfillConfigError('Missing value for --plan-id');
156
- }
157
- planId = nextValue;
158
- i += 1;
159
- }
133
+ export function parseStatusArgs(flags) {
134
+ const f = typedFlags(flags, PLAN_ID_FLAGS);
135
+ const planId = f['--plan-id'];
160
136
  if (!planId)
161
137
  throw new BackfillConfigError('Missing required --plan-id <id>');
162
138
  return { planId: normalizePlanId(planId) };
163
139
  }
164
- export function parseCancelArgs(args) {
165
- return parseStatusArgs(args);
140
+ export function parseCancelArgs(flags) {
141
+ return parseStatusArgs(flags);
166
142
  }
167
- export function parseDoctorArgs(args) {
168
- return parseStatusArgs(args);
143
+ export function parseDoctorArgs(flags) {
144
+ return parseStatusArgs(flags);
169
145
  }
170
146
  //# sourceMappingURL=args.js.map
package/dist/args.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"args.js","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAUjD,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAgB;IACvD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,mBAAmB,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAA;IAChE,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,QAAQ,KAAK,GAAG,EAAE,CAAC,CAAA;IAC5E,CAAC;IAED,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;AAC3B,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,mBAAmB,CAAC,wDAAwD,CAAC,CAAA;IACzF,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,mBAAmB,CAAC,mEAAmE,CAAC,CAAA;IACpG,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAc;IAC1C,IAAI,MAA0B,CAAA;IAC9B,IAAI,IAAwB,CAAA;IAC5B,IAAI,EAAsB,CAAA;IAC1B,IAAI,UAA8B,CAAA;IAClC,IAAI,gBAAgB,GAAG,KAAK,CAAA;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACrB,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,IAAI,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAA;YACvB,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,eAAe,EAAE,CAAC;YAChG,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7B,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,mBAAmB,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;YAC7D,CAAC;YAED,IAAI,KAAK,KAAK,UAAU;gBAAE,MAAM,GAAG,SAAS,CAAA;YAC5C,IAAI,KAAK,KAAK,QAAQ;gBAAE,IAAI,GAAG,SAAS,CAAA;YACxC,IAAI,KAAK,KAAK,MAAM;gBAAE,EAAE,GAAG,SAAS,CAAA;YACpC,IAAI,KAAK,KAAK,eAAe,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC5C,MAAM,IAAI,mBAAmB,CAAC,8DAA8D,CAAC,CAAA;gBAC/F,CAAC;gBACD,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;YAED,CAAC,IAAI,CAAC,CAAA;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,4CAA4C,CAAC,CAAA;IACxF,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAA;IAC/E,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,mBAAmB,CAAC,mCAAmC,CAAC,CAAA;IAE3E,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxC,EAAE,EAAE,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC;QAClC,UAAU;QACV,gBAAgB;KACjB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAc;IACzC,IAAI,MAA0B,CAAA;IAC9B,IAAI,UAAU,GAAG,KAAK,CAAA;IACtB,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,IAAI,kBAAkB,GAAG,KAAK,CAAA;IAC9B,IAAI,iBAAqC,CAAA;IACzC,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACrB,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,IAAI,KAAK,KAAK,eAAe,EAAE,CAAC;YAC9B,UAAU,GAAG,IAAI,CAAA;YACjB,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,KAAK,iBAAiB,EAAE,CAAC;YAChC,YAAY,GAAG,IAAI,CAAA;YACnB,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,KAAK,iBAAiB,EAAE,CAAC;YAChC,YAAY,GAAG,IAAI,CAAA;YACnB,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,KAAK,uBAAuB,EAAE,CAAC;YACtC,kBAAkB,GAAG,IAAI,CAAA;YACzB,SAAQ;QACV,CAAC;QAED,IACE,KAAK,KAAK,WAAW;YACrB,KAAK,KAAK,uBAAuB;YACjC,KAAK,KAAK,uBAAuB,EACjC,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7B,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,mBAAmB,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;YAC7D,CAAC;YAED,IAAI,KAAK,KAAK,WAAW;gBAAE,MAAM,GAAG,SAAS,CAAA;YAC7C,IAAI,KAAK,KAAK,uBAAuB;gBAAE,iBAAiB,GAAG,SAAS,CAAA;YACpE,IAAI,KAAK,KAAK,uBAAuB,EAAE,CAAC;gBACtC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzE,MAAM,IAAI,mBAAmB,CAAC,gEAAgE,CAAC,CAAA;gBACjG,CAAC;gBACD,iBAAiB,GAAG,MAAM,CAAA;YAC5B,CAAC;YAED,CAAC,IAAI,CAAC,CAAA;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,iCAAiC,CAAC,CAAA;IAE7E,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,UAAU;QACV,YAAY;QACZ,YAAY;QACZ,kBAAkB;QAClB,iBAAiB;QACjB,iBAAiB;KAClB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAc;IAC5C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;KAC9C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAc;IAC5C,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACrB,IAAI,KAAK,KAAK,WAAW;YAAE,SAAQ;QAEnC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC7B,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,CAAC,CAAA;QAC9D,CAAC;QACD,MAAM,GAAG,SAAS,CAAA;QAClB,CAAC,IAAI,CAAC,CAAA;IACR,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,iCAAiC,CAAC,CAAA;IAE7E,OAAO,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAc;IAC5C,OAAO,eAAe,CAAC,IAAI,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAc;IAC5C,OAAO,eAAe,CAAC,IAAI,CAAC,CAAA;AAC9B,CAAC"}
1
+ {"version":3,"file":"args.js","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAoB,MAAM,aAAa,CAAA;AAEvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAUjD,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC;IACpC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,+BAA+B,EAAE,WAAW,EAAE,kBAAkB,EAAE;IACnH,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,aAAa,EAAE;IAC9F,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE;IAC1F,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE;IACjG,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE,WAAW,EAAE,UAAU,EAAE;IAC/G,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,+CAA+C,EAAE;IAC/G,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kDAAkD,EAAE;CAC7F,CAAC,CAAA;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAC;IACnC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,EAAE;IAC7F,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,qCAAqC,EAAE;IAC9F,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACrF,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACnF,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC5F,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;IACjG,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2BAA2B,EAAE,WAAW,EAAE,YAAY,EAAE;IACtH,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE,WAAW,EAAE,SAAS,EAAE;CAC9G,CAAC,CAAA;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;IACtC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,EAAE;IAC5F,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,qCAAqC,EAAE;IAC9F,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACrF,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACnF,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC5F,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;CACzF,CAAC,CAAA;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG,WAAW,CAAC;IACvC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE;CAC1E,CAAC,CAAA;AAEX,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAgB;IACvD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,mBAAmB,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAA;IAChE,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,QAAQ,KAAK,GAAG,EAAE,CAAC,CAAA;IAC5E,CAAC;IAED,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;AAC3B,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,mBAAmB,CAAC,wDAAwD,CAAC,CAAA;IACzF,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,mBAAmB,CAAC,mEAAmE,CAAC,CAAA;IACpG,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAkB;IAC9C,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAA;IAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAA;IACxB,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IACpB,MAAM,aAAa,GAAG,CAAC,CAAC,eAAe,CAAC,CAAA;IACxC,MAAM,UAAU,GAAG,CAAC,CAAC,eAAe,CAAC,CAAA;IACrC,MAAM,gBAAgB,GAAG,CAAC,CAAC,sBAAsB,CAAC,KAAK,IAAI,CAAA;IAC3D,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,IAAI,CAAA;IAEnC,IAAI,UAA8B,CAAA;IAClC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,mBAAmB,CAAC,8DAA8D,CAAC,CAAA;QAC/F,CAAC;QACD,UAAU,GAAG,MAAM,CAAA;IACrB,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,4CAA4C,CAAC,CAAA;IACxF,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAA;IAC/E,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,mBAAmB,CAAC,mCAAmC,CAAC,CAAA;IAE3E,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxC,EAAE,EAAE,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC;QAClC,UAAU;QACV,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,SAAS;QAC3C,gBAAgB;QAChB,KAAK;KACN,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkB;IAC7C,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,CAAA;IAC7B,MAAM,UAAU,GAAG,CAAC,CAAC,eAAe,CAAC,KAAK,IAAI,CAAA;IAC9C,MAAM,YAAY,GAAG,CAAC,CAAC,iBAAiB,CAAC,KAAK,IAAI,CAAA;IAClD,MAAM,YAAY,GAAG,CAAC,CAAC,iBAAiB,CAAC,KAAK,IAAI,CAAA;IAClD,MAAM,kBAAkB,GAAG,CAAC,CAAC,uBAAuB,CAAC,KAAK,IAAI,CAAA;IAC9D,MAAM,gBAAgB,GAAG,CAAC,CAAC,qBAAqB,CAAC,KAAK,IAAI,CAAA;IAC1D,MAAM,iBAAiB,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAA;IAEpD,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAA;IACvD,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAA;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,mBAAmB,CAAC,gEAAgE,CAAC,CAAA;QACjG,CAAC;QACD,iBAAiB,GAAG,MAAM,CAAA;IAC5B,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,iCAAiC,CAAC,CAAA;IAE7E,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,UAAU;QACV,YAAY;QACZ,YAAY;QACZ,kBAAkB;QAClB,gBAAgB;QAChB,iBAAiB;QACjB,iBAAiB;KAClB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;KAC1C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,CAAA;IAC7B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,iCAAiC,CAAC,CAAA;IAC7E,OAAO,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC"}
@@ -0,0 +1,64 @@
1
+ import type { ClickHouseExecutor } from '@chkit/clickhouse';
2
+ export interface BackfillOptions {
3
+ /** The executor to submit queries to (target ClickHouse) */
4
+ executor: ClickHouseExecutor;
5
+ /** Plan ID used as a namespace in deterministic query IDs */
6
+ planId: string;
7
+ /** The chunks to process (from buildChunks) */
8
+ chunks: Array<{
9
+ id: string;
10
+ from?: string;
11
+ to?: string;
12
+ [key: string]: unknown;
13
+ }>;
14
+ /** Build the SQL for a given chunk. Called once per chunk at submit time. */
15
+ buildQuery: (chunk: {
16
+ id: string;
17
+ from?: string;
18
+ to?: string;
19
+ }) => string;
20
+ /** Max concurrent queries running on the server. Default: 3 */
21
+ concurrency?: number;
22
+ /** Polling interval in ms. Default: 5000 */
23
+ pollIntervalMs?: number;
24
+ /** Max consecutive poll errors before marking a chunk as failed. Default: 10 */
25
+ maxPollErrors?: number;
26
+ /** Called whenever progress changes. Use this to persist state. */
27
+ onProgress?: (progress: BackfillProgress) => void | Promise<void>;
28
+ /** Previously saved progress to resume from. */
29
+ resumeFrom?: BackfillProgress;
30
+ /** When true, reset chunks confirmed failed (both locally and on server) back to pending. */
31
+ replayFailed?: boolean;
32
+ }
33
+ export interface BackfillChunkState {
34
+ status: 'pending' | 'submitted' | 'running' | 'done' | 'failed';
35
+ queryId?: string;
36
+ submittedAt?: string;
37
+ finishedAt?: string;
38
+ readRows?: number;
39
+ readBytes?: number;
40
+ writtenRows?: number;
41
+ writtenBytes?: number;
42
+ elapsedMs?: number;
43
+ durationMs?: number;
44
+ error?: string;
45
+ }
46
+ export type BackfillProgress = Record<string, BackfillChunkState>;
47
+ export interface BackfillResult {
48
+ total: number;
49
+ completed: number;
50
+ failed: number;
51
+ progress: BackfillProgress;
52
+ }
53
+ /**
54
+ * Reconcile local progress with server-side state.
55
+ *
56
+ * Queries system.processes and system.query_log for all chunk query IDs
57
+ * to discover queries that were submitted but whose status was never
58
+ * persisted locally (e.g. client crash between submit and state write).
59
+ */
60
+ export declare function syncProgress(executor: ClickHouseExecutor, planId: string, chunks: Array<{
61
+ id: string;
62
+ }>, progress: BackfillProgress): Promise<BackfillProgress>;
63
+ export declare function executeBackfill(options: BackfillOptions): Promise<BackfillResult>;
64
+ //# sourceMappingURL=async-backfill.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAA;AAGxE,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;IACjF,6EAA6E;IAC7E,UAAU,EAAE,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAA;IACzE,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AA2ED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAoF3B;AA2CD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAsGvF"}