@chkit/plugin-backfill 0.1.0-beta.2 → 0.1.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +170 -0
- package/dist/args.d.ts +109 -6
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +73 -97
- package/dist/args.js.map +1 -1
- package/dist/async-backfill.d.ts +64 -0
- package/dist/async-backfill.d.ts.map +1 -0
- package/dist/async-backfill.js +251 -0
- package/dist/async-backfill.js.map +1 -0
- package/dist/check.d.ts +9 -0
- package/dist/check.d.ts.map +1 -0
- package/dist/check.js +79 -0
- package/dist/check.js.map +1 -0
- package/dist/chunking/analyze.d.ts +8 -0
- package/dist/chunking/analyze.d.ts.map +1 -0
- package/dist/chunking/analyze.js +8 -0
- package/dist/chunking/analyze.js.map +1 -0
- package/dist/chunking/boundary-codec.d.ts +10 -0
- package/dist/chunking/boundary-codec.d.ts.map +1 -0
- package/dist/chunking/boundary-codec.js +79 -0
- package/dist/chunking/boundary-codec.js.map +1 -0
- package/dist/chunking/build.d.ts +11 -0
- package/dist/chunking/build.d.ts.map +1 -0
- package/dist/chunking/build.js +51 -0
- package/dist/chunking/build.js.map +1 -0
- package/dist/chunking/e2e/constants.d.ts +2 -0
- package/dist/chunking/e2e/constants.d.ts.map +1 -0
- package/dist/chunking/e2e/constants.js +2 -0
- package/dist/chunking/e2e/constants.js.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.js +134 -0
- package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
- package/dist/chunking/introspect.d.ts +40 -0
- package/dist/chunking/introspect.d.ts.map +1 -0
- package/dist/chunking/introspect.js +187 -0
- package/dist/chunking/introspect.js.map +1 -0
- package/dist/chunking/partition-slices.d.ts +14 -0
- package/dist/chunking/partition-slices.d.ts.map +1 -0
- package/dist/chunking/partition-slices.js +111 -0
- package/dist/chunking/partition-slices.js.map +1 -0
- package/dist/chunking/planner.d.ts +3 -0
- package/dist/chunking/planner.d.ts.map +1 -0
- package/dist/chunking/planner.js +343 -0
- package/dist/chunking/planner.js.map +1 -0
- package/dist/chunking/services/distribution-source.d.ts +11 -0
- package/dist/chunking/services/distribution-source.d.ts.map +1 -0
- package/dist/chunking/services/distribution-source.js +60 -0
- package/dist/chunking/services/distribution-source.js.map +1 -0
- package/dist/chunking/services/metadata-source.d.ts +4 -0
- package/dist/chunking/services/metadata-source.d.ts.map +1 -0
- package/dist/chunking/services/metadata-source.js +138 -0
- package/dist/chunking/services/metadata-source.js.map +1 -0
- package/dist/chunking/services/row-probe.d.ts +14 -0
- package/dist/chunking/services/row-probe.d.ts.map +1 -0
- package/dist/chunking/services/row-probe.js +62 -0
- package/dist/chunking/services/row-probe.js.map +1 -0
- package/dist/chunking/splitter.d.ts +20 -0
- package/dist/chunking/splitter.d.ts.map +1 -0
- package/dist/chunking/splitter.js +76 -0
- package/dist/chunking/splitter.js.map +1 -0
- package/dist/chunking/sql.d.ts +20 -0
- package/dist/chunking/sql.d.ts.map +1 -0
- package/dist/chunking/sql.js +304 -0
- package/dist/chunking/sql.js.map +1 -0
- package/dist/chunking/strategies/equal-width-split.d.ts +4 -0
- package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
- package/dist/chunking/strategies/equal-width-split.js +46 -0
- package/dist/chunking/strategies/equal-width-split.js.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.js +54 -0
- package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
- package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.js +132 -0
- package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
- package/dist/chunking/strategies/refinement.d.ts +4 -0
- package/dist/chunking/strategies/refinement.d.ts.map +1 -0
- package/dist/chunking/strategies/refinement.js +61 -0
- package/dist/chunking/strategies/refinement.js.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.js +73 -0
- package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts +4 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
- package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
- package/dist/chunking/strategy-policy.d.ts +3 -0
- package/dist/chunking/strategy-policy.d.ts.map +1 -0
- package/dist/chunking/strategy-policy.js +4 -0
- package/dist/chunking/strategy-policy.js.map +1 -0
- package/dist/chunking/types.d.ts +139 -0
- package/dist/chunking/types.d.ts.map +1 -0
- package/dist/chunking/types.js +2 -0
- package/dist/chunking/types.js.map +1 -0
- package/dist/chunking/utils/binary-string.d.ts +8 -0
- package/dist/chunking/utils/binary-string.d.ts.map +1 -0
- package/dist/chunking/utils/binary-string.js +52 -0
- package/dist/chunking/utils/binary-string.js.map +1 -0
- package/dist/chunking/utils/ids.d.ts +4 -0
- package/dist/chunking/utils/ids.d.ts.map +1 -0
- package/dist/chunking/utils/ids.js +11 -0
- package/dist/chunking/utils/ids.js.map +1 -0
- package/dist/chunking/utils/ranges.d.ts +5 -0
- package/dist/chunking/utils/ranges.d.ts.map +1 -0
- package/dist/chunking/utils/ranges.js +19 -0
- package/dist/chunking/utils/ranges.js.map +1 -0
- package/dist/detect.d.ts +13 -0
- package/dist/detect.d.ts.map +1 -0
- package/dist/detect.js +113 -0
- package/dist/detect.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/logging.d.ts +12 -0
- package/dist/logging.d.ts.map +1 -0
- package/dist/logging.js +61 -0
- package/dist/logging.js.map +1 -0
- package/dist/options.d.ts +151 -4
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +161 -109
- package/dist/options.js.map +1 -1
- package/dist/payload.d.ts +7 -17
- package/dist/payload.d.ts.map +1 -1
- package/dist/payload.js +7 -19
- package/dist/payload.js.map +1 -1
- package/dist/planner.d.ts +10 -8
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +76 -97
- package/dist/planner.js.map +1 -1
- package/dist/plugin.d.ts +4 -3
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +311 -215
- package/dist/plugin.js.map +1 -1
- package/dist/queries.d.ts +21 -0
- package/dist/queries.d.ts.map +1 -0
- package/dist/queries.js +113 -0
- package/dist/queries.js.map +1 -0
- package/dist/runtime.d.ts +14 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +162 -83
- package/dist/runtime.js.map +1 -1
- package/dist/sdk.d.ts +12 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +9 -0
- package/dist/sdk.js.map +1 -0
- package/dist/state.d.ts +16 -28
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +73 -127
- package/dist/state.js.map +1 -1
- package/dist/table-config.d.ts +9 -0
- package/dist/table-config.d.ts.map +1 -0
- package/dist/table-config.js +2 -0
- package/dist/table-config.js.map +1 -0
- package/dist/types.d.ts +49 -114
- package/dist/types.d.ts.map +1 -1
- package/package.json +31 -2
package/README.md
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# @chkit/plugin-backfill
|
|
2
|
+
|
|
3
|
+
Plugin for data backfill operations in chkit.
|
|
4
|
+
|
|
5
|
+
Part of the [chkit](https://github.com/obsessiondb/chkit) monorepo. This plugin extends the [`chkit`](https://www.npmjs.com/package/chkit) CLI with data backfill commands.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
bun add -d @chkit/plugin-backfill
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
Register the plugin in your config:
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
// clickhouse.config.ts
|
|
19
|
+
import { defineConfig } from '@chkit/core'
|
|
20
|
+
import { backfill } from '@chkit/plugin-backfill'
|
|
21
|
+
|
|
22
|
+
export default defineConfig({
|
|
23
|
+
schema: './src/db/schema/**/*.ts',
|
|
24
|
+
outDir: './chkit',
|
|
25
|
+
plugins: [
|
|
26
|
+
backfill(),
|
|
27
|
+
],
|
|
28
|
+
clickhouse: {
|
|
29
|
+
url: process.env.CLICKHOUSE_URL ?? 'http://localhost:8123',
|
|
30
|
+
},
|
|
31
|
+
})
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Documentation
|
|
35
|
+
|
|
36
|
+
See the [chkit documentation](https://chkit.obsessiondb.com).
|
|
37
|
+
|
|
38
|
+
## SDK
|
|
39
|
+
|
|
40
|
+
The package root is limited to the plugin registration API. Everything used by the CLI itself — the chunk planner, SQL builders, async executor, logging — is also exported from the `@chkit/plugin-backfill/sdk` subpath so you can build your own backfill scripts without going through the CLI.
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import {
|
|
44
|
+
generateChunkPlan,
|
|
45
|
+
buildChunkExecutionSql,
|
|
46
|
+
executeBackfill,
|
|
47
|
+
getBackfillLogger,
|
|
48
|
+
type ChunkPlan,
|
|
49
|
+
type PlannerQuery,
|
|
50
|
+
} from '@chkit/plugin-backfill/sdk'
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The pipeline has three stages, and you can use any subset:
|
|
54
|
+
|
|
55
|
+
1. **Plan** — `generateChunkPlan(...)` introspects a table and returns a `ChunkPlan` describing how to partition the work into roughly equal-sized chunks.
|
|
56
|
+
2. **Build SQL** — `buildChunkExecutionSql(...)` turns a single `Chunk` into an `INSERT … SELECT` statement.
|
|
57
|
+
3. **Execute** — `executeBackfill(...)` submits chunks against a real `ClickHouseExecutor` with deterministic query IDs, polling, and resume support.
|
|
58
|
+
|
|
59
|
+
### Plan a backfill
|
|
60
|
+
|
|
61
|
+
`generateChunkPlan` is decoupled from any ClickHouse client. You pass in a `query` function with the `PlannerQuery` shape and the planner uses it for every introspection / probe / split query. This makes the planner trivial to instrument or run against alternative clients.
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
import { createClient } from '@clickhouse/client'
|
|
65
|
+
import { generateChunkPlan, type PlannerQuery } from '@chkit/plugin-backfill/sdk'
|
|
66
|
+
|
|
67
|
+
const client = createClient({ url: process.env.CLICKHOUSE_URL })
|
|
68
|
+
|
|
69
|
+
const query: PlannerQuery = async (sql, settings) => {
|
|
70
|
+
const result = await client.query({
|
|
71
|
+
query: sql,
|
|
72
|
+
format: 'JSONEachRow',
|
|
73
|
+
clickhouse_settings: settings as Record<string, string | number | boolean>,
|
|
74
|
+
})
|
|
75
|
+
return result.json()
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const plan = await generateChunkPlan({
|
|
79
|
+
database: 'analytics',
|
|
80
|
+
table: 'events',
|
|
81
|
+
from: '2025-01-01T00:00:00Z',
|
|
82
|
+
to: '2025-02-01T00:00:00Z',
|
|
83
|
+
targetChunkBytes: 1_000_000_000, // ~1 GiB per chunk
|
|
84
|
+
query,
|
|
85
|
+
// 'count' is exact but slower; 'explain-estimate' is faster but approximate
|
|
86
|
+
rowProbeStrategy: 'count',
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
console.log(`${plan.chunks.length} chunks, ${plan.totalRows.toLocaleString()} rows`)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Execute chunks against a target
|
|
93
|
+
|
|
94
|
+
`buildChunkExecutionSql` produces the per-chunk `INSERT … SELECT` and `executeBackfill` runs them with concurrency, polling, and progress callbacks. Persist the `progress` argument anywhere you like to support resume.
|
|
95
|
+
|
|
96
|
+
```ts
|
|
97
|
+
import { createClickHouseExecutor } from '@chkit/clickhouse'
|
|
98
|
+
import {
|
|
99
|
+
buildChunkExecutionSql,
|
|
100
|
+
executeBackfill,
|
|
101
|
+
type BackfillProgress,
|
|
102
|
+
} from '@chkit/plugin-backfill/sdk'
|
|
103
|
+
|
|
104
|
+
const executor = createClickHouseExecutor({
|
|
105
|
+
url: process.env.CLICKHOUSE_URL!,
|
|
106
|
+
username: 'default',
|
|
107
|
+
password: process.env.CLICKHOUSE_PASSWORD!,
|
|
108
|
+
database: 'analytics',
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
const chunksById = new Map(plan.chunks.map((chunk) => [chunk.id, chunk]))
|
|
112
|
+
let saved: BackfillProgress | undefined // load from disk for resume
|
|
113
|
+
|
|
114
|
+
const result = await executeBackfill({
|
|
115
|
+
executor,
|
|
116
|
+
planId: plan.planId,
|
|
117
|
+
chunks: plan.chunks,
|
|
118
|
+
buildQuery: ({ id }) =>
|
|
119
|
+
buildChunkExecutionSql({
|
|
120
|
+
planId: plan.planId,
|
|
121
|
+
chunk: chunksById.get(id)!,
|
|
122
|
+
target: 'analytics.events_backfill',
|
|
123
|
+
table: plan.table,
|
|
124
|
+
}),
|
|
125
|
+
concurrency: 4,
|
|
126
|
+
pollIntervalMs: 5_000,
|
|
127
|
+
resumeFrom: saved,
|
|
128
|
+
onProgress: async (progress) => {
|
|
129
|
+
saved = progress
|
|
130
|
+
// persist to disk / state store
|
|
131
|
+
},
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
console.log(`done=${result.completed} failed=${result.failed}`)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Plan persistence
|
|
138
|
+
|
|
139
|
+
Plans contain string boundaries that may include non-UTF-8 bytes (the planner uses `latin1`-encoded byte ranges for string sort keys), so JSON-serializing a `ChunkPlan` directly will lose information. Use the codec helpers when you need to round-trip a plan through storage:
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
import {
|
|
143
|
+
encodeChunkPlanForPersistence,
|
|
144
|
+
decodeChunkPlanFromPersistence,
|
|
145
|
+
} from '@chkit/plugin-backfill/sdk'
|
|
146
|
+
|
|
147
|
+
const json = JSON.stringify(encodeChunkPlanForPersistence(plan))
|
|
148
|
+
// later …
|
|
149
|
+
const plan2 = decodeChunkPlanFromPersistence(JSON.parse(json))
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### Logging
|
|
153
|
+
|
|
154
|
+
The planner emits structured logs via [`@logtape/logtape`](https://logtape.org/) under the `['chkit', 'backfill']` category. Configure a sink at process start to see them — slow-query warnings (>5 s) are emitted at `warning` level, planning progress at `info`, and per-strategy decisions at `debug`.
|
|
155
|
+
|
|
156
|
+
```ts
|
|
157
|
+
import { configureSync, getConsoleSink, getTextFormatter } from '@chkit/plugin-backfill/sdk'
|
|
158
|
+
|
|
159
|
+
configureSync({
|
|
160
|
+
sinks: { console: getConsoleSink({ formatter: getTextFormatter({ timestamp: 'time' }) }) },
|
|
161
|
+
loggers: [{ category: 'chkit', sinks: ['console'], lowestLevel: 'info' }],
|
|
162
|
+
reset: true,
|
|
163
|
+
})
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
To capture every SQL statement the planner runs (with timing, server-side stats, and per-strategy classification), wrap your `query` function instead of relying solely on logging — the wrapper sees the raw SQL and settings on every call and can record query IDs, response headers, and durations alongside the structured logs.
|
|
167
|
+
|
|
168
|
+
## License
|
|
169
|
+
|
|
170
|
+
[MIT](../../LICENSE)
|
package/dist/args.d.ts
CHANGED
|
@@ -1,8 +1,111 @@
|
|
|
1
|
+
import { type ParsedFlags } from '@chkit/core';
|
|
1
2
|
import type { ParsedCancelArgs, ParsedDoctorArgs, ParsedPlanArgs, ParsedResumeArgs, ParsedRunArgs, ParsedStatusArgs } from './types.js';
|
|
2
|
-
export declare
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
export declare const PLAN_FLAGS: readonly [{
|
|
4
|
+
readonly name: "--target";
|
|
5
|
+
readonly type: "string";
|
|
6
|
+
readonly description: "Target table (database.table)";
|
|
7
|
+
readonly placeholder: "<database.table>";
|
|
8
|
+
}, {
|
|
9
|
+
readonly name: "--from";
|
|
10
|
+
readonly type: "string";
|
|
11
|
+
readonly description: "Start timestamp";
|
|
12
|
+
readonly placeholder: "<timestamp>";
|
|
13
|
+
}, {
|
|
14
|
+
readonly name: "--to";
|
|
15
|
+
readonly type: "string";
|
|
16
|
+
readonly description: "End timestamp";
|
|
17
|
+
readonly placeholder: "<timestamp>";
|
|
18
|
+
}, {
|
|
19
|
+
readonly name: "--chunk-hours";
|
|
20
|
+
readonly type: "string";
|
|
21
|
+
readonly description: "Hours per chunk";
|
|
22
|
+
readonly placeholder: "<hours>";
|
|
23
|
+
}, {
|
|
24
|
+
readonly name: "--time-column";
|
|
25
|
+
readonly type: "string";
|
|
26
|
+
readonly description: "Time column for WHERE clause";
|
|
27
|
+
readonly placeholder: "<column>";
|
|
28
|
+
}, {
|
|
29
|
+
readonly name: "--force-large-window";
|
|
30
|
+
readonly type: "boolean";
|
|
31
|
+
readonly description: "Allow large time windows without confirmation";
|
|
32
|
+
}, {
|
|
33
|
+
readonly name: "--force";
|
|
34
|
+
readonly type: "boolean";
|
|
35
|
+
readonly description: "Delete existing plan and regenerate from scratch";
|
|
36
|
+
}];
|
|
37
|
+
export declare const RUN_FLAGS: readonly [{
|
|
38
|
+
readonly name: "--plan-id";
|
|
39
|
+
readonly type: "string";
|
|
40
|
+
readonly description: "Plan ID to execute";
|
|
41
|
+
readonly placeholder: "<id>";
|
|
42
|
+
}, {
|
|
43
|
+
readonly name: "--replay-done";
|
|
44
|
+
readonly type: "boolean";
|
|
45
|
+
readonly description: "Re-execute already completed chunks";
|
|
46
|
+
}, {
|
|
47
|
+
readonly name: "--replay-failed";
|
|
48
|
+
readonly type: "boolean";
|
|
49
|
+
readonly description: "Re-execute failed chunks";
|
|
50
|
+
}, {
|
|
51
|
+
readonly name: "--force-overlap";
|
|
52
|
+
readonly type: "boolean";
|
|
53
|
+
readonly description: "Allow overlapping runs";
|
|
54
|
+
}, {
|
|
55
|
+
readonly name: "--force-compatibility";
|
|
56
|
+
readonly type: "boolean";
|
|
57
|
+
readonly description: "Skip compatibility checks";
|
|
58
|
+
}, {
|
|
59
|
+
readonly name: "--force-environment";
|
|
60
|
+
readonly type: "boolean";
|
|
61
|
+
readonly description: "Skip environment mismatch checks";
|
|
62
|
+
}, {
|
|
63
|
+
readonly name: "--simulate-fail-chunk";
|
|
64
|
+
readonly type: "string";
|
|
65
|
+
readonly description: "Simulate failure on chunk";
|
|
66
|
+
readonly placeholder: "<chunk-id>";
|
|
67
|
+
}, {
|
|
68
|
+
readonly name: "--simulate-fail-count";
|
|
69
|
+
readonly type: "string";
|
|
70
|
+
readonly description: "Number of simulated failures";
|
|
71
|
+
readonly placeholder: "<count>";
|
|
72
|
+
}];
|
|
73
|
+
export declare const RESUME_FLAGS: readonly [{
|
|
74
|
+
readonly name: "--plan-id";
|
|
75
|
+
readonly type: "string";
|
|
76
|
+
readonly description: "Plan ID to resume";
|
|
77
|
+
readonly placeholder: "<id>";
|
|
78
|
+
}, {
|
|
79
|
+
readonly name: "--replay-done";
|
|
80
|
+
readonly type: "boolean";
|
|
81
|
+
readonly description: "Re-execute already completed chunks";
|
|
82
|
+
}, {
|
|
83
|
+
readonly name: "--replay-failed";
|
|
84
|
+
readonly type: "boolean";
|
|
85
|
+
readonly description: "Re-execute failed chunks";
|
|
86
|
+
}, {
|
|
87
|
+
readonly name: "--force-overlap";
|
|
88
|
+
readonly type: "boolean";
|
|
89
|
+
readonly description: "Allow overlapping runs";
|
|
90
|
+
}, {
|
|
91
|
+
readonly name: "--force-compatibility";
|
|
92
|
+
readonly type: "boolean";
|
|
93
|
+
readonly description: "Skip compatibility checks";
|
|
94
|
+
}, {
|
|
95
|
+
readonly name: "--force-environment";
|
|
96
|
+
readonly type: "boolean";
|
|
97
|
+
readonly description: "Skip environment mismatch checks";
|
|
98
|
+
}];
|
|
99
|
+
export declare const PLAN_ID_FLAGS: readonly [{
|
|
100
|
+
readonly name: "--plan-id";
|
|
101
|
+
readonly type: "string";
|
|
102
|
+
readonly description: "Plan ID";
|
|
103
|
+
readonly placeholder: "<id>";
|
|
104
|
+
}];
|
|
105
|
+
export declare function parsePlanArgs(flags: ParsedFlags): ParsedPlanArgs;
|
|
106
|
+
export declare function parseRunArgs(flags: ParsedFlags): ParsedRunArgs;
|
|
107
|
+
export declare function parseResumeArgs(flags: ParsedFlags): ParsedResumeArgs;
|
|
108
|
+
export declare function parseStatusArgs(flags: ParsedFlags): ParsedStatusArgs;
|
|
109
|
+
export declare function parseCancelArgs(flags: ParsedFlags): ParsedCancelArgs;
|
|
110
|
+
export declare function parseDoctorArgs(flags: ParsedFlags): ParsedDoctorArgs;
|
|
8
111
|
//# sourceMappingURL=args.d.ts.map
|
package/dist/args.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,aAAa,CAAA;AAGvE,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EACjB,MAAM,YAAY,CAAA;AAEnB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQZ,CAAA;AAEX,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASX,CAAA;AAEX,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;EAOd,CAAA;AAEX,eAAO,MAAM,aAAa;;;;;EAEf,CAAA;AAgCX,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,cAAc,CAgChE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,aAAa,CAgC9D;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAUpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAKpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAEpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAEpE"}
|
package/dist/args.js
CHANGED
|
@@ -1,4 +1,35 @@
|
|
|
1
|
+
import { defineFlags, typedFlags } from '@chkit/core';
|
|
1
2
|
import { BackfillConfigError } from './errors.js';
|
|
3
|
+
export const PLAN_FLAGS = defineFlags([
|
|
4
|
+
{ name: '--target', type: 'string', description: 'Target table (database.table)', placeholder: '<database.table>' },
|
|
5
|
+
{ name: '--from', type: 'string', description: 'Start timestamp', placeholder: '<timestamp>' },
|
|
6
|
+
{ name: '--to', type: 'string', description: 'End timestamp', placeholder: '<timestamp>' },
|
|
7
|
+
{ name: '--chunk-hours', type: 'string', description: 'Hours per chunk', placeholder: '<hours>' },
|
|
8
|
+
{ name: '--time-column', type: 'string', description: 'Time column for WHERE clause', placeholder: '<column>' },
|
|
9
|
+
{ name: '--force-large-window', type: 'boolean', description: 'Allow large time windows without confirmation' },
|
|
10
|
+
{ name: '--force', type: 'boolean', description: 'Delete existing plan and regenerate from scratch' },
|
|
11
|
+
]);
|
|
12
|
+
export const RUN_FLAGS = defineFlags([
|
|
13
|
+
{ name: '--plan-id', type: 'string', description: 'Plan ID to execute', placeholder: '<id>' },
|
|
14
|
+
{ name: '--replay-done', type: 'boolean', description: 'Re-execute already completed chunks' },
|
|
15
|
+
{ name: '--replay-failed', type: 'boolean', description: 'Re-execute failed chunks' },
|
|
16
|
+
{ name: '--force-overlap', type: 'boolean', description: 'Allow overlapping runs' },
|
|
17
|
+
{ name: '--force-compatibility', type: 'boolean', description: 'Skip compatibility checks' },
|
|
18
|
+
{ name: '--force-environment', type: 'boolean', description: 'Skip environment mismatch checks' },
|
|
19
|
+
{ name: '--simulate-fail-chunk', type: 'string', description: 'Simulate failure on chunk', placeholder: '<chunk-id>' },
|
|
20
|
+
{ name: '--simulate-fail-count', type: 'string', description: 'Number of simulated failures', placeholder: '<count>' },
|
|
21
|
+
]);
|
|
22
|
+
export const RESUME_FLAGS = defineFlags([
|
|
23
|
+
{ name: '--plan-id', type: 'string', description: 'Plan ID to resume', placeholder: '<id>' },
|
|
24
|
+
{ name: '--replay-done', type: 'boolean', description: 'Re-execute already completed chunks' },
|
|
25
|
+
{ name: '--replay-failed', type: 'boolean', description: 'Re-execute failed chunks' },
|
|
26
|
+
{ name: '--force-overlap', type: 'boolean', description: 'Allow overlapping runs' },
|
|
27
|
+
{ name: '--force-compatibility', type: 'boolean', description: 'Skip compatibility checks' },
|
|
28
|
+
{ name: '--force-environment', type: 'boolean', description: 'Skip environment mismatch checks' },
|
|
29
|
+
]);
|
|
30
|
+
export const PLAN_ID_FLAGS = defineFlags([
|
|
31
|
+
{ name: '--plan-id', type: 'string', description: 'Plan ID', placeholder: '<id>' },
|
|
32
|
+
]);
|
|
2
33
|
function normalizeTimestamp(raw, flagName) {
|
|
3
34
|
const value = raw.trim();
|
|
4
35
|
if (value.length === 0) {
|
|
@@ -24,40 +55,22 @@ function normalizePlanId(raw) {
|
|
|
24
55
|
}
|
|
25
56
|
return value;
|
|
26
57
|
}
|
|
27
|
-
export function parsePlanArgs(
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
58
|
+
export function parsePlanArgs(flags) {
|
|
59
|
+
const f = typedFlags(flags, PLAN_FLAGS);
|
|
60
|
+
const target = f['--target'];
|
|
61
|
+
const from = f['--from'];
|
|
62
|
+
const to = f['--to'];
|
|
63
|
+
const rawChunkHours = f['--chunk-hours'];
|
|
64
|
+
const timeColumn = f['--time-column'];
|
|
65
|
+
const forceLargeWindow = f['--force-large-window'] === true;
|
|
66
|
+
const force = f['--force'] === true;
|
|
31
67
|
let chunkHours;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
continue;
|
|
37
|
-
if (token === '--force-large-window') {
|
|
38
|
-
forceLargeWindow = true;
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
if (token === '--target' || token === '--from' || token === '--to' || token === '--chunk-hours') {
|
|
42
|
-
const nextValue = args[i + 1];
|
|
43
|
-
if (!nextValue || nextValue.startsWith('--')) {
|
|
44
|
-
throw new BackfillConfigError(`Missing value for ${token}`);
|
|
45
|
-
}
|
|
46
|
-
if (token === '--target')
|
|
47
|
-
target = nextValue;
|
|
48
|
-
if (token === '--from')
|
|
49
|
-
from = nextValue;
|
|
50
|
-
if (token === '--to')
|
|
51
|
-
to = nextValue;
|
|
52
|
-
if (token === '--chunk-hours') {
|
|
53
|
-
const parsed = Number(nextValue);
|
|
54
|
-
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
55
|
-
throw new BackfillConfigError('Invalid value for --chunk-hours. Expected a positive number.');
|
|
56
|
-
}
|
|
57
|
-
chunkHours = parsed;
|
|
58
|
-
}
|
|
59
|
-
i += 1;
|
|
68
|
+
if (rawChunkHours !== undefined) {
|
|
69
|
+
const parsed = Number(rawChunkHours);
|
|
70
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
71
|
+
throw new BackfillConfigError('Invalid value for --chunk-hours. Expected a positive number.');
|
|
60
72
|
}
|
|
73
|
+
chunkHours = parsed;
|
|
61
74
|
}
|
|
62
75
|
if (!target)
|
|
63
76
|
throw new BackfillConfigError('Missing required --target <database.table>');
|
|
@@ -70,57 +83,28 @@ export function parsePlanArgs(args) {
|
|
|
70
83
|
from: normalizeTimestamp(from, '--from'),
|
|
71
84
|
to: normalizeTimestamp(to, '--to'),
|
|
72
85
|
chunkHours,
|
|
86
|
+
timeColumn: timeColumn?.trim() || undefined,
|
|
73
87
|
forceLargeWindow,
|
|
88
|
+
force,
|
|
74
89
|
};
|
|
75
90
|
}
|
|
76
|
-
export function parseRunArgs(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
91
|
+
export function parseRunArgs(flags) {
|
|
92
|
+
const f = typedFlags(flags, RUN_FLAGS);
|
|
93
|
+
const planId = f['--plan-id'];
|
|
94
|
+
const replayDone = f['--replay-done'] === true;
|
|
95
|
+
const replayFailed = f['--replay-failed'] === true;
|
|
96
|
+
const forceOverlap = f['--force-overlap'] === true;
|
|
97
|
+
const forceCompatibility = f['--force-compatibility'] === true;
|
|
98
|
+
const forceEnvironment = f['--force-environment'] === true;
|
|
99
|
+
const simulateFailChunk = f['--simulate-fail-chunk'];
|
|
83
100
|
let simulateFailCount = 1;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
replayDone = true;
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
if (token === '--replay-failed') {
|
|
93
|
-
replayFailed = true;
|
|
94
|
-
continue;
|
|
95
|
-
}
|
|
96
|
-
if (token === '--force-overlap') {
|
|
97
|
-
forceOverlap = true;
|
|
98
|
-
continue;
|
|
99
|
-
}
|
|
100
|
-
if (token === '--force-compatibility') {
|
|
101
|
-
forceCompatibility = true;
|
|
102
|
-
continue;
|
|
103
|
-
}
|
|
104
|
-
if (token === '--plan-id' ||
|
|
105
|
-
token === '--simulate-fail-chunk' ||
|
|
106
|
-
token === '--simulate-fail-count') {
|
|
107
|
-
const nextValue = args[i + 1];
|
|
108
|
-
if (!nextValue || nextValue.startsWith('--')) {
|
|
109
|
-
throw new BackfillConfigError(`Missing value for ${token}`);
|
|
110
|
-
}
|
|
111
|
-
if (token === '--plan-id')
|
|
112
|
-
planId = nextValue;
|
|
113
|
-
if (token === '--simulate-fail-chunk')
|
|
114
|
-
simulateFailChunk = nextValue;
|
|
115
|
-
if (token === '--simulate-fail-count') {
|
|
116
|
-
const parsed = Number(nextValue);
|
|
117
|
-
if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
|
|
118
|
-
throw new BackfillConfigError('Invalid value for --simulate-fail-count. Expected integer > 0.');
|
|
119
|
-
}
|
|
120
|
-
simulateFailCount = parsed;
|
|
121
|
-
}
|
|
122
|
-
i += 1;
|
|
101
|
+
const rawSimulateFailCount = f['--simulate-fail-count'];
|
|
102
|
+
if (rawSimulateFailCount !== undefined) {
|
|
103
|
+
const parsed = Number(rawSimulateFailCount);
|
|
104
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
|
|
105
|
+
throw new BackfillConfigError('Invalid value for --simulate-fail-count. Expected integer > 0.');
|
|
123
106
|
}
|
|
107
|
+
simulateFailCount = parsed;
|
|
124
108
|
}
|
|
125
109
|
if (!planId)
|
|
126
110
|
throw new BackfillConfigError('Missing required --plan-id <id>');
|
|
@@ -130,41 +114,33 @@ export function parseRunArgs(args) {
|
|
|
130
114
|
replayFailed,
|
|
131
115
|
forceOverlap,
|
|
132
116
|
forceCompatibility,
|
|
117
|
+
forceEnvironment,
|
|
133
118
|
simulateFailChunk,
|
|
134
119
|
simulateFailCount,
|
|
135
120
|
};
|
|
136
121
|
}
|
|
137
|
-
export function parseResumeArgs(
|
|
138
|
-
const parsed = parseRunArgs(
|
|
122
|
+
export function parseResumeArgs(flags) {
|
|
123
|
+
const parsed = parseRunArgs(flags);
|
|
139
124
|
return {
|
|
140
125
|
planId: parsed.planId,
|
|
141
126
|
replayDone: parsed.replayDone,
|
|
142
127
|
replayFailed: parsed.replayFailed,
|
|
143
128
|
forceOverlap: parsed.forceOverlap,
|
|
144
129
|
forceCompatibility: parsed.forceCompatibility,
|
|
130
|
+
forceEnvironment: parsed.forceEnvironment,
|
|
145
131
|
};
|
|
146
132
|
}
|
|
147
|
-
export function parseStatusArgs(
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
const token = args[i];
|
|
151
|
-
if (token !== '--plan-id')
|
|
152
|
-
continue;
|
|
153
|
-
const nextValue = args[i + 1];
|
|
154
|
-
if (!nextValue || nextValue.startsWith('--')) {
|
|
155
|
-
throw new BackfillConfigError('Missing value for --plan-id');
|
|
156
|
-
}
|
|
157
|
-
planId = nextValue;
|
|
158
|
-
i += 1;
|
|
159
|
-
}
|
|
133
|
+
export function parseStatusArgs(flags) {
|
|
134
|
+
const f = typedFlags(flags, PLAN_ID_FLAGS);
|
|
135
|
+
const planId = f['--plan-id'];
|
|
160
136
|
if (!planId)
|
|
161
137
|
throw new BackfillConfigError('Missing required --plan-id <id>');
|
|
162
138
|
return { planId: normalizePlanId(planId) };
|
|
163
139
|
}
|
|
164
|
-
export function parseCancelArgs(
|
|
165
|
-
return parseStatusArgs(
|
|
140
|
+
export function parseCancelArgs(flags) {
|
|
141
|
+
return parseStatusArgs(flags);
|
|
166
142
|
}
|
|
167
|
-
export function parseDoctorArgs(
|
|
168
|
-
return parseStatusArgs(
|
|
143
|
+
export function parseDoctorArgs(flags) {
|
|
144
|
+
return parseStatusArgs(flags);
|
|
169
145
|
}
|
|
170
146
|
//# sourceMappingURL=args.js.map
|
package/dist/args.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.js","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAUjD,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAgB;IACvD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,mBAAmB,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAA;IAChE,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,QAAQ,KAAK,GAAG,EAAE,CAAC,CAAA;IAC5E,CAAC;IAED,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;AAC3B,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,mBAAmB,CAAC,wDAAwD,CAAC,CAAA;IACzF,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,mBAAmB,CAAC,mEAAmE,CAAC,CAAA;IACpG,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,
|
|
1
|
+
{"version":3,"file":"args.js","sourceRoot":"","sources":["../src/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAoB,MAAM,aAAa,CAAA;AAEvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAUjD,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC;IACpC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,+BAA+B,EAAE,WAAW,EAAE,kBAAkB,EAAE;IACnH,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,aAAa,EAAE;IAC9F,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE;IAC1F,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE;IACjG,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE,WAAW,EAAE,UAAU,EAAE;IAC/G,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,+CAA+C,EAAE;IAC/G,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kDAAkD,EAAE;CAC7F,CAAC,CAAA;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAC;IACnC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,EAAE;IAC7F,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,qCAAqC,EAAE;IAC9F,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACrF,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACnF,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC5F,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;IACjG,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2BAA2B,EAAE,WAAW,EAAE,YAAY,EAAE;IACtH,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE,WAAW,EAAE,SAAS,EAAE;CAC9G,CAAC,CAAA;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;IACtC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,EAAE;IAC5F,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,qCAAqC,EAAE;IAC9F,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACrF,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACnF,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC5F,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;CACzF,CAAC,CAAA;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG,WAAW,CAAC;IACvC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE;CAC1E,CAAC,CAAA;AAEX,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAgB;IACvD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,mBAAmB,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAA;IAChE,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,QAAQ,KAAK,GAAG,EAAE,CAAC,CAAA;IAC5E,CAAC;IAED,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;AAC3B,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,mBAAmB,CAAC,wDAAwD,CAAC,CAAA;IACzF,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,mBAAmB,CAAC,mEAAmE,CAAC,CAAA;IACpG,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAkB;IAC9C,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAA;IAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAA;IACxB,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IACpB,MAAM,aAAa,GAAG,CAAC,CAAC,eAAe,CAAC,CAAA;IACxC,MAAM,UAAU,GAAG,CAAC,CAAC,eAAe,CAAC,CAAA;IACrC,MAAM,gBAAgB,GAAG,CAAC,CAAC,sBAAsB,CAAC,KAAK,IAAI,CAAA;IAC3D,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,IAAI,CAAA;IAEnC,IAAI,UAA8B,CAAA;IAClC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,mBAAmB,CAAC,8DAA8D,CAAC,CAAA;QAC/F,CAAC;QACD,UAAU,GAAG,MAAM,CAAA;IACrB,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,4CAA4C,CAAC,CAAA;IACxF,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,mBAAmB,CAAC,qCAAqC,CAAC,CAAA;IAC/E,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,mBAAmB,CAAC,mCAAmC,CAAC,CAAA;IAE3E,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxC,EAAE,EAAE,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC;QAClC,UAAU;QACV,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,SAAS;QAC3C,gBAAgB;QAChB,KAAK;KACN,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkB;IAC7C,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,CAAA;IAC7B,MAAM,UAAU,GAAG,CAAC,CAAC,eAAe,CAAC,KAAK,IAAI,CAAA;IAC9C,MAAM,YAAY,GAAG,CAAC,CAAC,iBAAiB,CAAC,KAAK,IAAI,CAAA;IAClD,MAAM,YAAY,GAAG,CAAC,CAAC,iBAAiB,CAAC,KAAK,IAAI,CAAA;IAClD,MAAM,kBAAkB,GAAG,CAAC,CAAC,uBAAuB,CAAC,KAAK,IAAI,CAAA;IAC9D,MAAM,gBAAgB,GAAG,CAAC,CAAC,qBAAqB,CAAC,KAAK,IAAI,CAAA;IAC1D,MAAM,iBAAiB,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAA;IAEpD,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAA;IACvD,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAA;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,mBAAmB,CAAC,gEAAgE,CAAC,CAAA;QACjG,CAAC;QACD,iBAAiB,GAAG,MAAM,CAAA;IAC5B,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,iCAAiC,CAAC,CAAA;IAE7E,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,UAAU;QACV,YAAY;QACZ,YAAY;QACZ,kBAAkB;QAClB,gBAAgB;QAChB,iBAAiB;QACjB,iBAAiB;KAClB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;KAC1C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,CAAA;IAC7B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,iCAAiC,CAAC,CAAA;IAC7E,OAAO,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { ClickHouseExecutor } from '@chkit/clickhouse';
|
|
2
|
+
export interface BackfillOptions {
|
|
3
|
+
/** The executor to submit queries to (target ClickHouse) */
|
|
4
|
+
executor: ClickHouseExecutor;
|
|
5
|
+
/** Plan ID used as a namespace in deterministic query IDs */
|
|
6
|
+
planId: string;
|
|
7
|
+
/** The chunks to process (from buildChunks) */
|
|
8
|
+
chunks: Array<{
|
|
9
|
+
id: string;
|
|
10
|
+
from?: string;
|
|
11
|
+
to?: string;
|
|
12
|
+
[key: string]: unknown;
|
|
13
|
+
}>;
|
|
14
|
+
/** Build the SQL for a given chunk. Called once per chunk at submit time. */
|
|
15
|
+
buildQuery: (chunk: {
|
|
16
|
+
id: string;
|
|
17
|
+
from?: string;
|
|
18
|
+
to?: string;
|
|
19
|
+
}) => string;
|
|
20
|
+
/** Max concurrent queries running on the server. Default: 3 */
|
|
21
|
+
concurrency?: number;
|
|
22
|
+
/** Polling interval in ms. Default: 5000 */
|
|
23
|
+
pollIntervalMs?: number;
|
|
24
|
+
/** Max consecutive poll errors before marking a chunk as failed. Default: 10 */
|
|
25
|
+
maxPollErrors?: number;
|
|
26
|
+
/** Called whenever progress changes. Use this to persist state. */
|
|
27
|
+
onProgress?: (progress: BackfillProgress) => void | Promise<void>;
|
|
28
|
+
/** Previously saved progress to resume from. */
|
|
29
|
+
resumeFrom?: BackfillProgress;
|
|
30
|
+
/** When true, reset chunks confirmed failed (both locally and on server) back to pending. */
|
|
31
|
+
replayFailed?: boolean;
|
|
32
|
+
}
|
|
33
|
+
export interface BackfillChunkState {
|
|
34
|
+
status: 'pending' | 'submitted' | 'running' | 'done' | 'failed';
|
|
35
|
+
queryId?: string;
|
|
36
|
+
submittedAt?: string;
|
|
37
|
+
finishedAt?: string;
|
|
38
|
+
readRows?: number;
|
|
39
|
+
readBytes?: number;
|
|
40
|
+
writtenRows?: number;
|
|
41
|
+
writtenBytes?: number;
|
|
42
|
+
elapsedMs?: number;
|
|
43
|
+
durationMs?: number;
|
|
44
|
+
error?: string;
|
|
45
|
+
}
|
|
46
|
+
export type BackfillProgress = Record<string, BackfillChunkState>;
|
|
47
|
+
export interface BackfillResult {
|
|
48
|
+
total: number;
|
|
49
|
+
completed: number;
|
|
50
|
+
failed: number;
|
|
51
|
+
progress: BackfillProgress;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Reconcile local progress with server-side state.
|
|
55
|
+
*
|
|
56
|
+
* Queries system.processes and system.query_log for all chunk query IDs
|
|
57
|
+
* to discover queries that were submitted but whose status was never
|
|
58
|
+
* persisted locally (e.g. client crash between submit and state write).
|
|
59
|
+
*/
|
|
60
|
+
export declare function syncProgress(executor: ClickHouseExecutor, planId: string, chunks: Array<{
|
|
61
|
+
id: string;
|
|
62
|
+
}>, progress: BackfillProgress): Promise<BackfillProgress>;
|
|
63
|
+
export declare function executeBackfill(options: BackfillOptions): Promise<BackfillResult>;
|
|
64
|
+
//# sourceMappingURL=async-backfill.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAA;AAGxE,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;IACjF,6EAA6E;IAC7E,UAAU,EAAE,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAA;IACzE,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AA2ED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAoF3B;AA2CD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAsGvF"}
|