@chkit/plugin-backfill 0.1.2-beta.7 → 0.2.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/async-backfill.d.ts +13 -2
- package/dist/async-backfill.d.ts.map +1 -1
- package/dist/async-backfill.js +38 -17
- package/dist/async-backfill.js.map +1 -1
- package/dist/check.d.ts +1 -1
- package/dist/check.d.ts.map +1 -1
- package/dist/check.js +42 -2
- package/dist/check.js.map +1 -1
- package/dist/chunking/e2e/seed-datasets.script.d.ts +2 -5
- package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -1
- package/dist/chunking/e2e/seed-datasets.script.js +4 -7
- package/dist/chunking/e2e/seed-datasets.script.js.map +1 -1
- package/dist/chunking/services/distribution-source.js.map +1 -1
- package/dist/chunking/services/metadata-source.d.ts.map +1 -1
- package/dist/chunking/services/metadata-source.js +10 -2
- package/dist/chunking/services/metadata-source.js.map +1 -1
- package/dist/chunking/services/row-probe.js.map +1 -1
- package/dist/chunking/sql.d.ts +2 -0
- package/dist/chunking/sql.d.ts.map +1 -1
- package/dist/chunking/sql.js +54 -7
- package/dist/chunking/sql.js.map +1 -1
- package/dist/options.d.ts +20 -2
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +18 -1
- package/dist/options.js.map +1 -1
- package/dist/planner.d.ts +31 -2
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +211 -9
- package/dist/planner.js.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +34 -3
- package/dist/plugin.js.map +1 -1
- package/dist/sdk.d.ts +2 -1
- package/dist/sdk.d.ts.map +1 -1
- package/dist/sdk.js +1 -0
- package/dist/sdk.js.map +1 -1
- package/dist/state.d.ts +43 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +58 -0
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +23 -3
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -6
- package/dist/args.d.ts +0 -111
- package/dist/args.d.ts.map +0 -1
- package/dist/args.js +0 -146
- package/dist/args.js.map +0 -1
- package/dist/chunking/build.d.ts +0 -11
- package/dist/chunking/build.d.ts.map +0 -1
- package/dist/chunking/build.js +0 -51
- package/dist/chunking/build.js.map +0 -1
- package/dist/chunking/introspect.d.ts +0 -40
- package/dist/chunking/introspect.d.ts.map +0 -1
- package/dist/chunking/introspect.js +0 -187
- package/dist/chunking/introspect.js.map +0 -1
- package/dist/chunking/splitter.d.ts +0 -20
- package/dist/chunking/splitter.d.ts.map +0 -1
- package/dist/chunking/splitter.js +0 -76
- package/dist/chunking/splitter.js.map +0 -1
- package/dist/runtime.d.ts +0 -54
- package/dist/runtime.d.ts.map +0 -1
- package/dist/runtime.js +0 -517
- package/dist/runtime.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
# @chkit/plugin-backfill
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Beta plugin for checkpointed SQL backfill operations in chkit. Its API, configuration, and CLI flags may change between releases.
|
|
4
4
|
|
|
5
5
|
Part of the [chkit](https://github.com/obsessiondb/chkit) monorepo. This plugin extends the [`chkit`](https://www.npmjs.com/package/chkit) CLI with data backfill commands.
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
9
|
-
```
|
|
9
|
+
```sh
|
|
10
10
|
bun add -d @chkit/plugin-backfill
|
|
11
11
|
```
|
|
12
12
|
|
|
@@ -33,7 +33,7 @@ export default defineConfig({
|
|
|
33
33
|
|
|
34
34
|
## Documentation
|
|
35
35
|
|
|
36
|
-
See the [
|
|
36
|
+
See the [backfill plugin documentation (beta)](https://chkit.obsessiondb.com/plugins/backfill/).
|
|
37
37
|
|
|
38
38
|
## SDK
|
|
39
39
|
|
package/dist/async-backfill.d.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type ClickHouseExecutor } from '@chkit/clickhouse';
|
|
2
2
|
export interface BackfillOptions {
|
|
3
3
|
/** The executor to submit queries to (target ClickHouse) */
|
|
4
4
|
executor: ClickHouseExecutor;
|
|
5
|
+
/** The configured `clickhouse.cluster`. Resume reads chunk status on every replica of
|
|
6
|
+
* this cluster, or of `default` (a multi-replica ObsessionDB service) when unset. */
|
|
7
|
+
cluster?: string;
|
|
5
8
|
/** Plan ID used as a namespace in deterministic query IDs */
|
|
6
9
|
planId: string;
|
|
7
10
|
/** The chunks to process (from buildChunks) */
|
|
@@ -56,9 +59,17 @@ export interface BackfillResult {
|
|
|
56
59
|
* Queries system.processes and system.query_log for all chunk query IDs
|
|
57
60
|
* to discover queries that were submitted but whose status was never
|
|
58
61
|
* persisted locally (e.g. client crash between submit and state write).
|
|
62
|
+
*
|
|
63
|
+
* A chunk query runs on the replica that received it, and a chunk this read
|
|
64
|
+
* does not find is submitted again (#289). So on a multi-replica target (see
|
|
65
|
+
* `resolveReplicaFanout`) both tables are read on every replica, also without
|
|
66
|
+
* `clickhouse.cluster`, and a replica that cannot be read fails the sync
|
|
67
|
+
* instead of being skipped.
|
|
59
68
|
*/
|
|
60
69
|
export declare function syncProgress(executor: ClickHouseExecutor, planId: string, chunks: Array<{
|
|
61
70
|
id: string;
|
|
62
|
-
}>, progress: BackfillProgress
|
|
71
|
+
}>, progress: BackfillProgress, options?: {
|
|
72
|
+
cluster?: string;
|
|
73
|
+
}): Promise<BackfillProgress>;
|
|
63
74
|
export declare function executeBackfill(options: BackfillOptions): Promise<BackfillResult>;
|
|
64
75
|
//# sourceMappingURL=async-backfill.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,kBAAkB,EAGxB,MAAM,mBAAmB,CAAA;AAG1B,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B;0FACsF;IACtF,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;IACjF,6EAA6E;IAC7E,UAAU,EAAE,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAA;IACzE,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AAuED;;;;;;;;;;;;GAYG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7B,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7B,OAAO,CAAC,gBAAgB,CAAC,CAmE3B;AAoFD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAuGvF"}
|
package/dist/async-backfill.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
2
|
+
import { allReplicas, observableSystemTable, resolveReplicaFanout, } from '@chkit/clickhouse';
|
|
1
3
|
import pMap from 'p-map';
|
|
2
|
-
function sleep(ms) {
|
|
3
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
|
-
}
|
|
5
4
|
/** Build the deterministic query ID for a chunk. */
|
|
6
5
|
function chunkQueryId(planId, chunkId) {
|
|
7
6
|
return `backfill-${planId}-${chunkId}`;
|
|
@@ -66,8 +65,14 @@ function updateChunk(progress, id, next) {
|
|
|
66
65
|
* Queries system.processes and system.query_log for all chunk query IDs
|
|
67
66
|
* to discover queries that were submitted but whose status was never
|
|
68
67
|
* persisted locally (e.g. client crash between submit and state write).
|
|
68
|
+
*
|
|
69
|
+
* A chunk query runs on the replica that received it, and a chunk this read
|
|
70
|
+
* does not find is submitted again (#289). So on a multi-replica target (see
|
|
71
|
+
* `resolveReplicaFanout`) both tables are read on every replica, also without
|
|
72
|
+
* `clickhouse.cluster`, and a replica that cannot be read fails the sync
|
|
73
|
+
* instead of being skipped.
|
|
69
74
|
*/
|
|
70
|
-
export async function syncProgress(executor, planId, chunks, progress) {
|
|
75
|
+
export async function syncProgress(executor, planId, chunks, progress, options) {
|
|
71
76
|
const prefix = `backfill-${planId}-`;
|
|
72
77
|
// Collect query IDs for non-terminal chunks that need reconciliation
|
|
73
78
|
const chunkIdsToSync = [];
|
|
@@ -79,18 +84,15 @@ export async function syncProgress(executor, planId, chunks, progress) {
|
|
|
79
84
|
}
|
|
80
85
|
if (chunkIdsToSync.length === 0)
|
|
81
86
|
return progress;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
87
|
+
const safePrefix = prefix.replace(/[\\%_']/g, (character) => {
|
|
88
|
+
if (character === "'")
|
|
89
|
+
return "''";
|
|
90
|
+
if (character === '\\')
|
|
91
|
+
return '\\\\\\\\';
|
|
92
|
+
return '\\\\' + character;
|
|
93
|
+
});
|
|
94
|
+
const { runningRows, logRows } = await readChunkQueries(executor, planId, safePrefix, options?.cluster);
|
|
85
95
|
const runningSet = new Set(runningRows.map((r) => r.query_id));
|
|
86
|
-
const logRows = await executor.query(`SELECT query_id, type, written_rows, written_bytes, query_duration_ms, exception
|
|
87
|
-
FROM clusterAllReplicas('cluster', system.query_log)
|
|
88
|
-
WHERE user = currentUser()
|
|
89
|
-
AND query_id LIKE '${safePrefix}%'
|
|
90
|
-
AND type IN ('QueryFinish', 'ExceptionWhileProcessing')
|
|
91
|
-
AND is_initial_query = 1
|
|
92
|
-
ORDER BY event_time DESC
|
|
93
|
-
SETTINGS skip_unavailable_shards = 1`);
|
|
94
96
|
// Deduplicate: take the latest log entry per query_id (results are ordered by event_time DESC)
|
|
95
97
|
const latestLogByQueryId = new Map();
|
|
96
98
|
for (const row of logRows) {
|
|
@@ -136,6 +138,25 @@ SETTINGS skip_unavailable_shards = 1`);
|
|
|
136
138
|
}
|
|
137
139
|
return updated;
|
|
138
140
|
}
|
|
141
|
+
async function readChunkQueries(executor, planId, safePrefix, cluster) {
|
|
142
|
+
const fanout = await resolveReplicaFanout(executor, cluster);
|
|
143
|
+
const from = (table) => fanout ? allReplicas(fanout, `system.${table}`) : observableSystemTable(executor, table);
|
|
144
|
+
try {
|
|
145
|
+
const runningRows = await executor.query(`SELECT query_id FROM ${from('processes')} WHERE user = currentUser() AND query_id LIKE '${safePrefix}%' SETTINGS skip_unavailable_shards = 0`);
|
|
146
|
+
const logRows = await executor.query(`SELECT query_id, type, written_rows, written_bytes, query_duration_ms, exception
|
|
147
|
+
FROM ${from('query_log')}
|
|
148
|
+
WHERE user = currentUser()
|
|
149
|
+
AND query_id LIKE '${safePrefix}%'
|
|
150
|
+
AND type IN ('QueryFinish', 'ExceptionWhileProcessing')
|
|
151
|
+
AND is_initial_query = 1
|
|
152
|
+
ORDER BY event_time DESC
|
|
153
|
+
SETTINGS skip_unavailable_shards = 0`);
|
|
154
|
+
return { runningRows, logRows };
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
throw new Error(`Backfill plan ${planId}: could not check which chunks are still running on the server (${error instanceof Error ? error.message : String(error)}). Nothing was submitted — re-run the resume.`, { cause: error });
|
|
158
|
+
}
|
|
159
|
+
}
|
|
139
160
|
async function pollChunk(executor, initial, pollIntervalMs, maxPollErrors, onChanged) {
|
|
140
161
|
let state = initial;
|
|
141
162
|
let consecutiveErrors = 0;
|
|
@@ -173,7 +194,7 @@ async function pollChunk(executor, initial, pollIntervalMs, maxPollErrors, onCha
|
|
|
173
194
|
return state;
|
|
174
195
|
}
|
|
175
196
|
export async function executeBackfill(options) {
|
|
176
|
-
const { executor, planId, chunks, buildQuery, concurrency = 3, pollIntervalMs = 5000, maxPollErrors = 10, onProgress, resumeFrom, replayFailed, } = options;
|
|
197
|
+
const { executor, cluster, planId, chunks, buildQuery, concurrency = 3, pollIntervalMs = 5000, maxPollErrors = 10, onProgress, resumeFrom, replayFailed, } = options;
|
|
177
198
|
let progress = Object.fromEntries(chunks.map((chunk) => {
|
|
178
199
|
const resumed = resumeFrom?.[chunk.id];
|
|
179
200
|
return [chunk.id, resumed ? { ...resumed } : { status: 'pending' }];
|
|
@@ -182,7 +203,7 @@ export async function executeBackfill(options) {
|
|
|
182
203
|
// This catches queries that were submitted but whose status was never
|
|
183
204
|
// persisted (e.g. client crash between submit() and state file write).
|
|
184
205
|
if (resumeFrom) {
|
|
185
|
-
progress = await syncProgress(executor, planId, chunks, progress);
|
|
206
|
+
progress = await syncProgress(executor, planId, chunks, progress, { cluster });
|
|
186
207
|
}
|
|
187
208
|
// Reset confirmed-failed chunks to pending AFTER sync so we operate on
|
|
188
209
|
// ground truth. The deterministic query_id is reused; the afterTime filter
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EACL,WAAW,EACX,qBAAqB,EACrB,oBAAoB,GAIrB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,IAAI,MAAM,OAAO,CAAA;AAmDxB,oDAAoD;AACpD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe;IACnD,OAAO,YAAY,MAAM,IAAI,OAAO,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAyB,EACzB,EAAe;IAEf,IAAI,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAuB;YAC/B,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAA;QACD,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,KAAK,SAAS;YAC1B,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,QAAQ;YAC9B,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,WAAW;YACpC,KAAK,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAA;QAClC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;IACjD,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;aAC9B;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,KAAK,EAAE,EAAE,CAAC,KAAK;aAChB;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,sEAAsE;IACtE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,QAA0B,EAAE,EAAU;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAA;IAChE,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,EAAU,EACV,IAAwB;IAExB,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA4B,EAC5B,MAAc,EACd,MAA6B,EAC7B,QAA0B,EAC1B,OAA8B;IAE9B,MAAM,MAAM,GAAG,YAAY,MAAM,GAAG,CAAA;IAEpC,qEAAqE;IACrE,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;YAAE,SAAQ;QAC/C,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,EAAE;QAC1D,IAAI,SAAS,KAAK,GAAG;YAAE,OAAO,IAAI,CAAA;QAClC,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,UAAU,CAAA;QACzC,OAAO,MAAM,GAAG,SAAS,CAAA;IAC3B,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IACvG,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE9D,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAA;IACjE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;IAE7B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAChC,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;QACrF,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAChD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACpC,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,MAAM;wBACd,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAC1C,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;wBAC5C,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;qBAC/C,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,QAAQ;wBAChB,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;wBAC9C,KAAK,EAAE,QAAQ,CAAC,SAAS;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAWD,KAAK,UAAU,gBAAgB,CAC7B,QAA4B,EAC5B,MAAc,EACd,UAAkB,EAClB,OAA2B;IAE3B,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC5D,MAAM,IAAI,GAAG,CAAC,KAA4B,EAAE,EAAE,CAC5C,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IAC1F,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CACtC,wBAAwB,IAAI,CAAC,WAAW,CAAC,kDAAkD,UAAU,yCAAyC,CAC/I,CAAA;QACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAClC;OACC,IAAI,CAAC,WAAW,CAAC;;uBAED,UAAU;;;;qCAII,CAChC,CAAA;QACD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,iBAAiB,MAAM,mEAAmE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAC/L,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAA;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAA4B,EAC5B,OAA2B,EAC3B,cAAsB,EACtB,aAAqB,EACrB,SAA8D;IAE9D,IAAI,KAAK,GAAG,OAAO,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,MAAK;QACzB,IAAI,EAAe,CAAA;QACnB,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;gBAC7C,SAAS,EAAE,KAAK,CAAC,WAAW;aAC7B,CAAC,CAAA;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAA;YACnB,IAAI,iBAAiB,IAAI,aAAa,EAAE,CAAC;gBACvC,KAAK,GAAG;oBACN,GAAG,KAAK;oBACR,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,KAAK,EAAE,iCAAiC,iBAAiB,0BAA0B;iBACpF,CAAA;gBACD,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;gBACtB,MAAK;YACP,CAAC;YACD,SAAQ;QACV,CAAC;QACD,iBAAiB,GAAG,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;YACpB,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAwB;IAC5D,MAAM,EACJ,QAAQ,EACR,OAAO,EACP,MAAM,EACN,MAAM,EACN,UAAU,EACV,WAAW,GAAG,CAAC,EACf,cAAc,GAAG,IAAI,EACrB,aAAa,GAAG,EAAE,EAClB,UAAU,EACV,UAAU,EACV,YAAY,GACb,GAAG,OAAO,CAAA;IAEX,IAAI,QAAQ,GAAqB,MAAM,CAAC,WAAW,CACjD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAkB,EAAE,CAAC,CAAA;IAC9E,CAAC,CAAC,CACH,CAAA;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,uEAAuE;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;IAChF,CAAC;IAED,uEAAuE;IACvE,2EAA2E;IAC3E,gFAAgF;IAChF,IAAI,YAAY,EAAE,CAAC;QACjB,iEAAiE;QACjE,oEAAoE;QACpE,iEAAiE;QACjE,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACnE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAChC,IAAI,KAAK,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAA;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,IAAwB,EAAE,EAAE;QACxD,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QAC1C,OAAO,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC,CAAA;IAED,MAAM,IAAI,CACR,MAAM,EACN,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1C,uCAAuC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAM;QAEhE,wCAAwC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;gBAC7F,OAAM;YACR,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACnC,MAAM,SAAS,GAAuB;YACpC,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,WAAW;YACnB,OAAO;SACR,CAAA;QACD,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAEnC,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACnG,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAA;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;IAExF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,SAAS;QACT,MAAM;QACN,QAAQ;KACT,CAAA;AACH,CAAC"}
|
package/dist/check.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { ResolvedChxConfig } from '@chkit/core';
|
|
|
2
2
|
import type { BackfillPluginCheckResult } from './types.js';
|
|
3
3
|
export declare function evaluateBackfillCheck(input: {
|
|
4
4
|
configPath: string;
|
|
5
|
-
config: Pick<ResolvedChxConfig, 'metaDir'>;
|
|
5
|
+
config: Pick<ResolvedChxConfig, 'metaDir' | 'plugins'>;
|
|
6
6
|
stateDir?: string;
|
|
7
7
|
failCheckOnRequiredPendingBackfill: boolean;
|
|
8
8
|
}): Promise<BackfillPluginCheckResult>;
|
package/dist/check.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"check.d.ts","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"check.d.ts","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAQpD,OAAO,KAAK,EACV,yBAAyB,EAC1B,MAAM,YAAY,CAAA;AAEnB,wBAAsB,qBAAqB,CAAC,KAAK,EAAE;IACjD,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,GAAG,SAAS,CAAC,CAAA;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kCAAkC,EAAE,OAAO,CAAA;CAC5C,GAAG,OAAO,CAAC,yBAAyB,CAAC,CA6GrC"}
|
package/dist/check.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
-
import { computeBackfillStateDir, listPlanIds, readRun, } from './state.js';
|
|
2
|
+
import { computeBackfillStateDir, listPlanIds, readRun, readSubmission, } from './state.js';
|
|
3
3
|
export async function evaluateBackfillCheck(input) {
|
|
4
4
|
const stateDir = computeBackfillStateDir(input.config, input.configPath, input.stateDir);
|
|
5
5
|
const plansDir = join(stateDir, 'plans');
|
|
6
6
|
const runsDir = join(stateDir, 'runs');
|
|
7
|
+
const submissionsDir = join(stateDir, 'submissions');
|
|
7
8
|
const planIds = await listPlanIds(plansDir);
|
|
8
9
|
if (planIds.length === 0) {
|
|
9
10
|
return {
|
|
@@ -15,17 +16,30 @@ export async function evaluateBackfillCheck(input) {
|
|
|
15
16
|
requiredCount: 0,
|
|
16
17
|
activeRuns: 0,
|
|
17
18
|
failedRuns: 0,
|
|
19
|
+
submittedCount: 0,
|
|
18
20
|
},
|
|
19
21
|
};
|
|
20
22
|
}
|
|
21
23
|
let requiredCount = 0;
|
|
22
24
|
let activeRuns = 0;
|
|
23
25
|
let failedRuns = 0;
|
|
26
|
+
let submittedCount = 0;
|
|
27
|
+
let unconfirmedCount = 0;
|
|
24
28
|
for (const planId of planIds) {
|
|
25
29
|
const runPath = join(runsDir, `${planId}.json`);
|
|
26
30
|
const run = await readRun(runPath);
|
|
27
31
|
if (!run) {
|
|
28
|
-
|
|
32
|
+
// A managed job runs on the backend and never writes a local run. Its
|
|
33
|
+
// backend's plugin (e.g. ObsessionDB) checks whether the job completed,
|
|
34
|
+
// so it is not pending here. A local run, if any, still wins.
|
|
35
|
+
const submission = await readSubmission(join(submissionsDir, `${planId}.json`));
|
|
36
|
+
if (!submission)
|
|
37
|
+
requiredCount += 1;
|
|
38
|
+
else {
|
|
39
|
+
submittedCount += 1;
|
|
40
|
+
if (submission.status !== 'completed')
|
|
41
|
+
unconfirmedCount += 1;
|
|
42
|
+
}
|
|
29
43
|
continue;
|
|
30
44
|
}
|
|
31
45
|
if (run.status === 'running')
|
|
@@ -46,6 +60,20 @@ export async function evaluateBackfillCheck(input) {
|
|
|
46
60
|
},
|
|
47
61
|
});
|
|
48
62
|
}
|
|
63
|
+
// Only the ObsessionDB plugin asks the jobs API whether a submitted job
|
|
64
|
+
// completed. Without it (e.g. on CI, where no login session adds it to the
|
|
65
|
+
// config) an unconfirmed submitted plan would otherwise pass unverified.
|
|
66
|
+
if (unconfirmedCount > 0 && !hasObsessionDBPlugin(input.config.plugins)) {
|
|
67
|
+
findings.push({
|
|
68
|
+
code: 'backfill_submitted_unverified',
|
|
69
|
+
message: `Submitted backfills not verified: ${unconfirmedCount}. Only the ObsessionDB plugin checks their jobs, and it is ` +
|
|
70
|
+
'not loaded: register obsessiondb() in the config plugins and log in with chkit obsessiondb login.',
|
|
71
|
+
severity: input.failCheckOnRequiredPendingBackfill ? 'error' : 'warn',
|
|
72
|
+
metadata: {
|
|
73
|
+
unconfirmedCount,
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
}
|
|
49
77
|
if (failedRuns > 0) {
|
|
50
78
|
findings.push({
|
|
51
79
|
code: 'backfill_chunk_failed_retry_exhausted',
|
|
@@ -73,7 +101,19 @@ export async function evaluateBackfillCheck(input) {
|
|
|
73
101
|
requiredCount,
|
|
74
102
|
activeRuns,
|
|
75
103
|
failedRuns,
|
|
104
|
+
submittedCount,
|
|
76
105
|
},
|
|
77
106
|
};
|
|
78
107
|
}
|
|
108
|
+
function hasObsessionDBPlugin(plugins) {
|
|
109
|
+
return plugins.some((entry) => entry.enabled !== false && pluginName(entry.plugin) === 'obsessiondb');
|
|
110
|
+
}
|
|
111
|
+
function pluginName(plugin) {
|
|
112
|
+
if (typeof plugin !== 'object' || plugin === null || !('manifest' in plugin))
|
|
113
|
+
return undefined;
|
|
114
|
+
const { manifest } = plugin;
|
|
115
|
+
if (typeof manifest !== 'object' || manifest === null || !('name' in manifest))
|
|
116
|
+
return undefined;
|
|
117
|
+
return typeof manifest.name === 'string' ? manifest.name : undefined;
|
|
118
|
+
}
|
|
79
119
|
//# sourceMappingURL=check.js.map
|
package/dist/check.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"check.js","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAIhC,OAAO,EACL,uBAAuB,EACvB,WAAW,EACX,OAAO,
|
|
1
|
+
{"version":3,"file":"check.js","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAIhC,OAAO,EACL,uBAAuB,EACvB,WAAW,EACX,OAAO,EACP,cAAc,GACf,MAAM,YAAY,CAAA;AAKnB,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAK3C;IACC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACtC,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;IAEpD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,IAAI;YACf,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE;gBACR,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,cAAc,EAAE,CAAC;aAClB;SACF,CAAA;IACH,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IAExB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,CAAA;QAC/C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;QAClC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,sEAAsE;YACtE,wEAAwE;YACxE,8DAA8D;YAC9D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC,CAAA;YAC/E,IAAI,CAAC,UAAU;gBAAE,aAAa,IAAI,CAAC,CAAA;iBAC9B,CAAC;gBACJ,cAAc,IAAI,CAAC,CAAA;gBACnB,IAAI,UAAU,CAAC,MAAM,KAAK,WAAW;oBAAE,gBAAgB,IAAI,CAAC,CAAA;YAC9D,CAAC;YACD,SAAQ;QACV,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,UAAU,IAAI,CAAC,CAAA;QAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,UAAU,IAAI,CAAC,CAAA;QAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,aAAa,IAAI,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,QAAQ,GAA0C,EAAE,CAAA;IAC1D,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,2BAA2B;YACjC,OAAO,EAAE,0CAA0C,aAAa,EAAE;YAClE,QAAQ,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YACrE,QAAQ,EAAE;gBACR,aAAa;aACd;SACF,CAAC,CAAA;IACJ,CAAC;IAED,wEAAwE;IACxE,2EAA2E;IAC3E,yEAAyE;IACzE,IAAI,gBAAgB,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxE,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,+BAA+B;YACrC,OAAO,EACL,qCAAqC,gBAAgB,6DAA6D;gBAClH,mGAAmG;YACrG,QAAQ,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YACrE,QAAQ,EAAE;gBACR,gBAAgB;aACjB;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,uCAAuC;YAC7C,OAAO,EAAE,4CAA4C,UAAU,EAAE;YACjE,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE;gBACR,UAAU;aACX;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,kCAAkC,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,yBAAyB;YAC/B,OAAO,EAAE,6EAA6E;YACtF,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IACpE,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,SAAS,EAAE,IAAI;QACf,EAAE;QACF,QAAQ;QACR,QAAQ,EAAE;YACR,aAAa;YACb,UAAU;YACV,UAAU;YACV,cAAc;SACf;KACF,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAqC;IACjE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,aAAa,CAAC,CAAA;AACvG,CAAC;AAED,SAAS,UAAU,CAAC,MAAe;IACjC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC;QAAE,OAAO,SAAS,CAAA;IAC9F,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAA;IAC3B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;QAAE,OAAO,SAAS,CAAA;IAChG,OAAO,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;AACtE,CAAC"}
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
/**
|
|
3
|
-
* Seeds ClickHouse tables for smart-chunking E2E tests.
|
|
3
|
+
* Seeds ClickHouse tables for smart-chunking E2E tests. The package's `test`
|
|
4
|
+
* script runs it before the tests, against the same target (see getLiveEnv).
|
|
4
5
|
*
|
|
5
|
-
* Run once manually:
|
|
6
|
-
* bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
|
|
7
|
-
*
|
|
8
|
-
* Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
|
|
9
6
|
* Creates tables if they don't exist, truncates them, and re-inserts data.
|
|
10
7
|
*/
|
|
11
8
|
interface DatasetConfig {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"seed-datasets.script.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA
|
|
1
|
+
{"version":3,"file":"seed-datasets.script.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;GAKG;AAOH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;CAC1C;AAUD,eAAO,MAAM,QAAQ,EAAE,aAAa,EA+EnC,CAAA"}
|
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
/**
|
|
3
|
-
* Seeds ClickHouse tables for smart-chunking E2E tests.
|
|
3
|
+
* Seeds ClickHouse tables for smart-chunking E2E tests. The package's `test`
|
|
4
|
+
* script runs it before the tests, against the same target (see getLiveEnv).
|
|
4
5
|
*
|
|
5
|
-
* Run once manually:
|
|
6
|
-
* bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
|
|
7
|
-
*
|
|
8
|
-
* Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
|
|
9
6
|
* Creates tables if they don't exist, truncates them, and re-inserts data.
|
|
10
7
|
*/
|
|
11
8
|
import { randomBytes } from 'node:crypto';
|
|
12
|
-
import {
|
|
9
|
+
import { getLiveEnv, createLiveExecutor } from '@chkit/clickhouse/e2e-testkit';
|
|
13
10
|
import { TABLE_PREFIX } from './constants.js';
|
|
14
11
|
function pad(bytes) {
|
|
15
12
|
return randomBytes(bytes).toString('hex');
|
|
@@ -93,7 +90,7 @@ export const datasets = [
|
|
|
93
90
|
];
|
|
94
91
|
const BATCH_SIZE = 5000;
|
|
95
92
|
async function seed() {
|
|
96
|
-
const env =
|
|
93
|
+
const env = getLiveEnv();
|
|
97
94
|
const executor = createLiveExecutor(env);
|
|
98
95
|
const db = env.clickhouseDatabase;
|
|
99
96
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"seed-datasets.script.js","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA
|
|
1
|
+
{"version":3,"file":"seed-datasets.script.js","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAA;AAE9E,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAU7C,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,IAAY;IACxC,OAAO,WAAW,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAA;AACzF,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAoB;IACvC;QACE,IAAI,EAAE,kBAAkB;QACxB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC;oBACR,SAAS,EAAE,WAAW;oBACtB,GAAG,EAAE,CAAC;oBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;oBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;iBAClB,CAAC,CAAA;YACJ,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBACjD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,0DAA0D;YAC1D,KAAK,MAAM,MAAM,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,MAAM;wBACjB,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;wBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,SAAS,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBAChD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;CACF,CAAA;AAED,MAAM,UAAU,GAAG,IAAI,CAAA;AAEvB,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,UAAU,EAAE,CAAA;IACxB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,EAAE,GAAG,GAAG,CAAC,kBAAkB,CAAA;IAEjC,IAAI,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,IAAI,EAAE,CAAA;YAC/C,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,KAAK,EAAE,CAAA;YAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,CAAA;YAEvC,MAAM,QAAQ,CAAC,OAAO,CAAC;qCACQ,GAAG;YAC5B,OAAO,CAAC,OAAO;;uBAEJ,OAAO,CAAC,WAAW;mBACvB,OAAO,CAAC,OAAO;OAC3B,CAAC,CAAA;YACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;YAE/B,MAAM,QAAQ,CAAC,OAAO,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;YAC/C,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;YAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAA;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC3C,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;gBACpD,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,OAAO,CAAC,CAAA;YAC1F,CAAC;YAED,SAAS;YACT,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CACnC,8BAA8B,GAAG,6CAA6C,CAC/E,CAAA;YACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,EAAE,GAAG,OAAO,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AACxB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distribution-source.js","sourceRoot":"","sources":["../../../src/chunking/services/distribution-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAWtD,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAAqB,EACrB,WAAmB,EACnB,MAAoB,EACpB,OAAgB,EAChB,cAAsB,EACtB,KAAa,EACb,QAAmB;IAEnB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,KAAK,cAAc,CAAC,CAAA;IACrF,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,CAAA;IAExC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,
|
|
1
|
+
{"version":3,"file":"distribution-source.js","sourceRoot":"","sources":["../../../src/chunking/services/distribution-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAWtD,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAAqB,EACrB,WAAmB,EACnB,MAAoB,EACpB,OAAgB,EAChB,cAAsB,EACtB,KAAa,EACb,QAAmB;IAEnB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,KAAK,cAAc,CAAC,CAAA;IACrF,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,CAAA;IAExC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAA2C;;cAE/D,OAAO,CAAC,IAAI,QAAQ,KAAK;;OAEhC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK;QAChC,0BAA0B,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC;;gBAEjD,EACZ,OAAO,CAAC,aAAa,CACtB,CAAA;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,KAAK,EAAE,GAAG,CAAC,MAAM;QACjB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACzB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,KAAK;KAC/D,CAAC,CAAC,CAAA;AACL,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,OAAqB,EACrB,WAAmB,EACnB,MAAoB,EACpB,OAAgB,EAChB,cAAsB,EACtB,QAAmB,EACnB,KAAa;IAEb,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,KAAK,cAAc,CAAC,CAAA;IACrF,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,SAAS,CAAA;IAE/C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAwC;;IAEtE,OAAO,CAAC,IAAI;;OAET,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK;QAChC,0BAA0B,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC;;;QAGzD,KAAK,GAAG,CAAC,EAAE,EACf,OAAO,CAAC,aAAa,CACtB,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,SAAS,CAAA;IAEzC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,KAAK,EAAE,GAAG,CAAC,GAAG;QACd,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;KAC1B,CAAC,CAAC,CAAA;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,OAAqB,EACrB,WAAmB,EACnB,MAAoB,EACpB,QAAmB,EACnB,cAAsB,EACtB,KAAqB;IAErB,MAAM,OAAO,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAA;IACxC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,UAAU;QAAE,OAAO,EAAE,CAAA;IAE1D,MAAM,gBAAgB,GAAG,KAAK,KAAK,KAAK;QACtC,CAAC,CAAC,gBAAgB,OAAO,CAAC,IAAI,GAAG;QACjC,CAAC,CAAC,iBAAiB,OAAO,CAAC,IAAI,GAAG,CAAA;IAEpC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAA2C;;mBAE1D,gBAAgB;;OAE5B,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK;QAChC,0BAA0B,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC;;gBAEjD,EACZ,OAAO,CAAC,aAAa,CACtB,CAAA;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,KAAK,EAAE,GAAG,CAAC,MAAM;QACjB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;KAC1B,CAAC,CAAC,CAAA;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metadata-source.d.ts","sourceRoot":"","sources":["../../../src/chunking/services/metadata-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAmB,MAAM,aAAa,CAAA;AAwFtF,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"metadata-source.d.ts","sourceRoot":"","sources":["../../../src/chunking/services/metadata-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAmB,MAAM,aAAa,CAAA;AAwFtF,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAiDxF;AAED,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAsCpF"}
|
|
@@ -75,13 +75,21 @@ function resolveSortKeyColumn(expression, knownColumns) {
|
|
|
75
75
|
}
|
|
76
76
|
export async function introspectPartitions(context) {
|
|
77
77
|
await context.query(`SELECT 1 FROM ${context.database}.${context.table} LIMIT 1 SETTINGS select_sequential_consistency = 1`);
|
|
78
|
+
// system.parts fills min_time/max_time only for a DateTime partition column. For
|
|
79
|
+
// a Date column it fills min_date/max_date instead, so fall back to those, with
|
|
80
|
+
// max_date ending at that day's last second like an inclusive max_time.
|
|
78
81
|
const rows = await context.query(`SELECT
|
|
79
82
|
partition_id,
|
|
80
83
|
toString(sum(rows)) AS total_rows,
|
|
81
84
|
toString(sum(bytes_on_disk)) AS total_bytes,
|
|
82
85
|
toString(sum(data_uncompressed_bytes)) AS total_uncompressed_bytes,
|
|
83
|
-
toString(
|
|
84
|
-
|
|
86
|
+
toString(minIf(
|
|
87
|
+
if(min_time > 0, min_time, toDateTime(min_date)),
|
|
88
|
+
min_time > 0 OR min_date > toDate(0)
|
|
89
|
+
)) AS min_time,
|
|
90
|
+
toString(max(
|
|
91
|
+
if(max_time = 0 AND max_date > toDate(0), toDateTime(max_date + 1) - 1, max_time)
|
|
92
|
+
)) AS max_time
|
|
85
93
|
FROM system.parts
|
|
86
94
|
WHERE database = '${context.database}'
|
|
87
95
|
AND table = '${context.table}'
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metadata-source.js","sourceRoot":"","sources":["../../../src/chunking/services/metadata-source.ts"],"names":[],"mappings":"AAEA,4EAA4E;AAC5E,SAAS,2BAA2B,CAAC,KAAa;IAChD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAA;IACpF,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;AACzF,CAAC;AAED,MAAM,aAAa,GAAG,wBAAwB,CAAA;AAC9C,MAAM,cAAc,GAAG,kBAAkB,CAAA;AAEzC,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAC9C,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAChD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,2BAA2B,CAAC,QAAyB;IAC5D,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAA;AACzD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,KAA6B,CAAA;IAEjC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;QACzB,IAAI,IAAI,KAAK,SAAS;YAAE,SAAQ;QAEhC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,IAAI,IAAI,CAAA;YACf,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI;gBAAE,KAAK,GAAG,SAAS,CAAA;YAClE,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAClC,KAAK,GAAG,IAAI,CAAA;YACZ,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;YAC9B,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1B,OAAO,GAAG,EAAE,CAAA;YACZ,SAAQ;QACV,CAAC;QAED,OAAO,IAAI,IAAI,CAAA;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,UAAkB,EAAE,YAAyB;IACzE,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,EAAE,CAAA;IACjC,IAAI,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAA;IAE7C,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;SAC5E,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,UAAU,EAAwB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;IACrG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,CAAA;IAC3C,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,OAAO,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC5C,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAAuB;IAChE,MAAM,OAAO,CAAC,KAAK,CACjB,iBAAiB,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,qDAAqD,CACxG,CAAA;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAO7B
|
|
1
|
+
{"version":3,"file":"metadata-source.js","sourceRoot":"","sources":["../../../src/chunking/services/metadata-source.ts"],"names":[],"mappings":"AAEA,4EAA4E;AAC5E,SAAS,2BAA2B,CAAC,KAAa;IAChD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAA;IACpF,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;AACzF,CAAC;AAED,MAAM,aAAa,GAAG,wBAAwB,CAAA;AAC9C,MAAM,cAAc,GAAG,kBAAkB,CAAA;AAEzC,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAC9C,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAChD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,2BAA2B,CAAC,QAAyB;IAC5D,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAA;AACzD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,KAA6B,CAAA;IAEjC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;QACzB,IAAI,IAAI,KAAK,SAAS;YAAE,SAAQ;QAEhC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,IAAI,IAAI,CAAA;YACf,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI;gBAAE,KAAK,GAAG,SAAS,CAAA;YAClE,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAClC,KAAK,GAAG,IAAI,CAAA;YACZ,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;YAC9B,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1B,OAAO,GAAG,EAAE,CAAA;YACZ,SAAQ;QACV,CAAC;QAED,OAAO,IAAI,IAAI,CAAA;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,UAAkB,EAAE,YAAyB;IACzE,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,EAAE,CAAA;IACjC,IAAI,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAA;IAE7C,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;SAC5E,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,UAAU,EAAwB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;IACrG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,CAAA;IAC3C,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,OAAO,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC5C,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAAuB;IAChE,MAAM,OAAO,CAAC,KAAK,CACjB,iBAAiB,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,qDAAqD,CACxG,CAAA;IAED,iFAAiF;IACjF,gFAAgF;IAChF,wEAAwE;IACxE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAO7B;;;;;;;;;;;;;oBAae,OAAO,CAAC,QAAQ;iBACnB,OAAO,CAAC,KAAK;;;;2CAIa,CAAC,CAAA;IAE1C,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACb,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;QAC5B,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;QACxC,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,WAAW,CAAC;QAC1E,OAAO,EAAE,2BAA2B,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClD,OAAO,EAAE,2BAA2B,CAAC,GAAG,CAAC,QAAQ,CAAC;KACnD,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACpB,IAAI,OAAO,CAAC,IAAI,IAAI,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,OAAO,CAAC,EAAE,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE;YAAE,OAAO,KAAK,CAAA;QAC/D,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAuB;IAC9D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,KAAK,CACnC,2DAA2D,OAAO,CAAC,QAAQ,iBAAiB,OAAO,CAAC,KAAK,GAAG,CAC7G,CAAA;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,CAAA;IAC5C,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAA;IAE1B,MAAM,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAA;IAChD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAEvC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CACpC,2DAA2D,OAAO,CAAC,QAAQ,kBAAkB,OAAO,CAAC,KAAK,GAAG,CAC9G,CAAA;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,UAAU;SACP,MAAM,CAAC,CAAC,GAAG,EAAyC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;SACzE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CACtC,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;IAE/C,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QAC/C,MAAM,MAAM,GAAG,oBAAoB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QAC7D,MAAM,IAAI,GAAG,MAAM;YACjB,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI;YAC1E,CAAC,CAAC,SAAS,CAAA;QACb,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAA;QAE/B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAC1C,OAAO,CAAC;gBACN,IAAI,EAAE,MAAM;gBACZ,IAAI;gBACJ,QAAQ;gBACR,gBAAgB,EAAE,2BAA2B,CAAC,QAAQ,CAAC;aACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"row-probe.js","sourceRoot":"","sources":["../../../src/chunking/services/row-probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAWvF,MAAM,UAAU,mBAAmB,CAAC,OAAiD;IACnF,OAAO,OAAO,CAAC,gBAAgB,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAuB,EACvB,MAAsB,EACtB,QAAmB;IAEnB,IAAI,mBAAmB,CAAC,OAAO,CAAC,KAAK,OAAO,EAAE,CAAC;QAC7C,OAAO,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAC9B,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC,EACzE,OAAO,CAAC,aAAa,CACtB,CAAA;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACxB,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAA;IAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAQ;QAChD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAA;IAC5C,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,CAAA;AACV,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAqB,EACrB,MAAsB,EACtB,QAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,
|
|
1
|
+
{"version":3,"file":"row-probe.js","sourceRoot":"","sources":["../../../src/chunking/services/row-probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAWvF,MAAM,UAAU,mBAAmB,CAAC,OAAiD;IACnF,OAAO,OAAO,CAAC,gBAAgB,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAuB,EACvB,MAAsB,EACtB,QAAmB;IAEnB,IAAI,mBAAmB,CAAC,OAAO,CAAC,KAAK,OAAO,EAAE,CAAC;QAC7C,OAAO,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAC9B,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC,EACzE,OAAO,CAAC,aAAa,CACtB,CAAA;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACxB,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAA;IAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAQ;QAChD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAA;IAC5C,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,CAAA;AACV,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAqB,EACrB,MAAsB,EACtB,QAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAA2B,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;IAC3H,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAqB,EACrB,WAAmB,EACnB,MAAoB,EACpB,QAAmB,EACnB,OAAgB;IAEhB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAqC;;iBAEtD,OAAO,CAAC,IAAI;iBACZ,OAAO,CAAC,IAAI;OACtB,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK;QAChC,0BAA0B,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,EAC/D,OAAO,CAAC,aAAa,CACtB,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IACvC,OAAO;QACL,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE;QAC1B,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE;KAC3B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IACxE,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,CAAA;AAC7E,CAAC"}
|
package/dist/chunking/sql.d.ts
CHANGED
|
@@ -8,6 +8,8 @@ export declare function buildChunkExecutionSql(input: {
|
|
|
8
8
|
mvReplayQueries?: string[];
|
|
9
9
|
targetColumns?: string[];
|
|
10
10
|
idempotencyToken?: string;
|
|
11
|
+
/** cutover_copy: skip rows whose key the target already holds. */
|
|
12
|
+
cutoverKey?: string[];
|
|
11
13
|
}): string;
|
|
12
14
|
export declare function buildWhereClauseFromRanges(partitionId: string, ranges: ChunkRange[], sortKeys: SortKey[]): string;
|
|
13
15
|
export declare function buildWhereClauseFromChunk(chunk: Pick<Chunk, 'partitionId' | 'ranges'>, table: Pick<TableProfile, 'sortKeys'>): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../src/chunking/sql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,KAAK,EACL,UAAU,EACV,cAAc,EACd,cAAc,EACd,gBAAgB,EAChB,OAAO,EACP,YAAY,EACb,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../src/chunking/sql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,KAAK,EACL,UAAU,EACV,cAAc,EACd,cAAc,EACd,gBAAgB,EAChB,OAAO,EACP,YAAY,EACb,MAAM,YAAY,CAAA;AA8BnB,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,KAAK,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;IACrC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAC1B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;CACtB,GAAG,MAAM,CAyCT;AA4CD,wBAAgB,0BAA0B,CACxC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,UAAU,EAAE,EACpB,QAAQ,EAAE,OAAO,EAAE,GAClB,MAAM,CAUR;AAED,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa,GAAG,QAAQ,CAAC,EAC5C,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GACpC,MAAM,CAER;AAED,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,cAAc,EACvB,gBAAgB,EAAE,gBAAgB,GACjC,MAAM,CAMR;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,UAAU,GAAG,OAAO,CAAC,GAClD,MAAM,CAER;AAID;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAenF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,GACZ;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQlD;AAiED,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAC7B,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,GACT,MAAM,CAkBR;AAoFD,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;CACjB;AAwCD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,UAAU,EAAE,CAa3F;AASD,gFAAgF;AAChF,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAatE"}
|
package/dist/chunking/sql.js
CHANGED
|
@@ -13,26 +13,39 @@ const TRAILING_CLAUSE_KEYWORDS = [
|
|
|
13
13
|
'LIMIT',
|
|
14
14
|
'SETTINGS',
|
|
15
15
|
];
|
|
16
|
+
/**
|
|
17
|
+
* `INSERT INTO t SELECT *` fills only `t`'s insertable columns (no MATERIALIZED,
|
|
18
|
+
* ALIAS or EPHEMERAL), so `*` must expand to the same set. A user profile with
|
|
19
|
+
* `asterisk_include_*_columns = 1` adds MATERIALIZED/ALIAS columns to `*`: copy
|
|
20
|
+
* mode then fails on a column-count mismatch, and an mv_replay `SELECT *` can
|
|
21
|
+
* shift values into the wrong target columns. Query-level SETTINGS override the
|
|
22
|
+
* profile and cover every `*` in the query, including each UNION ALL branch.
|
|
23
|
+
*/
|
|
24
|
+
const ASTERISK_SETTINGS = 'asterisk_include_materialized_columns=0, asterisk_include_alias_columns=0';
|
|
16
25
|
// ── Chunk execution SQL ──────────────────────────────────────────────────────
|
|
17
26
|
export function buildChunkExecutionSql(input) {
|
|
18
27
|
const sourceTarget = input.sourceTarget ?? input.target;
|
|
19
28
|
const header = `/* chkit backfill plan=${input.planId} chunk=${input.chunk.id} token=${input.idempotencyToken ?? ''} */`;
|
|
20
29
|
const settings = buildSettingsClause(input.idempotencyToken ?? '');
|
|
21
30
|
const chunkConditions = buildChunkConditions(input.chunk, input.table.sortKeys);
|
|
31
|
+
if (input.cutoverKey?.length && input.targetColumns?.length) {
|
|
32
|
+
return buildCutoverChunkSql({ ...input, cutoverKey: input.cutoverKey, targetColumns: input.targetColumns, sourceTarget, header, chunkConditions });
|
|
33
|
+
}
|
|
22
34
|
if (input.mvReplayQueries?.length) {
|
|
23
35
|
// Each MV feeding the target becomes a filtered SELECT; UNION ALL replays
|
|
24
36
|
// them in one INSERT so a single query_id and dedup token cover the chunk.
|
|
37
|
+
// With the plan's column list, both sides name the columns and each view's
|
|
38
|
+
// output is read by name, as ClickHouse does for a view's TO table.
|
|
39
|
+
const columns = input.targetColumns?.length ? input.targetColumns.map(quoteIdentifier).join(', ') : undefined;
|
|
25
40
|
const selects = input.mvReplayQueries.map((query) => {
|
|
26
41
|
let filtered = injectPartitionFilter(query, input.chunk.partitionId);
|
|
27
42
|
for (const condition of chunkConditions) {
|
|
28
43
|
filtered = injectWhereCondition(filtered, condition);
|
|
29
44
|
}
|
|
30
|
-
|
|
31
|
-
filtered = rewriteSelectColumns(filtered, input.targetColumns);
|
|
32
|
-
}
|
|
33
|
-
return filtered;
|
|
45
|
+
return columns ? `SELECT ${columns}\nFROM (\n${filtered}\n)` : filtered;
|
|
34
46
|
});
|
|
35
|
-
|
|
47
|
+
const insert = columns ? `INSERT INTO ${input.target} (${columns})` : `INSERT INTO ${input.target}`;
|
|
48
|
+
return [header, insert, selects.join('\nUNION ALL\n'), settings].join('\n');
|
|
36
49
|
}
|
|
37
50
|
const lines = [
|
|
38
51
|
header,
|
|
@@ -47,6 +60,37 @@ export function buildChunkExecutionSql(input) {
|
|
|
47
60
|
lines.push(settings);
|
|
48
61
|
return lines.join('\n');
|
|
49
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* A cutover chunk copies the snapshot rows of the chunk whose key the shadow
|
|
65
|
+
* table does not hold yet: rows the mirror already wrote, and rows an earlier
|
|
66
|
+
* attempt of this chunk wrote, are skipped, so a retry never duplicates rows.
|
|
67
|
+
* The key set is narrowed to the chunk's range when the range is on columns
|
|
68
|
+
* the copy writes unchanged, so a chunk does not load every key of the shadow.
|
|
69
|
+
* `select_sequential_consistency` makes a SharedMergeTree replica read the
|
|
70
|
+
* parts other replicas committed, including mirrored rows.
|
|
71
|
+
*/
|
|
72
|
+
function buildCutoverChunkSql(input) {
|
|
73
|
+
const columns = input.targetColumns.map(quoteIdentifier).join(', ');
|
|
74
|
+
const key = input.cutoverKey.map(quoteIdentifier).join(', ');
|
|
75
|
+
const copied = new Set(input.targetColumns);
|
|
76
|
+
const shadowConditions = input.chunk.ranges.flatMap((range) => {
|
|
77
|
+
const sortKey = input.table.sortKeys[range.dimensionIndex];
|
|
78
|
+
if (!sortKey || !copied.has(sortKey.name.replace(/^`|`$/g, '')))
|
|
79
|
+
return [];
|
|
80
|
+
return buildRangeBoundConditions(range, sortKey);
|
|
81
|
+
});
|
|
82
|
+
const lines = [
|
|
83
|
+
input.header,
|
|
84
|
+
`INSERT INTO ${input.target} (${columns})`,
|
|
85
|
+
`SELECT ${columns}`,
|
|
86
|
+
`FROM ${input.sourceTarget}`,
|
|
87
|
+
`WHERE _partition_id = ${quoteSqlString(input.chunk.partitionId)}`,
|
|
88
|
+
...input.chunkConditions.map((condition) => ` AND ${condition}`),
|
|
89
|
+
` AND (${key}) NOT IN (SELECT ${key} FROM ${input.target}${shadowConditions.length > 0 ? ` WHERE ${shadowConditions.join(' AND ')}` : ''})`,
|
|
90
|
+
`SETTINGS async_insert=0, ${ASTERISK_SETTINGS}, insert_deduplicate=0, select_sequential_consistency=1`,
|
|
91
|
+
];
|
|
92
|
+
return lines.join('\n');
|
|
93
|
+
}
|
|
50
94
|
// ── WHERE-clause builders ────────────────────────────────────────────────────
|
|
51
95
|
export function buildWhereClauseFromRanges(partitionId, ranges, sortKeys) {
|
|
52
96
|
const conditions = [`_partition_id = ${quoteSqlString(partitionId)}`];
|
|
@@ -331,6 +375,9 @@ export function splitTopLevel(sql, delimiter) {
|
|
|
331
375
|
return segments;
|
|
332
376
|
}
|
|
333
377
|
// ── Value formatting & condition helpers ─────────────────────────────────────
|
|
378
|
+
function quoteIdentifier(name) {
|
|
379
|
+
return `\`${name.replaceAll('\\', '\\\\').replaceAll('`', '\\`')}\``;
|
|
380
|
+
}
|
|
334
381
|
function quoteSqlString(value) {
|
|
335
382
|
return `'${value.replaceAll('\\', '\\\\').replaceAll('\'', '\\\'')}'`;
|
|
336
383
|
}
|
|
@@ -356,9 +403,9 @@ function buildRangeBoundConditions(range, sortKey) {
|
|
|
356
403
|
}
|
|
357
404
|
function buildSettingsClause(token) {
|
|
358
405
|
if (token) {
|
|
359
|
-
return `SETTINGS async_insert=0, insert_deduplication_token='${token}'`;
|
|
406
|
+
return `SETTINGS async_insert=0, ${ASTERISK_SETTINGS}, insert_deduplication_token='${token}'`;
|
|
360
407
|
}
|
|
361
|
-
return
|
|
408
|
+
return `SETTINGS async_insert=0, ${ASTERISK_SETTINGS}`;
|
|
362
409
|
}
|
|
363
410
|
function buildChunkConditions(chunk, sortKeys) {
|
|
364
411
|
return chunk.ranges.flatMap((range) => {
|