@chkit/plugin-backfill 0.1.0-beta.8 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -0
- package/dist/args.d.ts +17 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +11 -0
- package/dist/args.js.map +1 -1
- package/dist/async-backfill.d.ts +64 -0
- package/dist/async-backfill.d.ts.map +1 -0
- package/dist/async-backfill.js +251 -0
- package/dist/async-backfill.js.map +1 -0
- package/dist/check.d.ts +9 -0
- package/dist/check.d.ts.map +1 -0
- package/dist/check.js +79 -0
- package/dist/check.js.map +1 -0
- package/dist/chunking/analyze.d.ts +8 -0
- package/dist/chunking/analyze.d.ts.map +1 -0
- package/dist/chunking/analyze.js +8 -0
- package/dist/chunking/analyze.js.map +1 -0
- package/dist/chunking/boundary-codec.d.ts +4 -0
- package/dist/chunking/boundary-codec.d.ts.map +1 -0
- package/dist/chunking/boundary-codec.js +79 -0
- package/dist/chunking/boundary-codec.js.map +1 -0
- package/dist/chunking/build.d.ts +11 -0
- package/dist/chunking/build.d.ts.map +1 -0
- package/dist/chunking/build.js +51 -0
- package/dist/chunking/build.js.map +1 -0
- package/dist/chunking/e2e/constants.d.ts +2 -0
- package/dist/chunking/e2e/constants.d.ts.map +1 -0
- package/dist/chunking/e2e/constants.js +2 -0
- package/dist/chunking/e2e/constants.js.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.js +134 -0
- package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
- package/dist/chunking/introspect.d.ts +40 -0
- package/dist/chunking/introspect.d.ts.map +1 -0
- package/dist/chunking/introspect.js +187 -0
- package/dist/chunking/introspect.js.map +1 -0
- package/dist/chunking/partition-slices.d.ts +14 -0
- package/dist/chunking/partition-slices.d.ts.map +1 -0
- package/dist/chunking/partition-slices.js +111 -0
- package/dist/chunking/partition-slices.js.map +1 -0
- package/dist/chunking/planner.d.ts +3 -0
- package/dist/chunking/planner.d.ts.map +1 -0
- package/dist/chunking/planner.js +343 -0
- package/dist/chunking/planner.js.map +1 -0
- package/dist/chunking/services/distribution-source.d.ts +11 -0
- package/dist/chunking/services/distribution-source.d.ts.map +1 -0
- package/dist/chunking/services/distribution-source.js +60 -0
- package/dist/chunking/services/distribution-source.js.map +1 -0
- package/dist/chunking/services/metadata-source.d.ts +4 -0
- package/dist/chunking/services/metadata-source.d.ts.map +1 -0
- package/dist/chunking/services/metadata-source.js +138 -0
- package/dist/chunking/services/metadata-source.js.map +1 -0
- package/dist/chunking/services/row-probe.d.ts +12 -0
- package/dist/chunking/services/row-probe.d.ts.map +1 -0
- package/dist/chunking/services/row-probe.js +49 -0
- package/dist/chunking/services/row-probe.js.map +1 -0
- package/dist/chunking/splitter.d.ts +20 -0
- package/dist/chunking/splitter.d.ts.map +1 -0
- package/dist/chunking/splitter.js +76 -0
- package/dist/chunking/splitter.js.map +1 -0
- package/dist/chunking/sql.d.ts +18 -0
- package/dist/chunking/sql.d.ts.map +1 -0
- package/dist/chunking/sql.js +304 -0
- package/dist/chunking/sql.js.map +1 -0
- package/dist/chunking/strategies/equal-width-split.d.ts +3 -0
- package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
- package/dist/chunking/strategies/equal-width-split.js +46 -0
- package/dist/chunking/strategies/equal-width-split.js.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.js +54 -0
- package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
- package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.js +132 -0
- package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
- package/dist/chunking/strategies/refinement.d.ts +3 -0
- package/dist/chunking/strategies/refinement.d.ts.map +1 -0
- package/dist/chunking/strategies/refinement.js +61 -0
- package/dist/chunking/strategies/refinement.js.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.js +73 -0
- package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts +3 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
- package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
- package/dist/chunking/strategy-policy.d.ts +3 -0
- package/dist/chunking/strategy-policy.d.ts.map +1 -0
- package/dist/chunking/strategy-policy.js +4 -0
- package/dist/chunking/strategy-policy.js.map +1 -0
- package/dist/chunking/types.d.ts +137 -0
- package/dist/chunking/types.d.ts.map +1 -0
- package/dist/chunking/types.js +2 -0
- package/dist/chunking/types.js.map +1 -0
- package/dist/chunking/utils/binary-string.d.ts +8 -0
- package/dist/chunking/utils/binary-string.d.ts.map +1 -0
- package/dist/chunking/utils/binary-string.js +52 -0
- package/dist/chunking/utils/binary-string.js.map +1 -0
- package/dist/chunking/utils/ids.d.ts +4 -0
- package/dist/chunking/utils/ids.d.ts.map +1 -0
- package/dist/chunking/utils/ids.js +11 -0
- package/dist/chunking/utils/ids.js.map +1 -0
- package/dist/chunking/utils/ranges.d.ts +5 -0
- package/dist/chunking/utils/ranges.d.ts.map +1 -0
- package/dist/chunking/utils/ranges.js +19 -0
- package/dist/chunking/utils/ranges.js.map +1 -0
- package/dist/detect.d.ts +8 -0
- package/dist/detect.d.ts.map +1 -0
- package/dist/detect.js +72 -0
- package/dist/detect.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/logging.d.ts +12 -0
- package/dist/logging.d.ts.map +1 -0
- package/dist/logging.js +61 -0
- package/dist/logging.js.map +1 -0
- package/dist/options.d.ts +140 -4
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +145 -111
- package/dist/options.js.map +1 -1
- package/dist/payload.d.ts +7 -17
- package/dist/payload.d.ts.map +1 -1
- package/dist/payload.js +7 -19
- package/dist/payload.js.map +1 -1
- package/dist/planner.d.ts +10 -8
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +76 -97
- package/dist/planner.js.map +1 -1
- package/dist/plugin.d.ts +4 -3
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +351 -201
- package/dist/plugin.js.map +1 -1
- package/dist/queries.d.ts +21 -0
- package/dist/queries.d.ts.map +1 -0
- package/dist/queries.js +113 -0
- package/dist/queries.js.map +1 -0
- package/dist/runtime.d.ts +14 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +162 -83
- package/dist/runtime.js.map +1 -1
- package/dist/sdk.d.ts +12 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +9 -0
- package/dist/sdk.js.map +1 -0
- package/dist/state.d.ts +16 -28
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +73 -127
- package/dist/state.js.map +1 -1
- package/dist/table-config.d.ts +9 -0
- package/dist/table-config.d.ts.map +1 -0
- package/dist/table-config.js +2 -0
- package/dist/table-config.js.map +1 -0
- package/dist/types.d.ts +48 -118
- package/dist/types.d.ts.map +1 -1
- package/package.json +25 -2
package/dist/check.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { computeBackfillStateDir, listPlanIds, readRun, } from './state.js';
|
|
3
|
+
export async function evaluateBackfillCheck(input) {
|
|
4
|
+
const stateDir = computeBackfillStateDir(input.config, input.configPath, input.stateDir);
|
|
5
|
+
const plansDir = join(stateDir, 'plans');
|
|
6
|
+
const runsDir = join(stateDir, 'runs');
|
|
7
|
+
const planIds = await listPlanIds(plansDir);
|
|
8
|
+
if (planIds.length === 0) {
|
|
9
|
+
return {
|
|
10
|
+
plugin: 'backfill',
|
|
11
|
+
evaluated: true,
|
|
12
|
+
ok: true,
|
|
13
|
+
findings: [],
|
|
14
|
+
metadata: {
|
|
15
|
+
requiredCount: 0,
|
|
16
|
+
activeRuns: 0,
|
|
17
|
+
failedRuns: 0,
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
let requiredCount = 0;
|
|
22
|
+
let activeRuns = 0;
|
|
23
|
+
let failedRuns = 0;
|
|
24
|
+
for (const planId of planIds) {
|
|
25
|
+
const runPath = join(runsDir, `${planId}.json`);
|
|
26
|
+
const run = await readRun(runPath);
|
|
27
|
+
if (!run) {
|
|
28
|
+
requiredCount += 1;
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
if (run.status === 'running')
|
|
32
|
+
activeRuns += 1;
|
|
33
|
+
if (run.status === 'failed')
|
|
34
|
+
failedRuns += 1;
|
|
35
|
+
if (run.status !== 'completed')
|
|
36
|
+
requiredCount += 1;
|
|
37
|
+
}
|
|
38
|
+
const findings = [];
|
|
39
|
+
if (requiredCount > 0) {
|
|
40
|
+
findings.push({
|
|
41
|
+
code: 'backfill_required_pending',
|
|
42
|
+
message: `Required backfills pending completion: ${requiredCount}`,
|
|
43
|
+
severity: input.failCheckOnRequiredPendingBackfill ? 'error' : 'warn',
|
|
44
|
+
metadata: {
|
|
45
|
+
requiredCount,
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
if (failedRuns > 0) {
|
|
50
|
+
findings.push({
|
|
51
|
+
code: 'backfill_chunk_failed_retry_exhausted',
|
|
52
|
+
message: `Backfill runs failed after retry budget: ${failedRuns}`,
|
|
53
|
+
severity: 'error',
|
|
54
|
+
metadata: {
|
|
55
|
+
failedRuns,
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (!input.failCheckOnRequiredPendingBackfill) {
|
|
60
|
+
findings.push({
|
|
61
|
+
code: 'backfill_policy_relaxed',
|
|
62
|
+
message: 'Backfill check policy is relaxed: failCheckOnRequiredPendingBackfill=false.',
|
|
63
|
+
severity: 'warn',
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
const ok = findings.every((finding) => finding.severity !== 'error');
|
|
67
|
+
return {
|
|
68
|
+
plugin: 'backfill',
|
|
69
|
+
evaluated: true,
|
|
70
|
+
ok,
|
|
71
|
+
findings,
|
|
72
|
+
metadata: {
|
|
73
|
+
requiredCount,
|
|
74
|
+
activeRuns,
|
|
75
|
+
failedRuns,
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=check.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"check.js","sourceRoot":"","sources":["../src/check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAIhC,OAAO,EACL,uBAAuB,EACvB,WAAW,EACX,OAAO,GACR,MAAM,YAAY,CAAA;AAKnB,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAK3C;IACC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAEtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,IAAI;YACf,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE;gBACR,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;aACd;SACF,CAAA;IACH,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,CAAA;QAC/C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;QAClC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,aAAa,IAAI,CAAC,CAAA;YAClB,SAAQ;QACV,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,UAAU,IAAI,CAAC,CAAA;QAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,UAAU,IAAI,CAAC,CAAA;QAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,aAAa,IAAI,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,QAAQ,GAA0C,EAAE,CAAA;IAC1D,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,2BAA2B;YACjC,OAAO,EAAE,0CAA0C,aAAa,EAAE;YAClE,QAAQ,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YACrE,QAAQ,EAAE;gBACR,aAAa;aACd;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,uCAAuC;YAC7C,OAAO,EAAE,4CAA4C,UAAU,EAAE;YACjE,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE;gBACR,UAAU;aACX;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,kCAAkC,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,yBAAyB;YAC/B,OAAO,EAAE,6EAA6E;YACtF,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IACpE,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,SAAS,EAAE,IAAI;QACf,EAAE;QACF,QAAQ;QACR,QAAQ,EAAE;YACR,aAAa;YACb,UAAU;YACV,UAAU;SACX;KACF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ChunkPlan, GenerateChunkPlanInput } from './types.js';
|
|
2
|
+
export type AnalyzeAndChunkInput = GenerateChunkPlanInput;
|
|
3
|
+
export type AnalyzeAndChunkResult = ChunkPlan;
|
|
4
|
+
export type AnalyzeTableInput = GenerateChunkPlanInput;
|
|
5
|
+
export type AnalyzeTableResult = ChunkPlan;
|
|
6
|
+
export declare function analyzeAndChunk(input: AnalyzeAndChunkInput): Promise<AnalyzeAndChunkResult>;
|
|
7
|
+
export declare function analyzeTable(input: AnalyzeTableInput): Promise<AnalyzeTableResult>;
|
|
8
|
+
//# sourceMappingURL=analyze.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,MAAM,oBAAoB,GAAG,sBAAsB,CAAA;AACzD,MAAM,MAAM,qBAAqB,GAAG,SAAS,CAAA;AAC7C,MAAM,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AACtD,MAAM,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAE1C,wBAAsB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAEjG;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAExF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAQhD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAA2B;IAC/D,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAwB;IACzD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"boundary-codec.d.ts","sourceRoot":"","sources":["../../src/chunking/boundary-codec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,SAAS,EAIV,MAAM,YAAY,CAAA;AA0FnB,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,CAKxE;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,CAKzE"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
function encodeBoundary(value, sortKey) {
|
|
2
|
+
if (value === undefined || sortKey === undefined)
|
|
3
|
+
return value;
|
|
4
|
+
if (sortKey.boundaryEncoding === 'hex-latin1') {
|
|
5
|
+
return Buffer.from(value, 'latin1').toString('hex');
|
|
6
|
+
}
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
function decodeBoundary(value, sortKey) {
|
|
10
|
+
if (value === undefined || sortKey === undefined)
|
|
11
|
+
return value;
|
|
12
|
+
if (sortKey.boundaryEncoding === 'hex-latin1') {
|
|
13
|
+
return Buffer.from(value, 'hex').toString('latin1');
|
|
14
|
+
}
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
function encodeRangesForPlan(ranges, sortKeys) {
|
|
18
|
+
return ranges.map((range) => ({
|
|
19
|
+
dimensionIndex: range.dimensionIndex,
|
|
20
|
+
from: encodeBoundary(range.from, sortKeys[range.dimensionIndex]),
|
|
21
|
+
to: encodeBoundary(range.to, sortKeys[range.dimensionIndex]),
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
24
|
+
function decodeRangesFromPlan(ranges, sortKeys) {
|
|
25
|
+
return ranges.map((range) => ({
|
|
26
|
+
dimensionIndex: range.dimensionIndex,
|
|
27
|
+
from: decodeBoundary(range.from, sortKeys[range.dimensionIndex]),
|
|
28
|
+
to: decodeBoundary(range.to, sortKeys[range.dimensionIndex]),
|
|
29
|
+
}));
|
|
30
|
+
}
|
|
31
|
+
function encodeFocusedValue(focusedValue, sortKeys) {
|
|
32
|
+
if (!focusedValue)
|
|
33
|
+
return undefined;
|
|
34
|
+
return {
|
|
35
|
+
dimensionIndex: focusedValue.dimensionIndex,
|
|
36
|
+
value: encodeBoundary(focusedValue.value, sortKeys[focusedValue.dimensionIndex]) ?? focusedValue.value,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function decodeFocusedValue(focusedValue, sortKeys) {
|
|
40
|
+
if (!focusedValue)
|
|
41
|
+
return undefined;
|
|
42
|
+
return {
|
|
43
|
+
dimensionIndex: focusedValue.dimensionIndex,
|
|
44
|
+
value: decodeBoundary(focusedValue.value, sortKeys[focusedValue.dimensionIndex]) ?? focusedValue.value,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function encodeChunkForPlan(chunk, sortKeys) {
|
|
48
|
+
return {
|
|
49
|
+
...chunk,
|
|
50
|
+
ranges: encodeRangesForPlan(chunk.ranges, sortKeys),
|
|
51
|
+
analysis: {
|
|
52
|
+
...chunk.analysis,
|
|
53
|
+
focusedValue: encodeFocusedValue(chunk.analysis.focusedValue, sortKeys),
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
function decodeChunkFromPlan(chunk, sortKeys) {
|
|
58
|
+
return {
|
|
59
|
+
...chunk,
|
|
60
|
+
ranges: decodeRangesFromPlan(chunk.ranges, sortKeys),
|
|
61
|
+
analysis: {
|
|
62
|
+
...chunk.analysis,
|
|
63
|
+
focusedValue: decodeFocusedValue(chunk.analysis.focusedValue, sortKeys),
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export function encodeChunkPlanForPersistence(plan) {
|
|
68
|
+
return {
|
|
69
|
+
...plan,
|
|
70
|
+
chunks: plan.chunks.map((chunk) => encodeChunkForPlan(chunk, plan.table.sortKeys)),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
export function decodeChunkPlanFromPersistence(plan) {
|
|
74
|
+
return {
|
|
75
|
+
...plan,
|
|
76
|
+
chunks: plan.chunks.map((chunk) => decodeChunkFromPlan(chunk, plan.table.sortKeys)),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=boundary-codec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"boundary-codec.js","sourceRoot":"","sources":["../../src/chunking/boundary-codec.ts"],"names":[],"mappings":"AAQA,SAAS,cAAc,CACrB,KAAyB,EACzB,OAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,OAAO,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,cAAc,CACrB,KAAyB,EACzB,OAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,OAAO,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAoB,EACpB,QAAmB;IAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChE,EAAE,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;KAC7D,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAAoB,EACpB,QAAmB;IAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChE,EAAE,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;KAC7D,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAsC,EACtC,QAAmB;IAEnB,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,KAAK,EAAE,cAAc,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK;KACvG,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAsC,EACtC,QAAmB;IAEnB,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,KAAK,EAAE,cAAc,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK;KACvG,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAY,EAAE,QAAmB;IAC3D,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,mBAAmB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;QACnD,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,YAAY,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;SACxE;KACF,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAY,EAAE,QAAmB;IAC5D,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,oBAAoB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;QACpD,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,YAAY,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;SACxE;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,IAAe;IAC3D,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACnF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,IAAe;IAC5D,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACpF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ChunkBoundary, PartitionInfo, SortKeyInfo } from './types.js';
|
|
2
|
+
export declare function buildChunkBoundaries(input: {
|
|
3
|
+
partitions: PartitionInfo[];
|
|
4
|
+
maxChunkBytes: number;
|
|
5
|
+
sortKey?: SortKeyInfo;
|
|
6
|
+
sortKeyRanges?: Map<string, {
|
|
7
|
+
min: string;
|
|
8
|
+
max: string;
|
|
9
|
+
}>;
|
|
10
|
+
}): ChunkBoundary[];
|
|
11
|
+
//# sourceMappingURL=build.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/chunking/build.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE3E,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAC1D,GAAG,aAAa,EAAE,CAmDlB"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { splitSortKeyRange } from './splitter.js';
|
|
2
|
+
export function buildChunkBoundaries(input) {
|
|
3
|
+
const boundaries = [];
|
|
4
|
+
for (const partition of input.partitions) {
|
|
5
|
+
if (partition.bytesOnDisk <= input.maxChunkBytes) {
|
|
6
|
+
boundaries.push({
|
|
7
|
+
partitionId: partition.partitionId,
|
|
8
|
+
estimatedBytes: partition.bytesOnDisk,
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
else if (input.sortKey && input.sortKeyRanges) {
|
|
12
|
+
const range = input.sortKeyRanges.get(partition.partitionId);
|
|
13
|
+
if (!range) {
|
|
14
|
+
// No range data — emit as single chunk
|
|
15
|
+
boundaries.push({
|
|
16
|
+
partitionId: partition.partitionId,
|
|
17
|
+
estimatedBytes: partition.bytesOnDisk,
|
|
18
|
+
});
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
// If min === max, splitting would produce empty sub-ranges; emit as single chunk
|
|
22
|
+
if (range.min === range.max) {
|
|
23
|
+
boundaries.push({
|
|
24
|
+
partitionId: partition.partitionId,
|
|
25
|
+
estimatedBytes: partition.bytesOnDisk,
|
|
26
|
+
});
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
const subCount = Math.ceil(partition.bytesOnDisk / input.maxChunkBytes);
|
|
30
|
+
const subRanges = splitSortKeyRange(input.sortKey.category, range.min, range.max, subCount);
|
|
31
|
+
const estimatedBytesPerSub = Math.ceil(partition.bytesOnDisk / subCount);
|
|
32
|
+
for (const sub of subRanges) {
|
|
33
|
+
boundaries.push({
|
|
34
|
+
partitionId: partition.partitionId,
|
|
35
|
+
sortKeyFrom: sub.from,
|
|
36
|
+
sortKeyTo: sub.to,
|
|
37
|
+
estimatedBytes: estimatedBytesPerSub,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
// No sort key info — emit as single chunk despite being oversized
|
|
43
|
+
boundaries.push({
|
|
44
|
+
partitionId: partition.partitionId,
|
|
45
|
+
estimatedBytes: partition.bytesOnDisk,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return boundaries;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=build.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/chunking/build.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAGjD,MAAM,UAAU,oBAAoB,CAAC,KAKpC;IACC,MAAM,UAAU,GAAoB,EAAE,CAAA;IAEtC,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,SAAS,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACjD,UAAU,CAAC,IAAI,CAAC;gBACd,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;aACtC,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAA;YAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,uCAAuC;gBACvC,UAAU,CAAC,IAAI,CAAC;oBACd,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;iBACtC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,iFAAiF;YACjF,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,EAAE,CAAC;gBAC5B,UAAU,CAAC,IAAI,CAAC;oBACd,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;iBACtC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;YACvE,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YAC3F,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,GAAG,QAAQ,CAAC,CAAA;YAExE,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,UAAU,CAAC,IAAI,CAAC;oBACd,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,WAAW,EAAE,GAAG,CAAC,IAAI;oBACrB,SAAS,EAAE,GAAG,CAAC,EAAE;oBACjB,cAAc,EAAE,oBAAoB;iBACrC,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,kEAAkE;YAClE,UAAU,CAAC,IAAI,CAAC;gBACd,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,cAAc,EAAE,SAAS,CAAC,WAAW;aACtC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAA;AACnB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,uBAAuB,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../src/chunking/e2e/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,oBAAoB,CAAA"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Seeds ClickHouse tables for smart-chunking E2E tests.
|
|
4
|
+
*
|
|
5
|
+
* Run once manually:
|
|
6
|
+
* bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
|
|
7
|
+
*
|
|
8
|
+
* Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
|
|
9
|
+
* Creates tables if they don't exist, truncates them, and re-inserts data.
|
|
10
|
+
*/
|
|
11
|
+
interface DatasetConfig {
|
|
12
|
+
name: string;
|
|
13
|
+
columns: string;
|
|
14
|
+
orderBy: string;
|
|
15
|
+
partitionBy: string;
|
|
16
|
+
generate: () => Record<string, unknown>[];
|
|
17
|
+
}
|
|
18
|
+
export declare const datasets: DatasetConfig[];
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=seed-datasets.script.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed-datasets.script.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;;;;GAQG;AAOH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;CAC1C;AAUD,eAAO,MAAM,QAAQ,EAAE,aAAa,EA+EnC,CAAA"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Seeds ClickHouse tables for smart-chunking E2E tests.
|
|
4
|
+
*
|
|
5
|
+
* Run once manually:
|
|
6
|
+
* bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
|
|
7
|
+
*
|
|
8
|
+
* Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
|
|
9
|
+
* Creates tables if they don't exist, truncates them, and re-inserts data.
|
|
10
|
+
*/
|
|
11
|
+
import { randomBytes } from 'node:crypto';
|
|
12
|
+
import { getRequiredEnv, createLiveExecutor } from '@chkit/clickhouse/e2e-testkit';
|
|
13
|
+
import { TABLE_PREFIX } from './constants.js';
|
|
14
|
+
function pad(bytes) {
|
|
15
|
+
return randomBytes(bytes).toString('hex');
|
|
16
|
+
}
|
|
17
|
+
function dayHour(day, hour) {
|
|
18
|
+
return `2026-01-${String(day).padStart(2, '0')} ${String(hour).padStart(2, '0')}:00:00`;
|
|
19
|
+
}
|
|
20
|
+
export const datasets = [
|
|
21
|
+
{
|
|
22
|
+
name: 'skewed_power_law',
|
|
23
|
+
columns: [
|
|
24
|
+
'tenant_id String',
|
|
25
|
+
'seq UInt64',
|
|
26
|
+
'event_time DateTime',
|
|
27
|
+
'padding String',
|
|
28
|
+
].join(', '),
|
|
29
|
+
orderBy: '(tenant_id, seq)',
|
|
30
|
+
partitionBy: 'toYYYYMM(event_time)',
|
|
31
|
+
generate() {
|
|
32
|
+
const rows = [];
|
|
33
|
+
// 80%: single dominant tenant — 8,000 rows
|
|
34
|
+
for (let i = 0; i < 8000; i++) {
|
|
35
|
+
rows.push({
|
|
36
|
+
tenant_id: 'mega-corp',
|
|
37
|
+
seq: i,
|
|
38
|
+
event_time: dayHour(1 + (i % 28), i % 24),
|
|
39
|
+
padding: pad(512),
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
// 20%: 200 small tenants, 10 rows each — 2,000 rows
|
|
43
|
+
for (let t = 0; t < 200; t++) {
|
|
44
|
+
for (let i = 0; i < 10; i++) {
|
|
45
|
+
rows.push({
|
|
46
|
+
tenant_id: `tenant-${String(t).padStart(4, '0')}`,
|
|
47
|
+
seq: i,
|
|
48
|
+
event_time: dayHour(1 + ((t * 10 + i) % 28), (t + i) % 24),
|
|
49
|
+
padding: pad(512),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return rows;
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
name: 'multiple_hot_keys',
|
|
58
|
+
columns: [
|
|
59
|
+
'tenant_id String',
|
|
60
|
+
'seq UInt64',
|
|
61
|
+
'event_time DateTime',
|
|
62
|
+
'padding String',
|
|
63
|
+
].join(', '),
|
|
64
|
+
orderBy: '(tenant_id, seq)',
|
|
65
|
+
partitionBy: 'toYYYYMM(event_time)',
|
|
66
|
+
generate() {
|
|
67
|
+
const rows = [];
|
|
68
|
+
// 3 hot tenants, ~30% each — 3,000 rows each = 9,000 rows
|
|
69
|
+
for (const tenant of ['alpha-corp', 'beta-corp', 'gamma-corp']) {
|
|
70
|
+
for (let i = 0; i < 3000; i++) {
|
|
71
|
+
rows.push({
|
|
72
|
+
tenant_id: tenant,
|
|
73
|
+
seq: i,
|
|
74
|
+
event_time: dayHour(1 + (i % 28), i % 24),
|
|
75
|
+
padding: pad(512),
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// 10%: 100 small tenants, 10 rows each — 1,000 rows
|
|
80
|
+
for (let t = 0; t < 100; t++) {
|
|
81
|
+
for (let i = 0; i < 10; i++) {
|
|
82
|
+
rows.push({
|
|
83
|
+
tenant_id: `small-${String(t).padStart(4, '0')}`,
|
|
84
|
+
seq: i,
|
|
85
|
+
event_time: dayHour(1 + ((t * 10 + i) % 28), (t + i) % 24),
|
|
86
|
+
padding: pad(512),
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return rows;
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
];
|
|
94
|
+
const BATCH_SIZE = 5000;
|
|
95
|
+
async function seed() {
|
|
96
|
+
const env = getRequiredEnv();
|
|
97
|
+
const executor = createLiveExecutor(env);
|
|
98
|
+
const db = env.clickhouseDatabase;
|
|
99
|
+
try {
|
|
100
|
+
for (const dataset of datasets) {
|
|
101
|
+
const table = `${TABLE_PREFIX}_${dataset.name}`;
|
|
102
|
+
const fqn = `${db}.${table}`;
|
|
103
|
+
console.log(`\n--- Seeding ${fqn} ---`);
|
|
104
|
+
await executor.command(`
|
|
105
|
+
CREATE TABLE IF NOT EXISTS ${fqn} (
|
|
106
|
+
${dataset.columns}
|
|
107
|
+
) ENGINE = MergeTree()
|
|
108
|
+
PARTITION BY ${dataset.partitionBy}
|
|
109
|
+
ORDER BY ${dataset.orderBy}
|
|
110
|
+
`);
|
|
111
|
+
console.log(' Table ensured.');
|
|
112
|
+
await executor.command(`TRUNCATE TABLE ${fqn}`);
|
|
113
|
+
console.log(' Truncated.');
|
|
114
|
+
const rows = dataset.generate();
|
|
115
|
+
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
|
|
116
|
+
const batch = rows.slice(i, i + BATCH_SIZE);
|
|
117
|
+
await executor.insert({ table: fqn, values: batch });
|
|
118
|
+
console.log(` Inserted ${Math.min(i + BATCH_SIZE, rows.length)} / ${rows.length} rows`);
|
|
119
|
+
}
|
|
120
|
+
// Verify
|
|
121
|
+
const [result] = await executor.query(`SELECT count() AS cnt FROM ${fqn} SETTINGS select_sequential_consistency = 1`);
|
|
122
|
+
console.log(` Verified: ${result?.cnt} rows`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
await executor.close();
|
|
127
|
+
}
|
|
128
|
+
console.log('\nDone!');
|
|
129
|
+
}
|
|
130
|
+
seed().catch((error) => {
|
|
131
|
+
console.error(error);
|
|
132
|
+
process.exit(1);
|
|
133
|
+
});
|
|
134
|
+
//# sourceMappingURL=seed-datasets.script.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed-datasets.script.js","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;;;;GAQG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAA;AAElF,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAU7C,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,IAAY;IACxC,OAAO,WAAW,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAA;AACzF,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAoB;IACvC;QACE,IAAI,EAAE,kBAAkB;QACxB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC;oBACR,SAAS,EAAE,WAAW;oBACtB,GAAG,EAAE,CAAC;oBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;oBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;iBAClB,CAAC,CAAA;YACJ,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBACjD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,0DAA0D;YAC1D,KAAK,MAAM,MAAM,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,MAAM;wBACjB,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;wBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,SAAS,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBAChD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;CACF,CAAA;AAED,MAAM,UAAU,GAAG,IAAI,CAAA;AAEvB,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,cAAc,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,EAAE,GAAG,GAAG,CAAC,kBAAkB,CAAA;IAEjC,IAAI,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,IAAI,EAAE,CAAA;YAC/C,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,KAAK,EAAE,CAAA;YAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,CAAA;YAEvC,MAAM,QAAQ,CAAC,OAAO,CAAC;qCACQ,GAAG;YAC5B,OAAO,CAAC,OAAO;;uBAEJ,OAAO,CAAC,WAAW;mBACvB,OAAO,CAAC,OAAO;OAC3B,CAAC,CAAA;YACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;YAE/B,MAAM,QAAQ,CAAC,OAAO,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;YAC/C,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;YAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAA;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC3C,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;gBACpD,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,OAAO,CAAC,CAAA;YAC1F,CAAC;YAED,SAAS;YACT,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CACnC,8BAA8B,GAAG,6CAA6C,CAC/E,CAAA;YACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,EAAE,GAAG,OAAO,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AACxB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { PartitionInfo, SortKeyInfo } from './types.js';
|
|
2
|
+
export declare function queryPartitionInfo(input: {
|
|
3
|
+
database: string;
|
|
4
|
+
table: string;
|
|
5
|
+
from?: string;
|
|
6
|
+
to?: string;
|
|
7
|
+
query: <T>(sql: string) => Promise<T[]>;
|
|
8
|
+
}): Promise<PartitionInfo[]>;
|
|
9
|
+
export declare function querySortKeys(input: {
|
|
10
|
+
database: string;
|
|
11
|
+
table: string;
|
|
12
|
+
query: <T>(sql: string) => Promise<T[]>;
|
|
13
|
+
}): Promise<SortKeyInfo[]>;
|
|
14
|
+
export declare function querySortKeyInfo(input: {
|
|
15
|
+
database: string;
|
|
16
|
+
table: string;
|
|
17
|
+
query: <T>(sql: string) => Promise<T[]>;
|
|
18
|
+
}): Promise<SortKeyInfo | undefined>;
|
|
19
|
+
export declare function querySortKeyRanges(input: {
|
|
20
|
+
database: string;
|
|
21
|
+
table: string;
|
|
22
|
+
sortKeyColumn: string;
|
|
23
|
+
partitionIds: string[];
|
|
24
|
+
query: <T>(sql: string) => Promise<T[]>;
|
|
25
|
+
}): Promise<Map<string, {
|
|
26
|
+
min: string;
|
|
27
|
+
max: string;
|
|
28
|
+
}>>;
|
|
29
|
+
export declare function introspectTable(input: {
|
|
30
|
+
database: string;
|
|
31
|
+
table: string;
|
|
32
|
+
from?: string;
|
|
33
|
+
to?: string;
|
|
34
|
+
query: <T>(sql: string) => Promise<T[]>;
|
|
35
|
+
}): Promise<{
|
|
36
|
+
partitions: PartitionInfo[];
|
|
37
|
+
sortKey?: SortKeyInfo;
|
|
38
|
+
sortKeys: SortKeyInfo[];
|
|
39
|
+
}>;
|
|
40
|
+
//# sourceMappingURL=introspect.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"introspect.d.ts","sourceRoot":"","sources":["../../src/chunking/introspect.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAkB5D,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CA+C3B;AA2FD,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CA6CzB;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE;IAC5C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAEnC;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAiBrD;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,aAAa,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,WAAW,CAAC;IAAC,QAAQ,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC,CAS3F"}
|