@chkit/plugin-backfill 0.1.0-beta.20 → 0.1.0-beta.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +130 -0
  2. package/dist/async-backfill.d.ts +4 -4
  3. package/dist/async-backfill.d.ts.map +1 -1
  4. package/dist/async-backfill.js +2 -2
  5. package/dist/async-backfill.js.map +1 -1
  6. package/dist/chunking/analyze.d.ts +5 -35
  7. package/dist/chunking/analyze.d.ts.map +1 -1
  8. package/dist/chunking/analyze.js +3 -71
  9. package/dist/chunking/analyze.js.map +1 -1
  10. package/dist/chunking/boundary-codec.d.ts +4 -0
  11. package/dist/chunking/boundary-codec.d.ts.map +1 -0
  12. package/dist/chunking/boundary-codec.js +79 -0
  13. package/dist/chunking/boundary-codec.js.map +1 -0
  14. package/dist/chunking/e2e/constants.d.ts +2 -0
  15. package/dist/chunking/e2e/constants.d.ts.map +1 -0
  16. package/dist/chunking/e2e/constants.js +2 -0
  17. package/dist/chunking/e2e/constants.js.map +1 -0
  18. package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
  19. package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
  20. package/dist/chunking/e2e/seed-datasets.script.js +134 -0
  21. package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
  22. package/dist/chunking/introspect.d.ts +6 -0
  23. package/dist/chunking/introspect.d.ts.map +1 -1
  24. package/dist/chunking/introspect.js +113 -22
  25. package/dist/chunking/introspect.js.map +1 -1
  26. package/dist/chunking/partition-slices.d.ts +14 -0
  27. package/dist/chunking/partition-slices.d.ts.map +1 -0
  28. package/dist/chunking/partition-slices.js +111 -0
  29. package/dist/chunking/partition-slices.js.map +1 -0
  30. package/dist/chunking/planner.d.ts +3 -0
  31. package/dist/chunking/planner.d.ts.map +1 -0
  32. package/dist/chunking/planner.js +343 -0
  33. package/dist/chunking/planner.js.map +1 -0
  34. package/dist/chunking/services/distribution-source.d.ts +11 -0
  35. package/dist/chunking/services/distribution-source.d.ts.map +1 -0
  36. package/dist/chunking/services/distribution-source.js +60 -0
  37. package/dist/chunking/services/distribution-source.js.map +1 -0
  38. package/dist/chunking/services/metadata-source.d.ts +4 -0
  39. package/dist/chunking/services/metadata-source.d.ts.map +1 -0
  40. package/dist/chunking/services/metadata-source.js +138 -0
  41. package/dist/chunking/services/metadata-source.js.map +1 -0
  42. package/dist/chunking/services/row-probe.d.ts +12 -0
  43. package/dist/chunking/services/row-probe.d.ts.map +1 -0
  44. package/dist/chunking/services/row-probe.js +49 -0
  45. package/dist/chunking/services/row-probe.js.map +1 -0
  46. package/dist/chunking/sql.d.ts +11 -5
  47. package/dist/chunking/sql.d.ts.map +1 -1
  48. package/dist/chunking/sql.js +190 -107
  49. package/dist/chunking/sql.js.map +1 -1
  50. package/dist/chunking/strategies/equal-width-split.d.ts +3 -0
  51. package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
  52. package/dist/chunking/strategies/equal-width-split.js +46 -0
  53. package/dist/chunking/strategies/equal-width-split.js.map +1 -0
  54. package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
  55. package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
  56. package/dist/chunking/strategies/group-by-key-split.js +54 -0
  57. package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
  58. package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
  59. package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
  60. package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
  61. package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
  62. package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
  63. package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
  64. package/dist/chunking/strategies/quantile-range-split.js +132 -0
  65. package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
  66. package/dist/chunking/strategies/refinement.d.ts +3 -0
  67. package/dist/chunking/strategies/refinement.d.ts.map +1 -0
  68. package/dist/chunking/strategies/refinement.js +61 -0
  69. package/dist/chunking/strategies/refinement.js.map +1 -0
  70. package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
  71. package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
  72. package/dist/chunking/strategies/string-prefix-split.js +73 -0
  73. package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
  74. package/dist/chunking/strategies/temporal-bucket-split.d.ts +3 -0
  75. package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
  76. package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
  77. package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
  78. package/dist/chunking/strategy-policy.d.ts +3 -0
  79. package/dist/chunking/strategy-policy.d.ts.map +1 -0
  80. package/dist/chunking/strategy-policy.js +4 -0
  81. package/dist/chunking/strategy-policy.js.map +1 -0
  82. package/dist/chunking/types.d.ts +126 -18
  83. package/dist/chunking/types.d.ts.map +1 -1
  84. package/dist/chunking/utils/binary-string.d.ts +8 -0
  85. package/dist/chunking/utils/binary-string.d.ts.map +1 -0
  86. package/dist/chunking/utils/binary-string.js +52 -0
  87. package/dist/chunking/utils/binary-string.js.map +1 -0
  88. package/dist/chunking/utils/ids.d.ts +4 -0
  89. package/dist/chunking/utils/ids.d.ts.map +1 -0
  90. package/dist/chunking/utils/ids.js +11 -0
  91. package/dist/chunking/utils/ids.js.map +1 -0
  92. package/dist/chunking/utils/ranges.d.ts +5 -0
  93. package/dist/chunking/utils/ranges.d.ts.map +1 -0
  94. package/dist/chunking/utils/ranges.js +19 -0
  95. package/dist/chunking/utils/ranges.js.map +1 -0
  96. package/dist/detect.d.ts +0 -5
  97. package/dist/detect.d.ts.map +1 -1
  98. package/dist/detect.js +0 -41
  99. package/dist/detect.js.map +1 -1
  100. package/dist/index.d.ts +0 -5
  101. package/dist/index.d.ts.map +1 -1
  102. package/dist/index.js +0 -2
  103. package/dist/index.js.map +1 -1
  104. package/dist/logging.d.ts +12 -0
  105. package/dist/logging.d.ts.map +1 -0
  106. package/dist/logging.js +61 -0
  107. package/dist/logging.js.map +1 -0
  108. package/dist/options.d.ts +6 -15
  109. package/dist/options.d.ts.map +1 -1
  110. package/dist/options.js +1 -3
  111. package/dist/options.js.map +1 -1
  112. package/dist/payload.d.ts.map +1 -1
  113. package/dist/payload.js +4 -6
  114. package/dist/payload.js.map +1 -1
  115. package/dist/planner.d.ts +2 -1
  116. package/dist/planner.d.ts.map +1 -1
  117. package/dist/planner.js +31 -52
  118. package/dist/planner.js.map +1 -1
  119. package/dist/plugin.d.ts.map +1 -1
  120. package/dist/plugin.js +27 -11
  121. package/dist/plugin.js.map +1 -1
  122. package/dist/queries.js +3 -3
  123. package/dist/queries.js.map +1 -1
  124. package/dist/sdk.d.ts +12 -0
  125. package/dist/sdk.d.ts.map +1 -0
  126. package/dist/sdk.js +9 -0
  127. package/dist/sdk.js.map +1 -0
  128. package/dist/state.d.ts.map +1 -1
  129. package/dist/state.js +16 -5
  130. package/dist/state.js.map +1 -1
  131. package/dist/types.d.ts +12 -25
  132. package/dist/types.d.ts.map +1 -1
  133. package/package.json +11 -3
package/README.md CHANGED
@@ -35,6 +35,136 @@ export default defineConfig({
35
35
 
36
36
  See the [chkit documentation](https://chkit.obsessiondb.com).
37
37
 
38
+ ## SDK
39
+
40
+ The package root is limited to the plugin registration API. Everything used by the CLI itself — the chunk planner, SQL builders, async executor, logging — is also exported from the `@chkit/plugin-backfill/sdk` subpath so you can build your own backfill scripts without going through the CLI.
41
+
42
+ ```ts
43
+ import {
44
+ generateChunkPlan,
45
+ buildChunkExecutionSql,
46
+ executeBackfill,
47
+ getBackfillLogger,
48
+ type ChunkPlan,
49
+ type PlannerQuery,
50
+ } from '@chkit/plugin-backfill/sdk'
51
+ ```
52
+
53
+ The pipeline has three stages, and you can use any subset:
54
+
55
+ 1. **Plan** — `generateChunkPlan(...)` introspects a table and returns a `ChunkPlan` describing how to partition the work into roughly equal-sized chunks.
56
+ 2. **Build SQL** — `buildChunkExecutionSql(...)` turns a single `Chunk` into an `INSERT … SELECT` statement.
57
+ 3. **Execute** — `executeBackfill(...)` submits chunks against a real `ClickHouseExecutor` with deterministic query IDs, polling, and resume support.
58
+
59
+ ### Plan a backfill
60
+
61
+ `generateChunkPlan` is decoupled from any ClickHouse client. You pass in a `query` function with the `PlannerQuery` shape and the planner uses it for every introspection / probe / split query. This makes the planner trivial to instrument or run against alternative clients.
62
+
63
+ ```ts
64
+ import { createClient } from '@clickhouse/client'
65
+ import { generateChunkPlan, type PlannerQuery } from '@chkit/plugin-backfill/sdk'
66
+
67
+ const client = createClient({ url: process.env.CLICKHOUSE_URL })
68
+
69
+ const query: PlannerQuery = async (sql, settings) => {
70
+ const result = await client.query({
71
+ query: sql,
72
+ format: 'JSONEachRow',
73
+ clickhouse_settings: settings as Record<string, string | number | boolean>,
74
+ })
75
+ return result.json()
76
+ }
77
+
78
+ const plan = await generateChunkPlan({
79
+ database: 'analytics',
80
+ table: 'events',
81
+ from: '2025-01-01T00:00:00Z',
82
+ to: '2025-02-01T00:00:00Z',
83
+ targetChunkBytes: 1_000_000_000, // ~1 GiB per chunk
84
+ query,
85
+ // 'count' is exact but slower; 'explain-estimate' is faster but approximate
86
+ rowProbeStrategy: 'count',
87
+ })
88
+
89
+ console.log(`${plan.chunks.length} chunks, ${plan.totalRows.toLocaleString()} rows`)
90
+ ```
91
+
92
+ ### Execute chunks against a target
93
+
94
+ `buildChunkExecutionSql` produces the per-chunk `INSERT … SELECT` and `executeBackfill` runs them with concurrency, polling, and progress callbacks. Persist the `progress` argument anywhere you like to support resume.
95
+
96
+ ```ts
97
+ import { createClickHouseExecutor } from '@chkit/clickhouse'
98
+ import {
99
+ buildChunkExecutionSql,
100
+ executeBackfill,
101
+ type BackfillProgress,
102
+ } from '@chkit/plugin-backfill/sdk'
103
+
104
+ const executor = createClickHouseExecutor({
105
+ url: process.env.CLICKHOUSE_URL!,
106
+ username: 'default',
107
+ password: process.env.CLICKHOUSE_PASSWORD!,
108
+ database: 'analytics',
109
+ })
110
+
111
+ const chunksById = new Map(plan.chunks.map((chunk) => [chunk.id, chunk]))
112
+ let saved: BackfillProgress | undefined // load from disk for resume
113
+
114
+ const result = await executeBackfill({
115
+ executor,
116
+ planId: plan.planId,
117
+ chunks: plan.chunks,
118
+ buildQuery: ({ id }) =>
119
+ buildChunkExecutionSql({
120
+ planId: plan.planId,
121
+ chunk: chunksById.get(id)!,
122
+ target: 'analytics.events_backfill',
123
+ table: plan.table,
124
+ }),
125
+ concurrency: 4,
126
+ pollIntervalMs: 5_000,
127
+ resumeFrom: saved,
128
+ onProgress: async (progress) => {
129
+ saved = progress
130
+ // persist to disk / state store
131
+ },
132
+ })
133
+
134
+ console.log(`done=${result.completed} failed=${result.failed}`)
135
+ ```
136
+
137
+ ### Plan persistence
138
+
139
+ Plans contain string boundaries that may include non-UTF-8 bytes (the planner uses `latin1`-encoded byte ranges for string sort keys), so JSON-serializing a `ChunkPlan` directly will lose information. Use the codec helpers when you need to round-trip a plan through storage:
140
+
141
+ ```ts
142
+ import {
143
+ encodeChunkPlanForPersistence,
144
+ decodeChunkPlanFromPersistence,
145
+ } from '@chkit/plugin-backfill/sdk'
146
+
147
+ const json = JSON.stringify(encodeChunkPlanForPersistence(plan))
148
+ // later …
149
+ const plan2 = decodeChunkPlanFromPersistence(JSON.parse(json))
150
+ ```
151
+
152
+ ### Logging
153
+
154
+ The planner emits structured logs via [`@logtape/logtape`](https://logtape.org/) under the `['chkit', 'backfill']` category. Configure a sink at process start to see them — slow-query warnings (>5 s) are emitted at `warning` level, planning progress at `info`, and per-strategy decisions at `debug`.
155
+
156
+ ```ts
157
+ import { configureSync, getConsoleSink, getTextFormatter } from '@chkit/plugin-backfill/sdk'
158
+
159
+ configureSync({
160
+ sinks: { console: getConsoleSink({ formatter: getTextFormatter({ timestamp: 'time' }) }) },
161
+ loggers: [{ category: 'chkit', sinks: ['console'], lowestLevel: 'info' }],
162
+ reset: true,
163
+ })
164
+ ```
165
+
166
+ To capture every SQL statement the planner runs (with timing, server-side stats, and per-strategy classification), wrap your `query` function instead of relying solely on logging — the wrapper sees the raw SQL and settings on every call and can record query IDs, response headers, and durations alongside the structured logs.
167
+
38
168
  ## License
39
169
 
40
170
  [MIT](../../LICENSE)
@@ -7,15 +7,15 @@ export interface BackfillOptions {
7
7
  /** The chunks to process (from buildChunks) */
8
8
  chunks: Array<{
9
9
  id: string;
10
- from: string;
11
- to: string;
10
+ from?: string;
11
+ to?: string;
12
12
  [key: string]: unknown;
13
13
  }>;
14
14
  /** Build the SQL for a given chunk. Called once per chunk at submit time. */
15
15
  buildQuery: (chunk: {
16
16
  id: string;
17
- from: string;
18
- to: string;
17
+ from?: string;
18
+ to?: string;
19
19
  }) => string;
20
20
  /** Max concurrent queries running on the server. Default: 3 */
21
21
  concurrency?: number;
@@ -1 +1 @@
1
- {"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAA;AAGxE,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;IAC/E,6EAA6E;IAC7E,UAAU,EAAE,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAA;IACvE,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AA2ED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAoF3B;AA2CD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAsGvF"}
1
+ {"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAA;AAGxE,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;IACjF,6EAA6E;IAC7E,UAAU,EAAE,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAA;IACzE,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AA2ED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAoF3B;AA2CD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAsGvF"}
@@ -81,10 +81,10 @@ export async function syncProgress(executor, planId, chunks, progress) {
81
81
  return progress;
82
82
  // Escape single-quotes in the prefix for safe SQL embedding
83
83
  const safePrefix = prefix.replace(/'/g, "''").replace(/%/g, '\\%').replace(/_/g, '\\_');
84
- const runningRows = await executor.query(`SELECT query_id FROM clusterAllReplicas('parallel_replicas', system.processes) WHERE user = currentUser() AND query_id LIKE '${safePrefix}%' SETTINGS skip_unavailable_shards = 1`);
84
+ const runningRows = await executor.query(`SELECT query_id FROM clusterAllReplicas('cluster', system.processes) WHERE user = currentUser() AND query_id LIKE '${safePrefix}%' SETTINGS skip_unavailable_shards = 1`);
85
85
  const runningSet = new Set(runningRows.map((r) => r.query_id));
86
86
  const logRows = await executor.query(`SELECT query_id, type, written_rows, written_bytes, query_duration_ms, exception
87
- FROM clusterAllReplicas('parallel_replicas', system.query_log)
87
+ FROM clusterAllReplicas('cluster', system.query_log)
88
88
  WHERE user = currentUser()
89
89
  AND query_id LIKE '${safePrefix}%'
90
90
  AND type IN ('QueryFinish', 'ExceptionWhileProcessing')
@@ -1 +1 @@
1
- {"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,OAAO,CAAA;AAgDxB,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,oDAAoD;AACpD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe;IACnD,OAAO,YAAY,MAAM,IAAI,OAAO,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAyB,EACzB,EAAe;IAEf,IAAI,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAuB;YAC/B,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAA;QACD,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,KAAK,SAAS;YAC1B,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,QAAQ;YAC9B,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,WAAW;YACpC,KAAK,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAA;QAClC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;IACjD,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;aAC9B;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,KAAK,EAAE,EAAE,CAAC,KAAK;aAChB;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,sEAAsE;IACtE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,QAA0B,EAAE,EAAU;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAA;IAChE,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,EAAU,EACV,IAAwB;IAExB,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA4B,EAC5B,MAAc,EACd,MAA6B,EAC7B,QAA0B;IAE1B,MAAM,MAAM,GAAG,YAAY,MAAM,GAAG,CAAA;IAEpC,qEAAqE;IACrE,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;YAAE,SAAQ;QAC/C,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhD,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEvF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CACtC,gIAAgI,UAAU,yCAAyC,CACpL,CAAA;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAQlC;;;uBAGmB,UAAU;;;;qCAII,CAClC,CAAA;IAED,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAA;IACjE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;IAE7B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAChC,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;QACrF,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAChD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACpC,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,MAAM;wBACd,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAC1C,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;wBAC5C,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;qBAC/C,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,QAAQ;wBAChB,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;wBAC9C,KAAK,EAAE,QAAQ,CAAC,SAAS;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAA4B,EAC5B,OAA2B,EAC3B,cAAsB,EACtB,aAAqB,EACrB,SAA8D;IAE9D,IAAI,KAAK,GAAG,OAAO,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,MAAK;QACzB,IAAI,EAAe,CAAA;QACnB,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;gBAC7C,SAAS,EAAE,KAAK,CAAC,WAAW;aAC7B,CAAC,CAAA;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAA;YACnB,IAAI,iBAAiB,IAAI,aAAa,EAAE,CAAC;gBACvC,KAAK,GAAG;oBACN,GAAG,KAAK;oBACR,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,KAAK,EAAE,iCAAiC,iBAAiB,0BAA0B;iBACpF,CAAA;gBACD,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;gBACtB,MAAK;YACP,CAAC;YACD,SAAQ;QACV,CAAC;QACD,iBAAiB,GAAG,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;YACpB,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAwB;IAC5D,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,MAAM,EACN,UAAU,EACV,WAAW,GAAG,CAAC,EACf,cAAc,GAAG,IAAI,EACrB,aAAa,GAAG,EAAE,EAClB,UAAU,EACV,UAAU,EACV,YAAY,GACb,GAAG,OAAO,CAAA;IAEX,IAAI,QAAQ,GAAqB,MAAM,CAAC,WAAW,CACjD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAkB,EAAE,CAAC,CAAA;IAC9E,CAAC,CAAC,CACH,CAAA;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,uEAAuE;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;IACnE,CAAC;IAED,uEAAuE;IACvE,2EAA2E;IAC3E,gFAAgF;IAChF,IAAI,YAAY,EAAE,CAAC;QACjB,iEAAiE;QACjE,oEAAoE;QACpE,iEAAiE;QACjE,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACnE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAChC,IAAI,KAAK,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAA;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,IAAwB,EAAE,EAAE;QACxD,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QAC1C,OAAO,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC,CAAA;IAED,MAAM,IAAI,CACR,MAAM,EACN,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1C,uCAAuC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAM;QAEhE,wCAAwC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;gBAC7F,OAAM;YACR,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACnC,MAAM,SAAS,GAAuB;YACpC,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,WAAW;YACnB,OAAO;SACR,CAAA;QACD,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAEnC,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACnG,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAA;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;IAExF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,SAAS;QACT,MAAM;QACN,QAAQ;KACT,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,OAAO,CAAA;AAgDxB,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,oDAAoD;AACpD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe;IACnD,OAAO,YAAY,MAAM,IAAI,OAAO,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAyB,EACzB,EAAe;IAEf,IAAI,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAuB;YAC/B,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAA;QACD,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,KAAK,SAAS;YAC1B,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,QAAQ;YAC9B,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,WAAW;YACpC,KAAK,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAA;QAClC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;IACjD,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;aAC9B;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,KAAK,EAAE,EAAE,CAAC,KAAK;aAChB;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,sEAAsE;IACtE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,QAA0B,EAAE,EAAU;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAA;IAChE,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,EAAU,EACV,IAAwB;IAExB,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA4B,EAC5B,MAAc,EACd,MAA6B,EAC7B,QAA0B;IAE1B,MAAM,MAAM,GAAG,YAAY,MAAM,GAAG,CAAA;IAEpC,qEAAqE;IACrE,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;YAAE,SAAQ;QAC/C,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhD,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEvF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CACtC,sHAAsH,UAAU,yCAAyC,CAC1K,CAAA;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAQlC;;;uBAGmB,UAAU;;;;qCAII,CAClC,CAAA;IAED,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAA;IACjE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;IAE7B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAChC,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;QACrF,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAChD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACpC,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,MAAM;wBACd,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAC1C,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;wBAC5C,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;qBAC/C,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,QAAQ;wBAChB,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;wBAC9C,KAAK,EAAE,QAAQ,CAAC,SAAS;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAA4B,EAC5B,OAA2B,EAC3B,cAAsB,EACtB,aAAqB,EACrB,SAA8D;IAE9D,IAAI,KAAK,GAAG,OAAO,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,MAAK;QACzB,IAAI,EAAe,CAAA;QACnB,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;gBAC7C,SAAS,EAAE,KAAK,CAAC,WAAW;aAC7B,CAAC,CAAA;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAA;YACnB,IAAI,iBAAiB,IAAI,aAAa,EAAE,CAAC;gBACvC,KAAK,GAAG;oBACN,GAAG,KAAK;oBACR,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,KAAK,EAAE,iCAAiC,iBAAiB,0BAA0B;iBACpF,CAAA;gBACD,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;gBACtB,MAAK;YACP,CAAC;YACD,SAAQ;QACV,CAAC;QACD,iBAAiB,GAAG,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;YACpB,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAwB;IAC5D,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,MAAM,EACN,UAAU,EACV,WAAW,GAAG,CAAC,EACf,cAAc,GAAG,IAAI,EACrB,aAAa,GAAG,EAAE,EAClB,UAAU,EACV,UAAU,EACV,YAAY,GACb,GAAG,OAAO,CAAA;IAEX,IAAI,QAAQ,GAAqB,MAAM,CAAC,WAAW,CACjD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAkB,EAAE,CAAC,CAAA;IAC9E,CAAC,CAAC,CACH,CAAA;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,uEAAuE;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;IACnE,CAAC;IAED,uEAAuE;IACvE,2EAA2E;IAC3E,gFAAgF;IAChF,IAAI,YAAY,EAAE,CAAC;QACjB,iEAAiE;QACjE,oEAAoE;QACpE,iEAAiE;QACjE,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACnE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAChC,IAAI,KAAK,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAA;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,IAAwB,EAAE,EAAE;QACxD,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QAC1C,OAAO,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC,CAAA;IAED,MAAM,IAAI,CACR,MAAM,EACN,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1C,uCAAuC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAM;QAEhE,wCAAwC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;gBAC7F,OAAM;YACR,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACnC,MAAM,SAAS,GAAuB;YACpC,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,WAAW;YACnB,OAAO;SACR,CAAA;QACD,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAEnC,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACnG,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAA;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;IAExF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,SAAS;QACT,MAAM;QACN,QAAQ;KACT,CAAA;AACH,CAAC"}
@@ -1,38 +1,8 @@
1
- import type { ChunkBoundary, PartitionInfo, PlannedChunk, SortKeyInfo } from './types.js';
2
- export interface AnalyzeAndChunkInput {
3
- database: string;
4
- table: string;
5
- from?: string;
6
- to?: string;
7
- maxChunkBytes: number;
8
- requireIdempotencyToken: boolean;
9
- query: <T>(sql: string) => Promise<T[]>;
10
- }
11
- export interface AnalyzeAndChunkResult {
12
- planId: string;
13
- partitions: PartitionInfo[];
14
- sortKey?: SortKeyInfo;
15
- chunks: PlannedChunk[];
16
- }
1
+ import type { ChunkPlan, GenerateChunkPlanInput } from './types.js';
2
+ export type AnalyzeAndChunkInput = GenerateChunkPlanInput;
3
+ export type AnalyzeAndChunkResult = ChunkPlan;
4
+ export type AnalyzeTableInput = GenerateChunkPlanInput;
5
+ export type AnalyzeTableResult = ChunkPlan;
17
6
  export declare function analyzeAndChunk(input: AnalyzeAndChunkInput): Promise<AnalyzeAndChunkResult>;
18
- export interface AnalyzeTableInput {
19
- database: string;
20
- table: string;
21
- from?: string;
22
- to?: string;
23
- maxChunkBytes: number;
24
- query: <T>(sql: string) => Promise<T[]>;
25
- }
26
- export interface AnalyzeTableResult {
27
- partitions: PartitionInfo[];
28
- sortKey?: SortKeyInfo;
29
- boundaries: ChunkBoundary[];
30
- }
31
7
  export declare function analyzeTable(input: AnalyzeTableInput): Promise<AnalyzeTableResult>;
32
- export declare function buildPlannedChunks(input: {
33
- planId: string;
34
- partitions: PartitionInfo[];
35
- boundaries: ChunkBoundary[];
36
- requireIdempotencyToken: boolean;
37
- }): PlannedChunk[];
38
8
  //# sourceMappingURL=analyze.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAEzF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,aAAa,EAAE,MAAM,CAAA;IACrB,uBAAuB,EAAE,OAAO,CAAA;IAChC,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,MAAM,EAAE,YAAY,EAAE,CAAA;CACvB;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAoBjG;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACxC;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,UAAU,EAAE,aAAa,EAAE,CAAA;CAC5B;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgCxF;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACxC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,uBAAuB,EAAE,OAAO,CAAA;CACjC,GAAG,YAAY,EAAE,CA6BjB"}
1
+ {"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,MAAM,oBAAoB,GAAG,sBAAsB,CAAA;AACzD,MAAM,MAAM,qBAAqB,GAAG,SAAS,CAAA;AAC7C,MAAM,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AACtD,MAAM,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAE1C,wBAAsB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAEjG;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAExF"}
@@ -1,76 +1,8 @@
1
- import { hashId, randomPlanId } from '../state.js';
2
- import { buildChunkBoundaries } from './build.js';
3
- import { introspectTable, querySortKeyRanges } from './introspect.js';
1
+ import { generateChunkPlan } from './planner.js';
4
2
  export async function analyzeAndChunk(input) {
5
- const { partitions, sortKey, boundaries } = await analyzeTable({
6
- database: input.database,
7
- table: input.table,
8
- from: input.from,
9
- to: input.to,
10
- maxChunkBytes: input.maxChunkBytes,
11
- query: input.query,
12
- });
13
- const planId = randomPlanId();
14
- const chunks = buildPlannedChunks({
15
- planId,
16
- partitions,
17
- boundaries,
18
- requireIdempotencyToken: input.requireIdempotencyToken,
19
- });
20
- return { planId, partitions, sortKey, chunks };
3
+ return generateChunkPlan(input);
21
4
  }
22
5
  export async function analyzeTable(input) {
23
- const { partitions, sortKey } = await introspectTable({
24
- database: input.database,
25
- table: input.table,
26
- from: input.from,
27
- to: input.to,
28
- query: input.query,
29
- });
30
- const oversizedPartitionIds = partitions
31
- .filter(p => p.bytesOnDisk > input.maxChunkBytes)
32
- .map(p => p.partitionId);
33
- let sortKeyRanges;
34
- if (sortKey && oversizedPartitionIds.length > 0) {
35
- sortKeyRanges = await querySortKeyRanges({
36
- database: input.database,
37
- table: input.table,
38
- sortKeyColumn: sortKey.column,
39
- partitionIds: oversizedPartitionIds,
40
- query: input.query,
41
- });
42
- }
43
- const boundaries = buildChunkBoundaries({
44
- partitions,
45
- maxChunkBytes: input.maxChunkBytes,
46
- sortKey,
47
- sortKeyRanges,
48
- });
49
- return { partitions, sortKey, boundaries };
50
- }
51
- export function buildPlannedChunks(input) {
52
- const chunks = [];
53
- const partitionIndex = new Map();
54
- for (const boundary of input.boundaries) {
55
- const idx = partitionIndex.get(boundary.partitionId) ?? 0;
56
- partitionIndex.set(boundary.partitionId, idx + 1);
57
- const idSeed = `${input.planId}:${boundary.partitionId}:${idx}`;
58
- const chunkId = hashId(`chunk:${idSeed}`).slice(0, 16);
59
- const token = input.requireIdempotencyToken ? hashId(`token:${idSeed}`) : '';
60
- const partition = input.partitions.find(p => p.partitionId === boundary.partitionId);
61
- const from = boundary.sortKeyFrom ?? partition?.minTime ?? '';
62
- const to = boundary.sortKeyTo ?? partition?.maxTime ?? '';
63
- chunks.push({
64
- id: chunkId,
65
- partitionId: boundary.partitionId,
66
- sortKeyFrom: boundary.sortKeyFrom,
67
- sortKeyTo: boundary.sortKeyTo,
68
- estimatedBytes: boundary.estimatedBytes,
69
- idempotencyToken: token,
70
- from,
71
- to,
72
- });
73
- }
74
- return chunks;
6
+ return analyzeAndChunk(input);
75
7
  }
76
8
  //# sourceMappingURL=analyze.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAElD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAoBrE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAA2B;IAC/D,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,YAAY,CAAC;QAC7D,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;IAE7B,MAAM,MAAM,GAAG,kBAAkB,CAAC;QAChC,MAAM;QACN,UAAU;QACV,UAAU;QACV,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;KACvD,CAAC,CAAA;IAEF,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC;AAiBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAwB;IACzD,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,MAAM,eAAe,CAAC;QACpD,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAA;IAEF,MAAM,qBAAqB,GAAG,UAAU;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAA;IAE1B,IAAI,aAAoE,CAAA;IACxE,IAAI,OAAO,IAAI,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,aAAa,GAAG,MAAM,kBAAkB,CAAC;YACvC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,YAAY,EAAE,qBAAqB;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC;QACtC,UAAU;QACV,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO;QACP,aAAa;KACd,CAAC,CAAA;IAEF,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAKlC;IACC,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEhD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QACzD,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA;QAEjD,MAAM,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,WAAW,IAAI,GAAG,EAAE,CAAA;QAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAE5E,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,CAAC,CAAA;QACpF,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,IAAI,SAAS,EAAE,OAAO,IAAI,EAAE,CAAA;QAC7D,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,IAAI,SAAS,EAAE,OAAO,IAAI,EAAE,CAAA;QAEzD,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,OAAO;YACX,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,EAAE;SACH,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}
1
+ {"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAQhD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAA2B;IAC/D,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAwB;IACzD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC"}
@@ -0,0 +1,4 @@
1
+ import type { ChunkPlan } from './types.js';
2
+ export declare function encodeChunkPlanForPersistence(plan: ChunkPlan): ChunkPlan;
3
+ export declare function decodeChunkPlanFromPersistence(plan: ChunkPlan): ChunkPlan;
4
+ //# sourceMappingURL=boundary-codec.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"boundary-codec.d.ts","sourceRoot":"","sources":["../../src/chunking/boundary-codec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,SAAS,EAIV,MAAM,YAAY,CAAA;AA0FnB,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,CAKxE;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,CAKzE"}
@@ -0,0 +1,79 @@
1
+ function encodeBoundary(value, sortKey) {
2
+ if (value === undefined || sortKey === undefined)
3
+ return value;
4
+ if (sortKey.boundaryEncoding === 'hex-latin1') {
5
+ return Buffer.from(value, 'latin1').toString('hex');
6
+ }
7
+ return value;
8
+ }
9
+ function decodeBoundary(value, sortKey) {
10
+ if (value === undefined || sortKey === undefined)
11
+ return value;
12
+ if (sortKey.boundaryEncoding === 'hex-latin1') {
13
+ return Buffer.from(value, 'hex').toString('latin1');
14
+ }
15
+ return value;
16
+ }
17
+ function encodeRangesForPlan(ranges, sortKeys) {
18
+ return ranges.map((range) => ({
19
+ dimensionIndex: range.dimensionIndex,
20
+ from: encodeBoundary(range.from, sortKeys[range.dimensionIndex]),
21
+ to: encodeBoundary(range.to, sortKeys[range.dimensionIndex]),
22
+ }));
23
+ }
24
+ function decodeRangesFromPlan(ranges, sortKeys) {
25
+ return ranges.map((range) => ({
26
+ dimensionIndex: range.dimensionIndex,
27
+ from: decodeBoundary(range.from, sortKeys[range.dimensionIndex]),
28
+ to: decodeBoundary(range.to, sortKeys[range.dimensionIndex]),
29
+ }));
30
+ }
31
+ function encodeFocusedValue(focusedValue, sortKeys) {
32
+ if (!focusedValue)
33
+ return undefined;
34
+ return {
35
+ dimensionIndex: focusedValue.dimensionIndex,
36
+ value: encodeBoundary(focusedValue.value, sortKeys[focusedValue.dimensionIndex]) ?? focusedValue.value,
37
+ };
38
+ }
39
+ function decodeFocusedValue(focusedValue, sortKeys) {
40
+ if (!focusedValue)
41
+ return undefined;
42
+ return {
43
+ dimensionIndex: focusedValue.dimensionIndex,
44
+ value: decodeBoundary(focusedValue.value, sortKeys[focusedValue.dimensionIndex]) ?? focusedValue.value,
45
+ };
46
+ }
47
+ function encodeChunkForPlan(chunk, sortKeys) {
48
+ return {
49
+ ...chunk,
50
+ ranges: encodeRangesForPlan(chunk.ranges, sortKeys),
51
+ analysis: {
52
+ ...chunk.analysis,
53
+ focusedValue: encodeFocusedValue(chunk.analysis.focusedValue, sortKeys),
54
+ },
55
+ };
56
+ }
57
+ function decodeChunkFromPlan(chunk, sortKeys) {
58
+ return {
59
+ ...chunk,
60
+ ranges: decodeRangesFromPlan(chunk.ranges, sortKeys),
61
+ analysis: {
62
+ ...chunk.analysis,
63
+ focusedValue: decodeFocusedValue(chunk.analysis.focusedValue, sortKeys),
64
+ },
65
+ };
66
+ }
67
+ export function encodeChunkPlanForPersistence(plan) {
68
+ return {
69
+ ...plan,
70
+ chunks: plan.chunks.map((chunk) => encodeChunkForPlan(chunk, plan.table.sortKeys)),
71
+ };
72
+ }
73
+ export function decodeChunkPlanFromPersistence(plan) {
74
+ return {
75
+ ...plan,
76
+ chunks: plan.chunks.map((chunk) => decodeChunkFromPlan(chunk, plan.table.sortKeys)),
77
+ };
78
+ }
79
+ //# sourceMappingURL=boundary-codec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"boundary-codec.js","sourceRoot":"","sources":["../../src/chunking/boundary-codec.ts"],"names":[],"mappings":"AAQA,SAAS,cAAc,CACrB,KAAyB,EACzB,OAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,OAAO,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,cAAc,CACrB,KAAyB,EACzB,OAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,OAAO,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAoB,EACpB,QAAmB;IAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChE,EAAE,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;KAC7D,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAAoB,EACpB,QAAmB;IAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChE,EAAE,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;KAC7D,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAsC,EACtC,QAAmB;IAEnB,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,KAAK,EAAE,cAAc,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK;KACvG,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAsC,EACtC,QAAmB;IAEnB,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,KAAK,EAAE,cAAc,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK;KACvG,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAY,EAAE,QAAmB;IAC3D,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,mBAAmB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;QACnD,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,YAAY,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;SACxE;KACF,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAY,EAAE,QAAmB;IAC5D,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,oBAAoB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;QACpD,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,YAAY,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;SACxE;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,IAAe;IAC3D,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACnF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,IAAe;IAC5D,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACpF,CAAA;AACH,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare const TABLE_PREFIX = "chkit_e2e_chunking";
2
+ //# sourceMappingURL=constants.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,uBAAuB,CAAA"}
@@ -0,0 +1,2 @@
1
+ export const TABLE_PREFIX = 'chkit_e2e_chunking';
2
+ //# sourceMappingURL=constants.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../src/chunking/e2e/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,oBAAoB,CAAA"}
@@ -0,0 +1,20 @@
1
+ #!/usr/bin/env bun
2
+ /**
3
+ * Seeds ClickHouse tables for smart-chunking E2E tests.
4
+ *
5
+ * Run once manually:
6
+ * bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
7
+ *
8
+ * Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
9
+ * Creates tables if they don't exist, truncates them, and re-inserts data.
10
+ */
11
+ interface DatasetConfig {
12
+ name: string;
13
+ columns: string;
14
+ orderBy: string;
15
+ partitionBy: string;
16
+ generate: () => Record<string, unknown>[];
17
+ }
18
+ export declare const datasets: DatasetConfig[];
19
+ export {};
20
+ //# sourceMappingURL=seed-datasets.script.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"seed-datasets.script.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;;;;GAQG;AAOH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;CAC1C;AAUD,eAAO,MAAM,QAAQ,EAAE,aAAa,EA+EnC,CAAA"}
@@ -0,0 +1,134 @@
1
+ #!/usr/bin/env bun
2
+ /**
3
+ * Seeds ClickHouse tables for smart-chunking E2E tests.
4
+ *
5
+ * Run once manually:
6
+ * bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
7
+ *
8
+ * Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
9
+ * Creates tables if they don't exist, truncates them, and re-inserts data.
10
+ */
11
+ import { randomBytes } from 'node:crypto';
12
+ import { getRequiredEnv, createLiveExecutor } from '@chkit/clickhouse/e2e-testkit';
13
+ import { TABLE_PREFIX } from './constants.js';
14
+ function pad(bytes) {
15
+ return randomBytes(bytes).toString('hex');
16
+ }
17
+ function dayHour(day, hour) {
18
+ return `2026-01-${String(day).padStart(2, '0')} ${String(hour).padStart(2, '0')}:00:00`;
19
+ }
20
+ export const datasets = [
21
+ {
22
+ name: 'skewed_power_law',
23
+ columns: [
24
+ 'tenant_id String',
25
+ 'seq UInt64',
26
+ 'event_time DateTime',
27
+ 'padding String',
28
+ ].join(', '),
29
+ orderBy: '(tenant_id, seq)',
30
+ partitionBy: 'toYYYYMM(event_time)',
31
+ generate() {
32
+ const rows = [];
33
+ // 80%: single dominant tenant — 8,000 rows
34
+ for (let i = 0; i < 8000; i++) {
35
+ rows.push({
36
+ tenant_id: 'mega-corp',
37
+ seq: i,
38
+ event_time: dayHour(1 + (i % 28), i % 24),
39
+ padding: pad(512),
40
+ });
41
+ }
42
+ // 20%: 200 small tenants, 10 rows each — 2,000 rows
43
+ for (let t = 0; t < 200; t++) {
44
+ for (let i = 0; i < 10; i++) {
45
+ rows.push({
46
+ tenant_id: `tenant-${String(t).padStart(4, '0')}`,
47
+ seq: i,
48
+ event_time: dayHour(1 + ((t * 10 + i) % 28), (t + i) % 24),
49
+ padding: pad(512),
50
+ });
51
+ }
52
+ }
53
+ return rows;
54
+ },
55
+ },
56
+ {
57
+ name: 'multiple_hot_keys',
58
+ columns: [
59
+ 'tenant_id String',
60
+ 'seq UInt64',
61
+ 'event_time DateTime',
62
+ 'padding String',
63
+ ].join(', '),
64
+ orderBy: '(tenant_id, seq)',
65
+ partitionBy: 'toYYYYMM(event_time)',
66
+ generate() {
67
+ const rows = [];
68
+ // 3 hot tenants, ~30% each — 3,000 rows each = 9,000 rows
69
+ for (const tenant of ['alpha-corp', 'beta-corp', 'gamma-corp']) {
70
+ for (let i = 0; i < 3000; i++) {
71
+ rows.push({
72
+ tenant_id: tenant,
73
+ seq: i,
74
+ event_time: dayHour(1 + (i % 28), i % 24),
75
+ padding: pad(512),
76
+ });
77
+ }
78
+ }
79
+ // 10%: 100 small tenants, 10 rows each — 1,000 rows
80
+ for (let t = 0; t < 100; t++) {
81
+ for (let i = 0; i < 10; i++) {
82
+ rows.push({
83
+ tenant_id: `small-${String(t).padStart(4, '0')}`,
84
+ seq: i,
85
+ event_time: dayHour(1 + ((t * 10 + i) % 28), (t + i) % 24),
86
+ padding: pad(512),
87
+ });
88
+ }
89
+ }
90
+ return rows;
91
+ },
92
+ },
93
+ ];
94
+ const BATCH_SIZE = 5000;
95
+ async function seed() {
96
+ const env = getRequiredEnv();
97
+ const executor = createLiveExecutor(env);
98
+ const db = env.clickhouseDatabase;
99
+ try {
100
+ for (const dataset of datasets) {
101
+ const table = `${TABLE_PREFIX}_${dataset.name}`;
102
+ const fqn = `${db}.${table}`;
103
+ console.log(`\n--- Seeding ${fqn} ---`);
104
+ await executor.command(`
105
+ CREATE TABLE IF NOT EXISTS ${fqn} (
106
+ ${dataset.columns}
107
+ ) ENGINE = MergeTree()
108
+ PARTITION BY ${dataset.partitionBy}
109
+ ORDER BY ${dataset.orderBy}
110
+ `);
111
+ console.log(' Table ensured.');
112
+ await executor.command(`TRUNCATE TABLE ${fqn}`);
113
+ console.log(' Truncated.');
114
+ const rows = dataset.generate();
115
+ for (let i = 0; i < rows.length; i += BATCH_SIZE) {
116
+ const batch = rows.slice(i, i + BATCH_SIZE);
117
+ await executor.insert({ table: fqn, values: batch });
118
+ console.log(` Inserted ${Math.min(i + BATCH_SIZE, rows.length)} / ${rows.length} rows`);
119
+ }
120
+ // Verify
121
+ const [result] = await executor.query(`SELECT count() AS cnt FROM ${fqn} SETTINGS select_sequential_consistency = 1`);
122
+ console.log(` Verified: ${result?.cnt} rows`);
123
+ }
124
+ }
125
+ finally {
126
+ await executor.close();
127
+ }
128
+ console.log('\nDone!');
129
+ }
130
+ seed().catch((error) => {
131
+ console.error(error);
132
+ process.exit(1);
133
+ });
134
+ //# sourceMappingURL=seed-datasets.script.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"seed-datasets.script.js","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;;;;GAQG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAA;AAElF,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAU7C,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,IAAY;IACxC,OAAO,WAAW,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAA;AACzF,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAoB;IACvC;QACE,IAAI,EAAE,kBAAkB;QACxB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC;oBACR,SAAS,EAAE,WAAW;oBACtB,GAAG,EAAE,CAAC;oBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;oBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;iBAClB,CAAC,CAAA;YACJ,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBACjD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,0DAA0D;YAC1D,KAAK,MAAM,MAAM,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,MAAM;wBACjB,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;wBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,SAAS,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBAChD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;CACF,CAAA;AAED,MAAM,UAAU,GAAG,IAAI,CAAA;AAEvB,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,cAAc,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,EAAE,GAAG,GAAG,CAAC,kBAAkB,CAAA;IAEjC,IAAI,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,IAAI,EAAE,CAAA;YAC/C,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,KAAK,EAAE,CAAA;YAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,CAAA;YAEvC,MAAM,QAAQ,CAAC,OAAO,CAAC;qCACQ,GAAG;YAC5B,OAAO,CAAC,OAAO;;uBAEJ,OAAO,CAAC,WAAW;mBACvB,OAAO,CAAC,OAAO;OAC3B,CAAC,CAAA;YACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;YAE/B,MAAM,QAAQ,CAAC,OAAO,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;YAC/C,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;YAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAA;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC3C,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;gBACpD,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,OAAO,CAAC,CAAA;YAC1F,CAAC;YAED,SAAS;YACT,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CACnC,8BAA8B,GAAG,6CAA6C,CAC/E,CAAA;YACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,EAAE,GAAG,OAAO,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AACxB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}