@chkit/plugin-backfill 0.1.0-beta.20 → 0.1.0-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -0
- package/dist/async-backfill.d.ts +4 -4
- package/dist/async-backfill.d.ts.map +1 -1
- package/dist/async-backfill.js +2 -2
- package/dist/async-backfill.js.map +1 -1
- package/dist/chunking/analyze.d.ts +5 -35
- package/dist/chunking/analyze.d.ts.map +1 -1
- package/dist/chunking/analyze.js +3 -71
- package/dist/chunking/analyze.js.map +1 -1
- package/dist/chunking/boundary-codec.d.ts +4 -0
- package/dist/chunking/boundary-codec.d.ts.map +1 -0
- package/dist/chunking/boundary-codec.js +79 -0
- package/dist/chunking/boundary-codec.js.map +1 -0
- package/dist/chunking/e2e/constants.d.ts +2 -0
- package/dist/chunking/e2e/constants.d.ts.map +1 -0
- package/dist/chunking/e2e/constants.js +2 -0
- package/dist/chunking/e2e/constants.js.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts +20 -0
- package/dist/chunking/e2e/seed-datasets.script.d.ts.map +1 -0
- package/dist/chunking/e2e/seed-datasets.script.js +134 -0
- package/dist/chunking/e2e/seed-datasets.script.js.map +1 -0
- package/dist/chunking/introspect.d.ts +6 -0
- package/dist/chunking/introspect.d.ts.map +1 -1
- package/dist/chunking/introspect.js +113 -22
- package/dist/chunking/introspect.js.map +1 -1
- package/dist/chunking/partition-slices.d.ts +14 -0
- package/dist/chunking/partition-slices.d.ts.map +1 -0
- package/dist/chunking/partition-slices.js +111 -0
- package/dist/chunking/partition-slices.js.map +1 -0
- package/dist/chunking/planner.d.ts +3 -0
- package/dist/chunking/planner.d.ts.map +1 -0
- package/dist/chunking/planner.js +343 -0
- package/dist/chunking/planner.js.map +1 -0
- package/dist/chunking/services/distribution-source.d.ts +11 -0
- package/dist/chunking/services/distribution-source.d.ts.map +1 -0
- package/dist/chunking/services/distribution-source.js +60 -0
- package/dist/chunking/services/distribution-source.js.map +1 -0
- package/dist/chunking/services/metadata-source.d.ts +4 -0
- package/dist/chunking/services/metadata-source.d.ts.map +1 -0
- package/dist/chunking/services/metadata-source.js +138 -0
- package/dist/chunking/services/metadata-source.js.map +1 -0
- package/dist/chunking/services/row-probe.d.ts +12 -0
- package/dist/chunking/services/row-probe.d.ts.map +1 -0
- package/dist/chunking/services/row-probe.js +49 -0
- package/dist/chunking/services/row-probe.js.map +1 -0
- package/dist/chunking/sql.d.ts +11 -5
- package/dist/chunking/sql.d.ts.map +1 -1
- package/dist/chunking/sql.js +190 -107
- package/dist/chunking/sql.js.map +1 -1
- package/dist/chunking/strategies/equal-width-split.d.ts +3 -0
- package/dist/chunking/strategies/equal-width-split.d.ts.map +1 -0
- package/dist/chunking/strategies/equal-width-split.js +46 -0
- package/dist/chunking/strategies/equal-width-split.js.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts +3 -0
- package/dist/chunking/strategies/group-by-key-split.d.ts.map +1 -0
- package/dist/chunking/strategies/group-by-key-split.js +54 -0
- package/dist/chunking/strategies/group-by-key-split.js.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts +3 -0
- package/dist/chunking/strategies/metadata-single-chunk.d.ts.map +1 -0
- package/dist/chunking/strategies/metadata-single-chunk.js +5 -0
- package/dist/chunking/strategies/metadata-single-chunk.js.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts +5 -0
- package/dist/chunking/strategies/quantile-range-split.d.ts.map +1 -0
- package/dist/chunking/strategies/quantile-range-split.js +132 -0
- package/dist/chunking/strategies/quantile-range-split.js.map +1 -0
- package/dist/chunking/strategies/refinement.d.ts +3 -0
- package/dist/chunking/strategies/refinement.d.ts.map +1 -0
- package/dist/chunking/strategies/refinement.js +61 -0
- package/dist/chunking/strategies/refinement.js.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts +4 -0
- package/dist/chunking/strategies/string-prefix-split.d.ts.map +1 -0
- package/dist/chunking/strategies/string-prefix-split.js +73 -0
- package/dist/chunking/strategies/string-prefix-split.js.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts +3 -0
- package/dist/chunking/strategies/temporal-bucket-split.d.ts.map +1 -0
- package/dist/chunking/strategies/temporal-bucket-split.js +67 -0
- package/dist/chunking/strategies/temporal-bucket-split.js.map +1 -0
- package/dist/chunking/strategy-policy.d.ts +3 -0
- package/dist/chunking/strategy-policy.d.ts.map +1 -0
- package/dist/chunking/strategy-policy.js +4 -0
- package/dist/chunking/strategy-policy.js.map +1 -0
- package/dist/chunking/types.d.ts +126 -18
- package/dist/chunking/types.d.ts.map +1 -1
- package/dist/chunking/utils/binary-string.d.ts +8 -0
- package/dist/chunking/utils/binary-string.d.ts.map +1 -0
- package/dist/chunking/utils/binary-string.js +52 -0
- package/dist/chunking/utils/binary-string.js.map +1 -0
- package/dist/chunking/utils/ids.d.ts +4 -0
- package/dist/chunking/utils/ids.d.ts.map +1 -0
- package/dist/chunking/utils/ids.js +11 -0
- package/dist/chunking/utils/ids.js.map +1 -0
- package/dist/chunking/utils/ranges.d.ts +5 -0
- package/dist/chunking/utils/ranges.d.ts.map +1 -0
- package/dist/chunking/utils/ranges.js +19 -0
- package/dist/chunking/utils/ranges.js.map +1 -0
- package/dist/detect.d.ts +0 -5
- package/dist/detect.d.ts.map +1 -1
- package/dist/detect.js +0 -41
- package/dist/detect.js.map +1 -1
- package/dist/index.d.ts +0 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/logging.d.ts +12 -0
- package/dist/logging.d.ts.map +1 -0
- package/dist/logging.js +61 -0
- package/dist/logging.js.map +1 -0
- package/dist/options.d.ts +6 -15
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +1 -3
- package/dist/options.js.map +1 -1
- package/dist/payload.d.ts.map +1 -1
- package/dist/payload.js +4 -6
- package/dist/payload.js.map +1 -1
- package/dist/planner.d.ts +2 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +31 -52
- package/dist/planner.js.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +27 -11
- package/dist/plugin.js.map +1 -1
- package/dist/queries.js +3 -3
- package/dist/queries.js.map +1 -1
- package/dist/sdk.d.ts +12 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +9 -0
- package/dist/sdk.js.map +1 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +16 -5
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +12 -25
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -3
package/README.md
CHANGED
|
@@ -35,6 +35,136 @@ export default defineConfig({
|
|
|
35
35
|
|
|
36
36
|
See the [chkit documentation](https://chkit.obsessiondb.com).
|
|
37
37
|
|
|
38
|
+
## SDK
|
|
39
|
+
|
|
40
|
+
The package root is limited to the plugin registration API. Everything used by the CLI itself — the chunk planner, SQL builders, async executor, logging — is also exported from the `@chkit/plugin-backfill/sdk` subpath so you can build your own backfill scripts without going through the CLI.
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import {
|
|
44
|
+
generateChunkPlan,
|
|
45
|
+
buildChunkExecutionSql,
|
|
46
|
+
executeBackfill,
|
|
47
|
+
getBackfillLogger,
|
|
48
|
+
type ChunkPlan,
|
|
49
|
+
type PlannerQuery,
|
|
50
|
+
} from '@chkit/plugin-backfill/sdk'
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The pipeline has three stages, and you can use any subset:
|
|
54
|
+
|
|
55
|
+
1. **Plan** — `generateChunkPlan(...)` introspects a table and returns a `ChunkPlan` describing how to partition the work into roughly equal-sized chunks.
|
|
56
|
+
2. **Build SQL** — `buildChunkExecutionSql(...)` turns a single `Chunk` into an `INSERT … SELECT` statement.
|
|
57
|
+
3. **Execute** — `executeBackfill(...)` submits chunks against a real `ClickHouseExecutor` with deterministic query IDs, polling, and resume support.
|
|
58
|
+
|
|
59
|
+
### Plan a backfill
|
|
60
|
+
|
|
61
|
+
`generateChunkPlan` is decoupled from any ClickHouse client. You pass in a `query` function with the `PlannerQuery` shape and the planner uses it for every introspection / probe / split query. This makes the planner trivial to instrument or run against alternative clients.
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
import { createClient } from '@clickhouse/client'
|
|
65
|
+
import { generateChunkPlan, type PlannerQuery } from '@chkit/plugin-backfill/sdk'
|
|
66
|
+
|
|
67
|
+
const client = createClient({ url: process.env.CLICKHOUSE_URL })
|
|
68
|
+
|
|
69
|
+
const query: PlannerQuery = async (sql, settings) => {
|
|
70
|
+
const result = await client.query({
|
|
71
|
+
query: sql,
|
|
72
|
+
format: 'JSONEachRow',
|
|
73
|
+
clickhouse_settings: settings as Record<string, string | number | boolean>,
|
|
74
|
+
})
|
|
75
|
+
return result.json()
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const plan = await generateChunkPlan({
|
|
79
|
+
database: 'analytics',
|
|
80
|
+
table: 'events',
|
|
81
|
+
from: '2025-01-01T00:00:00Z',
|
|
82
|
+
to: '2025-02-01T00:00:00Z',
|
|
83
|
+
targetChunkBytes: 1_000_000_000, // ~1 GiB per chunk
|
|
84
|
+
query,
|
|
85
|
+
// 'count' is exact but slower; 'explain-estimate' is faster but approximate
|
|
86
|
+
rowProbeStrategy: 'count',
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
console.log(`${plan.chunks.length} chunks, ${plan.totalRows.toLocaleString()} rows`)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Execute chunks against a target
|
|
93
|
+
|
|
94
|
+
`buildChunkExecutionSql` produces the per-chunk `INSERT … SELECT` and `executeBackfill` runs them with concurrency, polling, and progress callbacks. Persist the `progress` argument anywhere you like to support resume.
|
|
95
|
+
|
|
96
|
+
```ts
|
|
97
|
+
import { createClickHouseExecutor } from '@chkit/clickhouse'
|
|
98
|
+
import {
|
|
99
|
+
buildChunkExecutionSql,
|
|
100
|
+
executeBackfill,
|
|
101
|
+
type BackfillProgress,
|
|
102
|
+
} from '@chkit/plugin-backfill/sdk'
|
|
103
|
+
|
|
104
|
+
const executor = createClickHouseExecutor({
|
|
105
|
+
url: process.env.CLICKHOUSE_URL!,
|
|
106
|
+
username: 'default',
|
|
107
|
+
password: process.env.CLICKHOUSE_PASSWORD!,
|
|
108
|
+
database: 'analytics',
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
const chunksById = new Map(plan.chunks.map((chunk) => [chunk.id, chunk]))
|
|
112
|
+
let saved: BackfillProgress | undefined // load from disk for resume
|
|
113
|
+
|
|
114
|
+
const result = await executeBackfill({
|
|
115
|
+
executor,
|
|
116
|
+
planId: plan.planId,
|
|
117
|
+
chunks: plan.chunks,
|
|
118
|
+
buildQuery: ({ id }) =>
|
|
119
|
+
buildChunkExecutionSql({
|
|
120
|
+
planId: plan.planId,
|
|
121
|
+
chunk: chunksById.get(id)!,
|
|
122
|
+
target: 'analytics.events_backfill',
|
|
123
|
+
table: plan.table,
|
|
124
|
+
}),
|
|
125
|
+
concurrency: 4,
|
|
126
|
+
pollIntervalMs: 5_000,
|
|
127
|
+
resumeFrom: saved,
|
|
128
|
+
onProgress: async (progress) => {
|
|
129
|
+
saved = progress
|
|
130
|
+
// persist to disk / state store
|
|
131
|
+
},
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
console.log(`done=${result.completed} failed=${result.failed}`)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Plan persistence
|
|
138
|
+
|
|
139
|
+
Plans contain string boundaries that may include non-UTF-8 bytes (the planner uses `latin1`-encoded byte ranges for string sort keys), so JSON-serializing a `ChunkPlan` directly will lose information. Use the codec helpers when you need to round-trip a plan through storage:
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
import {
|
|
143
|
+
encodeChunkPlanForPersistence,
|
|
144
|
+
decodeChunkPlanFromPersistence,
|
|
145
|
+
} from '@chkit/plugin-backfill/sdk'
|
|
146
|
+
|
|
147
|
+
const json = JSON.stringify(encodeChunkPlanForPersistence(plan))
|
|
148
|
+
// later …
|
|
149
|
+
const plan2 = decodeChunkPlanFromPersistence(JSON.parse(json))
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### Logging
|
|
153
|
+
|
|
154
|
+
The planner emits structured logs via [`@logtape/logtape`](https://logtape.org/) under the `['chkit', 'backfill']` category. Configure a sink at process start to see them — slow-query warnings (>5 s) are emitted at `warning` level, planning progress at `info`, and per-strategy decisions at `debug`.
|
|
155
|
+
|
|
156
|
+
```ts
|
|
157
|
+
import { configureSync, getConsoleSink, getTextFormatter } from '@chkit/plugin-backfill/sdk'
|
|
158
|
+
|
|
159
|
+
configureSync({
|
|
160
|
+
sinks: { console: getConsoleSink({ formatter: getTextFormatter({ timestamp: 'time' }) }) },
|
|
161
|
+
loggers: [{ category: 'chkit', sinks: ['console'], lowestLevel: 'info' }],
|
|
162
|
+
reset: true,
|
|
163
|
+
})
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
To capture every SQL statement the planner runs (with timing, server-side stats, and per-strategy classification), wrap your `query` function instead of relying solely on logging — the wrapper sees the raw SQL and settings on every call and can record query IDs, response headers, and durations alongside the structured logs.
|
|
167
|
+
|
|
38
168
|
## License
|
|
39
169
|
|
|
40
170
|
[MIT](../../LICENSE)
|
package/dist/async-backfill.d.ts
CHANGED
|
@@ -7,15 +7,15 @@ export interface BackfillOptions {
|
|
|
7
7
|
/** The chunks to process (from buildChunks) */
|
|
8
8
|
chunks: Array<{
|
|
9
9
|
id: string;
|
|
10
|
-
from
|
|
11
|
-
to
|
|
10
|
+
from?: string;
|
|
11
|
+
to?: string;
|
|
12
12
|
[key: string]: unknown;
|
|
13
13
|
}>;
|
|
14
14
|
/** Build the SQL for a given chunk. Called once per chunk at submit time. */
|
|
15
15
|
buildQuery: (chunk: {
|
|
16
16
|
id: string;
|
|
17
|
-
from
|
|
18
|
-
to
|
|
17
|
+
from?: string;
|
|
18
|
+
to?: string;
|
|
19
19
|
}) => string;
|
|
20
20
|
/** Max concurrent queries running on the server. Default: 3 */
|
|
21
21
|
concurrency?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAA;AAGxE,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"async-backfill.d.ts","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAA;AAGxE,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAA;IACd,+CAA+C;IAC/C,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAA;IACjF,6EAA6E;IAC7E,UAAU,EAAE,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAA;IACzE,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6FAA6F;IAC7F,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AA2ED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAoF3B;AA2CD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAsGvF"}
|
package/dist/async-backfill.js
CHANGED
|
@@ -81,10 +81,10 @@ export async function syncProgress(executor, planId, chunks, progress) {
|
|
|
81
81
|
return progress;
|
|
82
82
|
// Escape single-quotes in the prefix for safe SQL embedding
|
|
83
83
|
const safePrefix = prefix.replace(/'/g, "''").replace(/%/g, '\\%').replace(/_/g, '\\_');
|
|
84
|
-
const runningRows = await executor.query(`SELECT query_id FROM clusterAllReplicas('
|
|
84
|
+
const runningRows = await executor.query(`SELECT query_id FROM clusterAllReplicas('cluster', system.processes) WHERE user = currentUser() AND query_id LIKE '${safePrefix}%' SETTINGS skip_unavailable_shards = 1`);
|
|
85
85
|
const runningSet = new Set(runningRows.map((r) => r.query_id));
|
|
86
86
|
const logRows = await executor.query(`SELECT query_id, type, written_rows, written_bytes, query_duration_ms, exception
|
|
87
|
-
FROM clusterAllReplicas('
|
|
87
|
+
FROM clusterAllReplicas('cluster', system.query_log)
|
|
88
88
|
WHERE user = currentUser()
|
|
89
89
|
AND query_id LIKE '${safePrefix}%'
|
|
90
90
|
AND type IN ('QueryFinish', 'ExceptionWhileProcessing')
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,OAAO,CAAA;AAgDxB,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,oDAAoD;AACpD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe;IACnD,OAAO,YAAY,MAAM,IAAI,OAAO,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAyB,EACzB,EAAe;IAEf,IAAI,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAuB;YAC/B,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAA;QACD,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,KAAK,SAAS;YAC1B,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,QAAQ;YAC9B,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,WAAW;YACpC,KAAK,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAA;QAClC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;IACjD,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;aAC9B;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,KAAK,EAAE,EAAE,CAAC,KAAK;aAChB;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,sEAAsE;IACtE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,QAA0B,EAAE,EAAU;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAA;IAChE,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,EAAU,EACV,IAAwB;IAExB,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA4B,EAC5B,MAAc,EACd,MAA6B,EAC7B,QAA0B;IAE1B,MAAM,MAAM,GAAG,YAAY,MAAM,GAAG,CAAA;IAEpC,qEAAqE;IACrE,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;YAAE,SAAQ;QAC/C,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhD,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEvF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CACtC,
|
|
1
|
+
{"version":3,"file":"async-backfill.js","sourceRoot":"","sources":["../src/async-backfill.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,OAAO,CAAA;AAgDxB,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,oDAAoD;AACpD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe;IACnD,OAAO,YAAY,MAAM,IAAI,OAAO,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAyB,EACzB,EAAe;IAEf,IAAI,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAuB;YAC/B,GAAG,KAAK;YACR,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAA;QACD,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,KAAK,SAAS;YAC1B,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,QAAQ;YAC9B,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,WAAW;YACpC,KAAK,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAA;QAClC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;IACjD,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,YAAY,EAAE,EAAE,CAAC,YAAY;aAC9B;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,KAAK,EAAE,EAAE,CAAC,KAAK;aAChB;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;IACD,sEAAsE;IACtE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,QAA0B,EAAE,EAAU;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAA;IAChE,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,EAAU,EACV,IAAwB;IAExB,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA4B,EAC5B,MAAc,EACd,MAA6B,EAC7B,QAA0B;IAE1B,MAAM,MAAM,GAAG,YAAY,MAAM,GAAG,CAAA;IAEpC,qEAAqE;IACrE,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;YAAE,SAAQ;QAC/C,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhD,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEvF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CACtC,sHAAsH,UAAU,yCAAyC,CAC1K,CAAA;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAQlC;;;uBAGmB,UAAU;;;;qCAII,CAClC,CAAA;IAED,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAA;IACjE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;IAE7B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAChC,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;QACrF,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAChD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACpC,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,MAAM;wBACd,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAC1C,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;wBAC5C,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;qBAC/C,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE;wBACtC,GAAG,OAAO;wBACV,MAAM,EAAE,QAAQ;wBAChB,OAAO;wBACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;wBAC9C,KAAK,EAAE,QAAQ,CAAC,SAAS;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAA4B,EAC5B,OAA2B,EAC3B,cAAsB,EACtB,aAAqB,EACrB,SAA8D;IAE9D,IAAI,KAAK,GAAG,OAAO,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,MAAK;QACzB,IAAI,EAAe,CAAA;QACnB,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;gBAC7C,SAAS,EAAE,KAAK,CAAC,WAAW;aAC7B,CAAC,CAAA;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAA;YACnB,IAAI,iBAAiB,IAAI,aAAa,EAAE,CAAC;gBACvC,KAAK,GAAG;oBACN,GAAG,KAAK;oBACR,MAAM,EAAE,QAAQ;oBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,KAAK,EAAE,iCAAiC,iBAAiB,0BAA0B;iBACpF,CAAA;gBACD,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;gBACtB,MAAK;YACP,CAAC;YACD,SAAQ;QACV,CAAC;QACD,iBAAiB,GAAG,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;YACpB,MAAM,SAAS,CAAC,KAAK,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAwB;IAC5D,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,MAAM,EACN,UAAU,EACV,WAAW,GAAG,CAAC,EACf,cAAc,GAAG,IAAI,EACrB,aAAa,GAAG,EAAE,EAClB,UAAU,EACV,UAAU,EACV,YAAY,GACb,GAAG,OAAO,CAAA;IAEX,IAAI,QAAQ,GAAqB,MAAM,CAAC,WAAW,CACjD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAkB,EAAE,CAAC,CAAA;IAC9E,CAAC,CAAC,CACH,CAAA;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,uEAAuE;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;IACnE,CAAC;IAED,uEAAuE;IACvE,2EAA2E;IAC3E,gFAAgF;IAChF,IAAI,YAAY,EAAE,CAAC;QACjB,iEAAiE;QACjE,oEAAoE;QACpE,iEAAiE;QACjE,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;QACnE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAChC,IAAI,KAAK,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAA;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,IAAwB,EAAE,EAAE;QACxD,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QAC1C,OAAO,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC,CAAA;IAED,MAAM,IAAI,CACR,MAAM,EACN,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1C,uCAAuC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAM;QAEhE,wCAAwC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;gBAC7F,OAAM;YACR,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACnC,MAAM,SAAS,GAAuB;YACpC,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,WAAW;YACnB,OAAO;SACR,CAAA;QACD,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAEnC,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACnG,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAA;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;IAExF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,SAAS;QACT,MAAM;QACN,QAAQ;KACT,CAAA;AACH,CAAC"}
|
|
@@ -1,38 +1,8 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
to?: string;
|
|
7
|
-
maxChunkBytes: number;
|
|
8
|
-
requireIdempotencyToken: boolean;
|
|
9
|
-
query: <T>(sql: string) => Promise<T[]>;
|
|
10
|
-
}
|
|
11
|
-
export interface AnalyzeAndChunkResult {
|
|
12
|
-
planId: string;
|
|
13
|
-
partitions: PartitionInfo[];
|
|
14
|
-
sortKey?: SortKeyInfo;
|
|
15
|
-
chunks: PlannedChunk[];
|
|
16
|
-
}
|
|
1
|
+
import type { ChunkPlan, GenerateChunkPlanInput } from './types.js';
|
|
2
|
+
export type AnalyzeAndChunkInput = GenerateChunkPlanInput;
|
|
3
|
+
export type AnalyzeAndChunkResult = ChunkPlan;
|
|
4
|
+
export type AnalyzeTableInput = GenerateChunkPlanInput;
|
|
5
|
+
export type AnalyzeTableResult = ChunkPlan;
|
|
17
6
|
export declare function analyzeAndChunk(input: AnalyzeAndChunkInput): Promise<AnalyzeAndChunkResult>;
|
|
18
|
-
export interface AnalyzeTableInput {
|
|
19
|
-
database: string;
|
|
20
|
-
table: string;
|
|
21
|
-
from?: string;
|
|
22
|
-
to?: string;
|
|
23
|
-
maxChunkBytes: number;
|
|
24
|
-
query: <T>(sql: string) => Promise<T[]>;
|
|
25
|
-
}
|
|
26
|
-
export interface AnalyzeTableResult {
|
|
27
|
-
partitions: PartitionInfo[];
|
|
28
|
-
sortKey?: SortKeyInfo;
|
|
29
|
-
boundaries: ChunkBoundary[];
|
|
30
|
-
}
|
|
31
7
|
export declare function analyzeTable(input: AnalyzeTableInput): Promise<AnalyzeTableResult>;
|
|
32
|
-
export declare function buildPlannedChunks(input: {
|
|
33
|
-
planId: string;
|
|
34
|
-
partitions: PartitionInfo[];
|
|
35
|
-
boundaries: ChunkBoundary[];
|
|
36
|
-
requireIdempotencyToken: boolean;
|
|
37
|
-
}): PlannedChunk[];
|
|
38
8
|
//# sourceMappingURL=analyze.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,MAAM,oBAAoB,GAAG,sBAAsB,CAAA;AACzD,MAAM,MAAM,qBAAqB,GAAG,SAAS,CAAA;AAC7C,MAAM,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AACtD,MAAM,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAE1C,wBAAsB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAEjG;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAExF"}
|
package/dist/chunking/analyze.js
CHANGED
|
@@ -1,76 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { buildChunkBoundaries } from './build.js';
|
|
3
|
-
import { introspectTable, querySortKeyRanges } from './introspect.js';
|
|
1
|
+
import { generateChunkPlan } from './planner.js';
|
|
4
2
|
export async function analyzeAndChunk(input) {
|
|
5
|
-
|
|
6
|
-
database: input.database,
|
|
7
|
-
table: input.table,
|
|
8
|
-
from: input.from,
|
|
9
|
-
to: input.to,
|
|
10
|
-
maxChunkBytes: input.maxChunkBytes,
|
|
11
|
-
query: input.query,
|
|
12
|
-
});
|
|
13
|
-
const planId = randomPlanId();
|
|
14
|
-
const chunks = buildPlannedChunks({
|
|
15
|
-
planId,
|
|
16
|
-
partitions,
|
|
17
|
-
boundaries,
|
|
18
|
-
requireIdempotencyToken: input.requireIdempotencyToken,
|
|
19
|
-
});
|
|
20
|
-
return { planId, partitions, sortKey, chunks };
|
|
3
|
+
return generateChunkPlan(input);
|
|
21
4
|
}
|
|
22
5
|
export async function analyzeTable(input) {
|
|
23
|
-
|
|
24
|
-
database: input.database,
|
|
25
|
-
table: input.table,
|
|
26
|
-
from: input.from,
|
|
27
|
-
to: input.to,
|
|
28
|
-
query: input.query,
|
|
29
|
-
});
|
|
30
|
-
const oversizedPartitionIds = partitions
|
|
31
|
-
.filter(p => p.bytesOnDisk > input.maxChunkBytes)
|
|
32
|
-
.map(p => p.partitionId);
|
|
33
|
-
let sortKeyRanges;
|
|
34
|
-
if (sortKey && oversizedPartitionIds.length > 0) {
|
|
35
|
-
sortKeyRanges = await querySortKeyRanges({
|
|
36
|
-
database: input.database,
|
|
37
|
-
table: input.table,
|
|
38
|
-
sortKeyColumn: sortKey.column,
|
|
39
|
-
partitionIds: oversizedPartitionIds,
|
|
40
|
-
query: input.query,
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
const boundaries = buildChunkBoundaries({
|
|
44
|
-
partitions,
|
|
45
|
-
maxChunkBytes: input.maxChunkBytes,
|
|
46
|
-
sortKey,
|
|
47
|
-
sortKeyRanges,
|
|
48
|
-
});
|
|
49
|
-
return { partitions, sortKey, boundaries };
|
|
50
|
-
}
|
|
51
|
-
export function buildPlannedChunks(input) {
|
|
52
|
-
const chunks = [];
|
|
53
|
-
const partitionIndex = new Map();
|
|
54
|
-
for (const boundary of input.boundaries) {
|
|
55
|
-
const idx = partitionIndex.get(boundary.partitionId) ?? 0;
|
|
56
|
-
partitionIndex.set(boundary.partitionId, idx + 1);
|
|
57
|
-
const idSeed = `${input.planId}:${boundary.partitionId}:${idx}`;
|
|
58
|
-
const chunkId = hashId(`chunk:${idSeed}`).slice(0, 16);
|
|
59
|
-
const token = input.requireIdempotencyToken ? hashId(`token:${idSeed}`) : '';
|
|
60
|
-
const partition = input.partitions.find(p => p.partitionId === boundary.partitionId);
|
|
61
|
-
const from = boundary.sortKeyFrom ?? partition?.minTime ?? '';
|
|
62
|
-
const to = boundary.sortKeyTo ?? partition?.maxTime ?? '';
|
|
63
|
-
chunks.push({
|
|
64
|
-
id: chunkId,
|
|
65
|
-
partitionId: boundary.partitionId,
|
|
66
|
-
sortKeyFrom: boundary.sortKeyFrom,
|
|
67
|
-
sortKeyTo: boundary.sortKeyTo,
|
|
68
|
-
estimatedBytes: boundary.estimatedBytes,
|
|
69
|
-
idempotencyToken: token,
|
|
70
|
-
from,
|
|
71
|
-
to,
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
return chunks;
|
|
6
|
+
return analyzeAndChunk(input);
|
|
75
7
|
}
|
|
76
8
|
//# sourceMappingURL=analyze.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/chunking/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAQhD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAA2B;IAC/D,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAwB;IACzD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAA;AAC/B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"boundary-codec.d.ts","sourceRoot":"","sources":["../../src/chunking/boundary-codec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,SAAS,EAIV,MAAM,YAAY,CAAA;AA0FnB,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,CAKxE;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,CAKzE"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
function encodeBoundary(value, sortKey) {
|
|
2
|
+
if (value === undefined || sortKey === undefined)
|
|
3
|
+
return value;
|
|
4
|
+
if (sortKey.boundaryEncoding === 'hex-latin1') {
|
|
5
|
+
return Buffer.from(value, 'latin1').toString('hex');
|
|
6
|
+
}
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
function decodeBoundary(value, sortKey) {
|
|
10
|
+
if (value === undefined || sortKey === undefined)
|
|
11
|
+
return value;
|
|
12
|
+
if (sortKey.boundaryEncoding === 'hex-latin1') {
|
|
13
|
+
return Buffer.from(value, 'hex').toString('latin1');
|
|
14
|
+
}
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
function encodeRangesForPlan(ranges, sortKeys) {
|
|
18
|
+
return ranges.map((range) => ({
|
|
19
|
+
dimensionIndex: range.dimensionIndex,
|
|
20
|
+
from: encodeBoundary(range.from, sortKeys[range.dimensionIndex]),
|
|
21
|
+
to: encodeBoundary(range.to, sortKeys[range.dimensionIndex]),
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
24
|
+
function decodeRangesFromPlan(ranges, sortKeys) {
|
|
25
|
+
return ranges.map((range) => ({
|
|
26
|
+
dimensionIndex: range.dimensionIndex,
|
|
27
|
+
from: decodeBoundary(range.from, sortKeys[range.dimensionIndex]),
|
|
28
|
+
to: decodeBoundary(range.to, sortKeys[range.dimensionIndex]),
|
|
29
|
+
}));
|
|
30
|
+
}
|
|
31
|
+
function encodeFocusedValue(focusedValue, sortKeys) {
|
|
32
|
+
if (!focusedValue)
|
|
33
|
+
return undefined;
|
|
34
|
+
return {
|
|
35
|
+
dimensionIndex: focusedValue.dimensionIndex,
|
|
36
|
+
value: encodeBoundary(focusedValue.value, sortKeys[focusedValue.dimensionIndex]) ?? focusedValue.value,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function decodeFocusedValue(focusedValue, sortKeys) {
|
|
40
|
+
if (!focusedValue)
|
|
41
|
+
return undefined;
|
|
42
|
+
return {
|
|
43
|
+
dimensionIndex: focusedValue.dimensionIndex,
|
|
44
|
+
value: decodeBoundary(focusedValue.value, sortKeys[focusedValue.dimensionIndex]) ?? focusedValue.value,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function encodeChunkForPlan(chunk, sortKeys) {
|
|
48
|
+
return {
|
|
49
|
+
...chunk,
|
|
50
|
+
ranges: encodeRangesForPlan(chunk.ranges, sortKeys),
|
|
51
|
+
analysis: {
|
|
52
|
+
...chunk.analysis,
|
|
53
|
+
focusedValue: encodeFocusedValue(chunk.analysis.focusedValue, sortKeys),
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
function decodeChunkFromPlan(chunk, sortKeys) {
|
|
58
|
+
return {
|
|
59
|
+
...chunk,
|
|
60
|
+
ranges: decodeRangesFromPlan(chunk.ranges, sortKeys),
|
|
61
|
+
analysis: {
|
|
62
|
+
...chunk.analysis,
|
|
63
|
+
focusedValue: decodeFocusedValue(chunk.analysis.focusedValue, sortKeys),
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export function encodeChunkPlanForPersistence(plan) {
|
|
68
|
+
return {
|
|
69
|
+
...plan,
|
|
70
|
+
chunks: plan.chunks.map((chunk) => encodeChunkForPlan(chunk, plan.table.sortKeys)),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
export function decodeChunkPlanFromPersistence(plan) {
|
|
74
|
+
return {
|
|
75
|
+
...plan,
|
|
76
|
+
chunks: plan.chunks.map((chunk) => decodeChunkFromPlan(chunk, plan.table.sortKeys)),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=boundary-codec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"boundary-codec.js","sourceRoot":"","sources":["../../src/chunking/boundary-codec.ts"],"names":[],"mappings":"AAQA,SAAS,cAAc,CACrB,KAAyB,EACzB,OAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,OAAO,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,cAAc,CACrB,KAAyB,EACzB,OAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,OAAO,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAoB,EACpB,QAAmB;IAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChE,EAAE,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;KAC7D,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAAoB,EACpB,QAAmB;IAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChE,EAAE,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;KAC7D,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAsC,EACtC,QAAmB;IAEnB,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,KAAK,EAAE,cAAc,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK;KACvG,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAsC,EACtC,QAAmB;IAEnB,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,KAAK,EAAE,cAAc,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK;KACvG,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAY,EAAE,QAAmB;IAC3D,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,mBAAmB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;QACnD,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,YAAY,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;SACxE;KACF,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAY,EAAE,QAAmB;IAC5D,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,oBAAoB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;QACpD,QAAQ,EAAE;YACR,GAAG,KAAK,CAAC,QAAQ;YACjB,YAAY,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;SACxE;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,IAAe;IAC3D,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACnF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,IAAe;IAC5D,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACpF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,uBAAuB,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../src/chunking/e2e/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,oBAAoB,CAAA"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Seeds ClickHouse tables for smart-chunking E2E tests.
|
|
4
|
+
*
|
|
5
|
+
* Run once manually:
|
|
6
|
+
* bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
|
|
7
|
+
*
|
|
8
|
+
* Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
|
|
9
|
+
* Creates tables if they don't exist, truncates them, and re-inserts data.
|
|
10
|
+
*/
|
|
11
|
+
interface DatasetConfig {
|
|
12
|
+
name: string;
|
|
13
|
+
columns: string;
|
|
14
|
+
orderBy: string;
|
|
15
|
+
partitionBy: string;
|
|
16
|
+
generate: () => Record<string, unknown>[];
|
|
17
|
+
}
|
|
18
|
+
export declare const datasets: DatasetConfig[];
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=seed-datasets.script.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed-datasets.script.d.ts","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;;;;GAQG;AAOH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;CAC1C;AAUD,eAAO,MAAM,QAAQ,EAAE,aAAa,EA+EnC,CAAA"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Seeds ClickHouse tables for smart-chunking E2E tests.
|
|
4
|
+
*
|
|
5
|
+
* Run once manually:
|
|
6
|
+
* bun run packages/plugin-backfill/src/chunking/e2e/seed.ts
|
|
7
|
+
*
|
|
8
|
+
* Requires CLICKHOUSE_HOST/CLICKHOUSE_URL + CLICKHOUSE_PASSWORD env vars.
|
|
9
|
+
* Creates tables if they don't exist, truncates them, and re-inserts data.
|
|
10
|
+
*/
|
|
11
|
+
import { randomBytes } from 'node:crypto';
|
|
12
|
+
import { getRequiredEnv, createLiveExecutor } from '@chkit/clickhouse/e2e-testkit';
|
|
13
|
+
import { TABLE_PREFIX } from './constants.js';
|
|
14
|
+
function pad(bytes) {
|
|
15
|
+
return randomBytes(bytes).toString('hex');
|
|
16
|
+
}
|
|
17
|
+
function dayHour(day, hour) {
|
|
18
|
+
return `2026-01-${String(day).padStart(2, '0')} ${String(hour).padStart(2, '0')}:00:00`;
|
|
19
|
+
}
|
|
20
|
+
export const datasets = [
|
|
21
|
+
{
|
|
22
|
+
name: 'skewed_power_law',
|
|
23
|
+
columns: [
|
|
24
|
+
'tenant_id String',
|
|
25
|
+
'seq UInt64',
|
|
26
|
+
'event_time DateTime',
|
|
27
|
+
'padding String',
|
|
28
|
+
].join(', '),
|
|
29
|
+
orderBy: '(tenant_id, seq)',
|
|
30
|
+
partitionBy: 'toYYYYMM(event_time)',
|
|
31
|
+
generate() {
|
|
32
|
+
const rows = [];
|
|
33
|
+
// 80%: single dominant tenant — 8,000 rows
|
|
34
|
+
for (let i = 0; i < 8000; i++) {
|
|
35
|
+
rows.push({
|
|
36
|
+
tenant_id: 'mega-corp',
|
|
37
|
+
seq: i,
|
|
38
|
+
event_time: dayHour(1 + (i % 28), i % 24),
|
|
39
|
+
padding: pad(512),
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
// 20%: 200 small tenants, 10 rows each — 2,000 rows
|
|
43
|
+
for (let t = 0; t < 200; t++) {
|
|
44
|
+
for (let i = 0; i < 10; i++) {
|
|
45
|
+
rows.push({
|
|
46
|
+
tenant_id: `tenant-${String(t).padStart(4, '0')}`,
|
|
47
|
+
seq: i,
|
|
48
|
+
event_time: dayHour(1 + ((t * 10 + i) % 28), (t + i) % 24),
|
|
49
|
+
padding: pad(512),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return rows;
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
name: 'multiple_hot_keys',
|
|
58
|
+
columns: [
|
|
59
|
+
'tenant_id String',
|
|
60
|
+
'seq UInt64',
|
|
61
|
+
'event_time DateTime',
|
|
62
|
+
'padding String',
|
|
63
|
+
].join(', '),
|
|
64
|
+
orderBy: '(tenant_id, seq)',
|
|
65
|
+
partitionBy: 'toYYYYMM(event_time)',
|
|
66
|
+
generate() {
|
|
67
|
+
const rows = [];
|
|
68
|
+
// 3 hot tenants, ~30% each — 3,000 rows each = 9,000 rows
|
|
69
|
+
for (const tenant of ['alpha-corp', 'beta-corp', 'gamma-corp']) {
|
|
70
|
+
for (let i = 0; i < 3000; i++) {
|
|
71
|
+
rows.push({
|
|
72
|
+
tenant_id: tenant,
|
|
73
|
+
seq: i,
|
|
74
|
+
event_time: dayHour(1 + (i % 28), i % 24),
|
|
75
|
+
padding: pad(512),
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// 10%: 100 small tenants, 10 rows each — 1,000 rows
|
|
80
|
+
for (let t = 0; t < 100; t++) {
|
|
81
|
+
for (let i = 0; i < 10; i++) {
|
|
82
|
+
rows.push({
|
|
83
|
+
tenant_id: `small-${String(t).padStart(4, '0')}`,
|
|
84
|
+
seq: i,
|
|
85
|
+
event_time: dayHour(1 + ((t * 10 + i) % 28), (t + i) % 24),
|
|
86
|
+
padding: pad(512),
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return rows;
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
];
|
|
94
|
+
const BATCH_SIZE = 5000;
|
|
95
|
+
async function seed() {
|
|
96
|
+
const env = getRequiredEnv();
|
|
97
|
+
const executor = createLiveExecutor(env);
|
|
98
|
+
const db = env.clickhouseDatabase;
|
|
99
|
+
try {
|
|
100
|
+
for (const dataset of datasets) {
|
|
101
|
+
const table = `${TABLE_PREFIX}_${dataset.name}`;
|
|
102
|
+
const fqn = `${db}.${table}`;
|
|
103
|
+
console.log(`\n--- Seeding ${fqn} ---`);
|
|
104
|
+
await executor.command(`
|
|
105
|
+
CREATE TABLE IF NOT EXISTS ${fqn} (
|
|
106
|
+
${dataset.columns}
|
|
107
|
+
) ENGINE = MergeTree()
|
|
108
|
+
PARTITION BY ${dataset.partitionBy}
|
|
109
|
+
ORDER BY ${dataset.orderBy}
|
|
110
|
+
`);
|
|
111
|
+
console.log(' Table ensured.');
|
|
112
|
+
await executor.command(`TRUNCATE TABLE ${fqn}`);
|
|
113
|
+
console.log(' Truncated.');
|
|
114
|
+
const rows = dataset.generate();
|
|
115
|
+
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
|
|
116
|
+
const batch = rows.slice(i, i + BATCH_SIZE);
|
|
117
|
+
await executor.insert({ table: fqn, values: batch });
|
|
118
|
+
console.log(` Inserted ${Math.min(i + BATCH_SIZE, rows.length)} / ${rows.length} rows`);
|
|
119
|
+
}
|
|
120
|
+
// Verify
|
|
121
|
+
const [result] = await executor.query(`SELECT count() AS cnt FROM ${fqn} SETTINGS select_sequential_consistency = 1`);
|
|
122
|
+
console.log(` Verified: ${result?.cnt} rows`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
await executor.close();
|
|
127
|
+
}
|
|
128
|
+
console.log('\nDone!');
|
|
129
|
+
}
|
|
130
|
+
seed().catch((error) => {
|
|
131
|
+
console.error(error);
|
|
132
|
+
process.exit(1);
|
|
133
|
+
});
|
|
134
|
+
//# sourceMappingURL=seed-datasets.script.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed-datasets.script.js","sourceRoot":"","sources":["../../../src/chunking/e2e/seed-datasets.script.ts"],"names":[],"mappings":";AAEA;;;;;;;;GAQG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAA;AAElF,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAU7C,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,IAAY;IACxC,OAAO,WAAW,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAA;AACzF,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAoB;IACvC;QACE,IAAI,EAAE,kBAAkB;QACxB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC;oBACR,SAAS,EAAE,WAAW;oBACtB,GAAG,EAAE,CAAC;oBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;oBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;iBAClB,CAAC,CAAA;YACJ,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBACjD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;YACP,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;YACrB,gBAAgB;SACjB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,kBAAkB;QAC3B,WAAW,EAAE,sBAAsB;QACnC,QAAQ;YACN,MAAM,IAAI,GAA8B,EAAE,CAAA;YAE1C,0DAA0D;YAC1D,KAAK,MAAM,MAAM,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,MAAM;wBACjB,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC;wBACzC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC;wBACR,SAAS,EAAE,SAAS,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;wBAChD,GAAG,EAAE,CAAC;wBACN,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;wBAC1D,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;qBAClB,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;KACF;CACF,CAAA;AAED,MAAM,UAAU,GAAG,IAAI,CAAA;AAEvB,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,cAAc,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,EAAE,GAAG,GAAG,CAAC,kBAAkB,CAAA;IAEjC,IAAI,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,IAAI,EAAE,CAAA;YAC/C,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,KAAK,EAAE,CAAA;YAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,CAAA;YAEvC,MAAM,QAAQ,CAAC,OAAO,CAAC;qCACQ,GAAG;YAC5B,OAAO,CAAC,OAAO;;uBAEJ,OAAO,CAAC,WAAW;mBACvB,OAAO,CAAC,OAAO;OAC3B,CAAC,CAAA;YACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;YAE/B,MAAM,QAAQ,CAAC,OAAO,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;YAC/C,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;YAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAA;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC3C,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;gBACpD,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,OAAO,CAAC,CAAA;YAC1F,CAAC;YAED,SAAS;YACT,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CACnC,8BAA8B,GAAG,6CAA6C,CAC/E,CAAA;YACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,EAAE,GAAG,OAAO,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AACxB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}
|