@chkit/clickhouse 0.1.2-beta.7 → 0.2.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alter-column-statement.d.ts +32 -0
- package/dist/alter-column-statement.d.ts.map +1 -0
- package/dist/alter-column-statement.js +137 -0
- package/dist/alter-column-statement.js.map +1 -0
- package/dist/async-insert-guard.d.ts +31 -0
- package/dist/async-insert-guard.d.ts.map +1 -0
- package/dist/async-insert-guard.js +55 -0
- package/dist/async-insert-guard.js.map +1 -0
- package/dist/canonicalize.d.ts +21 -0
- package/dist/canonicalize.d.ts.map +1 -0
- package/dist/canonicalize.js +70 -0
- package/dist/canonicalize.js.map +1 -0
- package/dist/create-table-parser.d.ts +3 -0
- package/dist/create-table-parser.d.ts.map +1 -1
- package/dist/create-table-parser.js +54 -43
- package/dist/create-table-parser.js.map +1 -1
- package/dist/ddl-propagation.d.ts +62 -6
- package/dist/ddl-propagation.d.ts.map +1 -1
- package/dist/ddl-propagation.js +459 -49
- package/dist/ddl-propagation.js.map +1 -1
- package/dist/e2e-testkit.d.ts +26 -4
- package/dist/e2e-testkit.d.ts.map +1 -1
- package/dist/e2e-testkit.js +56 -14
- package/dist/e2e-testkit.js.map +1 -1
- package/dist/exact-json.d.ts +27 -0
- package/dist/exact-json.d.ts.map +1 -0
- package/dist/exact-json.js +128 -0
- package/dist/exact-json.js.map +1 -0
- package/dist/index.d.ts +52 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +220 -64
- package/dist/index.js.map +1 -1
- package/dist/insert-barrier.d.ts +32 -0
- package/dist/insert-barrier.d.ts.map +1 -0
- package/dist/insert-barrier.js +57 -0
- package/dist/insert-barrier.js.map +1 -0
- package/dist/replicas.d.ts +21 -0
- package/dist/replicas.d.ts.map +1 -0
- package/dist/replicas.js +39 -0
- package/dist/replicas.js.map +1 -0
- package/dist/view-details.d.ts +30 -0
- package/dist/view-details.d.ts.map +1 -0
- package/dist/view-details.js +73 -0
- package/dist/view-details.js.map +1 -0
- package/package.json +5 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"insert-barrier.d.ts","sourceRoot":"","sources":["../src/insert-barrier.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAOxE,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,kBAAkB,CAAA;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAC5B,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACrC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CACnB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAmChC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { setTimeout as defaultSleep } from 'node:timers/promises';
|
|
2
|
+
import { allReplicas, resolveReplicaFanout } from './replicas.js';
|
|
3
|
+
const DEFAULT_TIMEOUT_MS = 10 * 60_000;
|
|
4
|
+
const DEFAULT_POLL_INTERVAL_MS = 1_000;
|
|
5
|
+
const PROGRESS_EVERY_MS = 30_000;
|
|
6
|
+
/**
|
|
7
|
+
* Runs `sql` (a SELECT over `system.processes` that returns a `query_id`
|
|
8
|
+
* column) on every replica once, then again until none of the query ids it
|
|
9
|
+
* returned the first time is still listed.
|
|
10
|
+
*
|
|
11
|
+
* An INSERT builds its pipeline, including the materialized views it feeds,
|
|
12
|
+
* when it starts. After a mirror view is created, the INSERTs that were
|
|
13
|
+
* already running do not write to it. Every one of them is listed in
|
|
14
|
+
* `system.processes` until it ends, so once the ids listed after the view
|
|
15
|
+
* existed on every replica are gone, every INSERT without the mirror has
|
|
16
|
+
* committed. No clock is involved: the first read only has to happen after
|
|
17
|
+
* the view exists, which is why a resumed run can simply read again.
|
|
18
|
+
*
|
|
19
|
+
* On a target with several replicas `system.processes` is read through
|
|
20
|
+
* `clusterAllReplicas`, because an INSERT runs on the replica that received it.
|
|
21
|
+
*/
|
|
22
|
+
export async function waitForInsertBarrier(executor, sql, options = {}) {
|
|
23
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
24
|
+
const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
25
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
26
|
+
const now = options.now ?? Date.now;
|
|
27
|
+
const log = options.log ?? (() => { });
|
|
28
|
+
const fanout = await resolveReplicaFanout(executor, options.cluster);
|
|
29
|
+
const query = fanout ? sql.replace(/\bsystem\.processes\b/, allReplicas(fanout, 'system.processes')) : sql;
|
|
30
|
+
const read = async () => new Set((await executor.query(stripTrailingSemicolon(query), options.settings)).map((row) => row.query_id));
|
|
31
|
+
const initial = await read();
|
|
32
|
+
if (initial.size === 0)
|
|
33
|
+
return { waitedFor: 0 };
|
|
34
|
+
log(` waiting for ${initial.size} running INSERT ${initial.size === 1 ? 'query' : 'queries'} to finish`);
|
|
35
|
+
const started = now();
|
|
36
|
+
let lastProgress = started;
|
|
37
|
+
for (;;) {
|
|
38
|
+
await sleep(pollIntervalMs);
|
|
39
|
+
const current = await read();
|
|
40
|
+
const remaining = [...initial].filter((id) => current.has(id));
|
|
41
|
+
if (remaining.length === 0)
|
|
42
|
+
return { waitedFor: initial.size };
|
|
43
|
+
const elapsed = now() - started;
|
|
44
|
+
if (elapsed >= timeoutMs) {
|
|
45
|
+
throw new Error(`${remaining.length} INSERT ${remaining.length === 1 ? 'query' : 'queries'} that started before this step ${remaining.length === 1 ? 'is' : 'are'} still running after ${Math.round(elapsed / 1000)}s (query_id ${remaining.slice(0, 10).join(', ')}${remaining.length > 10 ? ', …' : ''}). ` +
|
|
46
|
+
'Their rows may not be mirrored or snapshotted yet. Let them finish, then run chkit migrate --apply again: this step reads the running INSERTs anew. A killed INSERT may have committed part of its rows, so inserting them again would reuse keys.');
|
|
47
|
+
}
|
|
48
|
+
if (now() - lastProgress >= PROGRESS_EVERY_MS) {
|
|
49
|
+
lastProgress = now();
|
|
50
|
+
log(` still waiting for ${remaining.length} INSERT ${remaining.length === 1 ? 'query' : 'queries'} (elapsed ${Math.round(elapsed / 1000)}s)`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function stripTrailingSemicolon(sql) {
|
|
55
|
+
return sql.replace(/;\s*$/, '');
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=insert-barrier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"insert-barrier.js","sourceRoot":"","sources":["../src/insert-barrier.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,YAAY,EAAE,MAAM,sBAAsB,CAAA;AAGjE,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAA;AAEjE,MAAM,kBAAkB,GAAG,EAAE,GAAG,MAAM,CAAA;AACtC,MAAM,wBAAwB,GAAG,KAAK,CAAA;AACtC,MAAM,iBAAiB,GAAG,MAAM,CAAA;AAchC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAA4B,EAC5B,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAA;IACzD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAA;IACzE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAA;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAErC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,uBAAuB,EAAE,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IAC1G,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CACtB,IAAI,GAAG,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAuB,sBAAsB,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;IAEnI,MAAM,OAAO,GAAG,MAAM,IAAI,EAAE,CAAA;IAC5B,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,CAAA;IAC/C,GAAG,CAAC,iBAAiB,OAAO,CAAC,IAAI,mBAAmB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,YAAY,CAAC,CAAA;IAEzG,MAAM,OAAO,GAAG,GAAG,EAAE,CAAA;IACrB,IAAI,YAAY,GAAG,OAAO,CAAA;IAC1B,SAAS,CAAC;QACR,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,EAAE,CAAA;QAC5B,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAC9D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,IAAI,EAAE,CAAA;QAC9D,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,OAAO,CAAA;QAC/B,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,GAAG,SAAS,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,kCAAkC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,wBAAwB,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;gBAC3R,oPAAoP,CACvP,CAAA;QACH,CAAC;QACD,IAAI,GAAG,EAAE,GAAG,YAAY,IAAI,iBAAiB,EAAE,CAAC;YAC9C,YAAY,GAAG,GAAG,EAAE,CAAA;YACpB,GAAG,CAAC,uBAAuB,SAAS,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,aAAa,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAA;QAChJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAW;IACzC,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;AACjC,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ClickHouseExecutor } from './index.js';
|
|
2
|
+
/**
|
|
3
|
+
* A target whose requests a load balancer spreads over several replicas, such
|
|
4
|
+
* as a multi-replica ObsessionDB service. A read there answers from whichever
|
|
5
|
+
* replica it lands on, which may not have applied the latest DDL or journal
|
|
6
|
+
* write yet, so chkit reads every replica through `clusterAllReplicas` instead.
|
|
7
|
+
*/
|
|
8
|
+
export interface ReplicaFanout {
|
|
9
|
+
cluster: string;
|
|
10
|
+
replicas: number;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Counts the replicas of `cluster` (default `default`) once per executor.
|
|
14
|
+
* Returns undefined for a single replica, or when the cluster does not exist
|
|
15
|
+
* or the user cannot query it, so callers keep their single-replica path.
|
|
16
|
+
*/
|
|
17
|
+
export declare function resolveReplicaFanout(executor: ClickHouseExecutor, cluster?: string): Promise<ReplicaFanout | undefined>;
|
|
18
|
+
/** `source` read on every replica, e.g. `allReplicas(fanout, 'system.tables')`. */
|
|
19
|
+
export declare function allReplicas(fanout: ReplicaFanout, source: string): string;
|
|
20
|
+
export declare function stringLiteral(value: string): string;
|
|
21
|
+
//# sourceMappingURL=replicas.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replicas.d.ts","sourceRoot":"","sources":["../src/replicas.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAEpD;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;CACjB;AAOD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,GAAE,MAAwB,GAChC,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAYpC;AAED,mFAAmF;AACnF,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEzE;AAiBD,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnD"}
|
package/dist/replicas.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/** ObsessionDB services expose their replicas as the cluster named `default`. */
|
|
2
|
+
const DEFAULT_CLUSTER = 'default';
|
|
3
|
+
const probes = new WeakMap();
|
|
4
|
+
/**
|
|
5
|
+
* Counts the replicas of `cluster` (default `default`) once per executor.
|
|
6
|
+
* Returns undefined for a single replica, or when the cluster does not exist
|
|
7
|
+
* or the user cannot query it, so callers keep their single-replica path.
|
|
8
|
+
*/
|
|
9
|
+
export function resolveReplicaFanout(executor, cluster = DEFAULT_CLUSTER) {
|
|
10
|
+
let byCluster = probes.get(executor);
|
|
11
|
+
if (!byCluster) {
|
|
12
|
+
byCluster = new Map();
|
|
13
|
+
probes.set(executor, byCluster);
|
|
14
|
+
}
|
|
15
|
+
let probe = byCluster.get(cluster);
|
|
16
|
+
if (!probe) {
|
|
17
|
+
probe = probeReplicas(executor, cluster);
|
|
18
|
+
byCluster.set(cluster, probe);
|
|
19
|
+
}
|
|
20
|
+
return probe;
|
|
21
|
+
}
|
|
22
|
+
/** `source` read on every replica, e.g. `allReplicas(fanout, 'system.tables')`. */
|
|
23
|
+
export function allReplicas(fanout, source) {
|
|
24
|
+
return `clusterAllReplicas(${stringLiteral(fanout.cluster)}, ${source})`;
|
|
25
|
+
}
|
|
26
|
+
async function probeReplicas(executor, cluster) {
|
|
27
|
+
try {
|
|
28
|
+
const rows = await executor.query(`SELECT count(DISTINCT hostName()) AS replicas FROM clusterAllReplicas(${stringLiteral(cluster)}, system.one)`);
|
|
29
|
+
const replicas = Number(rows[0]?.replicas ?? 0);
|
|
30
|
+
return replicas > 1 ? { cluster, replicas } : undefined;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return undefined;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export function stringLiteral(value) {
|
|
37
|
+
return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=replicas.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replicas.js","sourceRoot":"","sources":["../src/replicas.ts"],"names":[],"mappings":"AAaA,iFAAiF;AACjF,MAAM,eAAe,GAAG,SAAS,CAAA;AAEjC,MAAM,MAAM,GAAG,IAAI,OAAO,EAAuE,CAAA;AAEjG;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,QAA4B,EAC5B,UAAkB,eAAe;IAEjC,IAAI,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IACpC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,SAAS,GAAG,IAAI,GAAG,EAAE,CAAA;QACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;IACjC,CAAC;IACD,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAClC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACxC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,MAAqB,EAAE,MAAc;IAC/D,OAAO,sBAAsB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,MAAM,GAAG,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,QAA4B,EAC5B,OAAe;IAEf,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,CAC/B,yEAAyE,aAAa,CAAC,OAAO,CAAC,eAAe,CAC/G,CAAA;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAA;QAC/C,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAA;AACjE,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ClickHouseExecutor } from './index.js';
|
|
2
|
+
export interface IntrospectedView {
|
|
3
|
+
kind: 'view' | 'materialized_view';
|
|
4
|
+
database: string;
|
|
5
|
+
name: string;
|
|
6
|
+
/** The SELECT as ClickHouse stores it (`system.tables.as_select`). */
|
|
7
|
+
query: string;
|
|
8
|
+
/** The `TO` target of a materialized view, when it has one. */
|
|
9
|
+
to?: {
|
|
10
|
+
database: string;
|
|
11
|
+
name: string;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The views and materialized views in `databases`, with the query ClickHouse
|
|
16
|
+
* stores for each. ClickHouse qualifies unqualified table names in that query
|
|
17
|
+
* with the database the view was created from, so `FROM events` comes back as
|
|
18
|
+
* `FROM analytics.events`.
|
|
19
|
+
*/
|
|
20
|
+
export declare function listViewDetails(executor: Pick<ClickHouseExecutor, 'query'>, databases: string[]): Promise<IntrospectedView[]>;
|
|
21
|
+
/**
|
|
22
|
+
* The `TO [db.]name` of a `CREATE MATERIALIZED VIEW` statement: the first
|
|
23
|
+
* top-level `TO` after the view name and before `AS`. A materialized view with
|
|
24
|
+
* an inner table has none.
|
|
25
|
+
*/
|
|
26
|
+
export declare function parseMaterializedViewTarget(createQuery: string | undefined, fallbackDatabase: string): {
|
|
27
|
+
database: string;
|
|
28
|
+
name: string;
|
|
29
|
+
} | undefined;
|
|
30
|
+
//# sourceMappingURL=view-details.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"view-details.d.ts","sourceRoot":"","sources":["../src/view-details.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAEpD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAA;IAClC,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAA;IACb,+DAA+D;IAC/D,EAAE,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CACxC;AAUD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,IAAI,CAAC,kBAAkB,EAAE,OAAO,CAAC,EAC3C,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAuB7B;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,gBAAgB,EAAE,MAAM,GACvB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CA2BhD"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { identifierName, isTrivia, tokenizeSQL } from '@chkit/core';
|
|
2
|
+
/**
|
|
3
|
+
* The views and materialized views in `databases`, with the query ClickHouse
|
|
4
|
+
* stores for each. ClickHouse qualifies unqualified table names in that query
|
|
5
|
+
* with the database the view was created from, so `FROM events` comes back as
|
|
6
|
+
* `FROM analytics.events`.
|
|
7
|
+
*/
|
|
8
|
+
export async function listViewDetails(executor, databases) {
|
|
9
|
+
if (databases.length === 0)
|
|
10
|
+
return [];
|
|
11
|
+
const quotedDatabases = databases.map((db) => `'${db.replace(/'/g, "''")}'`).join(', ');
|
|
12
|
+
const rows = await executor.query(`SELECT database, name, engine, as_select, create_table_query
|
|
13
|
+
FROM system.tables
|
|
14
|
+
WHERE is_temporary = 0
|
|
15
|
+
AND database IN (${quotedDatabases})
|
|
16
|
+
AND engine IN ('View', 'MaterializedView')`);
|
|
17
|
+
return rows.map((row) => {
|
|
18
|
+
if (row.engine === 'View') {
|
|
19
|
+
return { kind: 'view', database: row.database, name: row.name, query: row.as_select };
|
|
20
|
+
}
|
|
21
|
+
const to = parseMaterializedViewTarget(row.create_table_query, row.database);
|
|
22
|
+
return {
|
|
23
|
+
kind: 'materialized_view',
|
|
24
|
+
database: row.database,
|
|
25
|
+
name: row.name,
|
|
26
|
+
query: row.as_select,
|
|
27
|
+
...(to ? { to } : {}),
|
|
28
|
+
};
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The `TO [db.]name` of a `CREATE MATERIALIZED VIEW` statement: the first
|
|
33
|
+
* top-level `TO` after the view name and before `AS`. A materialized view with
|
|
34
|
+
* an inner table has none.
|
|
35
|
+
*/
|
|
36
|
+
export function parseMaterializedViewTarget(createQuery, fallbackDatabase) {
|
|
37
|
+
if (!createQuery)
|
|
38
|
+
return undefined;
|
|
39
|
+
const tokens = tokenizeSQL(createQuery).filter((token) => !isTrivia(token));
|
|
40
|
+
const nameAt = (i) => {
|
|
41
|
+
const token = tokens[i];
|
|
42
|
+
return token ? identifierName(token) : undefined;
|
|
43
|
+
};
|
|
44
|
+
// Skip `CREATE MATERIALIZED VIEW <db>.<name>`.
|
|
45
|
+
let i = tokens[4]?.text === '.' ? 6 : 4;
|
|
46
|
+
let depth = 0;
|
|
47
|
+
for (; i < tokens.length; i++) {
|
|
48
|
+
const token = tokens[i];
|
|
49
|
+
if (!token)
|
|
50
|
+
continue;
|
|
51
|
+
if (token.text === '(')
|
|
52
|
+
depth++;
|
|
53
|
+
else if (token.text === ')')
|
|
54
|
+
depth--;
|
|
55
|
+
if (depth !== 0 || token.kind !== 'identifier')
|
|
56
|
+
continue;
|
|
57
|
+
const keyword = token.text.toUpperCase();
|
|
58
|
+
// A materialized view with an inner table has an engine and no `TO`.
|
|
59
|
+
if (keyword === 'AS' || keyword === 'ENGINE')
|
|
60
|
+
return undefined;
|
|
61
|
+
if (keyword !== 'TO')
|
|
62
|
+
continue;
|
|
63
|
+
const first = nameAt(i + 1);
|
|
64
|
+
if (first === undefined)
|
|
65
|
+
return undefined;
|
|
66
|
+
if (tokens[i + 2]?.text !== '.')
|
|
67
|
+
return { database: fallbackDatabase, name: first };
|
|
68
|
+
const second = nameAt(i + 3);
|
|
69
|
+
return second === undefined ? undefined : { database: first, name: second };
|
|
70
|
+
}
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=view-details.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"view-details.js","sourceRoot":"","sources":["../src/view-details.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAsBnE;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAA2C,EAC3C,SAAmB;IAEnB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IACrC,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,CAC/B;;;qBAGiB,eAAe;6CACS,CAC1C,CAAA;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,EAAE,CAAA;QACvF,CAAC;QACD,MAAM,EAAE,GAAG,2BAA2B,CAAC,GAAG,CAAC,kBAAkB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC5E,OAAO;YACL,IAAI,EAAE,mBAAmB;YACzB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,KAAK,EAAE,GAAG,CAAC,SAAS;YACpB,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtB,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CACzC,WAA+B,EAC/B,gBAAwB;IAExB,IAAI,CAAC,WAAW;QAAE,OAAO,SAAS,CAAA;IAClC,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;IAC3E,MAAM,MAAM,GAAG,CAAC,CAAS,EAAsB,EAAE;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QACvB,OAAO,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAClD,CAAC,CAAA;IACD,+CAA+C;IAC/C,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACvC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QACvB,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,IAAI,KAAK,CAAC,IAAI,KAAK,GAAG;YAAE,KAAK,EAAE,CAAA;aAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,GAAG;YAAE,KAAK,EAAE,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;YAAE,SAAQ;QACxD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;QACxC,qEAAqE;QACrE,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAA;QAC9D,IAAI,OAAO,KAAK,IAAI;YAAE,SAAQ;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,SAAS,CAAA;QACzC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,GAAG;YAAE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;QACnF,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC5B,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IAC7E,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chkit/clickhouse",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0-beta.12",
|
|
4
4
|
"description": "ClickHouse client wrapper for chkit",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "ObsessionDB",
|
|
@@ -36,6 +36,9 @@
|
|
|
36
36
|
"files": [
|
|
37
37
|
"dist"
|
|
38
38
|
],
|
|
39
|
+
"engines": {
|
|
40
|
+
"node": ">=22"
|
|
41
|
+
},
|
|
39
42
|
"scripts": {
|
|
40
43
|
"build": "tsc -p tsconfig.json",
|
|
41
44
|
"dev": "tsc -p tsconfig.json --watch",
|
|
@@ -45,7 +48,7 @@
|
|
|
45
48
|
"clean": "rm -rf dist"
|
|
46
49
|
},
|
|
47
50
|
"dependencies": {
|
|
48
|
-
"@chkit/core": "0.
|
|
51
|
+
"@chkit/core": "0.2.0-beta.12",
|
|
49
52
|
"@clickhouse/client": "^1.18.0",
|
|
50
53
|
"@logtape/logtape": "^2.0.5",
|
|
51
54
|
"p-retry": "^7.1.1"
|