@intentius/chant-lexicon-sql 0.114.1 → 0.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/apply/mutations.d.ts +7 -1
- package/dist/clickhouse/apply/mutations.d.ts.map +1 -1
- package/dist/clickhouse/container.d.ts +22 -7
- package/dist/clickhouse/container.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.d.ts +5 -0
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/backfill.d.ts +11 -5
- package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/op.d.ts +29 -0
- package/dist/clickhouse/rebuild/op.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/partitions.d.ts +8 -0
- package/dist/clickhouse/rebuild/partitions.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/receipts.d.ts +4 -6
- package/dist/clickhouse/rebuild/receipts.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/shards.d.ts +59 -0
- package/dist/clickhouse/rebuild/shards.d.ts.map +1 -0
- package/dist/clickhouse/rebuild/steps.d.ts +3 -0
- package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/verify.d.ts +4 -0
- package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
- package/dist/config.d.ts +14 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/core/receipts.d.ts +5 -3
- package/dist/core/receipts.d.ts.map +1 -1
- package/dist/integrity.json +2 -2
- package/dist/manifest.json +1 -1
- package/dist/op/activities/clickhouse-rebuild.d.ts.map +1 -1
- package/dist/op/activities/index.d.ts +13 -1
- package/dist/op/activities/index.d.ts.map +1 -1
- package/dist/op/activities/sql-exec.d.ts +41 -0
- package/dist/op/activities/sql-exec.d.ts.map +1 -0
- package/dist/op/activity-contracts.d.ts +20 -0
- package/dist/op/activity-contracts.d.ts.map +1 -1
- package/dist/postgres/live/catalog.d.ts +1 -1
- package/dist/postgres/live/catalog.d.ts.map +1 -1
- package/dist/postgres/migrate/receipts.d.ts +8 -5
- package/dist/postgres/migrate/receipts.d.ts.map +1 -1
- package/dist/receipts.d.ts +102 -0
- package/dist/receipts.d.ts.map +1 -0
- package/package.json +2 -2
- package/src/clickhouse/apply/mutations.ts +9 -3
- package/src/clickhouse/container.ts +39 -11
- package/src/clickhouse/plan/cluster-drift.e2e.test.ts +117 -0
- package/src/clickhouse/plan/deep.ts +102 -3
- package/src/clickhouse/plan/normalize.ts +10 -0
- package/src/clickhouse/plan/plan.test.ts +10 -0
- package/src/clickhouse/rebuild/backfill.ts +85 -42
- package/src/clickhouse/rebuild/op.ts +80 -35
- package/src/clickhouse/rebuild/partitions.ts +11 -0
- package/src/clickhouse/rebuild/rebuild-sharded.e2e.test.ts +161 -0
- package/src/clickhouse/rebuild/rebuild.e2e.test.ts +110 -4
- package/src/clickhouse/rebuild/rebuild.test.ts +56 -4
- package/src/clickhouse/rebuild/receipts.ts +4 -6
- package/src/clickhouse/rebuild/shards.ts +104 -0
- package/src/clickhouse/rebuild/steps.ts +4 -0
- package/src/clickhouse/rebuild/verify.ts +14 -5
- package/src/config.ts +7 -0
- package/src/core/receipts.ts +5 -3
- package/src/op/activities/clickhouse-rebuild.ts +32 -7
- package/src/op/activities/index.ts +21 -4
- package/src/op/activities/sql-exec.ts +84 -0
- package/src/op/activity-contracts.ts +15 -0
- package/src/op/effect-receipts.e2e.test.ts +277 -0
- package/src/postgres/live/catalog.ts +3 -1
- package/src/postgres/migrate/migrate.test.ts +3 -2
- package/src/postgres/migrate/receipts.ts +18 -10
- package/src/receipts.test.ts +91 -0
- package/src/receipts.ts +259 -0
- package/src/skills/chant-sql-rebuild.md +5 -1
|
@@ -30,6 +30,7 @@ export declare const clickhouseRebuildPlanContract: import("@intentius/chant/op"
|
|
|
30
30
|
stack?: string | undefined;
|
|
31
31
|
ownershipEnv?: string | undefined;
|
|
32
32
|
cwd?: string | undefined;
|
|
33
|
+
keepOnFailure?: boolean | undefined;
|
|
33
34
|
}, {
|
|
34
35
|
state: "done" | "rebuild" | "swapped";
|
|
35
36
|
table: string;
|
|
@@ -54,6 +55,7 @@ export declare const clickhouseRebuildCreateContract: import("@intentius/chant/o
|
|
|
54
55
|
stack?: string | undefined;
|
|
55
56
|
ownershipEnv?: string | undefined;
|
|
56
57
|
cwd?: string | undefined;
|
|
58
|
+
keepOnFailure?: boolean | undefined;
|
|
57
59
|
}, {
|
|
58
60
|
state: "done" | "rebuild" | "swapped";
|
|
59
61
|
table: string;
|
|
@@ -76,6 +78,7 @@ export declare const clickhouseRebuildDualWriteContract: import("@intentius/chan
|
|
|
76
78
|
stack?: string | undefined;
|
|
77
79
|
ownershipEnv?: string | undefined;
|
|
78
80
|
cwd?: string | undefined;
|
|
81
|
+
keepOnFailure?: boolean | undefined;
|
|
79
82
|
}, {
|
|
80
83
|
state: "done" | "rebuild" | "swapped";
|
|
81
84
|
mode: "app" | "materialized-view";
|
|
@@ -99,6 +102,7 @@ export declare const clickhouseRebuildBackfillContract: import("@intentius/chant
|
|
|
99
102
|
stack?: string | undefined;
|
|
100
103
|
ownershipEnv?: string | undefined;
|
|
101
104
|
cwd?: string | undefined;
|
|
105
|
+
keepOnFailure?: boolean | undefined;
|
|
102
106
|
}, {
|
|
103
107
|
state: "done" | "rebuild" | "swapped";
|
|
104
108
|
partitions: number;
|
|
@@ -123,6 +127,7 @@ export declare const clickhouseRebuildVerifyContract: import("@intentius/chant/o
|
|
|
123
127
|
stack?: string | undefined;
|
|
124
128
|
ownershipEnv?: string | undefined;
|
|
125
129
|
cwd?: string | undefined;
|
|
130
|
+
keepOnFailure?: boolean | undefined;
|
|
126
131
|
}, {
|
|
127
132
|
state: "done" | "rebuild" | "swapped";
|
|
128
133
|
partitions: number;
|
|
@@ -152,6 +157,7 @@ export declare const clickhouseRebuildSwapContract: import("@intentius/chant/op"
|
|
|
152
157
|
stack?: string | undefined;
|
|
153
158
|
ownershipEnv?: string | undefined;
|
|
154
159
|
cwd?: string | undefined;
|
|
160
|
+
keepOnFailure?: boolean | undefined;
|
|
155
161
|
}, {
|
|
156
162
|
state: "done" | "rebuild" | "swapped";
|
|
157
163
|
swapped: boolean;
|
|
@@ -176,6 +182,7 @@ export declare const clickhouseRebuildRetainContract: import("@intentius/chant/o
|
|
|
176
182
|
stack?: string | undefined;
|
|
177
183
|
ownershipEnv?: string | undefined;
|
|
178
184
|
cwd?: string | undefined;
|
|
185
|
+
keepOnFailure?: boolean | undefined;
|
|
179
186
|
}, {
|
|
180
187
|
state: "done" | "rebuild" | "swapped";
|
|
181
188
|
due: boolean;
|
|
@@ -200,6 +207,7 @@ export declare const clickhouseRebuildDropContract: import("@intentius/chant/op"
|
|
|
200
207
|
stack?: string | undefined;
|
|
201
208
|
ownershipEnv?: string | undefined;
|
|
202
209
|
cwd?: string | undefined;
|
|
210
|
+
keepOnFailure?: boolean | undefined;
|
|
203
211
|
}, {
|
|
204
212
|
state: "done" | "rebuild" | "swapped";
|
|
205
213
|
dropped: boolean;
|
|
@@ -223,6 +231,7 @@ export declare const clickhouseRebuildCompensateContract: import("@intentius/cha
|
|
|
223
231
|
stack?: string | undefined;
|
|
224
232
|
ownershipEnv?: string | undefined;
|
|
225
233
|
cwd?: string | undefined;
|
|
234
|
+
keepOnFailure?: boolean | undefined;
|
|
226
235
|
}, {
|
|
227
236
|
dropped: string[];
|
|
228
237
|
}>;
|
|
@@ -461,4 +470,15 @@ export declare const postgresMigrationCompensateContract: import("@intentius/cha
|
|
|
461
470
|
}, {
|
|
462
471
|
dropped: string[];
|
|
463
472
|
}>;
|
|
473
|
+
/** `SqlExecArgs` (`./activities/sql-exec.ts`): one batch's SQL on the environment's server. */
|
|
474
|
+
export declare const sqlExecContract: import("@intentius/chant/op").ActivityContract<{
|
|
475
|
+
sql: string;
|
|
476
|
+
environment?: string | undefined;
|
|
477
|
+
settings?: Record<string, string | number> | undefined;
|
|
478
|
+
cwd?: string | undefined;
|
|
479
|
+
}, {
|
|
480
|
+
dialect: "clickhouse" | "postgres";
|
|
481
|
+
source: string;
|
|
482
|
+
rows: number;
|
|
483
|
+
}>;
|
|
464
484
|
//# sourceMappingURL=activity-contracts.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"activity-contracts.d.ts","sourceRoot":"","sources":["../../src/op/activity-contracts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"activity-contracts.d.ts","sourceRoot":"","sources":["../../src/op/activity-contracts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA4BH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAY3C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;;EAK/C,CAAC;AAuBF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;EAK1C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;EAc3C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;EAK/C,CAAC;AAIF,+FAA+F;AAC/F,eAAO,MAAM,eAAe;;;;;;;;;EAS3B,CAAC"}
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* column's sequence, since the column prints as `serial`;
|
|
23
23
|
* - an index that backs a constraint (the constraint creates it), and an
|
|
24
24
|
* index partition (the partitioned index creates it);
|
|
25
|
-
* - chant's own receipts tables (`../../core/ownership.ts`), and the working
|
|
25
|
+
* - chant's own receipts tables and schema (`../../core/ownership.ts`), and the working
|
|
26
26
|
* columns, constraints and indexes of an expand-and-contract migration in
|
|
27
27
|
* progress (`../migrate/names.ts`, `../migrate/carry.ts`), so a table reads
|
|
28
28
|
* as it was before the migration until its switch, and as declared after it.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../../src/postgres/live/catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAyB,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAKjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,iEAAiE;AACjE,eAAO,MAAM,cAAc,UAAmD,CAAC;AAE/E,kFAAkF;AAClF,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAa3D,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,kBAAkB,CAAC;IACzB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,iDAAiD;IACjD,GAAG,EAAE,MAAM,CAAC;IACZ,uFAAuF;IACvF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC1B,gHAAgH;IAChH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7B;AAyCD,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CAChB;AAQD,yFAAyF;AACzF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,GAAG,OAAO,CAAC,CAKtH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,GAAG,MAAM,EAAE,CAa9E;AAcD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,GAAE,WAAgB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../../src/postgres/live/catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAyB,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAKjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,iEAAiE;AACjE,eAAO,MAAM,cAAc,UAAmD,CAAC;AAE/E,kFAAkF;AAClF,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAa3D,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,kBAAkB,CAAC;IACzB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,iDAAiD;IACjD,GAAG,EAAE,MAAM,CAAC;IACZ,uFAAuF;IACvF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC1B,gHAAgH;IAChH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7B;AAyCD,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CAChB;AAQD,yFAAyF;AACzF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,GAAG,OAAO,CAAC,CAKtH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,GAAG,MAAM,EAAE,CAa9E;AAcD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,GAAE,WAAgB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAqX7G;AAED,yEAAyE;AACzE,eAAO,MAAM,OAAO,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,GAAG,SAAS,EAAE,MAAM,MAAM,KAAG,MAA2C,CAAC;AAE7H;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,EAAE,QAAQ,EAAE,gBAAgB,GAAG,SAAS,GAAG,YAAY,EAAE,CAW1H"}
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
* stale; onFailure and the contract also delete the migration's receipts.
|
|
28
28
|
*
|
|
29
29
|
* The store implements the shared core's `SqlReceiptStore`
|
|
30
|
-
* (`../../core/receipts.ts`) over the migration's own connection
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* (`../../core/receipts.ts`) over the migration's own connection, which the
|
|
31
|
+
* backfill reaches directly. A sql project's `effect()` receipts use the same
|
|
32
|
+
* store over `chant_receipts.receipts` (`../../receipts.ts`).
|
|
33
33
|
*/
|
|
34
34
|
import { type SqlReceiptStore } from "../../core/receipts.js";
|
|
35
35
|
import type { PostgresClient } from "../live/client.js";
|
|
@@ -38,7 +38,7 @@ export declare const POSTGRES_RECEIPTS_TABLE = "__chant_receipts";
|
|
|
38
38
|
export { receiptAddress } from "../../core/receipts.js";
|
|
39
39
|
/** A receipts table chant did not make: a user's table under the same name. */
|
|
40
40
|
export declare class ReceiptsTableConflict extends Error {
|
|
41
|
-
constructor(schema: string);
|
|
41
|
+
constructor(schema: string, table?: string);
|
|
42
42
|
}
|
|
43
43
|
export interface PostgresReceiptStore extends SqlReceiptStore {
|
|
44
44
|
/** The qualified, quoted table name. */
|
|
@@ -52,12 +52,15 @@ export interface PostgresReceiptStore extends SqlReceiptStore {
|
|
|
52
52
|
* A receipt store in `<schema>.__chant_receipts`, over `client`. `write` runs
|
|
53
53
|
* on the client as it is, so a write inside the caller's transaction commits
|
|
54
54
|
* or rolls back with it. `identity` is the project's ownership stack and env,
|
|
55
|
-
* which every address starts with.
|
|
55
|
+
* which every address starts with. `table` names the table in `schema`,
|
|
56
|
+
* `__chant_receipts` by default (`../../receipts.ts` keeps `effect()`
|
|
57
|
+
* receipts in `chant_receipts.receipts`).
|
|
56
58
|
*/
|
|
57
59
|
export declare function postgresReceiptStore(client: PostgresClient, schema: string, identity: {
|
|
58
60
|
stack?: string;
|
|
59
61
|
env?: string;
|
|
60
62
|
}, opts?: {
|
|
61
63
|
runId?: string;
|
|
64
|
+
table?: string;
|
|
62
65
|
}): PostgresReceiptStore;
|
|
63
66
|
//# sourceMappingURL=receipts.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"receipts.d.ts","sourceRoot":"","sources":["../../../src/postgres/migrate/receipts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,0DAA0D;AAC1D,eAAO,MAAM,uBAAuB,qBAAqB,CAAC;AAI1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,+EAA+E;AAC/E,qBAAa,qBAAsB,SAAQ,KAAK;gBAClC,MAAM,EAAE,MAAM;
|
|
1
|
+
{"version":3,"file":"receipts.d.ts","sourceRoot":"","sources":["../../../src/postgres/migrate/receipts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,0DAA0D;AAC1D,eAAO,MAAM,uBAAuB,qBAAqB,CAAC;AAI1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,+EAA+E;AAC/E,qBAAa,qBAAsB,SAAQ,KAAK;gBAClC,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAAgC;CAOpE;AAED,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,gGAAgG;IAChG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,iGAAiG;IACjG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,EAC1C,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,oBAAoB,CA6DtB"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sql lexicon's `effect()` receipt store (#3657): where the receipts of a
|
|
3
|
+
* project that uses only the sql lexicon are kept, on the database server the
|
|
4
|
+
* environment names.
|
|
5
|
+
*
|
|
6
|
+
* ClickHouse chant_receipts.receipts
|
|
7
|
+
* Postgres chant_receipts.receipts (or <receiptsSchema>.__chant_receipts)
|
|
8
|
+
*
|
|
9
|
+
* The op activities module binds it as `receiptRead`, `receiptWrite` and
|
|
10
|
+
* `receiptStaleness` (`./op/activities/index.ts`), listed as fallbacks: a
|
|
11
|
+
* project that also configures aws or k8s keeps its receipts in that
|
|
12
|
+
* lexicon's receipt row, in whichever order `lexicons` lists them.
|
|
13
|
+
*
|
|
14
|
+
* Which server: the environment is the one `chant run --env` sets
|
|
15
|
+
* (`CHANT_ENV`), else a literal `ownership.env`. Its `sql.profiles.<env>`
|
|
16
|
+
* binds the server, a `postgres://` URL making it Postgres; with no profile,
|
|
17
|
+
* `CLICKHOUSE_URL` or `POSTGRES_URL`, picked by `sql.dialect`. A receipt's
|
|
18
|
+
* address is `<stack>/<env>/<effect>`, from `ownership.stack` and that
|
|
19
|
+
* environment, the same identity the rebuild's and the column migration's
|
|
20
|
+
* receipts carry (`./core/receipts.ts`).
|
|
21
|
+
*
|
|
22
|
+
* Both tables carry chant's ownership trailer with the `receipts` key in their
|
|
23
|
+
* comment (the Postgres schema too), so plan, import and prune leave them out.
|
|
24
|
+
*
|
|
25
|
+
* A caller that runs its own Op over a server it has already bound (a
|
|
26
|
+
* migration runner, say) passes that server instead: a ClickHouse endpoint,
|
|
27
|
+
* or a Postgres connection, whose receipt writes then commit or roll back
|
|
28
|
+
* with whatever transaction the caller has open on it.
|
|
29
|
+
*/
|
|
30
|
+
import type { ChantConfig } from "@intentius/chant/config";
|
|
31
|
+
import { type ReceiptActivities, type ReceiptActivityOptions } from "@intentius/chant/op/receipt-store";
|
|
32
|
+
import type { SqlReceiptStore } from "./core/receipts.js";
|
|
33
|
+
import type { ClickHouseEndpoint } from "./clickhouse/http.js";
|
|
34
|
+
import type { Topology } from "./clickhouse/topology.js";
|
|
35
|
+
import type { PostgresClient } from "./postgres/live/client.js";
|
|
36
|
+
/** The database (ClickHouse) or schema (Postgres) the receipts are kept in. */
|
|
37
|
+
export declare const SQL_RECEIPTS_SCHEMA = "chant_receipts";
|
|
38
|
+
/** The table in it. */
|
|
39
|
+
export declare const SQL_RECEIPTS_TABLE = "receipts";
|
|
40
|
+
/** The table's name in a schema of the user's own (`receiptsSchema`), as the column migration names it. */
|
|
41
|
+
export declare const SQL_RECEIPTS_TABLE_BESIDE = "__chant_receipts";
|
|
42
|
+
export interface SqlReceiptStoreOptions {
|
|
43
|
+
/** The chant environment. Default: `CHANT_ENV`, then a literal `ownership.env`. */
|
|
44
|
+
environment?: string;
|
|
45
|
+
/** The address's stack segment. Default: `ownership.stack`. */
|
|
46
|
+
stack?: string;
|
|
47
|
+
/** The project directory chant.config.ts is read from. Default: the working directory. */
|
|
48
|
+
cwd?: string;
|
|
49
|
+
/** The project's config, instead of reading chant.config.ts. */
|
|
50
|
+
config?: Pick<ChantConfig, "sql" | "ownership">;
|
|
51
|
+
/** The process environment the bindings and `CHANT_ENV` are read from. Default: `process.env`. */
|
|
52
|
+
env?: Record<string, string | undefined>;
|
|
53
|
+
/** A ClickHouse server already bound: the receipts go there, and no profile is read. */
|
|
54
|
+
clickhouse?: {
|
|
55
|
+
endpoint: ClickHouseEndpoint;
|
|
56
|
+
topology?: Topology;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* A Postgres connection already open: the receipts go over it, and it is
|
|
60
|
+
* never closed. A write runs inside whatever transaction the caller has
|
|
61
|
+
* open, so it commits or rolls back with it. Call `ensure()` before such a
|
|
62
|
+
* transaction begins, so the receipts table does not roll back with it.
|
|
63
|
+
*/
|
|
64
|
+
postgres?: PostgresClient;
|
|
65
|
+
/** Postgres: the schema to keep the receipts in. Default: the profile's `receiptsSchema`, then `chant_receipts`. */
|
|
66
|
+
schema?: string;
|
|
67
|
+
/** Recorded on each receipt written. Default: the current Op run's id. */
|
|
68
|
+
runId?: string;
|
|
69
|
+
}
|
|
70
|
+
/** The store, and the step a caller with its own transaction takes before one opens. */
|
|
71
|
+
export interface SqlServerReceiptStore extends SqlReceiptStore {
|
|
72
|
+
/** Create the receipts table (and database or schema) if it is not there. */
|
|
73
|
+
ensure(): Promise<void>;
|
|
74
|
+
/** Where the receipts are, as resolved: `chant_receipts.receipts` and the server it is on. */
|
|
75
|
+
location(): Promise<{
|
|
76
|
+
dialect: "clickhouse" | "postgres";
|
|
77
|
+
table: string;
|
|
78
|
+
source: string;
|
|
79
|
+
address: string;
|
|
80
|
+
}>;
|
|
81
|
+
}
|
|
82
|
+
type Config = Pick<ChantConfig, "sql" | "ownership">;
|
|
83
|
+
/** The dialect an environment's receipts are kept in: its profile's URL, else `sql.dialect`, else whichever server variable is set. */
|
|
84
|
+
export declare function receiptDialect(config: Config | undefined, environment: string | undefined, env: Record<string, string | undefined>): "clickhouse" | "postgres";
|
|
85
|
+
/** The run's environment: the option, `CHANT_ENV`, or a literal `ownership.env`. */
|
|
86
|
+
export declare function environmentOf(options: {
|
|
87
|
+
environment?: string;
|
|
88
|
+
}, config: Pick<ChantConfig, "ownership"> | undefined, env: Record<string, string | undefined>): string | undefined;
|
|
89
|
+
/**
|
|
90
|
+
* The receipt store on the environment's database server. Nothing is read or
|
|
91
|
+
* connected until the first read or write; the environment, the server and
|
|
92
|
+
* the identity are resolved then, once.
|
|
93
|
+
*/
|
|
94
|
+
export declare function sqlReceiptStore(options?: SqlReceiptStoreOptions): SqlServerReceiptStore;
|
|
95
|
+
/**
|
|
96
|
+
* The `receiptRead`, `receiptWrite` and `receiptStaleness` activities over
|
|
97
|
+
* {@link sqlReceiptStore}: what the op activities module exports, and what a
|
|
98
|
+
* caller running an Op of `effect()` steps with its own activity map binds.
|
|
99
|
+
*/
|
|
100
|
+
export declare function sqlReceiptActivities(options?: SqlReceiptStoreOptions, activityOptions?: ReceiptActivityOptions): ReceiptActivities;
|
|
101
|
+
export {};
|
|
102
|
+
//# sourceMappingURL=receipts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"receipts.d.ts","sourceRoot":"","sources":["../src/receipts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAqB,KAAK,iBAAiB,EAAE,KAAK,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAE3H,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE7D,+EAA+E;AAC/E,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AACpD,uBAAuB;AACvB,eAAO,MAAM,kBAAkB,aAAa,CAAC;AAC7C,2GAA2G;AAC3G,eAAO,MAAM,yBAAyB,qBAAqB,CAAC;AAE5D,MAAM,WAAW,sBAAsB;IACrC,mFAAmF;IACnF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0FAA0F;IAC1F,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,MAAM,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,WAAW,CAAC,CAAC;IAChD,kGAAkG;IAClG,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,wFAAwF;IACxF,UAAU,CAAC,EAAE;QAAE,QAAQ,EAAE,kBAAkB,CAAC;QAAC,QAAQ,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IACnE;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,oHAAoH;IACpH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,6EAA6E;IAC7E,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,8FAA8F;IAC9F,QAAQ,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,YAAY,GAAG,UAAU,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7G;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,WAAW,CAAC,CAAC;AAqBrD,uIAAuI;AACvI,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,YAAY,GAAG,UAAU,CAO9J;AAED,oFAAoF;AACpF,wBAAgB,aAAa,CAAC,OAAO,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,GAAG,SAAS,CAKhL;AAgGD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,GAAE,sBAA2B,GAAG,qBAAqB,CA+B3F;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,GAAE,sBAA2B,EAAE,eAAe,CAAC,EAAE,sBAAsB,GAAG,iBAAiB,CAEtI"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@intentius/chant-lexicon-sql",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.115.0",
|
|
4
4
|
"description": "SQL lexicon for chant — database schema as typed declarations, one dialect at a time; ClickHouse first, with types read from a pinned server's catalog",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://intentius.io/chant",
|
|
@@ -70,7 +70,7 @@
|
|
|
70
70
|
"typescript": "^5.9.3"
|
|
71
71
|
},
|
|
72
72
|
"peerDependencies": {
|
|
73
|
-
"@intentius/chant": "^0.
|
|
73
|
+
"@intentius/chant": "^0.115.0",
|
|
74
74
|
"typescript": "^5.9.3"
|
|
75
75
|
}
|
|
76
76
|
}
|
|
@@ -49,10 +49,10 @@ export class MutationFailedError extends Error {
|
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
/** The table's mutations that are not done. */
|
|
52
|
-
export async function pendingMutations(endpoint: ClickHouseEndpoint, database: string, table: string): Promise<PendingMutation[]> {
|
|
52
|
+
export async function pendingMutations(endpoint: ClickHouseEndpoint, database: string, table: string, source = "system.mutations"): Promise<PendingMutation[]> {
|
|
53
53
|
return clickhouseQuery<PendingMutation>(
|
|
54
54
|
endpoint,
|
|
55
|
-
`SELECT mutation_id, command, latest_fail_reason FROM
|
|
55
|
+
`SELECT mutation_id, command, latest_fail_reason FROM ${source} ` +
|
|
56
56
|
`WHERE database = ${sqlString(database)} AND table = ${sqlString(table)} AND NOT is_done ORDER BY create_time`,
|
|
57
57
|
);
|
|
58
58
|
}
|
|
@@ -63,6 +63,12 @@ export interface WaitOptions {
|
|
|
63
63
|
/** First poll interval; it doubles up to two seconds. Default: 100ms. */
|
|
64
64
|
intervalMs?: number;
|
|
65
65
|
signal?: AbortSignal;
|
|
66
|
+
/**
|
|
67
|
+
* Where the mutations are listed. Default: `system.mutations`, this
|
|
68
|
+
* server's. A rebuild across the shards of a cluster waits on every
|
|
69
|
+
* server's, `clusterAllReplicas('<cluster>', system.mutations)` (#3663).
|
|
70
|
+
*/
|
|
71
|
+
source?: string;
|
|
66
72
|
}
|
|
67
73
|
|
|
68
74
|
/**
|
|
@@ -78,7 +84,7 @@ export async function waitForMutations(endpoint: ClickHouseEndpoint, database: s
|
|
|
78
84
|
const name = `${database}.${table}`;
|
|
79
85
|
for (;;) {
|
|
80
86
|
opts.signal?.throwIfAborted();
|
|
81
|
-
const pending = await pendingMutations(endpoint, database, table);
|
|
87
|
+
const pending = await pendingMutations(endpoint, database, table, opts.source);
|
|
82
88
|
for (const m of pending) seen.add(m.mutation_id);
|
|
83
89
|
if (pending.length === 0) return [...seen];
|
|
84
90
|
const failing = pending.find((m) => m.latest_fail_reason);
|
|
@@ -51,8 +51,12 @@ export async function startScratchServer(
|
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
export interface ScratchCluster {
|
|
54
|
-
/** One endpoint per
|
|
54
|
+
/** One endpoint per server, in order: shard 1's replicas first (`r1`, `r2`, ...), then shard 2's. */
|
|
55
55
|
replicas: ClickHouseEndpoint[];
|
|
56
|
+
/** Each server's shard and replica macros, in the order of `replicas`. */
|
|
57
|
+
layout: Array<{ shard: string; replica: string; host: string }>;
|
|
58
|
+
/** The cluster `remote_servers` defines over every server: `ON CLUSTER` and `cluster:<name>` name it. */
|
|
59
|
+
cluster: string;
|
|
56
60
|
/** The containers' common name prefix, for a log line. */
|
|
57
61
|
name: string;
|
|
58
62
|
/** Remove the containers and their network. Safe to call twice. */
|
|
@@ -72,23 +76,37 @@ export interface ScratchCluster {
|
|
|
72
76
|
}
|
|
73
77
|
|
|
74
78
|
/**
|
|
75
|
-
* Start
|
|
76
|
-
*
|
|
79
|
+
* Start servers of `image` on their own Docker network, and wait until each
|
|
80
|
+
* answers and reaches Keeper: `replicas` servers as one shard of replicas,
|
|
81
|
+
* or, with `shards`, that many replicas in each shard (`[2, 1]` is a shard of
|
|
82
|
+
* two replicas and a shard of one server).
|
|
77
83
|
*
|
|
78
84
|
* Keeper is the server's embedded one (`keeper_server`), run by the first
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
* a `Replicated` database's
|
|
82
|
-
*
|
|
85
|
+
* server, so the pinned server image and its digest pin Keeper too. Each
|
|
86
|
+
* server gets the macros `{shard}` = `s<n>` and `{replica}` = `r<n>` (the
|
|
87
|
+
* replica numbered within its shard), which a `Replicated` database's and a
|
|
88
|
+
* `Replicated*MergeTree`'s arguments name, and `remote_servers` defines the
|
|
89
|
+
* cluster `cluster` (default `chant`) over all of them, with
|
|
90
|
+
* `internal_replication`, so `ON CLUSTER` reaches every server. Configuration
|
|
91
|
+
* is copied into each container before it starts, with no bind mount.
|
|
83
92
|
*/
|
|
84
93
|
export async function startScratchCluster(
|
|
85
94
|
image: string,
|
|
86
|
-
options: { replicas?: number; namePrefix?: string; readyTimeoutMs?: number } = {},
|
|
95
|
+
options: { replicas?: number; shards?: number[]; cluster?: string; namePrefix?: string; readyTimeoutMs?: number } = {},
|
|
87
96
|
): Promise<ScratchCluster> {
|
|
88
|
-
const
|
|
97
|
+
const shards = options.shards ?? [options.replicas ?? 2];
|
|
98
|
+
const clusterName = options.cluster ?? "chant";
|
|
89
99
|
const name = uniqueContainerName(options.namePrefix ?? "chant-sql-cluster");
|
|
90
100
|
const network = `${name}-net`;
|
|
91
|
-
|
|
101
|
+
// One shard keeps the names the single-shard cluster always had (`<name>-r<n>`).
|
|
102
|
+
const layout = shards.flatMap((replicas, s) =>
|
|
103
|
+
Array.from({ length: replicas }, (_, r) => ({
|
|
104
|
+
shard: `s${s + 1}`,
|
|
105
|
+
replica: `r${r + 1}`,
|
|
106
|
+
host: shards.length === 1 ? `${name}-r${r + 1}` : `${name}-s${s + 1}r${r + 1}`,
|
|
107
|
+
})),
|
|
108
|
+
);
|
|
109
|
+
const nodes = layout.map((l) => l.host);
|
|
92
110
|
const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-"));
|
|
93
111
|
let removed = false;
|
|
94
112
|
const stop = async (): Promise<void> => {
|
|
@@ -100,6 +118,12 @@ export async function startScratchCluster(
|
|
|
100
118
|
};
|
|
101
119
|
|
|
102
120
|
const keeper = nodes[0]!;
|
|
121
|
+
const remoteServers = shards
|
|
122
|
+
.map((_, s) => {
|
|
123
|
+
const hosts = layout.filter((l) => l.shard === `s${s + 1}`).map((l) => `<replica><host>${l.host}</host><port>9000</port></replica>`);
|
|
124
|
+
return `<shard><internal_replication>true</internal_replication>${hosts.join("")}</shard>`;
|
|
125
|
+
})
|
|
126
|
+
.join("");
|
|
103
127
|
const config = (node: string, i: number) => `<clickhouse>
|
|
104
128
|
${
|
|
105
129
|
i === 0
|
|
@@ -113,7 +137,9 @@ ${
|
|
|
113
137
|
`
|
|
114
138
|
: ""
|
|
115
139
|
} <zookeeper><node><host>${keeper}</host><port>9181</port></node></zookeeper>
|
|
116
|
-
<
|
|
140
|
+
<distributed_ddl><path>/clickhouse/task_queue/ddl</path></distributed_ddl>
|
|
141
|
+
<remote_servers><${clusterName}>${remoteServers}</${clusterName}></remote_servers>
|
|
142
|
+
<macros><shard>${layout[i]!.shard}</shard><replica>${layout[i]!.replica}</replica></macros>
|
|
117
143
|
<interserver_http_host>${node}</interserver_http_host>
|
|
118
144
|
</clickhouse>
|
|
119
145
|
`;
|
|
@@ -156,6 +182,8 @@ ${
|
|
|
156
182
|
};
|
|
157
183
|
return {
|
|
158
184
|
replicas: endpoints,
|
|
185
|
+
layout,
|
|
186
|
+
cluster: clusterName,
|
|
159
187
|
name,
|
|
160
188
|
stop,
|
|
161
189
|
async stopReplica(index) {
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drift on a multi-shard cluster (#3664), against a scratch cluster of the
|
|
3
|
+
* pinned image: two shards (two replicas and one server) with Keeper and a
|
|
4
|
+
* cluster `chant` over all three, the profile naming the first server with
|
|
5
|
+
* topology `cluster:chant`.
|
|
6
|
+
*
|
|
7
|
+
* - The schema applied `ON CLUSTER`, with a `Distributed` table declared with
|
|
8
|
+
* unquoted identifiers, reads as no drift: the server's quoted arguments
|
|
9
|
+
* are the same arguments.
|
|
10
|
+
* - A TTL changed on shard 2's server alone, without `ON CLUSTER`, is drift,
|
|
11
|
+
* seen on that server, though the profile's server still holds the
|
|
12
|
+
* declaration.
|
|
13
|
+
*
|
|
14
|
+
* Runs the sql lexicon's deep reader and core's deep diff, as
|
|
15
|
+
* `chant lifecycle diff --live` does. Needs Docker; skips cleanly without it.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
19
|
+
import { tmpdir } from "node:os";
|
|
20
|
+
import { join } from "node:path";
|
|
21
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
22
|
+
import { diffDeepObservation } from "@intentius/chant/lifecycle/deep-observe";
|
|
23
|
+
import { normalizeDeepObservation } from "@intentius/chant/deep-observation";
|
|
24
|
+
import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
|
|
25
|
+
import { clickhouseQuery } from "../http";
|
|
26
|
+
import { clickhouseImage } from "../../spec/pin";
|
|
27
|
+
import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
|
|
28
|
+
import { normalizeApply } from "@intentius/chant/apply";
|
|
29
|
+
import { database, table } from "../entities";
|
|
30
|
+
import { observeResourcesDeep, sqlDeepNormalizationHooks } from "./deep";
|
|
31
|
+
|
|
32
|
+
const enabled = await dockerAvailable();
|
|
33
|
+
const DB = "chant_e2e_3664";
|
|
34
|
+
let cluster: ScratchCluster | undefined;
|
|
35
|
+
const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-drift-"));
|
|
36
|
+
|
|
37
|
+
const DB_DDL = `CREATE DATABASE ${DB}`;
|
|
38
|
+
const EVENTS_DDL = `CREATE TABLE ${DB}.events
|
|
39
|
+
(
|
|
40
|
+
id UInt64,
|
|
41
|
+
ts DateTime
|
|
42
|
+
)
|
|
43
|
+
ENGINE = MergeTree
|
|
44
|
+
ORDER BY id
|
|
45
|
+
TTL ts + INTERVAL 30 DAY`;
|
|
46
|
+
const EVENTS_ALL_DDL = `CREATE TABLE ${DB}.events_all
|
|
47
|
+
(
|
|
48
|
+
id UInt64,
|
|
49
|
+
ts DateTime
|
|
50
|
+
)
|
|
51
|
+
ENGINE = Distributed(chant, ${DB}, events, id)`;
|
|
52
|
+
|
|
53
|
+
const config = () => ({ ownership: { stack: "e2e", env: "test" }, sql: { profiles: { e2e: { url: cluster!.replicas[0]!.url, topology: "cluster:chant" } } } });
|
|
54
|
+
const tag = (fn: (strings: TemplateStringsArray) => unknown, ddl: string) => fn(Object.assign([ddl], { raw: [ddl] }) as unknown as TemplateStringsArray) as unknown as { entityType: string; props: Record<string, unknown> };
|
|
55
|
+
|
|
56
|
+
const ENTITIES = new Map([
|
|
57
|
+
["db", tag(database, DB_DDL)],
|
|
58
|
+
["events", tag(table, EVENTS_DDL)],
|
|
59
|
+
["eventsAll", tag(table, EVENTS_ALL_DDL)],
|
|
60
|
+
]);
|
|
61
|
+
|
|
62
|
+
async function drift() {
|
|
63
|
+
const entities = new Map([...ENTITIES].map(([k, v]) => [k, { entityType: v.entityType, props: v.props }]));
|
|
64
|
+
const live = await observeResourcesDeep({ environment: "e2e", entityNames: [...entities.keys()], entities, config: config(), env: {} });
|
|
65
|
+
return diffDeepObservation(entities, normalizeDeepObservation(live), sqlDeepNormalizationHooks).drifted;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
beforeAll(async () => {
|
|
69
|
+
if (!enabled) return;
|
|
70
|
+
cluster = await startScratchCluster(clickhouseImage(), { shards: [2, 1], cluster: "chant", namePrefix: "chant-sql-cluster-drift" });
|
|
71
|
+
}, 600_000);
|
|
72
|
+
|
|
73
|
+
afterAll(async () => {
|
|
74
|
+
await cluster?.stop();
|
|
75
|
+
rmSync(dir, { recursive: true, force: true });
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
describe.skipIf(!enabled)("drift on a two-shard cluster (#3664)", () => {
|
|
79
|
+
test("applied ON CLUSTER, with unquoted Distributed arguments, it reads as no drift", async () => {
|
|
80
|
+
const build = join(dir, "schema.json");
|
|
81
|
+
writeFileSync(
|
|
82
|
+
build,
|
|
83
|
+
JSON.stringify({
|
|
84
|
+
dialect: "clickhouse",
|
|
85
|
+
applyOrder: ["db", "events", "eventsAll"],
|
|
86
|
+
objects: [
|
|
87
|
+
{ export: "db", type: "ClickHouse::Database", ddl: DB_DDL, dependsOn: [] },
|
|
88
|
+
{ export: "events", type: "ClickHouse::Table", ddl: EVENTS_DDL, dependsOn: ["db"] },
|
|
89
|
+
{ export: "eventsAll", type: "ClickHouse::Table", ddl: EVENTS_ALL_DDL, dependsOn: ["events"] },
|
|
90
|
+
],
|
|
91
|
+
}),
|
|
92
|
+
);
|
|
93
|
+
const applied = normalizeApply(toApplyResult(await clickhouseApply({ buildPath: build, environment: "e2e" }, undefined, { config: config(), env: {}, log: () => undefined })));
|
|
94
|
+
expect(applied.notAttempted).toEqual([]);
|
|
95
|
+
// Every server has both tables.
|
|
96
|
+
for (const endpoint of cluster!.replicas) {
|
|
97
|
+
const rows = await clickhouseQuery<{ name: string }>(endpoint, `SELECT name FROM system.tables WHERE database = '${DB}' ORDER BY name`);
|
|
98
|
+
expect(rows.map((r) => r.name)).toEqual(["events", "events_all"]);
|
|
99
|
+
}
|
|
100
|
+
expect(await drift()).toEqual([]);
|
|
101
|
+
}, 300_000);
|
|
102
|
+
|
|
103
|
+
test("a TTL changed on shard 2 alone is drift, seen on that server", async () => {
|
|
104
|
+
const shard2 = cluster!.layout.findIndex((l) => l.shard === "s2");
|
|
105
|
+
await clickhouseQuery(cluster!.replicas[shard2]!, `ALTER TABLE ${DB}.events MODIFY TTL ts + INTERVAL 7 DAY`);
|
|
106
|
+
// The profile's server still holds the declaration.
|
|
107
|
+
const [mine] = await clickhouseQuery<{ q: string }>(cluster!.replicas[0]!, `SELECT create_table_query AS q FROM system.tables WHERE database = '${DB}' AND name = 'events'`);
|
|
108
|
+
expect(mine!.q).toContain("toIntervalDay(30)");
|
|
109
|
+
|
|
110
|
+
const drifted = await drift();
|
|
111
|
+
expect(drifted.map((d) => d.name)).toEqual(["events"]);
|
|
112
|
+
const ttl = drifted[0]!.changes.find((c) => c.path.startsWith("ttl"));
|
|
113
|
+
expect(ttl, JSON.stringify(drifted[0]!.changes)).toBeDefined();
|
|
114
|
+
expect(String(ttl!.live)).toMatch(/7/);
|
|
115
|
+
expect(ttl!.seenOn).toBe(cluster!.layout[shard2]!.host);
|
|
116
|
+
}, 300_000);
|
|
117
|
+
});
|