@intentius/chant-lexicon-sql 0.114.1 → 0.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/clickhouse/apply/mutations.d.ts +7 -1
  2. package/dist/clickhouse/apply/mutations.d.ts.map +1 -1
  3. package/dist/clickhouse/container.d.ts +22 -7
  4. package/dist/clickhouse/container.d.ts.map +1 -1
  5. package/dist/clickhouse/plan/deep.d.ts +5 -0
  6. package/dist/clickhouse/plan/deep.d.ts.map +1 -1
  7. package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
  8. package/dist/clickhouse/rebuild/backfill.d.ts +11 -5
  9. package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
  10. package/dist/clickhouse/rebuild/op.d.ts +29 -0
  11. package/dist/clickhouse/rebuild/op.d.ts.map +1 -1
  12. package/dist/clickhouse/rebuild/partitions.d.ts +8 -0
  13. package/dist/clickhouse/rebuild/partitions.d.ts.map +1 -1
  14. package/dist/clickhouse/rebuild/receipts.d.ts +4 -6
  15. package/dist/clickhouse/rebuild/receipts.d.ts.map +1 -1
  16. package/dist/clickhouse/rebuild/shards.d.ts +59 -0
  17. package/dist/clickhouse/rebuild/shards.d.ts.map +1 -0
  18. package/dist/clickhouse/rebuild/steps.d.ts +3 -0
  19. package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
  20. package/dist/clickhouse/rebuild/verify.d.ts +4 -0
  21. package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
  22. package/dist/config.d.ts +14 -0
  23. package/dist/config.d.ts.map +1 -1
  24. package/dist/core/receipts.d.ts +5 -3
  25. package/dist/core/receipts.d.ts.map +1 -1
  26. package/dist/integrity.json +2 -2
  27. package/dist/manifest.json +1 -1
  28. package/dist/op/activities/clickhouse-rebuild.d.ts.map +1 -1
  29. package/dist/op/activities/index.d.ts +13 -1
  30. package/dist/op/activities/index.d.ts.map +1 -1
  31. package/dist/op/activities/sql-exec.d.ts +41 -0
  32. package/dist/op/activities/sql-exec.d.ts.map +1 -0
  33. package/dist/op/activity-contracts.d.ts +20 -0
  34. package/dist/op/activity-contracts.d.ts.map +1 -1
  35. package/dist/postgres/live/catalog.d.ts +1 -1
  36. package/dist/postgres/live/catalog.d.ts.map +1 -1
  37. package/dist/postgres/migrate/receipts.d.ts +8 -5
  38. package/dist/postgres/migrate/receipts.d.ts.map +1 -1
  39. package/dist/receipts.d.ts +102 -0
  40. package/dist/receipts.d.ts.map +1 -0
  41. package/package.json +2 -2
  42. package/src/clickhouse/apply/mutations.ts +9 -3
  43. package/src/clickhouse/container.ts +39 -11
  44. package/src/clickhouse/plan/cluster-drift.e2e.test.ts +117 -0
  45. package/src/clickhouse/plan/deep.ts +102 -3
  46. package/src/clickhouse/plan/normalize.ts +10 -0
  47. package/src/clickhouse/plan/plan.test.ts +10 -0
  48. package/src/clickhouse/rebuild/backfill.ts +85 -42
  49. package/src/clickhouse/rebuild/op.ts +80 -35
  50. package/src/clickhouse/rebuild/partitions.ts +11 -0
  51. package/src/clickhouse/rebuild/rebuild-sharded.e2e.test.ts +161 -0
  52. package/src/clickhouse/rebuild/rebuild.e2e.test.ts +110 -4
  53. package/src/clickhouse/rebuild/rebuild.test.ts +56 -4
  54. package/src/clickhouse/rebuild/receipts.ts +4 -6
  55. package/src/clickhouse/rebuild/shards.ts +104 -0
  56. package/src/clickhouse/rebuild/steps.ts +4 -0
  57. package/src/clickhouse/rebuild/verify.ts +14 -5
  58. package/src/config.ts +7 -0
  59. package/src/core/receipts.ts +5 -3
  60. package/src/op/activities/clickhouse-rebuild.ts +32 -7
  61. package/src/op/activities/index.ts +21 -4
  62. package/src/op/activities/sql-exec.ts +84 -0
  63. package/src/op/activity-contracts.ts +15 -0
  64. package/src/op/effect-receipts.e2e.test.ts +277 -0
  65. package/src/postgres/live/catalog.ts +3 -1
  66. package/src/postgres/migrate/migrate.test.ts +3 -2
  67. package/src/postgres/migrate/receipts.ts +18 -10
  68. package/src/receipts.test.ts +91 -0
  69. package/src/receipts.ts +259 -0
  70. package/src/skills/chant-sql-rebuild.md +5 -1
@@ -30,6 +30,7 @@ export declare const clickhouseRebuildPlanContract: import("@intentius/chant/op"
30
30
  stack?: string | undefined;
31
31
  ownershipEnv?: string | undefined;
32
32
  cwd?: string | undefined;
33
+ keepOnFailure?: boolean | undefined;
33
34
  }, {
34
35
  state: "done" | "rebuild" | "swapped";
35
36
  table: string;
@@ -54,6 +55,7 @@ export declare const clickhouseRebuildCreateContract: import("@intentius/chant/o
54
55
  stack?: string | undefined;
55
56
  ownershipEnv?: string | undefined;
56
57
  cwd?: string | undefined;
58
+ keepOnFailure?: boolean | undefined;
57
59
  }, {
58
60
  state: "done" | "rebuild" | "swapped";
59
61
  table: string;
@@ -76,6 +78,7 @@ export declare const clickhouseRebuildDualWriteContract: import("@intentius/chan
76
78
  stack?: string | undefined;
77
79
  ownershipEnv?: string | undefined;
78
80
  cwd?: string | undefined;
81
+ keepOnFailure?: boolean | undefined;
79
82
  }, {
80
83
  state: "done" | "rebuild" | "swapped";
81
84
  mode: "app" | "materialized-view";
@@ -99,6 +102,7 @@ export declare const clickhouseRebuildBackfillContract: import("@intentius/chant
99
102
  stack?: string | undefined;
100
103
  ownershipEnv?: string | undefined;
101
104
  cwd?: string | undefined;
105
+ keepOnFailure?: boolean | undefined;
102
106
  }, {
103
107
  state: "done" | "rebuild" | "swapped";
104
108
  partitions: number;
@@ -123,6 +127,7 @@ export declare const clickhouseRebuildVerifyContract: import("@intentius/chant/o
123
127
  stack?: string | undefined;
124
128
  ownershipEnv?: string | undefined;
125
129
  cwd?: string | undefined;
130
+ keepOnFailure?: boolean | undefined;
126
131
  }, {
127
132
  state: "done" | "rebuild" | "swapped";
128
133
  partitions: number;
@@ -152,6 +157,7 @@ export declare const clickhouseRebuildSwapContract: import("@intentius/chant/op"
152
157
  stack?: string | undefined;
153
158
  ownershipEnv?: string | undefined;
154
159
  cwd?: string | undefined;
160
+ keepOnFailure?: boolean | undefined;
155
161
  }, {
156
162
  state: "done" | "rebuild" | "swapped";
157
163
  swapped: boolean;
@@ -176,6 +182,7 @@ export declare const clickhouseRebuildRetainContract: import("@intentius/chant/o
176
182
  stack?: string | undefined;
177
183
  ownershipEnv?: string | undefined;
178
184
  cwd?: string | undefined;
185
+ keepOnFailure?: boolean | undefined;
179
186
  }, {
180
187
  state: "done" | "rebuild" | "swapped";
181
188
  due: boolean;
@@ -200,6 +207,7 @@ export declare const clickhouseRebuildDropContract: import("@intentius/chant/op"
200
207
  stack?: string | undefined;
201
208
  ownershipEnv?: string | undefined;
202
209
  cwd?: string | undefined;
210
+ keepOnFailure?: boolean | undefined;
203
211
  }, {
204
212
  state: "done" | "rebuild" | "swapped";
205
213
  dropped: boolean;
@@ -223,6 +231,7 @@ export declare const clickhouseRebuildCompensateContract: import("@intentius/cha
223
231
  stack?: string | undefined;
224
232
  ownershipEnv?: string | undefined;
225
233
  cwd?: string | undefined;
234
+ keepOnFailure?: boolean | undefined;
226
235
  }, {
227
236
  dropped: string[];
228
237
  }>;
@@ -461,4 +470,15 @@ export declare const postgresMigrationCompensateContract: import("@intentius/cha
461
470
  }, {
462
471
  dropped: string[];
463
472
  }>;
473
+ /** `SqlExecArgs` (`./activities/sql-exec.ts`): one batch's SQL on the environment's server. */
474
+ export declare const sqlExecContract: import("@intentius/chant/op").ActivityContract<{
475
+ sql: string;
476
+ environment?: string | undefined;
477
+ settings?: Record<string, string | number> | undefined;
478
+ cwd?: string | undefined;
479
+ }, {
480
+ dialect: "clickhouse" | "postgres";
481
+ source: string;
482
+ rows: number;
483
+ }>;
464
484
  //# sourceMappingURL=activity-contracts.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"activity-contracts.d.ts","sourceRoot":"","sources":["../../src/op/activity-contracts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA2BH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAY3C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;EAK/C,CAAC;AAuBF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;EAK1C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;EAc3C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;EAK/C,CAAC"}
1
+ {"version":3,"file":"activity-contracts.d.ts","sourceRoot":"","sources":["../../src/op/activity-contracts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA4BH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAY3C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;;EAK/C,CAAC;AAuBF,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;EAKzC,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;EAK1C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;EAc3C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;EAK3C,CAAC;AAEF,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;EAK7C,CAAC;AAEF,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;EAK/C,CAAC;AAIF,+FAA+F;AAC/F,eAAO,MAAM,eAAe;;;;;;;;;EAS3B,CAAC"}
@@ -22,7 +22,7 @@
22
22
  * column's sequence, since the column prints as `serial`;
23
23
  * - an index that backs a constraint (the constraint creates it), and an
24
24
  * index partition (the partitioned index creates it);
25
- * - chant's own receipts tables (`../../core/ownership.ts`), and the working
25
+ * - chant's own receipts tables and schema (`../../core/ownership.ts`), and the working
26
26
  * columns, constraints and indexes of an expand-and-contract migration in
27
27
  * progress (`../migrate/names.ts`, `../migrate/carry.ts`), so a table reads
28
28
  * as it was before the migration until its switch, and as declared after it.
@@ -1 +1 @@
1
- {"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../../src/postgres/live/catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAyB,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAKjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,iEAAiE;AACjE,eAAO,MAAM,cAAc,UAAmD,CAAC;AAE/E,kFAAkF;AAClF,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAa3D,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,kBAAkB,CAAC;IACzB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,iDAAiD;IACjD,GAAG,EAAE,MAAM,CAAC;IACZ,uFAAuF;IACvF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC1B,gHAAgH;IAChH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7B;AAyCD,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CAChB;AAQD,yFAAyF;AACzF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,GAAG,OAAO,CAAC,CAKtH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,GAAG,MAAM,EAAE,CAa9E;AAcD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,GAAE,WAAgB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAmX7G;AAED,yEAAyE;AACzE,eAAO,MAAM,OAAO,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,GAAG,SAAS,EAAE,MAAM,MAAM,KAAG,MAA2C,CAAC;AAE7H;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,EAAE,QAAQ,EAAE,gBAAgB,GAAG,SAAS,GAAG,YAAY,EAAE,CAW1H"}
1
+ {"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../../src/postgres/live/catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAyB,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAKjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,iEAAiE;AACjE,eAAO,MAAM,cAAc,UAAmD,CAAC;AAE/E,kFAAkF;AAClF,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAa3D,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,kBAAkB,CAAC;IACzB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,iDAAiD;IACjD,GAAG,EAAE,MAAM,CAAC;IACZ,uFAAuF;IACvF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC1B,gHAAgH;IAChH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7B;AAyCD,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CAChB;AAQD,yFAAyF;AACzF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,GAAG,OAAO,CAAC,CAKtH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,GAAG,MAAM,EAAE,CAa9E;AAcD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,GAAE,WAAgB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAqX7G;AAED,yEAAyE;AACzE,eAAO,MAAM,OAAO,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,GAAG,SAAS,EAAE,MAAM,MAAM,KAAG,MAA2C,CAAC;AAE7H;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,EAAE,QAAQ,EAAE,gBAAgB,GAAG,SAAS,GAAG,YAAY,EAAE,CAW1H"}
@@ -27,9 +27,9 @@
27
27
  * stale; onFailure and the contract also delete the migration's receipts.
28
28
  *
29
29
  * The store implements the shared core's `SqlReceiptStore`
30
- * (`../../core/receipts.ts`) over the migration's own connection. It is not
31
- * exported as the `receiptRead` / `receiptWrite` activities, whose names are
32
- * global to a run and belong to an aws or k8s project's receipt row.
30
+ * (`../../core/receipts.ts`) over the migration's own connection, which the
31
+ * backfill reaches directly. A sql project's `effect()` receipts use the same
32
+ * store over `chant_receipts.receipts` (`../../receipts.ts`).
33
33
  */
34
34
  import { type SqlReceiptStore } from "../../core/receipts.js";
35
35
  import type { PostgresClient } from "../live/client.js";
@@ -38,7 +38,7 @@ export declare const POSTGRES_RECEIPTS_TABLE = "__chant_receipts";
38
38
  export { receiptAddress } from "../../core/receipts.js";
39
39
  /** A receipts table chant did not make: a user's table under the same name. */
40
40
  export declare class ReceiptsTableConflict extends Error {
41
- constructor(schema: string);
41
+ constructor(schema: string, table?: string);
42
42
  }
43
43
  export interface PostgresReceiptStore extends SqlReceiptStore {
44
44
  /** The qualified, quoted table name. */
@@ -52,12 +52,15 @@ export interface PostgresReceiptStore extends SqlReceiptStore {
52
52
  * A receipt store in `<schema>.__chant_receipts`, over `client`. `write` runs
53
53
  * on the client as it is, so a write inside the caller's transaction commits
54
54
  * or rolls back with it. `identity` is the project's ownership stack and env,
55
- * which every address starts with.
55
+ * which every address starts with. `table` names the table in `schema`,
56
+ * `__chant_receipts` by default (`../../receipts.ts` keeps `effect()`
57
+ * receipts in `chant_receipts.receipts`).
56
58
  */
57
59
  export declare function postgresReceiptStore(client: PostgresClient, schema: string, identity: {
58
60
  stack?: string;
59
61
  env?: string;
60
62
  }, opts?: {
61
63
  runId?: string;
64
+ table?: string;
62
65
  }): PostgresReceiptStore;
63
66
  //# sourceMappingURL=receipts.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"receipts.d.ts","sourceRoot":"","sources":["../../../src/postgres/migrate/receipts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,0DAA0D;AAC1D,eAAO,MAAM,uBAAuB,qBAAqB,CAAC;AAI1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,+EAA+E;AAC/E,qBAAa,qBAAsB,SAAQ,KAAK;gBAClC,MAAM,EAAE,MAAM;CAO3B;AAED,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,gGAAgG;IAChG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,iGAAiG;IACjG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,oBAAoB,CA4D5K"}
1
+ {"version":3,"file":"receipts.d.ts","sourceRoot":"","sources":["../../../src/postgres/migrate/receipts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,0DAA0D;AAC1D,eAAO,MAAM,uBAAuB,qBAAqB,CAAC;AAI1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,+EAA+E;AAC/E,qBAAa,qBAAsB,SAAQ,KAAK;gBAClC,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAAgC;CAOpE;AAED,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,gGAAgG;IAChG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,iGAAiG;IACjG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,EAC1C,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,oBAAoB,CA6DtB"}
@@ -0,0 +1,102 @@
1
+ /**
2
+ * The sql lexicon's `effect()` receipt store (#3657): where the receipts of a
3
+ * project that uses only the sql lexicon are kept, on the database server the
4
+ * environment names.
5
+ *
6
+ * ClickHouse chant_receipts.receipts
7
+ * Postgres chant_receipts.receipts (or <receiptsSchema>.__chant_receipts)
8
+ *
9
+ * The op activities module binds it as `receiptRead`, `receiptWrite` and
10
+ * `receiptStaleness` (`./op/activities/index.ts`), listed as fallbacks: a
11
+ * project that also configures aws or k8s keeps its receipts in that
12
+ * lexicon's receipt row, in whichever order `lexicons` lists them.
13
+ *
14
+ * Which server: the environment is the one `chant run --env` sets
15
+ * (`CHANT_ENV`), else a literal `ownership.env`. Its `sql.profiles.<env>`
16
+ * binds the server, a `postgres://` URL making it Postgres; with no profile,
17
+ * `CLICKHOUSE_URL` or `POSTGRES_URL`, picked by `sql.dialect`. A receipt's
18
+ * address is `<stack>/<env>/<effect>`, from `ownership.stack` and that
19
+ * environment, the same identity the rebuild's and the column migration's
20
+ * receipts carry (`./core/receipts.ts`).
21
+ *
22
+ * Both tables carry chant's ownership trailer with the `receipts` key in their
23
+ * comment (the Postgres schema too), so plan, import and prune leave them out.
24
+ *
25
+ * A caller that runs its own Op over a server it has already bound (a
26
+ * migration runner, say) passes that server instead: a ClickHouse endpoint,
27
+ * or a Postgres connection, whose receipt writes then commit or roll back
28
+ * with whatever transaction the caller has open on it.
29
+ */
30
+ import type { ChantConfig } from "@intentius/chant/config";
31
+ import { type ReceiptActivities, type ReceiptActivityOptions } from "@intentius/chant/op/receipt-store";
32
+ import type { SqlReceiptStore } from "./core/receipts.js";
33
+ import type { ClickHouseEndpoint } from "./clickhouse/http.js";
34
+ import type { Topology } from "./clickhouse/topology.js";
35
+ import type { PostgresClient } from "./postgres/live/client.js";
36
+ /** The database (ClickHouse) or schema (Postgres) the receipts are kept in. */
37
+ export declare const SQL_RECEIPTS_SCHEMA = "chant_receipts";
38
+ /** The table in it. */
39
+ export declare const SQL_RECEIPTS_TABLE = "receipts";
40
+ /** The table's name in a schema of the user's own (`receiptsSchema`), as the column migration names it. */
41
+ export declare const SQL_RECEIPTS_TABLE_BESIDE = "__chant_receipts";
42
+ export interface SqlReceiptStoreOptions {
43
+ /** The chant environment. Default: `CHANT_ENV`, then a literal `ownership.env`. */
44
+ environment?: string;
45
+ /** The address's stack segment. Default: `ownership.stack`. */
46
+ stack?: string;
47
+ /** The project directory chant.config.ts is read from. Default: the working directory. */
48
+ cwd?: string;
49
+ /** The project's config, instead of reading chant.config.ts. */
50
+ config?: Pick<ChantConfig, "sql" | "ownership">;
51
+ /** The process environment the bindings and `CHANT_ENV` are read from. Default: `process.env`. */
52
+ env?: Record<string, string | undefined>;
53
+ /** A ClickHouse server already bound: the receipts go there, and no profile is read. */
54
+ clickhouse?: {
55
+ endpoint: ClickHouseEndpoint;
56
+ topology?: Topology;
57
+ };
58
+ /**
59
+ * A Postgres connection already open: the receipts go over it, and it is
60
+ * never closed. A write runs inside whatever transaction the caller has
61
+ * open, so it commits or rolls back with it. Call `ensure()` before such a
62
+ * transaction begins, so the receipts table does not roll back with it.
63
+ */
64
+ postgres?: PostgresClient;
65
+ /** Postgres: the schema to keep the receipts in. Default: the profile's `receiptsSchema`, then `chant_receipts`. */
66
+ schema?: string;
67
+ /** Recorded on each receipt written. Default: the current Op run's id. */
68
+ runId?: string;
69
+ }
70
+ /** The store, and the step a caller with its own transaction takes before one opens. */
71
+ export interface SqlServerReceiptStore extends SqlReceiptStore {
72
+ /** Create the receipts table (and database or schema) if it is not there. */
73
+ ensure(): Promise<void>;
74
+ /** Where the receipts are, as resolved: `chant_receipts.receipts` and the server it is on. */
75
+ location(): Promise<{
76
+ dialect: "clickhouse" | "postgres";
77
+ table: string;
78
+ source: string;
79
+ address: string;
80
+ }>;
81
+ }
82
+ type Config = Pick<ChantConfig, "sql" | "ownership">;
83
+ /** The dialect an environment's receipts are kept in: its profile's URL, else `sql.dialect`, else whichever server variable is set. */
84
+ export declare function receiptDialect(config: Config | undefined, environment: string | undefined, env: Record<string, string | undefined>): "clickhouse" | "postgres";
85
+ /** The run's environment: the option, `CHANT_ENV`, or a literal `ownership.env`. */
86
+ export declare function environmentOf(options: {
87
+ environment?: string;
88
+ }, config: Pick<ChantConfig, "ownership"> | undefined, env: Record<string, string | undefined>): string | undefined;
89
+ /**
90
+ * The receipt store on the environment's database server. Nothing is read or
91
+ * connected until the first read or write; the environment, the server and
92
+ * the identity are resolved then, once.
93
+ */
94
+ export declare function sqlReceiptStore(options?: SqlReceiptStoreOptions): SqlServerReceiptStore;
95
+ /**
96
+ * The `receiptRead`, `receiptWrite` and `receiptStaleness` activities over
97
+ * {@link sqlReceiptStore}: what the op activities module exports, and what a
98
+ * caller running an Op of `effect()` steps with its own activity map binds.
99
+ */
100
+ export declare function sqlReceiptActivities(options?: SqlReceiptStoreOptions, activityOptions?: ReceiptActivityOptions): ReceiptActivities;
101
+ export {};
102
+ //# sourceMappingURL=receipts.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"receipts.d.ts","sourceRoot":"","sources":["../src/receipts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAqB,KAAK,iBAAiB,EAAE,KAAK,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAE3H,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE7D,+EAA+E;AAC/E,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AACpD,uBAAuB;AACvB,eAAO,MAAM,kBAAkB,aAAa,CAAC;AAC7C,2GAA2G;AAC3G,eAAO,MAAM,yBAAyB,qBAAqB,CAAC;AAE5D,MAAM,WAAW,sBAAsB;IACrC,mFAAmF;IACnF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0FAA0F;IAC1F,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,MAAM,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,WAAW,CAAC,CAAC;IAChD,kGAAkG;IAClG,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,wFAAwF;IACxF,UAAU,CAAC,EAAE;QAAE,QAAQ,EAAE,kBAAkB,CAAC;QAAC,QAAQ,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IACnE;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,oHAAoH;IACpH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,6EAA6E;IAC7E,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,8FAA8F;IAC9F,QAAQ,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,YAAY,GAAG,UAAU,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7G;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,WAAW,CAAC,CAAC;AAqBrD,uIAAuI;AACvI,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,YAAY,GAAG,UAAU,CAO9J;AAED,oFAAoF;AACpF,wBAAgB,aAAa,CAAC,OAAO,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,GAAG,SAAS,CAKhL;AAgGD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,GAAE,sBAA2B,GAAG,qBAAqB,CA+B3F;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,GAAE,sBAA2B,EAAE,eAAe,CAAC,EAAE,sBAAsB,GAAG,iBAAiB,CAEtI"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@intentius/chant-lexicon-sql",
3
- "version": "0.114.1",
3
+ "version": "0.115.0",
4
4
  "description": "SQL lexicon for chant — database schema as typed declarations, one dialect at a time; ClickHouse first, with types read from a pinned server's catalog",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://intentius.io/chant",
@@ -70,7 +70,7 @@
70
70
  "typescript": "^5.9.3"
71
71
  },
72
72
  "peerDependencies": {
73
- "@intentius/chant": "^0.114.1",
73
+ "@intentius/chant": "^0.115.0",
74
74
  "typescript": "^5.9.3"
75
75
  }
76
76
  }
@@ -49,10 +49,10 @@ export class MutationFailedError extends Error {
49
49
  }
50
50
 
51
51
  /** The table's mutations that are not done. */
52
- export async function pendingMutations(endpoint: ClickHouseEndpoint, database: string, table: string): Promise<PendingMutation[]> {
52
+ export async function pendingMutations(endpoint: ClickHouseEndpoint, database: string, table: string, source = "system.mutations"): Promise<PendingMutation[]> {
53
53
  return clickhouseQuery<PendingMutation>(
54
54
  endpoint,
55
- `SELECT mutation_id, command, latest_fail_reason FROM system.mutations ` +
55
+ `SELECT mutation_id, command, latest_fail_reason FROM ${source} ` +
56
56
  `WHERE database = ${sqlString(database)} AND table = ${sqlString(table)} AND NOT is_done ORDER BY create_time`,
57
57
  );
58
58
  }
@@ -63,6 +63,12 @@ export interface WaitOptions {
63
63
  /** First poll interval; it doubles up to two seconds. Default: 100ms. */
64
64
  intervalMs?: number;
65
65
  signal?: AbortSignal;
66
+ /**
67
+ * Where the mutations are listed. Default: `system.mutations`, this
68
+ * server's. A rebuild across the shards of a cluster waits on every
69
+ * server's, `clusterAllReplicas('<cluster>', system.mutations)` (#3663).
70
+ */
71
+ source?: string;
66
72
  }
67
73
 
68
74
  /**
@@ -78,7 +84,7 @@ export async function waitForMutations(endpoint: ClickHouseEndpoint, database: s
78
84
  const name = `${database}.${table}`;
79
85
  for (;;) {
80
86
  opts.signal?.throwIfAborted();
81
- const pending = await pendingMutations(endpoint, database, table);
87
+ const pending = await pendingMutations(endpoint, database, table, opts.source);
82
88
  for (const m of pending) seen.add(m.mutation_id);
83
89
  if (pending.length === 0) return [...seen];
84
90
  const failing = pending.find((m) => m.latest_fail_reason);
@@ -51,8 +51,12 @@ export async function startScratchServer(
51
51
  }
52
52
 
53
53
  export interface ScratchCluster {
54
- /** One endpoint per replica, in replica order (`r1`, `r2`, ...). */
54
+ /** One endpoint per server, in order: shard 1's replicas first (`r1`, `r2`, ...), then shard 2's. */
55
55
  replicas: ClickHouseEndpoint[];
56
+ /** Each server's shard and replica macros, in the order of `replicas`. */
57
+ layout: Array<{ shard: string; replica: string; host: string }>;
58
+ /** The cluster `remote_servers` defines over every server: `ON CLUSTER` and `cluster:<name>` name it. */
59
+ cluster: string;
56
60
  /** The containers' common name prefix, for a log line. */
57
61
  name: string;
58
62
  /** Remove the containers and their network. Safe to call twice. */
@@ -72,23 +76,37 @@ export interface ScratchCluster {
72
76
  }
73
77
 
74
78
  /**
75
- * Start `replicas` servers of `image` as one shard of replicas, on their own
76
- * Docker network, and wait until each answers and reaches Keeper.
79
+ * Start servers of `image` on their own Docker network, and wait until each
80
+ * answers and reaches Keeper: `replicas` servers as one shard of replicas,
81
+ * or, with `shards`, that many replicas in each shard (`[2, 1]` is a shard of
82
+ * two replicas and a shard of one server).
77
83
  *
78
84
  * Keeper is the server's embedded one (`keeper_server`), run by the first
79
- * replica, so the pinned server image and its digest pin Keeper too. Each
80
- * replica gets the macros `{shard}` = `s1` and `{replica}` = `r<n>`, which
81
- * a `Replicated` database's arguments name. Configuration is copied into each
82
- * container before it starts, with no bind mount.
85
+ * server, so the pinned server image and its digest pin Keeper too. Each
86
+ * server gets the macros `{shard}` = `s<n>` and `{replica}` = `r<n>` (the
87
+ * replica numbered within its shard), which a `Replicated` database's and a
88
+ * `Replicated*MergeTree`'s arguments name, and `remote_servers` defines the
89
+ * cluster `cluster` (default `chant`) over all of them, with
90
+ * `internal_replication`, so `ON CLUSTER` reaches every server. Configuration
91
+ * is copied into each container before it starts, with no bind mount.
83
92
  */
84
93
  export async function startScratchCluster(
85
94
  image: string,
86
- options: { replicas?: number; namePrefix?: string; readyTimeoutMs?: number } = {},
95
+ options: { replicas?: number; shards?: number[]; cluster?: string; namePrefix?: string; readyTimeoutMs?: number } = {},
87
96
  ): Promise<ScratchCluster> {
88
- const count = options.replicas ?? 2;
97
+ const shards = options.shards ?? [options.replicas ?? 2];
98
+ const clusterName = options.cluster ?? "chant";
89
99
  const name = uniqueContainerName(options.namePrefix ?? "chant-sql-cluster");
90
100
  const network = `${name}-net`;
91
- const nodes = Array.from({ length: count }, (_, i) => `${name}-r${i + 1}`);
101
+ // One shard keeps the names the single-shard cluster always had (`<name>-r<n>`).
102
+ const layout = shards.flatMap((replicas, s) =>
103
+ Array.from({ length: replicas }, (_, r) => ({
104
+ shard: `s${s + 1}`,
105
+ replica: `r${r + 1}`,
106
+ host: shards.length === 1 ? `${name}-r${r + 1}` : `${name}-s${s + 1}r${r + 1}`,
107
+ })),
108
+ );
109
+ const nodes = layout.map((l) => l.host);
92
110
  const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-"));
93
111
  let removed = false;
94
112
  const stop = async (): Promise<void> => {
@@ -100,6 +118,12 @@ export async function startScratchCluster(
100
118
  };
101
119
 
102
120
  const keeper = nodes[0]!;
121
+ const remoteServers = shards
122
+ .map((_, s) => {
123
+ const hosts = layout.filter((l) => l.shard === `s${s + 1}`).map((l) => `<replica><host>${l.host}</host><port>9000</port></replica>`);
124
+ return `<shard><internal_replication>true</internal_replication>${hosts.join("")}</shard>`;
125
+ })
126
+ .join("");
103
127
  const config = (node: string, i: number) => `<clickhouse>
104
128
  ${
105
129
  i === 0
@@ -113,7 +137,9 @@ ${
113
137
  `
114
138
  : ""
115
139
  } <zookeeper><node><host>${keeper}</host><port>9181</port></node></zookeeper>
116
- <macros><shard>s1</shard><replica>r${i + 1}</replica></macros>
140
+ <distributed_ddl><path>/clickhouse/task_queue/ddl</path></distributed_ddl>
141
+ <remote_servers><${clusterName}>${remoteServers}</${clusterName}></remote_servers>
142
+ <macros><shard>${layout[i]!.shard}</shard><replica>${layout[i]!.replica}</replica></macros>
117
143
  <interserver_http_host>${node}</interserver_http_host>
118
144
  </clickhouse>
119
145
  `;
@@ -156,6 +182,8 @@ ${
156
182
  };
157
183
  return {
158
184
  replicas: endpoints,
185
+ layout,
186
+ cluster: clusterName,
159
187
  name,
160
188
  stop,
161
189
  async stopReplica(index) {
@@ -0,0 +1,117 @@
1
+ /**
2
+ * Drift on a multi-shard cluster (#3664), against a scratch cluster of the
3
+ * pinned image: two shards (two replicas and one server) with Keeper and a
4
+ * cluster `chant` over all three, the profile naming the first server with
5
+ * topology `cluster:chant`.
6
+ *
7
+ * - The schema applied `ON CLUSTER`, with a `Distributed` table declared with
8
+ * unquoted identifiers, reads as no drift: the server's quoted arguments
9
+ * are the same arguments.
10
+ * - A TTL changed on shard 2's server alone, without `ON CLUSTER`, is drift,
11
+ * seen on that server, though the profile's server still holds the
12
+ * declaration.
13
+ *
14
+ * Runs the sql lexicon's deep reader and core's deep diff, as
15
+ * `chant lifecycle diff --live` does. Needs Docker; skips cleanly without it.
16
+ */
17
+
18
+ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
19
+ import { tmpdir } from "node:os";
20
+ import { join } from "node:path";
21
+ import { afterAll, beforeAll, describe, expect, test } from "vitest";
22
+ import { diffDeepObservation } from "@intentius/chant/lifecycle/deep-observe";
23
+ import { normalizeDeepObservation } from "@intentius/chant/deep-observation";
24
+ import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
25
+ import { clickhouseQuery } from "../http";
26
+ import { clickhouseImage } from "../../spec/pin";
27
+ import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
28
+ import { normalizeApply } from "@intentius/chant/apply";
29
+ import { database, table } from "../entities";
30
+ import { observeResourcesDeep, sqlDeepNormalizationHooks } from "./deep";
31
+
32
+ const enabled = await dockerAvailable();
33
+ const DB = "chant_e2e_3664";
34
+ let cluster: ScratchCluster | undefined;
35
+ const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-drift-"));
36
+
37
+ const DB_DDL = `CREATE DATABASE ${DB}`;
38
+ const EVENTS_DDL = `CREATE TABLE ${DB}.events
39
+ (
40
+ id UInt64,
41
+ ts DateTime
42
+ )
43
+ ENGINE = MergeTree
44
+ ORDER BY id
45
+ TTL ts + INTERVAL 30 DAY`;
46
+ const EVENTS_ALL_DDL = `CREATE TABLE ${DB}.events_all
47
+ (
48
+ id UInt64,
49
+ ts DateTime
50
+ )
51
+ ENGINE = Distributed(chant, ${DB}, events, id)`;
52
+
53
+ const config = () => ({ ownership: { stack: "e2e", env: "test" }, sql: { profiles: { e2e: { url: cluster!.replicas[0]!.url, topology: "cluster:chant" } } } });
54
+ const tag = (fn: (strings: TemplateStringsArray) => unknown, ddl: string) => fn(Object.assign([ddl], { raw: [ddl] }) as unknown as TemplateStringsArray) as unknown as { entityType: string; props: Record<string, unknown> };
55
+
56
+ const ENTITIES = new Map([
57
+ ["db", tag(database, DB_DDL)],
58
+ ["events", tag(table, EVENTS_DDL)],
59
+ ["eventsAll", tag(table, EVENTS_ALL_DDL)],
60
+ ]);
61
+
62
+ async function drift() {
63
+ const entities = new Map([...ENTITIES].map(([k, v]) => [k, { entityType: v.entityType, props: v.props }]));
64
+ const live = await observeResourcesDeep({ environment: "e2e", entityNames: [...entities.keys()], entities, config: config(), env: {} });
65
+ return diffDeepObservation(entities, normalizeDeepObservation(live), sqlDeepNormalizationHooks).drifted;
66
+ }
67
+
68
+ beforeAll(async () => {
69
+ if (!enabled) return;
70
+ cluster = await startScratchCluster(clickhouseImage(), { shards: [2, 1], cluster: "chant", namePrefix: "chant-sql-cluster-drift" });
71
+ }, 600_000);
72
+
73
+ afterAll(async () => {
74
+ await cluster?.stop();
75
+ rmSync(dir, { recursive: true, force: true });
76
+ });
77
+
78
+ describe.skipIf(!enabled)("drift on a two-shard cluster (#3664)", () => {
79
+ test("applied ON CLUSTER, with unquoted Distributed arguments, it reads as no drift", async () => {
80
+ const build = join(dir, "schema.json");
81
+ writeFileSync(
82
+ build,
83
+ JSON.stringify({
84
+ dialect: "clickhouse",
85
+ applyOrder: ["db", "events", "eventsAll"],
86
+ objects: [
87
+ { export: "db", type: "ClickHouse::Database", ddl: DB_DDL, dependsOn: [] },
88
+ { export: "events", type: "ClickHouse::Table", ddl: EVENTS_DDL, dependsOn: ["db"] },
89
+ { export: "eventsAll", type: "ClickHouse::Table", ddl: EVENTS_ALL_DDL, dependsOn: ["events"] },
90
+ ],
91
+ }),
92
+ );
93
+ const applied = normalizeApply(toApplyResult(await clickhouseApply({ buildPath: build, environment: "e2e" }, undefined, { config: config(), env: {}, log: () => undefined })));
94
+ expect(applied.notAttempted).toEqual([]);
95
+ // Every server has both tables.
96
+ for (const endpoint of cluster!.replicas) {
97
+ const rows = await clickhouseQuery<{ name: string }>(endpoint, `SELECT name FROM system.tables WHERE database = '${DB}' ORDER BY name`);
98
+ expect(rows.map((r) => r.name)).toEqual(["events", "events_all"]);
99
+ }
100
+ expect(await drift()).toEqual([]);
101
+ }, 300_000);
102
+
103
+ test("a TTL changed on shard 2 alone is drift, seen on that server", async () => {
104
+ const shard2 = cluster!.layout.findIndex((l) => l.shard === "s2");
105
+ await clickhouseQuery(cluster!.replicas[shard2]!, `ALTER TABLE ${DB}.events MODIFY TTL ts + INTERVAL 7 DAY`);
106
+ // The profile's server still holds the declaration.
107
+ const [mine] = await clickhouseQuery<{ q: string }>(cluster!.replicas[0]!, `SELECT create_table_query AS q FROM system.tables WHERE database = '${DB}' AND name = 'events'`);
108
+ expect(mine!.q).toContain("toIntervalDay(30)");
109
+
110
+ const drifted = await drift();
111
+ expect(drifted.map((d) => d.name)).toEqual(["events"]);
112
+ const ttl = drifted[0]!.changes.find((c) => c.path.startsWith("ttl"));
113
+ expect(ttl, JSON.stringify(drifted[0]!.changes)).toBeDefined();
114
+ expect(String(ttl!.live)).toMatch(/7/);
115
+ expect(ttl!.seenOn).toBe(cluster!.layout[shard2]!.host);
116
+ }, 300_000);
117
+ });