@wavehouse/chtypes 0.5.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +34 -38
  3. package/dist/abi1/buildinfo.d.ts +40 -0
  4. package/dist/abi1/buildinfo.d.ts.map +1 -0
  5. package/dist/abi1/buildinfo.js +83 -0
  6. package/dist/abi1/buildinfo.js.map +1 -0
  7. package/dist/abi1/calls.gen.d.ts +55 -0
  8. package/dist/abi1/calls.gen.d.ts.map +1 -0
  9. package/dist/abi1/calls.gen.js +110 -0
  10. package/dist/abi1/calls.gen.js.map +1 -0
  11. package/dist/abi1/decls.gen.d.ts +115 -0
  12. package/dist/abi1/decls.gen.d.ts.map +1 -0
  13. package/dist/abi1/decls.gen.js +1613 -0
  14. package/dist/abi1/decls.gen.js.map +1 -0
  15. package/dist/abi1/errmap.gen.d.ts +18 -0
  16. package/dist/abi1/errmap.gen.d.ts.map +1 -0
  17. package/dist/abi1/errmap.gen.js +66 -0
  18. package/dist/abi1/errmap.gen.js.map +1 -0
  19. package/dist/abi1/errors.d.ts +107 -0
  20. package/dist/abi1/errors.d.ts.map +1 -0
  21. package/dist/abi1/errors.js +134 -0
  22. package/dist/abi1/errors.js.map +1 -0
  23. package/dist/abi1/handles.d.ts +31 -0
  24. package/dist/abi1/handles.d.ts.map +1 -0
  25. package/dist/abi1/handles.js +76 -0
  26. package/dist/abi1/handles.js.map +1 -0
  27. package/dist/abi1/index.d.ts +17 -0
  28. package/dist/abi1/index.d.ts.map +1 -0
  29. package/dist/abi1/index.js +17 -0
  30. package/dist/abi1/index.js.map +1 -0
  31. package/dist/abi1/libc.d.ts +54 -0
  32. package/dist/abi1/libc.d.ts.map +1 -0
  33. package/dist/abi1/libc.gen.d.ts +12 -0
  34. package/dist/abi1/libc.gen.d.ts.map +1 -0
  35. package/dist/abi1/libc.gen.js +75 -0
  36. package/dist/abi1/libc.gen.js.map +1 -0
  37. package/dist/abi1/libc.js +109 -0
  38. package/dist/abi1/libc.js.map +1 -0
  39. package/dist/abi1/loader.d.ts +99 -0
  40. package/dist/abi1/loader.d.ts.map +1 -0
  41. package/dist/abi1/loader.js +261 -0
  42. package/dist/abi1/loader.js.map +1 -0
  43. package/dist/abi1/raw.d.ts +104 -0
  44. package/dist/abi1/raw.d.ts.map +1 -0
  45. package/dist/abi1/raw.js +296 -0
  46. package/dist/abi1/raw.js.map +1 -0
  47. package/dist/abi1/strictjson.d.ts +19 -0
  48. package/dist/abi1/strictjson.d.ts.map +1 -0
  49. package/dist/abi1/strictjson.js +78 -0
  50. package/dist/abi1/strictjson.js.map +1 -0
  51. package/dist/abi1/vocab.gen.d.ts +169 -0
  52. package/dist/abi1/vocab.gen.d.ts.map +1 -0
  53. package/dist/abi1/vocab.gen.js +306 -0
  54. package/dist/abi1/vocab.gen.js.map +1 -0
  55. package/dist/cli.d.ts +25 -26
  56. package/dist/cli.d.ts.map +1 -1
  57. package/dist/cli.js +206 -218
  58. package/dist/cli.js.map +1 -1
  59. package/dist/documents.d.ts +195 -0
  60. package/dist/documents.d.ts.map +1 -0
  61. package/dist/documents.js +389 -0
  62. package/dist/documents.js.map +1 -0
  63. package/dist/env.d.ts +16 -0
  64. package/dist/env.d.ts.map +1 -0
  65. package/dist/env.js +57 -0
  66. package/dist/env.js.map +1 -0
  67. package/dist/index.d.ts +19 -24
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +17 -23
  70. package/dist/index.js.map +1 -1
  71. package/dist/json.d.ts +11 -89
  72. package/dist/json.d.ts.map +1 -1
  73. package/dist/json.js +16 -180
  74. package/dist/json.js.map +1 -1
  75. package/dist/library.d.ts +53 -309
  76. package/dist/library.d.ts.map +1 -1
  77. package/dist/library.js +77 -315
  78. package/dist/library.js.map +1 -1
  79. package/dist/ocifetch/constants.gen.d.ts +84 -0
  80. package/dist/ocifetch/constants.gen.d.ts.map +1 -0
  81. package/dist/ocifetch/constants.gen.js +92 -0
  82. package/dist/ocifetch/constants.gen.js.map +1 -0
  83. package/dist/ocifetch/dsse.d.ts +114 -0
  84. package/dist/ocifetch/dsse.d.ts.map +1 -0
  85. package/dist/ocifetch/dsse.js +331 -0
  86. package/dist/ocifetch/dsse.js.map +1 -0
  87. package/dist/ocifetch/ensure.d.ts +65 -0
  88. package/dist/ocifetch/ensure.d.ts.map +1 -0
  89. package/dist/ocifetch/ensure.js +775 -0
  90. package/dist/ocifetch/ensure.js.map +1 -0
  91. package/dist/ocifetch/errors.d.ts +92 -0
  92. package/dist/ocifetch/errors.d.ts.map +1 -0
  93. package/dist/ocifetch/errors.js +101 -0
  94. package/dist/ocifetch/errors.js.map +1 -0
  95. package/dist/ocifetch/goldens.d.ts +44 -0
  96. package/dist/ocifetch/goldens.d.ts.map +1 -0
  97. package/dist/ocifetch/goldens.js +62 -0
  98. package/dist/ocifetch/goldens.js.map +1 -0
  99. package/dist/ocifetch/http.d.ts +124 -0
  100. package/dist/ocifetch/http.d.ts.map +1 -0
  101. package/dist/ocifetch/http.js +549 -0
  102. package/dist/ocifetch/http.js.map +1 -0
  103. package/dist/ocifetch/index.d.ts +17 -0
  104. package/dist/ocifetch/index.d.ts.map +1 -0
  105. package/dist/ocifetch/index.js +14 -0
  106. package/dist/ocifetch/index.js.map +1 -0
  107. package/dist/ocifetch/layout.d.ts +126 -0
  108. package/dist/ocifetch/layout.d.ts.map +1 -0
  109. package/dist/ocifetch/layout.js +375 -0
  110. package/dist/ocifetch/layout.js.map +1 -0
  111. package/dist/ocifetch/localverify.d.ts +43 -0
  112. package/dist/ocifetch/localverify.d.ts.map +1 -0
  113. package/dist/ocifetch/localverify.js +164 -0
  114. package/dist/ocifetch/localverify.js.map +1 -0
  115. package/dist/ocifetch/lock.d.ts +39 -0
  116. package/dist/ocifetch/lock.d.ts.map +1 -0
  117. package/dist/ocifetch/lock.js +140 -0
  118. package/dist/ocifetch/lock.js.map +1 -0
  119. package/dist/ocifetch/oci.d.ts +100 -0
  120. package/dist/ocifetch/oci.d.ts.map +1 -0
  121. package/dist/ocifetch/oci.js +323 -0
  122. package/dist/ocifetch/oci.js.map +1 -0
  123. package/dist/ocifetch/referrers.d.ts +55 -0
  124. package/dist/ocifetch/referrers.d.ts.map +1 -0
  125. package/dist/ocifetch/referrers.js +132 -0
  126. package/dist/ocifetch/referrers.js.map +1 -0
  127. package/dist/ocifetch/types.d.ts +167 -0
  128. package/dist/ocifetch/types.d.ts.map +1 -0
  129. package/dist/ocifetch/types.js +88 -0
  130. package/dist/ocifetch/types.js.map +1 -0
  131. package/dist/ocifetch/unpack.d.ts +55 -0
  132. package/dist/ocifetch/unpack.d.ts.map +1 -0
  133. package/dist/ocifetch/unpack.js +205 -0
  134. package/dist/ocifetch/unpack.js.map +1 -0
  135. package/dist/registry.d.ts +38 -387
  136. package/dist/registry.d.ts.map +1 -1
  137. package/dist/registry.js +94 -819
  138. package/dist/registry.js.map +1 -1
  139. package/dist/schema.d.ts +77 -588
  140. package/dist/schema.d.ts.map +1 -1
  141. package/dist/schema.js +87 -549
  142. package/dist/schema.js.map +1 -1
  143. package/dist/settings.d.ts +27 -36
  144. package/dist/settings.d.ts.map +1 -1
  145. package/dist/settings.js +75 -25
  146. package/dist/settings.js.map +1 -1
  147. package/dist/setup.d.ts +37 -0
  148. package/dist/setup.d.ts.map +1 -0
  149. package/dist/setup.js +60 -0
  150. package/dist/setup.js.map +1 -0
  151. package/dist/tar.d.ts +27 -11
  152. package/dist/tar.d.ts.map +1 -1
  153. package/dist/tar.js +54 -32
  154. package/dist/tar.js.map +1 -1
  155. package/package.json +2 -2
  156. package/dist/discover.d.ts +0 -142
  157. package/dist/discover.d.ts.map +0 -1
  158. package/dist/discover.js +0 -288
  159. package/dist/discover.js.map +0 -1
  160. package/dist/error-codes.d.ts +0 -80
  161. package/dist/error-codes.d.ts.map +0 -1
  162. package/dist/error-codes.js +0 -139
  163. package/dist/error-codes.js.map +0 -1
  164. package/dist/errors.d.ts +0 -284
  165. package/dist/errors.d.ts.map +0 -1
  166. package/dist/errors.js +0 -321
  167. package/dist/errors.js.map +0 -1
  168. package/dist/fetch.d.ts +0 -382
  169. package/dist/fetch.d.ts.map +0 -1
  170. package/dist/fetch.js +0 -1498
  171. package/dist/fetch.js.map +0 -1
  172. package/dist/ffi.d.ts +0 -410
  173. package/dist/ffi.d.ts.map +0 -1
  174. package/dist/ffi.js +0 -1154
  175. package/dist/ffi.js.map +0 -1
  176. package/dist/format.d.ts +0 -128
  177. package/dist/format.d.ts.map +0 -1
  178. package/dist/format.js +0 -132
  179. package/dist/format.js.map +0 -1
  180. package/dist/numeric.d.ts +0 -20
  181. package/dist/numeric.d.ts.map +0 -1
  182. package/dist/numeric.js +0 -107
  183. package/dist/numeric.js.map +0 -1
  184. package/dist/paths.d.ts +0 -55
  185. package/dist/paths.d.ts.map +0 -1
  186. package/dist/paths.js +0 -87
  187. package/dist/paths.js.map +0 -1
  188. package/dist/results.d.ts +0 -495
  189. package/dist/results.d.ts.map +0 -1
  190. package/dist/results.js +0 -406
  191. package/dist/results.js.map +0 -1
  192. package/dist/transform.d.ts +0 -73
  193. package/dist/transform.d.ts.map +0 -1
  194. package/dist/transform.js +0 -533
  195. package/dist/transform.js.map +0 -1
package/dist/fetch.js DELETED
@@ -1,1498 +0,0 @@
1
- /**
2
- * Fetching, verifying and installing artifacts — docs/guides/fetch.md, the contract
3
- * every SDK implements identically: the same source (§2), the same
4
- * verification chain (§3), the same signature policy (§4), the same lock
5
- * file (§5), the same function (§6) and the same codes (§7).
6
- *
7
- * Nothing here is a verdict but the chain. No exit code, no
8
- * `Content-Length`, no "download finished" ever is:
9
- *
10
- * 0. `SHA256SUMS.sig` — ed25519 over the exact bytes of `SHA256SUMS`, with a
11
- * trusted key. Failure stops everything; nothing is downloaded around it.
12
- * 1. `index.json` names the asset for the line/platform and its sha256 — only
13
- * ever a row at this binding's own `ABI_REVISION` (docs/guides/fetch.md §2).
14
- * 2. `SHA256SUMS` — now known-authentic — must list the same file with the
15
- * same sha256; a disagreement is a broken release, reported, not repaired.
16
- * 3. The tarball is hashed BEFORE it is unpacked.
17
- * 4. `manifest.json` inside names the library and its sha256; after the
18
- * move into `<registry>/<minor>/`, the installed library is hashed again
19
- * in place.
20
- *
21
- * The install is atomic — unpack into a temporary sibling, rename into place
22
- * — and idempotent: an installed line that hashes what the release says is
23
- * reported as installed and nothing is downloaded.
24
- *
25
- * Steady state is offline: once files are in a registry directory the loader
26
- * trusts the directory, exactly as a runtime trusts `node_modules`.
27
- * Verification is a fetch-time policy, not a load-time gate.
28
- */
29
- import { createHash, createPublicKey, randomBytes, verify as verifyRaw } from 'node:crypto';
30
- import { createReadStream, createWriteStream, existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
31
- import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
32
- import path from 'node:path';
33
- import { Readable, Transform } from 'node:stream';
34
- import { pipeline } from 'node:stream/promises';
35
- import { setTimeout as sleep } from 'node:timers/promises';
36
- import { fileURLToPath } from 'node:url';
37
- import { ABI_REVISION, ArtifactCorruptError, ArtifactPinnedError, ArtifactUnpublishedError, ArtifactUntrustedError, ChtypesError, FETCH_COMMAND, SourceUnreachableError, } from './errors.js';
38
- import { fetchDestination, hostPlatform, isPlatformKey } from './paths.js';
39
- import { extractTarGz } from './tar.js';
40
- // ------------------------------------------------------------- constants
41
- /** The public artifacts host (docs/guides/fetch.md §2); `CHTYPES_ARTIFACTS_URL` overrides it. */
42
- export const DEFAULT_ARTIFACTS_URL = 'https://artifacts.wavehouse.dev';
43
- /** The rolling release every fetch reads by default; `tag` picks a frozen one. */
44
- export const DEFAULT_RELEASE_TAG = 'artifacts';
45
- /**
46
- * The release signing key(s) — raw 32-byte ed25519 public keys, hex
47
- * (docs/guides/fetch.md §4). `CHTYPES_TRUSTED_KEYS` or the `trustedKeys` option
48
- * REPLACES this list; it is never extended silently.
49
- */
50
- export const RELEASE_PUBLIC_KEYS = [
51
- 'fdb5f06a8d4c9918d049a5f1748fa2e3b3238c3f2000986d5bb9e31beff778fc',
52
- ];
53
- /**
54
- * The lock-file schema this SDK writes, and the newest it reads
55
- * (docs/guides/fetch.md §5). Schema 1 (`<os>-<arch>/<minor>`) is still read
56
- * and is converted to schema 2 (`<os>-<arch>/<clickhouse_version>`) the first
57
- * time this SDK writes the file — schema 2 is what a two-patches-per-line
58
- * cache needs to key on, since two patches of one line share a schema-1 key.
59
- * SDKs through 0.4.x cannot read a schema-2 lock and refuse it (issue #284).
60
- */
61
- export const LOCK_SCHEMA = 2;
62
- /** The newest lock schema this SDK STILL READS besides its own (docs/guides/fetch.md §5). */
63
- const LOCK_SCHEMA_LEGACY = 1;
64
- /** The lock file `frozen` reads when no `lock` names one (docs/guides/fetch.md, Decisions); relative, so the working directory's. */
65
- export const DEFAULT_LOCK_FILE = 'chtypes.lock';
66
- const INDEX_SCHEMA = 1;
67
- const CHANNEL = /-(lts|stable|prestable|testing)$/;
68
- /**
69
- * `25.8` is a line; `25.8.28.1-lts`, `v25.8.28.1-lts` and `25.8.28.1` are an
70
- * exact patch and a hard requirement (a release that publishes another patch
71
- * of the line refuses it — ask for the line to take what was published).
72
- *
73
- * @throws {ChtypesError} when the spelling is not a ClickHouse version at all.
74
- */
75
- export function parseVersionSpelling(spelling) {
76
- const s = spelling.trim().replace(/^v/, '');
77
- const bare = s.replace(CHANNEL, '');
78
- const parts = bare.split('.');
79
- if (parts.length < 2 || !parts.every((p) => /^\d+$/.test(p))) {
80
- throw new ChtypesError(`chtypes: cannot make a ClickHouse version out of ${JSON.stringify(spelling)}`);
81
- }
82
- const line = `${parts[0]}.${parts[1]}`;
83
- return parts.length >= 4 ? { spelling, line, exact: s, bare } : { spelling, line, exact: null, bare: null };
84
- }
85
- /**
86
- * Decision 7 / R2 (issue #284): does `candidate` — a served or installed
87
- * `clickhouse_version` — match `req`? A spelled channel (`-lts` etc.) matches
88
- * only itself; an unspelled one matches that patch on any channel. This is
89
- * the ONE matching rule for every place a patch is matched: the registry, the
90
- * release, the lock and `scripts/fetch.sh`. `false` for a line request — a
91
- * line never "matches" a patch under this rule, it CONTAINS one.
92
- */
93
- export function patchMatches(candidate, req) {
94
- if (req.exact === null)
95
- return false;
96
- if (CHANNEL.test(req.exact))
97
- return candidate === req.exact;
98
- return candidate === req.exact || candidate.replace(CHANNEL, '') === req.bare;
99
- }
100
- /** Numeric order over dotted versions, channel suffix ignored: 25.10 > 25.8, 25.8.30.16 > 25.8.28.1. */
101
- export function compareVersions(a, b) {
102
- const pa = a.replace(CHANNEL, '').split('.').map((p) => Number.parseInt(p, 10));
103
- const pb = b.replace(CHANNEL, '').split('.').map((p) => Number.parseInt(p, 10));
104
- for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
105
- const x = pa[i] ?? -1;
106
- const y = pb[i] ?? -1;
107
- if (Number.isNaN(x) || Number.isNaN(y))
108
- return a < b ? -1 : a > b ? 1 : 0;
109
- if (x !== y)
110
- return x - y;
111
- }
112
- return 0;
113
- }
114
- // ------------------------------------------------------------ signature
115
- /** The 12-byte SubjectPublicKeyInfo header for an Ed25519 key (RFC 8410): SEQUENCE { AlgorithmIdentifier(1.3.101.112), BIT STRING }. */
116
- const ED25519_SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex');
117
- const RAW_KEY = /^[0-9a-fA-F]{64}$/;
118
- /** The key id docs/guides/fetch.md §4 defines: the first 16 hex characters of sha256 over the raw 32-byte public key. */
119
- export function keyId(rawKeyHex) {
120
- return createHash('sha256').update(Buffer.from(rawKeyHex, 'hex')).digest('hex').slice(0, 16);
121
- }
122
- /**
123
- * `keyId(RELEASE_PUBLIC_KEYS[0])`, precomputed — the id the signature file's
124
- * comment line names for the primary release key. `RELEASE_PUBLIC_KEYS` stays
125
- * the list to verify against (rotation adds a second entry); this is the one
126
- * short id go/python/rust each export as a scalar constant.
127
- */
128
- export const RELEASE_KEY_ID = keyId(RELEASE_PUBLIC_KEYS[0]);
129
- /**
130
- * Verify an ed25519 signature over `message` with a raw public key, through
131
- * `node:crypto` — the key wrapped as SPKI DER, the 64-byte signature as is.
132
- * False, never a throw, for a malformed key or signature.
133
- */
134
- export function verifyEd25519(message, signature, rawKeyHex) {
135
- if (!RAW_KEY.test(rawKeyHex) || signature.byteLength !== 64)
136
- return false;
137
- try {
138
- const key = createPublicKey({
139
- key: Buffer.concat([ED25519_SPKI_PREFIX, Buffer.from(rawKeyHex, 'hex')]),
140
- format: 'der',
141
- type: 'spki',
142
- });
143
- return verifyRaw(null, message, key, signature);
144
- }
145
- catch {
146
- return false;
147
- }
148
- }
149
- /**
150
- * `SHA256SUMS.sig`: an `untrusted comment:` line, then the base64 of the
151
- * 64-byte signature. Null when the file is not in that shape.
152
- */
153
- export function parseSignatureFile(bytes) {
154
- const lines = Buffer.from(bytes).toString('utf8').split(/\r?\n/);
155
- let comment = '';
156
- for (const raw of lines) {
157
- const line = raw.trim();
158
- if (line === '')
159
- continue;
160
- if (line.startsWith('untrusted comment:')) {
161
- comment = line.slice('untrusted comment:'.length).trim();
162
- continue;
163
- }
164
- if (!/^[A-Za-z0-9+/]+={0,2}$/.test(line))
165
- return null;
166
- const signature = Buffer.from(line, 'base64');
167
- return signature.length === 64 ? { comment, signature } : null;
168
- }
169
- return null;
170
- }
171
- /**
172
- * The keys a fetch trusts: the `trustedKeys` option, else `CHTYPES_TRUSTED_KEYS`
173
- * (`<hex>[,<hex>…]`), else the embedded release key. Each REPLACES the one
174
- * below it.
175
- *
176
- * @throws {ChtypesError} when an entry is not 64 hex characters.
177
- */
178
- export function trustedKeys(explicit) {
179
- let list;
180
- if (explicit !== undefined)
181
- list = explicit;
182
- else {
183
- const env = process.env['CHTYPES_TRUSTED_KEYS'];
184
- const fromEnv = env === undefined ? [] : env.split(',').map((k) => k.trim()).filter((k) => k !== '');
185
- list = fromEnv.length > 0 ? fromEnv : RELEASE_PUBLIC_KEYS;
186
- }
187
- return list.map((k) => {
188
- if (!RAW_KEY.test(k)) {
189
- throw new ChtypesError(`chtypes: ${JSON.stringify(k)} is not a raw ed25519 public key (64 hex characters) — check CHTYPES_TRUSTED_KEYS`);
190
- }
191
- return k.toLowerCase();
192
- });
193
- }
194
- class DirSource {
195
- dir;
196
- description;
197
- constructor(dir) {
198
- this.dir = dir;
199
- this.description = dir;
200
- }
201
- async get(name) {
202
- try {
203
- return await readFile(path.join(this.dir, name));
204
- }
205
- catch (err) {
206
- if (isNoEntry(err))
207
- return null;
208
- throw new SourceUnreachableError(`chtypes: cannot read ${path.join(this.dir, name)}: ${errorText(err)}`, {
209
- cause: err,
210
- });
211
- }
212
- }
213
- async download(name, to, onReceived) {
214
- const from = path.join(this.dir, name);
215
- if (!existsSync(from))
216
- return false;
217
- try {
218
- await pipeline(createReadStream(from), progressTap(onReceived), createWriteStream(to));
219
- }
220
- catch (err) {
221
- throw new SourceUnreachableError(`chtypes: cannot read ${from}: ${errorText(err)}`, { cause: err });
222
- }
223
- return true;
224
- }
225
- }
226
- class HttpSource {
227
- base;
228
- signal;
229
- description;
230
- constructor(base, signal) {
231
- this.base = base;
232
- this.signal = signal;
233
- this.description = base;
234
- }
235
- async request(name) {
236
- const url = `${this.base}/${name}`;
237
- const headers = {};
238
- const token = process.env['CHTYPES_DOWNLOAD_TOKEN'];
239
- if (token !== undefined && token !== '')
240
- headers['Authorization'] = `Bearer ${token}`;
241
- let last;
242
- for (let attempt = 1; attempt <= 3; attempt++) {
243
- try {
244
- const res = await fetch(url, { headers, redirect: 'follow', ...(this.signal ? { signal: this.signal } : {}) });
245
- if (res.status === 404) {
246
- await res.body?.cancel();
247
- return null;
248
- }
249
- if (res.ok)
250
- return res;
251
- await res.body?.cancel();
252
- last = new Error(`HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`);
253
- // Client errors do not get better on retry; server errors and throttling might.
254
- if (res.status < 500 && res.status !== 408 && res.status !== 429)
255
- break;
256
- }
257
- catch (err) {
258
- if (this.signal?.aborted)
259
- throw err;
260
- last = err;
261
- }
262
- if (attempt < 3)
263
- await sleep(500 * attempt);
264
- }
265
- throw new SourceUnreachableError(`chtypes: cannot fetch ${url}: ${errorText(last)}`, { cause: last });
266
- }
267
- async get(name) {
268
- const res = await this.request(name);
269
- if (res === null)
270
- return null;
271
- return Buffer.from(await res.arrayBuffer());
272
- }
273
- async download(name, to, onReceived) {
274
- const res = await this.request(name);
275
- if (res === null)
276
- return false;
277
- if (res.body === null) {
278
- throw new SourceUnreachableError(`chtypes: ${this.base}/${name} returned no body`);
279
- }
280
- try {
281
- await pipeline(Readable.fromWeb(res.body), progressTap(onReceived), createWriteStream(to));
282
- }
283
- catch (err) {
284
- throw new SourceUnreachableError(`chtypes: download of ${this.base}/${name} failed: ${errorText(err)}`, {
285
- cause: err,
286
- });
287
- }
288
- return true;
289
- }
290
- }
291
- /** §2: `url` is a full base (http(s), `file://`, or a directory); otherwise `<CHTYPES_ARTIFACTS_URL>/<tag>`. */
292
- function openSource(options) {
293
- if (options.url !== undefined && options.url !== '' && options.tag !== undefined && options.tag !== '') {
294
- throw new ChtypesError('chtypes: url names a full base; tag selects a release on the artifacts host — pass one');
295
- }
296
- let base;
297
- if (options.url !== undefined && options.url !== '')
298
- base = options.url;
299
- else {
300
- const host = process.env['CHTYPES_ARTIFACTS_URL'];
301
- const root = host !== undefined && host !== '' ? host : DEFAULT_ARTIFACTS_URL;
302
- const tag = options.tag !== undefined && options.tag !== '' ? options.tag : DEFAULT_RELEASE_TAG;
303
- base = `${root.replace(/\/+$/, '')}/${tag}`;
304
- }
305
- if (/^https?:\/\//i.test(base))
306
- return new HttpSource(base.replace(/\/+$/, ''), options.signal);
307
- const dir = base.startsWith('file://') ? fileURLToPath(base) : path.resolve(base);
308
- return new DirSource(dir);
309
- }
310
- /** Steps 0–2's inputs, verified in that order: the signature over SHA256SUMS, then the index. */
311
- async function loadReleaseOnce(source, options, emit) {
312
- const sumsBytes = await source.get('SHA256SUMS');
313
- if (sumsBytes === null)
314
- throw new SourceUnreachableError(`chtypes: no SHA256SUMS at ${source.description}`);
315
- let signed = false;
316
- let signer = null;
317
- if (options.allowUnsigned ?? envFlag('CHTYPES_ALLOW_UNSIGNED')) {
318
- warn(`chtypes: WARNING — the signature of ${source.description}/SHA256SUMS is NOT being verified ` +
319
- '(CHTYPES_ALLOW_UNSIGNED). Anything it lists will be installed on its hashes alone.');
320
- }
321
- else {
322
- const sigBytes = await source.get('SHA256SUMS.sig');
323
- if (sigBytes === null) {
324
- throw new ArtifactUntrustedError(`chtypes: the release at ${source.description} is unsigned (no SHA256SUMS.sig); nothing was downloaded. ` +
325
- 'CHTYPES_ALLOW_UNSIGNED=1 installs it anyway, loudly.');
326
- }
327
- const sig = parseSignatureFile(sigBytes);
328
- if (sig === null) {
329
- throw new ArtifactUntrustedError(`chtypes: ${source.description}/SHA256SUMS.sig is not an ed25519 signature file; nothing was downloaded.`);
330
- }
331
- const keys = trustedKeys(options.trustedKeys);
332
- const hit = keys.find((k) => verifyEd25519(sumsBytes, sig.signature, k));
333
- if (hit === undefined) {
334
- throw new ArtifactUntrustedError(`chtypes: ${source.description}/SHA256SUMS.sig does not verify with any trusted key ` +
335
- `(${keys.map(keyId).join(', ')}); the release is not trusted and nothing was downloaded.`);
336
- }
337
- signed = true;
338
- signer = keyId(hit);
339
- emit({ type: 'status', message: `SHA256SUMS signature verified (key ${signer})` });
340
- }
341
- const indexBytes = await source.get('index.json');
342
- if (indexBytes === null)
343
- throw new SourceUnreachableError(`chtypes: no index.json at ${source.description}`);
344
- const index = parseIndex(indexBytes, source.description);
345
- if (index.license !== undefined && index.license !== '') {
346
- emit({
347
- type: 'status',
348
- message: `artifacts are licensed under ${index.license}${index.license_url ? ` (${index.license_url})` : ''} — LICENSE and NOTICE ship beside them`,
349
- });
350
- }
351
- const release = { source, index, sums: parseSums(sumsBytes), signed, keyId: signer };
352
- // Step 2 for the whole release, not just the asset being installed. installOne
353
- // still checks its own asset; this runs here so a disagreement is seen while
354
- // loadRelease can still fix it by reading all three files again.
355
- for (const art of index.artifacts) {
356
- const listed = release.sums.get(art.file);
357
- if (listed !== undefined && listed !== art.sha256) {
358
- throw new ArtifactCorruptError(`chtypes: index.json says ${art.file} is ${art.sha256} but SHA256SUMS says ${listed} — the release disagrees with itself; not installing it`);
359
- }
360
- }
361
- return release;
362
- }
363
- /**
364
- * How many times {@link loadRelease} (and {@link installGoldens}'s own retry)
365
- * reads an HTTP release before giving up, and the BASE delay (ms) before the
366
- * first retry — each later retry doubles it, so the default 5 attempts sleep
367
- * 4+8+16+32 = 60s (~70s wall with network time). That is chosen to outlast
368
- * the artifacts host's edge cache (observed `Cache-Control: max-age=60` on
369
- * the mutable release objects, so a stale pairing of any of them can persist
370
- * up to 60s), while a genuine few-second mid-publish window still clears on
371
- * the second attempt.
372
- *
373
- * A mutable object, not two `const`s: an ES module's imported bindings are
374
- * read-only, so a plain exported `let` could not be reassigned from outside,
375
- * and the fetch test suite needs to shrink `delayMs` to near zero. Not
376
- * re-exported from `index.ts` — internal to this module and its test suite.
377
- */
378
- export const RELEASE_RETRY = { attempts: 5, delayMs: 4_000 };
379
- /**
380
- * {@link loadReleaseOnce}, retried through a publish window.
381
- *
382
- * A publish into the rolling release is three objects — `SHA256SUMS`,
383
- * `SHA256SUMS.sig`, `index.json` — and object storage cannot swap them
384
- * atomically. The edge cache in front of the artifacts host widens the unsafe
385
- * window from "between two uploads" to "as long as any one object can still be
386
- * served stale from cache", which measures the same as its `Cache-Control`
387
- * max-age.
388
- *
389
- * Three symptoms of reading inside that window are retried: a signature under
390
- * no trusted key, an index that disagrees with the sums, and (in
391
- * {@link installGoldens}) a release-level file whose hash disagrees with its
392
- * own SHA256SUMS row, or that the sums list but the source does not yet serve.
393
- * Every retry re-reads the WHOLE set from scratch — never one freshly
394
- * re-fetched object checked against another attempt's stale one. Nothing else
395
- * is retried, and neither are these three once the attempts run out: the same
396
- * error surfaces, with the same code, as it did before. A tarball whose hash
397
- * is wrong is never retried; that is the release lying about a byte, not a
398
- * half-finished upload.
399
- *
400
- * Only an HTTP source can be mid-publish, so a `file://` or directory source is
401
- * read exactly once and refuses on the first look.
402
- */
403
- async function loadRelease(source, options, emit) {
404
- const attempts = source instanceof HttpSource ? RELEASE_RETRY.attempts : 1;
405
- let delayMs = RELEASE_RETRY.delayMs;
406
- for (let attempt = 1;; attempt++) {
407
- try {
408
- return await loadReleaseOnce(source, options, emit);
409
- }
410
- catch (err) {
411
- const retryable = err instanceof ArtifactUntrustedError || err instanceof ArtifactCorruptError;
412
- if (!retryable || attempt >= attempts)
413
- throw err;
414
- emit({
415
- type: 'status',
416
- message: `${String(err)} (attempt ${attempt}/${attempts}) — this is what a release being ` +
417
- `published looks like from outside; retrying in ${delayMs / 1000}s`,
418
- });
419
- await new Promise((resolve) => setTimeout(resolve, delayMs));
420
- delayMs *= 2;
421
- }
422
- }
423
- }
424
- function parseIndex(bytes, where) {
425
- let doc;
426
- try {
427
- doc = JSON.parse(bytes.toString('utf8'));
428
- }
429
- catch (err) {
430
- throw new ArtifactCorruptError(`chtypes: index.json at ${where} is not JSON: ${errorText(err)}`, { cause: err });
431
- }
432
- if (typeof doc !== 'object' || doc === null)
433
- throw new ArtifactCorruptError(`chtypes: index.json at ${where} is not an object`);
434
- const d = doc;
435
- if (d['schema'] !== INDEX_SCHEMA) {
436
- throw new ArtifactCorruptError(`chtypes: index.json schema ${JSON.stringify(d['schema'])} is not ${INDEX_SCHEMA} — this SDK cannot read it`);
437
- }
438
- if (!Array.isArray(d['artifacts']))
439
- throw new ArtifactCorruptError(`chtypes: index.json at ${where} has no artifacts list`);
440
- const artifacts = d['artifacts'].map((row, i) => indexArtifact(row, i));
441
- return {
442
- schema: INDEX_SCHEMA,
443
- ...(typeof d['generated_at'] === 'string' ? { generated_at: d['generated_at'] } : {}),
444
- ...(typeof d['release_tag'] === 'string' ? { release_tag: d['release_tag'] } : {}),
445
- ...(typeof d['license'] === 'string' ? { license: d['license'] } : {}),
446
- ...(typeof d['license_url'] === 'string' ? { license_url: d['license_url'] } : {}),
447
- artifacts,
448
- };
449
- }
450
- function indexArtifact(row, i) {
451
- if (typeof row !== 'object' || row === null)
452
- throw new ArtifactCorruptError(`chtypes: index.json artifacts[${i}] is not an object`);
453
- const r = row;
454
- const str = (k) => {
455
- const v = r[k];
456
- if (typeof v !== 'string' || v === '')
457
- throw new ArtifactCorruptError(`chtypes: index.json entry for ${String(r['file'] ?? i)} is missing ${k}`);
458
- return v;
459
- };
460
- const bytes = r['bytes'];
461
- if (typeof bytes !== 'number' || !Number.isSafeInteger(bytes) || bytes <= 0) {
462
- throw new ArtifactCorruptError(`chtypes: index.json entry for ${String(r['file'] ?? i)} is missing bytes`);
463
- }
464
- const file = str('file');
465
- const version = str('clickhouse_version');
466
- const minor = typeof r['clickhouse_minor'] === 'string' && r['clickhouse_minor'] !== '' ? r['clickhouse_minor'] : minorOfVersion(version);
467
- return {
468
- os: str('os'),
469
- arch: str('arch'),
470
- file,
471
- sha256: str('sha256').toLowerCase(),
472
- bytes,
473
- clickhouse_version: version,
474
- clickhouse_minor: minor,
475
- library: str('library'),
476
- library_sha256: str('library_sha256').toLowerCase(),
477
- build: buildOf(r['build'], file),
478
- core_commit: typeof r['core_commit'] === 'string' ? r['core_commit'] : '',
479
- ...revisionOf(r['abi_revision']),
480
- };
481
- }
482
- /**
483
- * A row's `abi_revision` when it is an integer; otherwise the row is read as
484
- * declaring none — the field is left out, and never matches — rather than
485
- * failing the whole listing.
486
- */
487
- function revisionOf(field) {
488
- return typeof field === 'number' && Number.isSafeInteger(field) ? { abi_revision: field } : {};
489
- }
490
- /**
491
- * TEST-ONLY. When `override` is set, the ABI revision fetch selects release rows
492
- * at, in place of `ABI_REVISION` (docs/guides/fetch.md §2). The fetch-fixture
493
- * suite sets it to the revision its fixture release carries — derived from that
494
- * release's own `index.json` — so a binding whose own revision has moved ahead
495
- * of the fixtures still exercises the whole chain. It changes WHICH rows are
496
- * eligible and nothing else: the default registry directory stays
497
- * `abi<ABI_REVISION>/`, and the loader still refuses an artifact of another
498
- * revision.
499
- *
500
- * A mutable object for the same reason as {@link RELEASE_RETRY}: an imported
501
- * binding is read-only. Not re-exported from `index.ts` — internal to this
502
- * module and its test suite, never part of the public contract.
503
- */
504
- export const FETCH_ABI_REVISION = { override: null };
505
- /** The one ABI revision whose rows fetch may install: this binding's own. */
506
- function fetchAbiRevision() {
507
- return FETCH_ABI_REVISION.override ?? ABI_REVISION;
508
- }
509
- /**
510
- * What the release DOES have for `noun`, for an unpublished message: the ABI
511
- * revision(s) its rows carry, or that it has none at any revision.
512
- */
513
- /**
514
- * Rows that carry no `abi_revision`, named with the reason: the artifact
515
- * producer records the field from the revision that introduced it onward. Every
516
- * unpublished message, `ensureAll` warning and `list` note says this in these
517
- * words rather than that the release serves "none".
518
- */
519
- const BUILT_BEFORE = '(built before revisions were recorded)';
520
- const NO_RECORDED_REVISION = `rows that record no ABI revision ${BUILT_BEFORE}`;
521
- function served(rows, noun) {
522
- if (rows.length === 0)
523
- return `the release does not have ${noun} at any ABI revision`;
524
- const revs = declaredRevisions(rows);
525
- if (revs.length === 0)
526
- return `the release has ${noun} only in ${NO_RECORDED_REVISION}`;
527
- let said = revs.length === 1 ? `ABI revision ${revs[0]}` : `ABI revisions ${revs.join(', ')}`;
528
- if (rows.some((a) => a.abi_revision === undefined))
529
- said += ` and in ${NO_RECORDED_REVISION}`;
530
- return `the release has ${noun} only at ${said}`;
531
- }
532
- /** The distinct revisions rows declare, ascending. */
533
- function declaredRevisions(rows) {
534
- return [...new Set(rows.flatMap((a) => (a.abi_revision === undefined ? [] : [a.abi_revision])))].sort((a, b) => a - b);
535
- }
536
- /**
537
- * One message per line the release has for `platform` only at another ABI
538
- * revision, or only in rows that record none — the lines `ensureAll` installs
539
- * nothing for — oldest line first. Exported for the test suite only; not
540
- * re-exported from `index.ts`.
541
- */
542
- export function skippedLines(index, platform) {
543
- const [os, arch] = platform.split('-');
544
- const rev = fetchAbiRevision();
545
- const byLine = new Map();
546
- for (const a of index.artifacts) {
547
- if (a.os !== os || a.arch !== arch)
548
- continue;
549
- byLine.set(a.clickhouse_minor, [...(byLine.get(a.clickhouse_minor) ?? []), a]);
550
- }
551
- return [...byLine.entries()]
552
- .filter(([, rows]) => !rows.some((a) => a.abi_revision === rev))
553
- .sort(([a], [b]) => compareVersions(a, b))
554
- .map(([line, rows]) => `ClickHouse line ${line} on ${platform} is not installed: ${served(rows, `that line for ${platform}`)}, ` +
555
- `and this SDK speaks ABI revision ${rev}`);
556
- }
557
- /**
558
- * `list`'s one line naming the platform's rows at another ABI revision, which it
559
- * does not show (docs/guides/fetch.md §6); `undefined` when none were hidden.
560
- * Rows that carry no `abi_revision` are named for what they are — built before
561
- * revisions were recorded — never as a revision called "none". Exported for the
562
- * test suite only; not re-exported from `index.ts`.
563
- */
564
- export function notShown(index, platform) {
565
- const [os, arch] = platform.split('-');
566
- const rev = fetchAbiRevision();
567
- const hidden = index.artifacts.filter((a) => a.os === os && a.arch === arch && a.abi_revision !== rev);
568
- if (hidden.length === 0)
569
- return undefined;
570
- const undeclared = hidden.filter((a) => a.abi_revision === undefined).length;
571
- const declared = hidden.length - undeclared;
572
- const parts = [];
573
- if (declared > 0)
574
- parts.push(`${declared} row(s) at ABI revision(s) ${declaredRevisions(hidden).join(', ')}`);
575
- if (undeclared > 0)
576
- parts.push(`${undeclared} row(s) that record no ABI revision ${BUILT_BEFORE}`);
577
- return `${parts.join(' and ')} not shown; this SDK speaks ${rev}`;
578
- }
579
- /**
580
- * The wrapper build number: the row's own `build` when it has one, else the
581
- * `-b<N>` suffix of the file name, else 0 — an old row, published before builds
582
- * existed, which is build 0 by definition.
583
- */
584
- function buildOf(field, file) {
585
- if (typeof field === 'number' && Number.isSafeInteger(field) && field >= 0)
586
- return field;
587
- const m = /-b(\d+)\.tar\.gz$/.exec(file);
588
- return m ? Number(m[1]) : 0;
589
- }
590
- /** `sha256sum` format: `<hex> <file>` (or `<hex> *<file>`), one per line. */
591
- function parseSums(bytes) {
592
- const out = new Map();
593
- for (const raw of bytes.toString('utf8').split(/\r?\n/)) {
594
- const m = /^([0-9a-fA-F]{64})\s+\*?(\S.*)$/.exec(raw.trim());
595
- if (m)
596
- out.set(m[2], m[1].toLowerCase());
597
- }
598
- return out;
599
- }
600
- function minorOfVersion(version) {
601
- const parts = version.split('.');
602
- return parts.length < 2 ? version : `${parts[0]}.${parts[1]}`;
603
- }
604
- /**
605
- * The one row for a request, or `CHTYPES_ARTIFACT_UNPUBLISHED` naming what the
606
- * release does have. Only rows at this binding's ABI revision are considered,
607
- * FIRST (docs/guides/fetch.md §2): nothing at that revision is unpublished, never
608
- * another revision's row.
609
- */
610
- export function selectArtifact(index, platform, req) {
611
- const all = forPlatform(index, platform);
612
- const rev = fetchAbiRevision();
613
- const have = all.filter((a) => a.abi_revision === rev).map((a) => a.clickhouse_version).join(', ') || 'nothing';
614
- const anyRevision = req.exact !== null ? all.filter((a) => patchMatches(a.clickhouse_version, req)) : all.filter((a) => a.clickhouse_minor === req.line);
615
- const hit = anyRevision.filter((a) => a.abi_revision === rev);
616
- if (hit.length === 0) {
617
- throw new ArtifactUnpublishedError(req.exact !== null
618
- ? `chtypes: you asked for exactly ClickHouse ${req.exact} on ${platform} at ABI revision ${rev} (this SDK's) and this ` +
619
- `release does not publish it at that revision: ${served(anyRevision, `that patch for ${platform}`)}` +
620
- `; at ABI revision ${rev} the release has: ${have}. Ask for the line (${req.line}) to take what was published.`
621
- : `chtypes: no artifact for ClickHouse line ${req.line} on ${platform} at ABI revision ${rev} (this SDK's): ` +
622
- `${served(anyRevision, `that line for ${platform}`)}; at ABI revision ${rev} the release has: ${have}.`);
623
- }
624
- // A line can carry more than one row: two patches, or the same patch built
625
- // twice (a release lists every build it publishes). Take the newest
626
- // by version and then by BUILD — never by list order, which is what a
627
- // version-only comparison degrades to once two rows compare equal, because
628
- // Array.prototype.sort is stable.
629
- return hit.reduce((best, a) => (newerRow(a, best) ? a : best));
630
- }
631
- /**
632
- * Is `a` the row to prefer over `b`? Newest ClickHouse version first, and among
633
- * rows of the SAME version the highest wrapper build — a rebuild is a new row
634
- * beside the old one, so without the build tie-break the older build could win
635
- * on nothing but its position in the index.
636
- */
637
- function newerRow(a, b) {
638
- const byVersion = compareVersions(a.clickhouse_version, b.clickhouse_version);
639
- return byVersion !== 0 ? byVersion > 0 : a.build > b.build;
640
- }
641
- /**
642
- * Every line the release publishes for the platform at this binding's ABI
643
- * revision, newest patch (then highest build) per line, oldest line first.
644
- * Nothing at that revision is `CHTYPES_ARTIFACT_UNPUBLISHED`, naming the
645
- * revision(s) the release does serve (docs/guides/fetch.md §2).
646
- */
647
- export function selectAll(index, platform) {
648
- const all = forPlatform(index, platform);
649
- const rev = fetchAbiRevision();
650
- const rows = all.filter((a) => a.abi_revision === rev);
651
- if (rows.length === 0) {
652
- throw new ArtifactUnpublishedError(`chtypes: the release has nothing for ${platform} at ABI revision ${rev} (this SDK's): ${served(all, `rows for ${platform}`)}`);
653
- }
654
- const best = new Map();
655
- for (const a of rows) {
656
- const cur = best.get(a.clickhouse_minor);
657
- if (cur === undefined || newerRow(a, cur))
658
- best.set(a.clickhouse_minor, a);
659
- }
660
- return [...best.values()].sort((a, b) => compareVersions(a.clickhouse_minor, b.clickhouse_minor));
661
- }
662
- function forPlatform(index, platform) {
663
- const [os, arch] = platform.split('-');
664
- const rows = index.artifacts.filter((a) => a.os === os && a.arch === arch);
665
- if (rows.length === 0) {
666
- const have = [...new Set(index.artifacts.map((a) => `${a.os}-${a.arch}`))].sort();
667
- throw new ArtifactUnpublishedError(`chtypes: the release has nothing for ${platform} (it has: ${have.length > 0 ? have.join(', ') : 'nothing'}).`);
668
- }
669
- return rows;
670
- }
671
- // -------------------------------------------------------------- ensure
672
- /** Concurrent `ensure`s of one line in one process share one fetch (docs/guides/fetch.md §6). */
673
- const inFlight = new Map();
674
- /**
675
- * Make one ClickHouse line available in a registry directory, through the
676
- * verification chain, and say where it is. Idempotent: installed and
677
- * verified is a no-op that downloads nothing (`force` re-downloads).
678
- *
679
- * @param spelling - a line (`"25.8"`) or an exact patch (`"25.8.28.1-lts"`, a hard requirement).
680
- * @param options - see `EnsureOptions`; every §6 flag has a property here.
681
- * @returns where the line is and what it is — `dir` is `<registry>/<minor>`.
682
- * @throws {ArtifactUntrustedError} `CHTYPES_ARTIFACT_UNTRUSTED` — no signature, or none a trusted key verifies.
683
- * @throws {ArtifactCorruptError} `CHTYPES_ARTIFACT_CORRUPT` — any hash mismatch, a release that disagrees with itself.
684
- * @throws {ArtifactPinnedError} `CHTYPES_ARTIFACT_PINNED` — `frozen` and the lock says otherwise.
685
- * @throws {ArtifactUnpublishedError} `CHTYPES_ARTIFACT_UNPUBLISHED` — nothing for this platform/line/patch.
686
- * @throws {SourceUnreachableError} `CHTYPES_SOURCE_UNREACHABLE` — the source cannot be reached, or `offline` and not installed.
687
- * @throws {ChtypesError} for a spelling that is not a version, a bad platform key, or conflicting options.
688
- */
689
- export async function ensure(spelling, options = {}) {
690
- const req = parseVersionSpelling(spelling);
691
- const platform = resolvePlatform(options.platform);
692
- const dest = fetchDestination(options.dest, platform);
693
- const key = [dest, platform, req.exact ?? req.line, options.force ? 'force' : ''].join(' ');
694
- const running = inFlight.get(key);
695
- if (running !== undefined)
696
- return running;
697
- const p = ensureUncached(req, platform, dest, options).finally(() => {
698
- inFlight.delete(key);
699
- });
700
- inFlight.set(key, p);
701
- return p;
702
- }
703
- /** A synthetic line-shaped request, for the parts of `installOne` / the lock helpers that take one — `--all` never spells an exact patch. */
704
- function lineRequest(line) {
705
- return { spelling: line, line, exact: null, bare: null };
706
- }
707
- /**
708
- * `fetch --all`: every line the release publishes for the platform at this
709
- * binding's ABI revision, each through `ensure`'s chain — installed flat, as
710
- * a line request always is. A line it has only at another revision is not
711
- * installed and is named in one loud stderr line; the rest go on.
712
- *
713
- * Under `--frozen` (F6): every line the lock pins for this platform, at its
714
- * own newest pinned patch — never a line the release has that the lock does
715
- * not pin (that is not drift in anything that was pinned). Every candidate's
716
- * ABI revision is checked (F2) before the release is even read.
717
- */
718
- export async function ensureAll(options = {}) {
719
- const platform = resolvePlatform(options.platform);
720
- const dest = fetchDestination(options.dest, platform);
721
- const emit = options.onProgress ?? noop;
722
- if (options.offline)
723
- throw new ChtypesError('chtypes: --all needs the release listing; it cannot run offline');
724
- const ctx = installContext(platform, dest, options);
725
- let pinnedLines = [];
726
- if (ctx.frozen) {
727
- pinnedLines = linesPinnedForPlatform(ctx.lock, platform);
728
- for (const line of pinnedLines)
729
- checkLockRevisionEarly(ctx, lineRequest(line));
730
- }
731
- const source = openSource(options);
732
- emit({ type: 'status', message: `every published ClickHouse line, ${platform} -> ${dest}` });
733
- emit({ type: 'status', message: `source ${source.description}` });
734
- const release = await loadRelease(source, options, emit);
735
- if (ctx.frozen) {
736
- const out = [];
737
- for (const line of pinnedLines) {
738
- const req = lineRequest(line);
739
- const candidate = selectFromLock(ctx.lock, platform, req);
740
- const art = verifyFrozenCandidate(release, ctx, req, candidate);
741
- out.push(await installOne(release, art, ctx, req));
742
- }
743
- await installGoldens(release, ctx, options);
744
- return out;
745
- }
746
- const rows = selectAll(release.index, platform);
747
- // A line the release has only at another ABI revision is not installed —
748
- // and never silently: one loud line per such line, then the rest go on.
749
- for (const message of skippedLines(release.index, platform))
750
- warn(`chtypes: WARNING: ${message}`);
751
- const out = [];
752
- for (const art of rows)
753
- out.push(await installOne(release, art, ctx, lineRequest(art.clickhouse_minor)));
754
- await installGoldens(release, ctx, options);
755
- return out;
756
- }
757
- /**
758
- * The served golden set: a release-level file like `index.json`, and a row in
759
- * the signed `SHA256SUMS` like a tarball, so it verifies through the same chain
760
- * and installs beside the artifacts as `<registry>/sdk-goldens.json` — where
761
- * every binding's golden test reads it offline.
762
- */
763
- const GOLDENS_ASSET = 'sdk-goldens.json';
764
- /**
765
- * One look at the served golden set against an already-verified `sums` map.
766
- * Resolves `null` when the release simply does not list it (not an error).
767
- * Throws `ArtifactCorruptError`/`SourceUnreachableError` for the new window
768
- * symptom — a hash mismatch, or the sums listing it while the source does not
769
- * (yet) serve it — matching the codes {@link loadRelease}'s own retry
770
- * recognizes, so {@link installGoldens} can reuse the same retryable check.
771
- */
772
- async function readGoldensOnce(source, sums) {
773
- const want = sums.get(GOLDENS_ASSET);
774
- if (want === undefined)
775
- return null;
776
- const blob = await source.get(GOLDENS_ASSET);
777
- if (blob === null) {
778
- throw new ArtifactCorruptError(`chtypes: SHA256SUMS lists ${GOLDENS_ASSET} but ${source.description} does not serve it — the release disagrees with itself; not installing it`);
779
- }
780
- const got = createHash('sha256').update(blob).digest('hex');
781
- if (got !== want) {
782
- throw new ArtifactCorruptError(`chtypes: ${GOLDENS_ASSET} hashes to ${got} but the signed SHA256SUMS says ${want} — not installing it`);
783
- }
784
- return blob;
785
- }
786
- /**
787
- * Install the served golden set, if this release publishes one, and never
788
- * fail the fetch that called it: a release-level file here is best-effort.
789
- *
790
- * The first look reuses `release.sums` — already verified by {@link loadRelease}
791
- * — which is cheap and right on the overwhelmingly common case that nothing is
792
- * mid-publish. Only a disagreement is retried (the same publish-window symptom
793
- * as the signature and index.json), and a retry re-reads the WHOLE consistent
794
- * set fresh through {@link loadReleaseOnce} — SHA256SUMS, its signature,
795
- * index.json AND the golden set together — never the golden set alone checked
796
- * against this call's by-then possibly-stale `release.sums`.
797
- *
798
- * Never throws. A release with no such row simply predates the served set, and a
799
- * mismatch that never heals — or a set that cannot be written — leaves the
800
- * golden tests skipping loudly, which is their job when there is nothing
801
- * trustworthy to read.
802
- */
803
- async function installGoldens(release, ctx, options) {
804
- const attempts = release.source instanceof HttpSource ? RELEASE_RETRY.attempts : 1;
805
- let delayMs = RELEASE_RETRY.delayMs;
806
- let blob = null;
807
- for (let attempt = 1;; attempt++) {
808
- try {
809
- if (attempt === 1) {
810
- blob = await readGoldensOnce(release.source, release.sums);
811
- }
812
- else {
813
- // A stale first look: re-verify everything from scratch, not just the
814
- // golden set against sums that may themselves have moved on.
815
- const fresh = await loadReleaseOnce(release.source, options, ctx.emit);
816
- blob = await readGoldensOnce(fresh.source, fresh.sums);
817
- }
818
- break;
819
- }
820
- catch (err) {
821
- const retryable = err instanceof ArtifactUntrustedError || err instanceof ArtifactCorruptError;
822
- if (!retryable || attempt >= attempts) {
823
- ctx.emit({ type: 'status', message: `${String(err)} — the golden tests will skip` });
824
- return;
825
- }
826
- ctx.emit({
827
- type: 'status',
828
- message: `${String(err)} (attempt ${attempt}/${attempts}) — this is what a release being ` +
829
- `published looks like from outside; retrying in ${delayMs / 1000}s`,
830
- });
831
- await new Promise((resolve) => setTimeout(resolve, delayMs));
832
- delayMs *= 2;
833
- }
834
- }
835
- if (blob === null) {
836
- ctx.emit({ type: 'status', message: `this release does not publish ${GOLDENS_ASSET} (the SDKs' golden tests will skip until it does)` });
837
- return;
838
- }
839
- const out = path.join(ctx.dest, GOLDENS_ASSET);
840
- try {
841
- await mkdir(ctx.dest, { recursive: true });
842
- await writeFile(out, blob);
843
- }
844
- catch (err) {
845
- ctx.emit({ type: 'status', message: `could not write ${GOLDENS_ASSET}: ${String(err)} — the golden tests will skip` });
846
- return;
847
- }
848
- ctx.emit({ type: 'status', message: `golden set verified and installed: ${out}` });
849
- }
850
- function installContext(platform, dest, options) {
851
- const frozen = options.frozen ?? false;
852
- // frozen without a lock path reads ./chtypes.lock (docs/guides/fetch.md, Decisions).
853
- const lockPath = options.lock !== undefined && options.lock !== '' ? path.resolve(options.lock) : frozen ? path.resolve(DEFAULT_LOCK_FILE) : undefined;
854
- const lock = lockPath !== undefined ? readLock(lockPath) : null;
855
- // Decision 6: offline never reads the lock at all, so a missing lock file
856
- // is not refused here — `ensureUncached`'s offline branch never consults
857
- // `ctx.lock` and answers from the destination alone.
858
- if (frozen && lock === null && options.offline !== true) {
859
- throw new ArtifactPinnedError(`chtypes: ${lockPath} does not exist, and frozen refuses anything it does not pin`);
860
- }
861
- if (platform !== hostPlatform()) {
862
- warn(`chtypes: note — fetching ${platform} artifacts on a ${hostPlatform()} host. They are for a ${platform} process ` +
863
- `(a container, usually), not this one.`);
864
- }
865
- return { platform, dest, lockPath, lock, frozen, force: options.force ?? false, emit: options.onProgress ?? noop };
866
- }
867
- /**
868
- * Every lock entry (its full `<os>-<arch>/<clickhouse_version>` key) that
869
- * `req` — a patch or a line spelling, for `platform` — matches under R2.
870
- */
871
- function candidatesFromLock(lock, platform, req) {
872
- const prefix = `${platform}/`;
873
- const rows = [];
874
- for (const [key, entry] of Object.entries(lock.artifacts)) {
875
- if (!key.startsWith(prefix))
876
- continue;
877
- const version = key.slice(prefix.length);
878
- const matches = req.exact !== null ? patchMatches(version, req) : minorOfVersion(version) === req.line;
879
- if (matches)
880
- rows.push({ key, entry, version });
881
- }
882
- return rows;
883
- }
884
- /** F1: the newest-versioned candidate the lock pins for `req` — a patch spelling's own entry, or a line's newest pinned patch. */
885
- function selectFromLock(lock, platform, req) {
886
- const rows = candidatesFromLock(lock, platform, req);
887
- if (rows.length === 0)
888
- return undefined;
889
- return rows.reduce((best, cur) => (compareVersions(cur.version, best.version) > 0 ? cur : best));
890
- }
891
- /** Every line the lock pins at least one patch of, for `platform` — what `--all --frozen` (F6) installs. */
892
- function linesPinnedForPlatform(lock, platform) {
893
- const prefix = `${platform}/`;
894
- const lines = new Set();
895
- for (const key of Object.keys(lock.artifacts)) {
896
- if (key.startsWith(prefix))
897
- lines.add(minorOfVersion(key.slice(prefix.length)));
898
- }
899
- return [...lines].sort(compareVersions);
900
- }
901
- /**
902
- * Fails fast, before any network access, when `frozen`'s lock already names
903
- * an ABI revision for one of `req`'s candidates that is not this binding's own
904
- * (docs/guides/fetch.md §5, F2). A lock made for one ABI revision does not get
905
- * a second chance disguised as a drifted pin or an unpublished line once the
906
- * SDK moves to another: the fix is always the same re-lock, so the message
907
- * says that directly instead of waiting to see which of the two symptoms
908
- * selection would have produced.
909
- */
910
- function checkLockRevisionEarly(ctx, req) {
911
- if (!ctx.frozen || ctx.lock === null)
912
- return;
913
- for (const { key, entry } of candidatesFromLock(ctx.lock, ctx.platform, req)) {
914
- if (entry.abi_revision === undefined || entry.abi_revision === fetchAbiRevision())
915
- continue;
916
- throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins ${key} at ABI revision ${entry.abi_revision}; this SDK speaks ABI revision ` +
917
- `${fetchAbiRevision()} — re-lock with: ${FETCH_COMMAND} ${req.spelling} --lock ${ctx.lockPath}`);
918
- }
919
- }
920
- /**
921
- * The one sentence appended to a PINNED or UNPUBLISHED message when, under
922
- * `frozen`, one of the lock's own candidate entries for `req` exists but
923
- * names no ABI revision at all — written by an SDK before this field
924
- * existed. `''` when there is nothing to add.
925
- */
926
- function revisionNote(ctx, req) {
927
- if (!ctx.frozen || ctx.lock === null)
928
- return '';
929
- const entry = candidatesFromLock(ctx.lock, ctx.platform, req).find((r) => r.entry.abi_revision === undefined);
930
- if (entry === undefined)
931
- return '';
932
- return (` ${ctx.lockPath} records no ABI revision (written by an older SDK); this SDK speaks ABI revision ` +
933
- `${fetchAbiRevision()} — re-lock with: ${FETCH_COMMAND} ${req.spelling} --lock ${ctx.lockPath}`);
934
- }
935
- /** F5: the release's own row for a lock candidate, verified against SHA256SUMS, the pin's own hash, and the SDK's ABI revision — in that order. */
936
- function verifyFrozenCandidate(release, ctx, req, candidate) {
937
- const { key, entry, version } = candidate;
938
- const [os, arch] = ctx.platform.split('-');
939
- const row = release.index.artifacts.find((a) => a.file === entry.file && a.os === os && a.arch === arch);
940
- const listedSha = release.sums.get(entry.file);
941
- if (row === undefined || listedSha === undefined) {
942
- throw new ArtifactUnpublishedError(`chtypes: the release at ${release.source.description} does not list ${entry.file}, which ${ctx.lockPath} pins for ${key}.${revisionNote(ctx, req)}`);
943
- }
944
- if (row.sha256 !== listedSha) {
945
- throw new ArtifactCorruptError(`chtypes: index.json says ${entry.file} is ${row.sha256} but SHA256SUMS says ${listedSha} — the release disagrees with itself; not installing it`);
946
- }
947
- if (row.abi_revision !== fetchAbiRevision()) {
948
- throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins ${key} to ${entry.file}, which the release at ${release.source.description} serves at ABI revision ` +
949
- `${row.abi_revision ?? 'none recorded'}; this SDK speaks ABI revision ${fetchAbiRevision()} — re-lock with: ` +
950
- `${FETCH_COMMAND} ${version} --lock ${ctx.lockPath}${revisionNote(ctx, req)}`);
951
- }
952
- if (entry.sha256.toLowerCase() !== listedSha) {
953
- throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins ${key} to ${entry.file} (${entry.sha256}) but the release lists it as ${listedSha}; ` +
954
- `frozen refuses it.${revisionNote(ctx, req)}`);
955
- }
956
- return row;
957
- }
958
- /** F1-F5: install exactly what the lock pins for `req`, never a newer patch or build the release also serves. */
959
- async function installFrozen(release, req, ctx, options) {
960
- const lock = ctx.lock; // installContext refuses `frozen` with no lock before this is ever called.
961
- const candidate = selectFromLock(lock, ctx.platform, req);
962
- if (candidate === undefined) {
963
- throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins nothing for ${ctx.platform}/${req.spelling}, and frozen refuses anything it does not pin`);
964
- }
965
- const art = verifyFrozenCandidate(release, ctx, req, candidate);
966
- const installed = await installOne(release, art, ctx, req);
967
- await installGoldens(release, ctx, options);
968
- return installed;
969
- }
970
- async function ensureUncached(req, platform, dest, options) {
971
- const ctx = installContext(platform, dest, options);
972
- if (options.offline) {
973
- // Never touch the source, and never the lock (Decision 6): installed and
974
- // hashing what its own manifest says, at either slot, is the whole test.
975
- const patches = await installedPatchesForLine(dest, req.line);
976
- const match = req.exact !== null
977
- ? (patches.find((p) => patchMatches(p.version, req)) ?? null)
978
- : patches.reduce((best, cur) => (best === null || compareVersions(cur.version, best.version) > 0 ? cur : best), null);
979
- if (match !== null && match.ok) {
980
- ctx.emit({ type: 'status', message: `offline — already installed and verified: ${match.dir}` });
981
- return {
982
- dir: match.dir,
983
- registry: dest,
984
- line: req.line,
985
- version: match.version,
986
- platform,
987
- file: '',
988
- sha256: '',
989
- library: match.library,
990
- librarySha256: match.expected,
991
- installed: false,
992
- source: 'offline',
993
- signed: false,
994
- keyId: null,
995
- };
996
- }
997
- throw new SourceUnreachableError(`chtypes: offline — ClickHouse ${req.spelling} (${platform}) is not installed and verified in ${dest}` +
998
- (match !== null && !match.ok ? ` (${match.problem})` : '') +
999
- ', and offline forbids fetching it');
1000
- }
1001
- checkLockRevisionEarly(ctx, req);
1002
- const source = openSource(options);
1003
- ctx.emit({
1004
- type: 'status',
1005
- message: `ClickHouse ${req.spelling} -> line ${req.line}${req.exact !== null ? ` (exact ${req.exact})` : ''}, ${platform}`,
1006
- });
1007
- ctx.emit({ type: 'status', message: `source ${source.description}` });
1008
- const release = await loadRelease(source, options, ctx.emit);
1009
- if (ctx.frozen)
1010
- return installFrozen(release, req, ctx, options);
1011
- const art = selectArtifact(release.index, platform, req);
1012
- const installed = await installOne(release, art, ctx, req);
1013
- await installGoldens(release, ctx, options);
1014
- return installed;
1015
- }
1016
- /**
1017
- * Same-filesystem, atomic: move the current flat `<dest>/<minor>/` occupant
1018
- * aside to `<dest>/patches/<minor>/<its version>/` — NEVER deleted — before a
1019
- * different patch takes the flat slot (issue #284, the layout rule). A
1020
- * no-op when nothing is flat yet, or when the flat occupant already IS
1021
- * `incomingVersion` (the ordinary re-verify/rebuild path handles that case).
1022
- */
1023
- async function demoteFlatIfDifferent(dest, minor, incomingVersion) {
1024
- const flat = path.join(dest, minor);
1025
- if (!existsSync(flat))
1026
- return;
1027
- const existing = readInstalledManifest(flat);
1028
- if (existing === null || existing.version === incomingVersion)
1029
- return;
1030
- const patchesDir = path.join(dest, 'patches', minor);
1031
- const target = path.join(patchesDir, existing.version);
1032
- await mkdir(patchesDir, { recursive: true });
1033
- if (existsSync(target))
1034
- await rm(target, { recursive: true, force: true });
1035
- await rename(flat, target);
1036
- }
1037
- /** Does `dir` already hold `art`, byte-verified — a manifest and a library hashing `art.library_sha256`? */
1038
- async function verifiedInstall(dir, art) {
1039
- const libPath = path.join(dir, art.library);
1040
- if (!existsSync(path.join(dir, 'manifest.json')) || !existsSync(libPath))
1041
- return false;
1042
- return (await sha256File(libPath)) === art.library_sha256;
1043
- }
1044
- /**
1045
- * One index row, from the release into its slot — steps 2 through 4. A LINE
1046
- * request's pick installs flat, `<dest>/<minor>/`, demoting a different flat
1047
- * occupant first (never deleting it). Any other exact patch request installs
1048
- * at `<dest>/patches/<minor>/<clickhouse_version>/` — unless that exact
1049
- * version already sits flat, byte-verified, in which case it counts as
1050
- * installed there and no second copy is made (docs/guides/fetch.md §4).
1051
- */
1052
- async function installOne(release, art, ctx, req) {
1053
- const { dest, platform, emit } = ctx;
1054
- const minor = art.clickhouse_minor;
1055
- const version = art.clickhouse_version;
1056
- const wantsFlat = req.exact === null;
1057
- const flat = path.join(dest, minor);
1058
- const nested = path.join(dest, 'patches', minor, version);
1059
- const install = wantsFlat ? flat : nested;
1060
- const lockKey = `${platform}/${version}`;
1061
- emit({ type: 'status', message: `${art.file} (${art.bytes} bytes, ClickHouse ${art.clickhouse_version}, library ${art.library})` });
1062
- // Step 2: the authentic SHA256SUMS must agree with index.json, byte for byte.
1063
- const listed = release.sums.get(art.file);
1064
- if (listed === undefined)
1065
- throw new ArtifactCorruptError(`chtypes: SHA256SUMS has no line for ${art.file}`);
1066
- if (listed !== art.sha256) {
1067
- throw new ArtifactCorruptError(`chtypes: index.json says ${art.file} is ${art.sha256} but SHA256SUMS says ${listed} — the release disagrees with itself; not installing it`);
1068
- }
1069
- const result = (installed, dir) => ({
1070
- dir,
1071
- registry: dest,
1072
- line: minor,
1073
- version: art.clickhouse_version,
1074
- platform,
1075
- file: art.file,
1076
- sha256: art.sha256,
1077
- library: art.library,
1078
- librarySha256: art.library_sha256,
1079
- installed,
1080
- source: release.source.description,
1081
- signed: release.signed,
1082
- keyId: release.keyId,
1083
- });
1084
- const recordLock = () => {
1085
- if (ctx.lockPath !== undefined && !ctx.frozen) {
1086
- writeLockEntry(ctx.lockPath, lockKey, { file: art.file, sha256: art.sha256, abi_revision: fetchAbiRevision() });
1087
- }
1088
- };
1089
- // Already installed and intact? The same hash the install path ends with,
1090
- // so "already there" is a verified claim, not an inference from a directory.
1091
- if (!ctx.force) {
1092
- if (await verifiedInstall(install, art)) {
1093
- emit({ type: 'status', message: `already installed and verified: ${path.join(install, art.library)}` });
1094
- recordLock();
1095
- return result(false, install);
1096
- }
1097
- // §4: a patch request whose exact version already sits flat (put there by
1098
- // an earlier line fetch) is already installed — no second, nested copy.
1099
- if (!wantsFlat && (await verifiedInstall(flat, art))) {
1100
- emit({ type: 'status', message: `already installed and verified (flat): ${path.join(flat, art.library)}` });
1101
- recordLock();
1102
- return result(false, flat);
1103
- }
1104
- }
1105
- // A line request whose pick differs from the current flat occupant demotes
1106
- // that occupant to patches/ BEFORE the incoming patch takes the flat slot.
1107
- if (wantsFlat)
1108
- await demoteFlatIfDifferent(dest, minor, version);
1109
- await mkdir(dest, { recursive: true });
1110
- await mkdir(path.dirname(install), { recursive: true });
1111
- const tag = `${process.pid}.${randomBytes(4).toString('hex')}`;
1112
- // Hidden siblings: the loader skips dot-directories, so a half-written
1113
- // install is never scanned as a version.
1114
- const tarball = path.join(dest, `.${minor}.download.${tag}`);
1115
- const incoming = path.join(dest, `.${minor}.incoming.${tag}`);
1116
- try {
1117
- emit({ type: 'status', message: `downloading ${art.file}` });
1118
- const got = await release.source.download(art.file, tarball, (received) => emit({ type: 'download', file: art.file, received, total: art.bytes }));
1119
- if (!got)
1120
- throw new SourceUnreachableError(`chtypes: could not download ${art.file} from ${release.source.description}`);
1121
- // Step 3: size, then hash, before a byte is unpacked.
1122
- const size = statSync(tarball).size;
1123
- if (size !== art.bytes)
1124
- throw new ArtifactCorruptError(`chtypes: ${art.file} is ${size} bytes, index.json says ${art.bytes}`);
1125
- const sha = await sha256File(tarball);
1126
- if (sha !== art.sha256) {
1127
- throw new ArtifactCorruptError(`chtypes: ${art.file} FAILED its sha256: got ${sha}, want ${art.sha256} — not unpacking it`);
1128
- }
1129
- emit({ type: 'status', message: `sha256 verified before unpacking: ${sha}` });
1130
- await mkdir(incoming);
1131
- await extractTarGz(tarball, incoming);
1132
- const manifest = readInstalledManifest(incoming);
1133
- if (manifest === null)
1134
- throw new ArtifactCorruptError(`chtypes: ${art.file} contains no usable manifest.json at its root`);
1135
- // The index is a convenience; the manifest is the artifact's own claim
1136
- // about itself. They must agree, or the index was built from a different artifact.
1137
- if (manifest.library !== art.library ||
1138
- manifest.version !== art.clickhouse_version ||
1139
- manifest.minor !== minor ||
1140
- manifest.librarySha256 !== art.library_sha256) {
1141
- throw new ArtifactCorruptError(`chtypes: manifest.json inside ${art.file} disagrees with index.json ` +
1142
- `(${manifest.library}/${manifest.version}/${manifest.minor}/${manifest.librarySha256} vs ` +
1143
- `${art.library}/${art.clickhouse_version}/${minor}/${art.library_sha256})`);
1144
- }
1145
- if (!existsSync(path.join(incoming, manifest.library))) {
1146
- throw new ArtifactCorruptError(`chtypes: ${art.file} names library ${manifest.library} but does not contain it`);
1147
- }
1148
- // Into place through a sibling: an interrupted install never leaves a
1149
- // half-populated slot for the loader to dlopen.
1150
- const replaced = path.join(dest, `.${minor}.replaced.${tag}`);
1151
- if (existsSync(install))
1152
- await rename(install, replaced);
1153
- await rename(incoming, install);
1154
- await rm(replaced, { recursive: true, force: true });
1155
- // Step 4: re-hash the installed file, in place. Everything up to here
1156
- // proved the bytes were right somewhere else.
1157
- const installedLib = path.join(install, manifest.library);
1158
- const finalSha = await sha256File(installedLib);
1159
- if (finalSha !== manifest.librarySha256) {
1160
- const aside = path.join(dest, `.${minor}.corrupt.${tag}`);
1161
- await rename(install, aside);
1162
- throw new ArtifactCorruptError(`chtypes: installed ${installedLib} hashes ${finalSha}, manifest says ${manifest.librarySha256} — the install is bad; moved aside to ${aside}`);
1163
- }
1164
- emit({ type: 'status', message: `installed and verified: ${install} (${manifest.library} sha256 ${finalSha})` });
1165
- recordLock();
1166
- return result(true, install);
1167
- }
1168
- finally {
1169
- await rm(tarball, { force: true });
1170
- await rm(incoming, { recursive: true, force: true });
1171
- }
1172
- }
1173
- // ------------------------------------------------------ verify and list
1174
- /**
1175
- * `chtypes verify`: re-hash every installed PATCH in a registry directory —
1176
- * the flat slot and every `patches/<minor>/<version>/` sibling (issue #284)
1177
- * — against its own manifest. A directory whose manifest cannot be read is
1178
- * not a patch (a scratch directory) and is skipped, as the loader skips it.
1179
- */
1180
- export async function verifyInstalled(dest, platform) {
1181
- const dir = fetchDestination(dest, resolvePlatform(platform));
1182
- const out = [];
1183
- for (const sub of allInstalledDirs(dir)) {
1184
- const state = await installedPatch(sub);
1185
- if (state !== null)
1186
- out.push(state);
1187
- }
1188
- return out.sort((a, b) => compareVersions(a.version, b.version));
1189
- }
1190
- /** `chtypes list`: what is installed in the registry directory (one record per patch), and what the release offers for the platform. */
1191
- export async function listArtifacts(options = {}) {
1192
- const platform = resolvePlatform(options.platform);
1193
- const dir = fetchDestination(options.dest, platform);
1194
- const installed = [];
1195
- for (const sub of allInstalledDirs(dir)) {
1196
- const m = readInstalledManifest(sub);
1197
- if (m === null)
1198
- continue;
1199
- const present = existsSync(path.join(sub, m.library));
1200
- installed.push({
1201
- line: m.minor,
1202
- dir: sub,
1203
- version: m.version,
1204
- library: m.library,
1205
- expected: m.librarySha256,
1206
- actual: null,
1207
- ok: present,
1208
- problem: present ? null : `${m.library} is missing`,
1209
- });
1210
- }
1211
- installed.sort((a, b) => compareVersions(a.version, b.version));
1212
- if (options.offline)
1213
- return { registry: dir, platform, installed, offered: null, source: null };
1214
- const source = openSource(options);
1215
- const release = await loadRelease(source, options, options.onProgress ?? noop);
1216
- let offered;
1217
- try {
1218
- offered = selectAll(release.index, platform);
1219
- }
1220
- catch (err) {
1221
- if (!(err instanceof ArtifactUnpublishedError))
1222
- throw err;
1223
- offered = [];
1224
- }
1225
- const note = notShown(release.index, platform);
1226
- return { registry: dir, platform, installed, offered, source: source.description, ...(note !== undefined ? { notShown: note } : {}) };
1227
- }
1228
- // ------------------------------------------------------------ the lock
1229
- /** `chtypes-<version>-<os>-<arch>[-b<N>].tar.gz` (docs/guides/artifacts.md), version captured whole (it may itself contain dots and a channel). */
1230
- const ASSET_FILE_NAME = /^chtypes-(.+)-(linux|darwin)-(arm64|amd64)(?:-b\d+)?\.tar\.gz$/;
1231
- /** The ClickHouse version a served asset file name claims, or `null` when it does not parse. */
1232
- function versionFromAssetFile(file) {
1233
- const m = ASSET_FILE_NAME.exec(file);
1234
- return m === null ? null : m[1];
1235
- }
1236
- /** True for a PATCH spelling (four or more numeric components) — what schema 2's keys must name, never a bare line. */
1237
- function isPatchVersion(version) {
1238
- try {
1239
- return parseVersionSpelling(version).exact !== null;
1240
- }
1241
- catch {
1242
- return false;
1243
- }
1244
- }
1245
- /**
1246
- * Read a lock file; `null` when it does not exist. Accepts schema 1 and
1247
- * schema 2 (docs/guides/fetch.md §5); `.schema` reports the number that was
1248
- * actually on disk (so a caller can tell which it read), while `.artifacts`
1249
- * is ALWAYS schema-2-shaped: a schema-1 entry `<os>-<arch>/<minor>` is
1250
- * converted to the schema-2 entry `<os>-<arch>/<v>`, `<v>` being the
1251
- * ClickHouse version its own `file` names — the one schema-1 could not
1252
- * record, because it never had two patches of a line to distinguish. Every
1253
- * other reader in this module treats `.artifacts` as schema-2-keyed
1254
- * unconditionally; only this doc comment and `.schema` itself ever mention
1255
- * schema 1 again. Any other schema number is refused, naming both 1 and
1256
- * `LOCK_SCHEMA`. An entry that does not parse, under either schema — including
1257
- * a schema-2 key that names a bare line rather than an exact patch — makes the
1258
- * whole lock unreadable and the error names it.
1259
- */
1260
- export function readLock(file) {
1261
- let text;
1262
- try {
1263
- text = readFileSync(file, 'utf8');
1264
- }
1265
- catch (err) {
1266
- if (isNoEntry(err))
1267
- return null;
1268
- throw new ChtypesError(`chtypes: cannot read lock file ${file}: ${errorText(err)}`, { cause: err });
1269
- }
1270
- let doc;
1271
- try {
1272
- doc = JSON.parse(text);
1273
- }
1274
- catch (err) {
1275
- throw new ChtypesError(`chtypes: lock file ${file} is not JSON: ${errorText(err)}`, { cause: err });
1276
- }
1277
- if (typeof doc !== 'object' || doc === null) {
1278
- throw new ChtypesError(`chtypes: lock file ${file} is not schema ${LOCK_SCHEMA_LEGACY} or ${LOCK_SCHEMA}`);
1279
- }
1280
- const schema = doc['schema'];
1281
- if (schema !== LOCK_SCHEMA_LEGACY && schema !== LOCK_SCHEMA) {
1282
- throw new ChtypesError(`chtypes: lock file ${file} is schema ${JSON.stringify(schema)}, not ${LOCK_SCHEMA_LEGACY} or ${LOCK_SCHEMA}`);
1283
- }
1284
- const raw = doc['artifacts'];
1285
- const artifacts = {};
1286
- if (typeof raw === 'object' && raw !== null) {
1287
- for (const [k, v] of Object.entries(raw)) {
1288
- if (typeof v !== 'object' || v === null)
1289
- continue;
1290
- const e = v;
1291
- if (typeof e['file'] !== 'string' || typeof e['sha256'] !== 'string')
1292
- continue;
1293
- // abi_revision is optional and additive (docs/guides/fetch.md §5): an
1294
- // entry written before this SDK recorded it simply has none, read the
1295
- // same way an index row that carries none is — never a typed number
1296
- // (TS cannot tell 5.0 from 5, so anything not a safe integer is absent).
1297
- const entry = { file: e['file'], sha256: e['sha256'], ...revisionOf(e['abi_revision']) };
1298
- const slash = k.indexOf('/');
1299
- if (slash < 0) {
1300
- throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)} is not <os>-<arch>/<version>`);
1301
- }
1302
- const platformKey = k.slice(0, slash);
1303
- const versionOrMinor = k.slice(slash + 1);
1304
- if (schema === LOCK_SCHEMA_LEGACY) {
1305
- const version = versionFromAssetFile(entry.file);
1306
- if (version === null) {
1307
- throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)}'s file ${JSON.stringify(entry.file)} does not name a ` +
1308
- 'ClickHouse version — cannot convert this schema 1 lock to schema 2');
1309
- }
1310
- if (minorOfVersion(version) !== versionOrMinor) {
1311
- throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)} names line ${versionOrMinor} but its file ` +
1312
- `${JSON.stringify(entry.file)} is ${version} — cannot convert this schema 1 lock to schema 2`);
1313
- }
1314
- artifacts[`${platformKey}/${version}`] = entry;
1315
- }
1316
- else {
1317
- if (!isPatchVersion(versionOrMinor)) {
1318
- throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)} names a line, not an exact patch — schema ${LOCK_SCHEMA} keys are <os>-<arch>/<clickhouse_version>`);
1319
- }
1320
- artifacts[k] = entry;
1321
- }
1322
- }
1323
- }
1324
- return { schema, artifacts };
1325
- }
1326
- function writeLockEntry(file, key, entry) {
1327
- const current = readLock(file) ?? { schema: LOCK_SCHEMA, artifacts: {} };
1328
- const merged = { ...current.artifacts, [key]: entry };
1329
- const sorted = Object.fromEntries(Object.keys(merged).sort().map((k) => [k, merged[k]]));
1330
- writeFileSync(file, `${JSON.stringify({ schema: LOCK_SCHEMA, artifacts: sorted }, null, 2)}\n`);
1331
- }
1332
- // -------------------------------------------------------------- helpers
1333
- /** The platform a fetch is for: the option, else `CHTYPES_TARGET`, else this host. */
1334
- export function resolvePlatform(explicit) {
1335
- const env = process.env['CHTYPES_TARGET'];
1336
- const key = explicit !== undefined && explicit !== '' ? explicit : env !== undefined && env !== '' ? env : hostPlatform();
1337
- if (!isPlatformKey(key))
1338
- throw new ChtypesError(`chtypes: not a known platform key: ${key} ((linux|darwin)-(arm64|amd64))`);
1339
- return key;
1340
- }
1341
- function readInstalledManifest(dir) {
1342
- try {
1343
- const m = JSON.parse(readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
1344
- if (typeof m['library'] !== 'string' || m['library'] === '')
1345
- return null;
1346
- if (typeof m['library_sha256'] !== 'string' || m['library_sha256'] === '')
1347
- return null;
1348
- const version = typeof m['clickhouse_version'] === 'string' ? m['clickhouse_version'] : '';
1349
- // clickhouse_minor is absent from the first Linux artifacts; deriving it
1350
- // from clickhouse_version is what the loader does, so both agree.
1351
- const minor = typeof m['clickhouse_minor'] === 'string' && m['clickhouse_minor'] !== '' ? m['clickhouse_minor'] : minorOfVersion(version);
1352
- return { library: m['library'], version, minor, librarySha256: m['library_sha256'].toLowerCase() };
1353
- }
1354
- catch {
1355
- return null;
1356
- }
1357
- }
1358
- /**
1359
- * The state of one installed patch directory (flat or `patches/<minor>/<version>/`):
1360
- * null when it is not one (no usable manifest). `line` comes from the
1361
- * manifest's own claim, never from the directory name — a nested directory is
1362
- * named after the VERSION, not the line.
1363
- */
1364
- async function installedPatch(dir) {
1365
- const m = readInstalledManifest(dir);
1366
- if (m === null)
1367
- return null;
1368
- const libPath = path.join(dir, m.library);
1369
- if (!existsSync(libPath)) {
1370
- return { line: m.minor, dir, version: m.version, library: m.library, expected: m.librarySha256, actual: null, ok: false, problem: `${m.library} is missing` };
1371
- }
1372
- const actual = await sha256File(libPath);
1373
- const ok = actual === m.librarySha256;
1374
- return {
1375
- line: m.minor,
1376
- dir,
1377
- version: m.version,
1378
- library: m.library,
1379
- expected: m.librarySha256,
1380
- actual,
1381
- ok,
1382
- problem: ok ? null : `${m.library} hashes ${actual}, manifest says ${m.librarySha256}`,
1383
- };
1384
- }
1385
- /** Every installed-patch directory in a registry root: the flat slot per line, plus every `patches/<minor>/<version>/`. */
1386
- function allInstalledDirs(dir) {
1387
- const out = [];
1388
- let entries;
1389
- try {
1390
- entries = readdirSync(dir);
1391
- }
1392
- catch {
1393
- return out;
1394
- }
1395
- for (const entry of entries.sort()) {
1396
- if (entry.startsWith('.') || entry === 'patches')
1397
- continue;
1398
- if (existsSync(path.join(dir, entry, 'manifest.json')))
1399
- out.push(path.join(dir, entry));
1400
- }
1401
- const patchesRoot = path.join(dir, 'patches');
1402
- let minorEntries;
1403
- try {
1404
- minorEntries = readdirSync(patchesRoot);
1405
- }
1406
- catch {
1407
- return out;
1408
- }
1409
- for (const minorEntry of minorEntries.sort()) {
1410
- if (minorEntry.startsWith('.'))
1411
- continue;
1412
- const minorDir = path.join(patchesRoot, minorEntry);
1413
- let versionEntries;
1414
- try {
1415
- versionEntries = readdirSync(minorDir);
1416
- }
1417
- catch {
1418
- continue;
1419
- }
1420
- for (const versionEntry of versionEntries.sort()) {
1421
- if (versionEntry.startsWith('.'))
1422
- continue;
1423
- const sub = path.join(minorDir, versionEntry);
1424
- if (existsSync(path.join(sub, 'manifest.json')))
1425
- out.push(sub);
1426
- }
1427
- }
1428
- return out;
1429
- }
1430
- /**
1431
- * Every installed patch of `minor` in `dest` — the flat slot and every
1432
- * `patches/<minor>/<version>/` sibling — byte-verified against its own
1433
- * manifest. Used offline, where the source and the lock are both off limits
1434
- * (Decision 6): "installed" there means hashed and matching, not merely present.
1435
- */
1436
- async function installedPatchesForLine(dest, minor) {
1437
- const flat = path.join(dest, minor);
1438
- const dirs = existsSync(path.join(flat, 'manifest.json')) ? [flat] : [];
1439
- const patchesDir = path.join(dest, 'patches', minor);
1440
- let versionEntries = [];
1441
- try {
1442
- versionEntries = readdirSync(patchesDir);
1443
- }
1444
- catch {
1445
- versionEntries = [];
1446
- }
1447
- for (const versionEntry of versionEntries.sort()) {
1448
- if (versionEntry.startsWith('.'))
1449
- continue;
1450
- const sub = path.join(patchesDir, versionEntry);
1451
- if (existsSync(path.join(sub, 'manifest.json')))
1452
- dirs.push(sub);
1453
- }
1454
- const out = [];
1455
- for (const d of dirs) {
1456
- const state = await installedPatch(d);
1457
- if (state !== null)
1458
- out.push(state);
1459
- }
1460
- return out;
1461
- }
1462
- /** sha256 of a file, streamed — a 300 MB library never sits in memory whole. */
1463
- export async function sha256File(file) {
1464
- const hash = createHash('sha256');
1465
- await pipeline(createReadStream(file), async function* (chunks) {
1466
- for await (const chunk of chunks)
1467
- hash.update(chunk);
1468
- });
1469
- return hash.digest('hex');
1470
- }
1471
- function progressTap(onReceived) {
1472
- let received = 0;
1473
- return new Transform({
1474
- transform(chunk, _enc, cb) {
1475
- received += chunk.length;
1476
- onReceived(received);
1477
- cb(null, chunk);
1478
- },
1479
- });
1480
- }
1481
- function envFlag(name) {
1482
- const v = process.env[name];
1483
- return v !== undefined && ['1', 'true', 'yes', 'on'].includes(v.trim().toLowerCase());
1484
- }
1485
- function isNoEntry(err) {
1486
- return typeof err === 'object' && err !== null && err.code === 'ENOENT';
1487
- }
1488
- function errorText(err) {
1489
- if (!(err instanceof Error))
1490
- return String(err);
1491
- const cause = err.cause;
1492
- return cause instanceof Error && cause.message !== err.message ? `${err.message}: ${cause.message}` : err.message;
1493
- }
1494
- function warn(message) {
1495
- process.stderr.write(`${message}\n`);
1496
- }
1497
- const noop = () => { };
1498
- //# sourceMappingURL=fetch.js.map