@wavehouse/chtypes 0.5.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +34 -38
- package/dist/abi1/buildinfo.d.ts +40 -0
- package/dist/abi1/buildinfo.d.ts.map +1 -0
- package/dist/abi1/buildinfo.js +83 -0
- package/dist/abi1/buildinfo.js.map +1 -0
- package/dist/abi1/calls.gen.d.ts +55 -0
- package/dist/abi1/calls.gen.d.ts.map +1 -0
- package/dist/abi1/calls.gen.js +110 -0
- package/dist/abi1/calls.gen.js.map +1 -0
- package/dist/abi1/decls.gen.d.ts +115 -0
- package/dist/abi1/decls.gen.d.ts.map +1 -0
- package/dist/abi1/decls.gen.js +1613 -0
- package/dist/abi1/decls.gen.js.map +1 -0
- package/dist/abi1/errmap.gen.d.ts +18 -0
- package/dist/abi1/errmap.gen.d.ts.map +1 -0
- package/dist/abi1/errmap.gen.js +66 -0
- package/dist/abi1/errmap.gen.js.map +1 -0
- package/dist/abi1/errors.d.ts +107 -0
- package/dist/abi1/errors.d.ts.map +1 -0
- package/dist/abi1/errors.js +134 -0
- package/dist/abi1/errors.js.map +1 -0
- package/dist/abi1/handles.d.ts +31 -0
- package/dist/abi1/handles.d.ts.map +1 -0
- package/dist/abi1/handles.js +76 -0
- package/dist/abi1/handles.js.map +1 -0
- package/dist/abi1/index.d.ts +17 -0
- package/dist/abi1/index.d.ts.map +1 -0
- package/dist/abi1/index.js +17 -0
- package/dist/abi1/index.js.map +1 -0
- package/dist/abi1/libc.d.ts +54 -0
- package/dist/abi1/libc.d.ts.map +1 -0
- package/dist/abi1/libc.gen.d.ts +12 -0
- package/dist/abi1/libc.gen.d.ts.map +1 -0
- package/dist/abi1/libc.gen.js +75 -0
- package/dist/abi1/libc.gen.js.map +1 -0
- package/dist/abi1/libc.js +109 -0
- package/dist/abi1/libc.js.map +1 -0
- package/dist/abi1/loader.d.ts +99 -0
- package/dist/abi1/loader.d.ts.map +1 -0
- package/dist/abi1/loader.js +261 -0
- package/dist/abi1/loader.js.map +1 -0
- package/dist/abi1/raw.d.ts +104 -0
- package/dist/abi1/raw.d.ts.map +1 -0
- package/dist/abi1/raw.js +296 -0
- package/dist/abi1/raw.js.map +1 -0
- package/dist/abi1/strictjson.d.ts +19 -0
- package/dist/abi1/strictjson.d.ts.map +1 -0
- package/dist/abi1/strictjson.js +78 -0
- package/dist/abi1/strictjson.js.map +1 -0
- package/dist/abi1/vocab.gen.d.ts +169 -0
- package/dist/abi1/vocab.gen.d.ts.map +1 -0
- package/dist/abi1/vocab.gen.js +306 -0
- package/dist/abi1/vocab.gen.js.map +1 -0
- package/dist/cli.d.ts +25 -26
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +206 -218
- package/dist/cli.js.map +1 -1
- package/dist/documents.d.ts +195 -0
- package/dist/documents.d.ts.map +1 -0
- package/dist/documents.js +389 -0
- package/dist/documents.js.map +1 -0
- package/dist/env.d.ts +16 -0
- package/dist/env.d.ts.map +1 -0
- package/dist/env.js +57 -0
- package/dist/env.js.map +1 -0
- package/dist/index.d.ts +19 -24
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -23
- package/dist/index.js.map +1 -1
- package/dist/json.d.ts +11 -89
- package/dist/json.d.ts.map +1 -1
- package/dist/json.js +16 -180
- package/dist/json.js.map +1 -1
- package/dist/library.d.ts +53 -309
- package/dist/library.d.ts.map +1 -1
- package/dist/library.js +77 -315
- package/dist/library.js.map +1 -1
- package/dist/ocifetch/constants.gen.d.ts +84 -0
- package/dist/ocifetch/constants.gen.d.ts.map +1 -0
- package/dist/ocifetch/constants.gen.js +92 -0
- package/dist/ocifetch/constants.gen.js.map +1 -0
- package/dist/ocifetch/dsse.d.ts +114 -0
- package/dist/ocifetch/dsse.d.ts.map +1 -0
- package/dist/ocifetch/dsse.js +331 -0
- package/dist/ocifetch/dsse.js.map +1 -0
- package/dist/ocifetch/ensure.d.ts +65 -0
- package/dist/ocifetch/ensure.d.ts.map +1 -0
- package/dist/ocifetch/ensure.js +775 -0
- package/dist/ocifetch/ensure.js.map +1 -0
- package/dist/ocifetch/errors.d.ts +92 -0
- package/dist/ocifetch/errors.d.ts.map +1 -0
- package/dist/ocifetch/errors.js +101 -0
- package/dist/ocifetch/errors.js.map +1 -0
- package/dist/ocifetch/goldens.d.ts +44 -0
- package/dist/ocifetch/goldens.d.ts.map +1 -0
- package/dist/ocifetch/goldens.js +62 -0
- package/dist/ocifetch/goldens.js.map +1 -0
- package/dist/ocifetch/http.d.ts +124 -0
- package/dist/ocifetch/http.d.ts.map +1 -0
- package/dist/ocifetch/http.js +549 -0
- package/dist/ocifetch/http.js.map +1 -0
- package/dist/ocifetch/index.d.ts +17 -0
- package/dist/ocifetch/index.d.ts.map +1 -0
- package/dist/ocifetch/index.js +14 -0
- package/dist/ocifetch/index.js.map +1 -0
- package/dist/ocifetch/layout.d.ts +126 -0
- package/dist/ocifetch/layout.d.ts.map +1 -0
- package/dist/ocifetch/layout.js +375 -0
- package/dist/ocifetch/layout.js.map +1 -0
- package/dist/ocifetch/localverify.d.ts +43 -0
- package/dist/ocifetch/localverify.d.ts.map +1 -0
- package/dist/ocifetch/localverify.js +164 -0
- package/dist/ocifetch/localverify.js.map +1 -0
- package/dist/ocifetch/lock.d.ts +39 -0
- package/dist/ocifetch/lock.d.ts.map +1 -0
- package/dist/ocifetch/lock.js +140 -0
- package/dist/ocifetch/lock.js.map +1 -0
- package/dist/ocifetch/oci.d.ts +100 -0
- package/dist/ocifetch/oci.d.ts.map +1 -0
- package/dist/ocifetch/oci.js +323 -0
- package/dist/ocifetch/oci.js.map +1 -0
- package/dist/ocifetch/referrers.d.ts +55 -0
- package/dist/ocifetch/referrers.d.ts.map +1 -0
- package/dist/ocifetch/referrers.js +132 -0
- package/dist/ocifetch/referrers.js.map +1 -0
- package/dist/ocifetch/types.d.ts +167 -0
- package/dist/ocifetch/types.d.ts.map +1 -0
- package/dist/ocifetch/types.js +88 -0
- package/dist/ocifetch/types.js.map +1 -0
- package/dist/ocifetch/unpack.d.ts +55 -0
- package/dist/ocifetch/unpack.d.ts.map +1 -0
- package/dist/ocifetch/unpack.js +205 -0
- package/dist/ocifetch/unpack.js.map +1 -0
- package/dist/registry.d.ts +38 -387
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +94 -819
- package/dist/registry.js.map +1 -1
- package/dist/schema.d.ts +77 -588
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +87 -549
- package/dist/schema.js.map +1 -1
- package/dist/settings.d.ts +27 -36
- package/dist/settings.d.ts.map +1 -1
- package/dist/settings.js +75 -25
- package/dist/settings.js.map +1 -1
- package/dist/setup.d.ts +37 -0
- package/dist/setup.d.ts.map +1 -0
- package/dist/setup.js +60 -0
- package/dist/setup.js.map +1 -0
- package/dist/tar.d.ts +27 -11
- package/dist/tar.d.ts.map +1 -1
- package/dist/tar.js +54 -32
- package/dist/tar.js.map +1 -1
- package/package.json +2 -2
- package/dist/discover.d.ts +0 -142
- package/dist/discover.d.ts.map +0 -1
- package/dist/discover.js +0 -288
- package/dist/discover.js.map +0 -1
- package/dist/error-codes.d.ts +0 -80
- package/dist/error-codes.d.ts.map +0 -1
- package/dist/error-codes.js +0 -139
- package/dist/error-codes.js.map +0 -1
- package/dist/errors.d.ts +0 -284
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js +0 -321
- package/dist/errors.js.map +0 -1
- package/dist/fetch.d.ts +0 -382
- package/dist/fetch.d.ts.map +0 -1
- package/dist/fetch.js +0 -1498
- package/dist/fetch.js.map +0 -1
- package/dist/ffi.d.ts +0 -410
- package/dist/ffi.d.ts.map +0 -1
- package/dist/ffi.js +0 -1154
- package/dist/ffi.js.map +0 -1
- package/dist/format.d.ts +0 -128
- package/dist/format.d.ts.map +0 -1
- package/dist/format.js +0 -132
- package/dist/format.js.map +0 -1
- package/dist/numeric.d.ts +0 -20
- package/dist/numeric.d.ts.map +0 -1
- package/dist/numeric.js +0 -107
- package/dist/numeric.js.map +0 -1
- package/dist/paths.d.ts +0 -55
- package/dist/paths.d.ts.map +0 -1
- package/dist/paths.js +0 -87
- package/dist/paths.js.map +0 -1
- package/dist/results.d.ts +0 -495
- package/dist/results.d.ts.map +0 -1
- package/dist/results.js +0 -406
- package/dist/results.js.map +0 -1
- package/dist/transform.d.ts +0 -73
- package/dist/transform.d.ts.map +0 -1
- package/dist/transform.js +0 -533
- package/dist/transform.js.map +0 -1
package/dist/fetch.js
DELETED
|
@@ -1,1498 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Fetching, verifying and installing artifacts — docs/guides/fetch.md, the contract
|
|
3
|
-
* every SDK implements identically: the same source (§2), the same
|
|
4
|
-
* verification chain (§3), the same signature policy (§4), the same lock
|
|
5
|
-
* file (§5), the same function (§6) and the same codes (§7).
|
|
6
|
-
*
|
|
7
|
-
* Nothing here is a verdict but the chain. No exit code, no
|
|
8
|
-
* `Content-Length`, no "download finished" ever is:
|
|
9
|
-
*
|
|
10
|
-
* 0. `SHA256SUMS.sig` — ed25519 over the exact bytes of `SHA256SUMS`, with a
|
|
11
|
-
* trusted key. Failure stops everything; nothing is downloaded around it.
|
|
12
|
-
* 1. `index.json` names the asset for the line/platform and its sha256 — only
|
|
13
|
-
* ever a row at this binding's own `ABI_REVISION` (docs/guides/fetch.md §2).
|
|
14
|
-
* 2. `SHA256SUMS` — now known-authentic — must list the same file with the
|
|
15
|
-
* same sha256; a disagreement is a broken release, reported, not repaired.
|
|
16
|
-
* 3. The tarball is hashed BEFORE it is unpacked.
|
|
17
|
-
* 4. `manifest.json` inside names the library and its sha256; after the
|
|
18
|
-
* move into `<registry>/<minor>/`, the installed library is hashed again
|
|
19
|
-
* in place.
|
|
20
|
-
*
|
|
21
|
-
* The install is atomic — unpack into a temporary sibling, rename into place
|
|
22
|
-
* — and idempotent: an installed line that hashes what the release says is
|
|
23
|
-
* reported as installed and nothing is downloaded.
|
|
24
|
-
*
|
|
25
|
-
* Steady state is offline: once files are in a registry directory the loader
|
|
26
|
-
* trusts the directory, exactly as a runtime trusts `node_modules`.
|
|
27
|
-
* Verification is a fetch-time policy, not a load-time gate.
|
|
28
|
-
*/
|
|
29
|
-
import { createHash, createPublicKey, randomBytes, verify as verifyRaw } from 'node:crypto';
|
|
30
|
-
import { createReadStream, createWriteStream, existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
31
|
-
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
32
|
-
import path from 'node:path';
|
|
33
|
-
import { Readable, Transform } from 'node:stream';
|
|
34
|
-
import { pipeline } from 'node:stream/promises';
|
|
35
|
-
import { setTimeout as sleep } from 'node:timers/promises';
|
|
36
|
-
import { fileURLToPath } from 'node:url';
|
|
37
|
-
import { ABI_REVISION, ArtifactCorruptError, ArtifactPinnedError, ArtifactUnpublishedError, ArtifactUntrustedError, ChtypesError, FETCH_COMMAND, SourceUnreachableError, } from './errors.js';
|
|
38
|
-
import { fetchDestination, hostPlatform, isPlatformKey } from './paths.js';
|
|
39
|
-
import { extractTarGz } from './tar.js';
|
|
40
|
-
// ------------------------------------------------------------- constants
|
|
41
|
-
/** The public artifacts host (docs/guides/fetch.md §2); `CHTYPES_ARTIFACTS_URL` overrides it. */
|
|
42
|
-
export const DEFAULT_ARTIFACTS_URL = 'https://artifacts.wavehouse.dev';
|
|
43
|
-
/** The rolling release every fetch reads by default; `tag` picks a frozen one. */
|
|
44
|
-
export const DEFAULT_RELEASE_TAG = 'artifacts';
|
|
45
|
-
/**
|
|
46
|
-
* The release signing key(s) — raw 32-byte ed25519 public keys, hex
|
|
47
|
-
* (docs/guides/fetch.md §4). `CHTYPES_TRUSTED_KEYS` or the `trustedKeys` option
|
|
48
|
-
* REPLACES this list; it is never extended silently.
|
|
49
|
-
*/
|
|
50
|
-
export const RELEASE_PUBLIC_KEYS = [
|
|
51
|
-
'fdb5f06a8d4c9918d049a5f1748fa2e3b3238c3f2000986d5bb9e31beff778fc',
|
|
52
|
-
];
|
|
53
|
-
/**
|
|
54
|
-
* The lock-file schema this SDK writes, and the newest it reads
|
|
55
|
-
* (docs/guides/fetch.md §5). Schema 1 (`<os>-<arch>/<minor>`) is still read
|
|
56
|
-
* and is converted to schema 2 (`<os>-<arch>/<clickhouse_version>`) the first
|
|
57
|
-
* time this SDK writes the file — schema 2 is what a two-patches-per-line
|
|
58
|
-
* cache needs to key on, since two patches of one line share a schema-1 key.
|
|
59
|
-
* SDKs through 0.4.x cannot read a schema-2 lock and refuse it (issue #284).
|
|
60
|
-
*/
|
|
61
|
-
export const LOCK_SCHEMA = 2;
|
|
62
|
-
/** The newest lock schema this SDK STILL READS besides its own (docs/guides/fetch.md §5). */
|
|
63
|
-
const LOCK_SCHEMA_LEGACY = 1;
|
|
64
|
-
/** The lock file `frozen` reads when no `lock` names one (docs/guides/fetch.md, Decisions); relative, so the working directory's. */
|
|
65
|
-
export const DEFAULT_LOCK_FILE = 'chtypes.lock';
|
|
66
|
-
const INDEX_SCHEMA = 1;
|
|
67
|
-
const CHANNEL = /-(lts|stable|prestable|testing)$/;
|
|
68
|
-
/**
|
|
69
|
-
* `25.8` is a line; `25.8.28.1-lts`, `v25.8.28.1-lts` and `25.8.28.1` are an
|
|
70
|
-
* exact patch and a hard requirement (a release that publishes another patch
|
|
71
|
-
* of the line refuses it — ask for the line to take what was published).
|
|
72
|
-
*
|
|
73
|
-
* @throws {ChtypesError} when the spelling is not a ClickHouse version at all.
|
|
74
|
-
*/
|
|
75
|
-
export function parseVersionSpelling(spelling) {
|
|
76
|
-
const s = spelling.trim().replace(/^v/, '');
|
|
77
|
-
const bare = s.replace(CHANNEL, '');
|
|
78
|
-
const parts = bare.split('.');
|
|
79
|
-
if (parts.length < 2 || !parts.every((p) => /^\d+$/.test(p))) {
|
|
80
|
-
throw new ChtypesError(`chtypes: cannot make a ClickHouse version out of ${JSON.stringify(spelling)}`);
|
|
81
|
-
}
|
|
82
|
-
const line = `${parts[0]}.${parts[1]}`;
|
|
83
|
-
return parts.length >= 4 ? { spelling, line, exact: s, bare } : { spelling, line, exact: null, bare: null };
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Decision 7 / R2 (issue #284): does `candidate` — a served or installed
|
|
87
|
-
* `clickhouse_version` — match `req`? A spelled channel (`-lts` etc.) matches
|
|
88
|
-
* only itself; an unspelled one matches that patch on any channel. This is
|
|
89
|
-
* the ONE matching rule for every place a patch is matched: the registry, the
|
|
90
|
-
* release, the lock and `scripts/fetch.sh`. `false` for a line request — a
|
|
91
|
-
* line never "matches" a patch under this rule, it CONTAINS one.
|
|
92
|
-
*/
|
|
93
|
-
export function patchMatches(candidate, req) {
|
|
94
|
-
if (req.exact === null)
|
|
95
|
-
return false;
|
|
96
|
-
if (CHANNEL.test(req.exact))
|
|
97
|
-
return candidate === req.exact;
|
|
98
|
-
return candidate === req.exact || candidate.replace(CHANNEL, '') === req.bare;
|
|
99
|
-
}
|
|
100
|
-
/** Numeric order over dotted versions, channel suffix ignored: 25.10 > 25.8, 25.8.30.16 > 25.8.28.1. */
|
|
101
|
-
export function compareVersions(a, b) {
|
|
102
|
-
const pa = a.replace(CHANNEL, '').split('.').map((p) => Number.parseInt(p, 10));
|
|
103
|
-
const pb = b.replace(CHANNEL, '').split('.').map((p) => Number.parseInt(p, 10));
|
|
104
|
-
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
|
105
|
-
const x = pa[i] ?? -1;
|
|
106
|
-
const y = pb[i] ?? -1;
|
|
107
|
-
if (Number.isNaN(x) || Number.isNaN(y))
|
|
108
|
-
return a < b ? -1 : a > b ? 1 : 0;
|
|
109
|
-
if (x !== y)
|
|
110
|
-
return x - y;
|
|
111
|
-
}
|
|
112
|
-
return 0;
|
|
113
|
-
}
|
|
114
|
-
// ------------------------------------------------------------ signature
|
|
115
|
-
/** The 12-byte SubjectPublicKeyInfo header for an Ed25519 key (RFC 8410): SEQUENCE { AlgorithmIdentifier(1.3.101.112), BIT STRING }. */
|
|
116
|
-
const ED25519_SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex');
|
|
117
|
-
const RAW_KEY = /^[0-9a-fA-F]{64}$/;
|
|
118
|
-
/** The key id docs/guides/fetch.md §4 defines: the first 16 hex characters of sha256 over the raw 32-byte public key. */
|
|
119
|
-
export function keyId(rawKeyHex) {
|
|
120
|
-
return createHash('sha256').update(Buffer.from(rawKeyHex, 'hex')).digest('hex').slice(0, 16);
|
|
121
|
-
}
|
|
122
|
-
/**
|
|
123
|
-
* `keyId(RELEASE_PUBLIC_KEYS[0])`, precomputed — the id the signature file's
|
|
124
|
-
* comment line names for the primary release key. `RELEASE_PUBLIC_KEYS` stays
|
|
125
|
-
* the list to verify against (rotation adds a second entry); this is the one
|
|
126
|
-
* short id go/python/rust each export as a scalar constant.
|
|
127
|
-
*/
|
|
128
|
-
export const RELEASE_KEY_ID = keyId(RELEASE_PUBLIC_KEYS[0]);
|
|
129
|
-
/**
|
|
130
|
-
* Verify an ed25519 signature over `message` with a raw public key, through
|
|
131
|
-
* `node:crypto` — the key wrapped as SPKI DER, the 64-byte signature as is.
|
|
132
|
-
* False, never a throw, for a malformed key or signature.
|
|
133
|
-
*/
|
|
134
|
-
export function verifyEd25519(message, signature, rawKeyHex) {
|
|
135
|
-
if (!RAW_KEY.test(rawKeyHex) || signature.byteLength !== 64)
|
|
136
|
-
return false;
|
|
137
|
-
try {
|
|
138
|
-
const key = createPublicKey({
|
|
139
|
-
key: Buffer.concat([ED25519_SPKI_PREFIX, Buffer.from(rawKeyHex, 'hex')]),
|
|
140
|
-
format: 'der',
|
|
141
|
-
type: 'spki',
|
|
142
|
-
});
|
|
143
|
-
return verifyRaw(null, message, key, signature);
|
|
144
|
-
}
|
|
145
|
-
catch {
|
|
146
|
-
return false;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
/**
|
|
150
|
-
* `SHA256SUMS.sig`: an `untrusted comment:` line, then the base64 of the
|
|
151
|
-
* 64-byte signature. Null when the file is not in that shape.
|
|
152
|
-
*/
|
|
153
|
-
export function parseSignatureFile(bytes) {
|
|
154
|
-
const lines = Buffer.from(bytes).toString('utf8').split(/\r?\n/);
|
|
155
|
-
let comment = '';
|
|
156
|
-
for (const raw of lines) {
|
|
157
|
-
const line = raw.trim();
|
|
158
|
-
if (line === '')
|
|
159
|
-
continue;
|
|
160
|
-
if (line.startsWith('untrusted comment:')) {
|
|
161
|
-
comment = line.slice('untrusted comment:'.length).trim();
|
|
162
|
-
continue;
|
|
163
|
-
}
|
|
164
|
-
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(line))
|
|
165
|
-
return null;
|
|
166
|
-
const signature = Buffer.from(line, 'base64');
|
|
167
|
-
return signature.length === 64 ? { comment, signature } : null;
|
|
168
|
-
}
|
|
169
|
-
return null;
|
|
170
|
-
}
|
|
171
|
-
/**
|
|
172
|
-
* The keys a fetch trusts: the `trustedKeys` option, else `CHTYPES_TRUSTED_KEYS`
|
|
173
|
-
* (`<hex>[,<hex>…]`), else the embedded release key. Each REPLACES the one
|
|
174
|
-
* below it.
|
|
175
|
-
*
|
|
176
|
-
* @throws {ChtypesError} when an entry is not 64 hex characters.
|
|
177
|
-
*/
|
|
178
|
-
export function trustedKeys(explicit) {
|
|
179
|
-
let list;
|
|
180
|
-
if (explicit !== undefined)
|
|
181
|
-
list = explicit;
|
|
182
|
-
else {
|
|
183
|
-
const env = process.env['CHTYPES_TRUSTED_KEYS'];
|
|
184
|
-
const fromEnv = env === undefined ? [] : env.split(',').map((k) => k.trim()).filter((k) => k !== '');
|
|
185
|
-
list = fromEnv.length > 0 ? fromEnv : RELEASE_PUBLIC_KEYS;
|
|
186
|
-
}
|
|
187
|
-
return list.map((k) => {
|
|
188
|
-
if (!RAW_KEY.test(k)) {
|
|
189
|
-
throw new ChtypesError(`chtypes: ${JSON.stringify(k)} is not a raw ed25519 public key (64 hex characters) — check CHTYPES_TRUSTED_KEYS`);
|
|
190
|
-
}
|
|
191
|
-
return k.toLowerCase();
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
class DirSource {
|
|
195
|
-
dir;
|
|
196
|
-
description;
|
|
197
|
-
constructor(dir) {
|
|
198
|
-
this.dir = dir;
|
|
199
|
-
this.description = dir;
|
|
200
|
-
}
|
|
201
|
-
async get(name) {
|
|
202
|
-
try {
|
|
203
|
-
return await readFile(path.join(this.dir, name));
|
|
204
|
-
}
|
|
205
|
-
catch (err) {
|
|
206
|
-
if (isNoEntry(err))
|
|
207
|
-
return null;
|
|
208
|
-
throw new SourceUnreachableError(`chtypes: cannot read ${path.join(this.dir, name)}: ${errorText(err)}`, {
|
|
209
|
-
cause: err,
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
async download(name, to, onReceived) {
|
|
214
|
-
const from = path.join(this.dir, name);
|
|
215
|
-
if (!existsSync(from))
|
|
216
|
-
return false;
|
|
217
|
-
try {
|
|
218
|
-
await pipeline(createReadStream(from), progressTap(onReceived), createWriteStream(to));
|
|
219
|
-
}
|
|
220
|
-
catch (err) {
|
|
221
|
-
throw new SourceUnreachableError(`chtypes: cannot read ${from}: ${errorText(err)}`, { cause: err });
|
|
222
|
-
}
|
|
223
|
-
return true;
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
class HttpSource {
|
|
227
|
-
base;
|
|
228
|
-
signal;
|
|
229
|
-
description;
|
|
230
|
-
constructor(base, signal) {
|
|
231
|
-
this.base = base;
|
|
232
|
-
this.signal = signal;
|
|
233
|
-
this.description = base;
|
|
234
|
-
}
|
|
235
|
-
async request(name) {
|
|
236
|
-
const url = `${this.base}/${name}`;
|
|
237
|
-
const headers = {};
|
|
238
|
-
const token = process.env['CHTYPES_DOWNLOAD_TOKEN'];
|
|
239
|
-
if (token !== undefined && token !== '')
|
|
240
|
-
headers['Authorization'] = `Bearer ${token}`;
|
|
241
|
-
let last;
|
|
242
|
-
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
243
|
-
try {
|
|
244
|
-
const res = await fetch(url, { headers, redirect: 'follow', ...(this.signal ? { signal: this.signal } : {}) });
|
|
245
|
-
if (res.status === 404) {
|
|
246
|
-
await res.body?.cancel();
|
|
247
|
-
return null;
|
|
248
|
-
}
|
|
249
|
-
if (res.ok)
|
|
250
|
-
return res;
|
|
251
|
-
await res.body?.cancel();
|
|
252
|
-
last = new Error(`HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`);
|
|
253
|
-
// Client errors do not get better on retry; server errors and throttling might.
|
|
254
|
-
if (res.status < 500 && res.status !== 408 && res.status !== 429)
|
|
255
|
-
break;
|
|
256
|
-
}
|
|
257
|
-
catch (err) {
|
|
258
|
-
if (this.signal?.aborted)
|
|
259
|
-
throw err;
|
|
260
|
-
last = err;
|
|
261
|
-
}
|
|
262
|
-
if (attempt < 3)
|
|
263
|
-
await sleep(500 * attempt);
|
|
264
|
-
}
|
|
265
|
-
throw new SourceUnreachableError(`chtypes: cannot fetch ${url}: ${errorText(last)}`, { cause: last });
|
|
266
|
-
}
|
|
267
|
-
async get(name) {
|
|
268
|
-
const res = await this.request(name);
|
|
269
|
-
if (res === null)
|
|
270
|
-
return null;
|
|
271
|
-
return Buffer.from(await res.arrayBuffer());
|
|
272
|
-
}
|
|
273
|
-
async download(name, to, onReceived) {
|
|
274
|
-
const res = await this.request(name);
|
|
275
|
-
if (res === null)
|
|
276
|
-
return false;
|
|
277
|
-
if (res.body === null) {
|
|
278
|
-
throw new SourceUnreachableError(`chtypes: ${this.base}/${name} returned no body`);
|
|
279
|
-
}
|
|
280
|
-
try {
|
|
281
|
-
await pipeline(Readable.fromWeb(res.body), progressTap(onReceived), createWriteStream(to));
|
|
282
|
-
}
|
|
283
|
-
catch (err) {
|
|
284
|
-
throw new SourceUnreachableError(`chtypes: download of ${this.base}/${name} failed: ${errorText(err)}`, {
|
|
285
|
-
cause: err,
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
return true;
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
/** §2: `url` is a full base (http(s), `file://`, or a directory); otherwise `<CHTYPES_ARTIFACTS_URL>/<tag>`. */
|
|
292
|
-
function openSource(options) {
|
|
293
|
-
if (options.url !== undefined && options.url !== '' && options.tag !== undefined && options.tag !== '') {
|
|
294
|
-
throw new ChtypesError('chtypes: url names a full base; tag selects a release on the artifacts host — pass one');
|
|
295
|
-
}
|
|
296
|
-
let base;
|
|
297
|
-
if (options.url !== undefined && options.url !== '')
|
|
298
|
-
base = options.url;
|
|
299
|
-
else {
|
|
300
|
-
const host = process.env['CHTYPES_ARTIFACTS_URL'];
|
|
301
|
-
const root = host !== undefined && host !== '' ? host : DEFAULT_ARTIFACTS_URL;
|
|
302
|
-
const tag = options.tag !== undefined && options.tag !== '' ? options.tag : DEFAULT_RELEASE_TAG;
|
|
303
|
-
base = `${root.replace(/\/+$/, '')}/${tag}`;
|
|
304
|
-
}
|
|
305
|
-
if (/^https?:\/\//i.test(base))
|
|
306
|
-
return new HttpSource(base.replace(/\/+$/, ''), options.signal);
|
|
307
|
-
const dir = base.startsWith('file://') ? fileURLToPath(base) : path.resolve(base);
|
|
308
|
-
return new DirSource(dir);
|
|
309
|
-
}
|
|
310
|
-
/** Steps 0–2's inputs, verified in that order: the signature over SHA256SUMS, then the index. */
|
|
311
|
-
async function loadReleaseOnce(source, options, emit) {
|
|
312
|
-
const sumsBytes = await source.get('SHA256SUMS');
|
|
313
|
-
if (sumsBytes === null)
|
|
314
|
-
throw new SourceUnreachableError(`chtypes: no SHA256SUMS at ${source.description}`);
|
|
315
|
-
let signed = false;
|
|
316
|
-
let signer = null;
|
|
317
|
-
if (options.allowUnsigned ?? envFlag('CHTYPES_ALLOW_UNSIGNED')) {
|
|
318
|
-
warn(`chtypes: WARNING — the signature of ${source.description}/SHA256SUMS is NOT being verified ` +
|
|
319
|
-
'(CHTYPES_ALLOW_UNSIGNED). Anything it lists will be installed on its hashes alone.');
|
|
320
|
-
}
|
|
321
|
-
else {
|
|
322
|
-
const sigBytes = await source.get('SHA256SUMS.sig');
|
|
323
|
-
if (sigBytes === null) {
|
|
324
|
-
throw new ArtifactUntrustedError(`chtypes: the release at ${source.description} is unsigned (no SHA256SUMS.sig); nothing was downloaded. ` +
|
|
325
|
-
'CHTYPES_ALLOW_UNSIGNED=1 installs it anyway, loudly.');
|
|
326
|
-
}
|
|
327
|
-
const sig = parseSignatureFile(sigBytes);
|
|
328
|
-
if (sig === null) {
|
|
329
|
-
throw new ArtifactUntrustedError(`chtypes: ${source.description}/SHA256SUMS.sig is not an ed25519 signature file; nothing was downloaded.`);
|
|
330
|
-
}
|
|
331
|
-
const keys = trustedKeys(options.trustedKeys);
|
|
332
|
-
const hit = keys.find((k) => verifyEd25519(sumsBytes, sig.signature, k));
|
|
333
|
-
if (hit === undefined) {
|
|
334
|
-
throw new ArtifactUntrustedError(`chtypes: ${source.description}/SHA256SUMS.sig does not verify with any trusted key ` +
|
|
335
|
-
`(${keys.map(keyId).join(', ')}); the release is not trusted and nothing was downloaded.`);
|
|
336
|
-
}
|
|
337
|
-
signed = true;
|
|
338
|
-
signer = keyId(hit);
|
|
339
|
-
emit({ type: 'status', message: `SHA256SUMS signature verified (key ${signer})` });
|
|
340
|
-
}
|
|
341
|
-
const indexBytes = await source.get('index.json');
|
|
342
|
-
if (indexBytes === null)
|
|
343
|
-
throw new SourceUnreachableError(`chtypes: no index.json at ${source.description}`);
|
|
344
|
-
const index = parseIndex(indexBytes, source.description);
|
|
345
|
-
if (index.license !== undefined && index.license !== '') {
|
|
346
|
-
emit({
|
|
347
|
-
type: 'status',
|
|
348
|
-
message: `artifacts are licensed under ${index.license}${index.license_url ? ` (${index.license_url})` : ''} — LICENSE and NOTICE ship beside them`,
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
const release = { source, index, sums: parseSums(sumsBytes), signed, keyId: signer };
|
|
352
|
-
// Step 2 for the whole release, not just the asset being installed. installOne
|
|
353
|
-
// still checks its own asset; this runs here so a disagreement is seen while
|
|
354
|
-
// loadRelease can still fix it by reading all three files again.
|
|
355
|
-
for (const art of index.artifacts) {
|
|
356
|
-
const listed = release.sums.get(art.file);
|
|
357
|
-
if (listed !== undefined && listed !== art.sha256) {
|
|
358
|
-
throw new ArtifactCorruptError(`chtypes: index.json says ${art.file} is ${art.sha256} but SHA256SUMS says ${listed} — the release disagrees with itself; not installing it`);
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
return release;
|
|
362
|
-
}
|
|
363
|
-
/**
|
|
364
|
-
* How many times {@link loadRelease} (and {@link installGoldens}'s own retry)
|
|
365
|
-
* reads an HTTP release before giving up, and the BASE delay (ms) before the
|
|
366
|
-
* first retry — each later retry doubles it, so the default 5 attempts sleep
|
|
367
|
-
* 4+8+16+32 = 60s (~70s wall with network time). That is chosen to outlast
|
|
368
|
-
* the artifacts host's edge cache (observed `Cache-Control: max-age=60` on
|
|
369
|
-
* the mutable release objects, so a stale pairing of any of them can persist
|
|
370
|
-
* up to 60s), while a genuine few-second mid-publish window still clears on
|
|
371
|
-
* the second attempt.
|
|
372
|
-
*
|
|
373
|
-
* A mutable object, not two `const`s: an ES module's imported bindings are
|
|
374
|
-
* read-only, so a plain exported `let` could not be reassigned from outside,
|
|
375
|
-
* and the fetch test suite needs to shrink `delayMs` to near zero. Not
|
|
376
|
-
* re-exported from `index.ts` — internal to this module and its test suite.
|
|
377
|
-
*/
|
|
378
|
-
export const RELEASE_RETRY = { attempts: 5, delayMs: 4_000 };
|
|
379
|
-
/**
|
|
380
|
-
* {@link loadReleaseOnce}, retried through a publish window.
|
|
381
|
-
*
|
|
382
|
-
* A publish into the rolling release is three objects — `SHA256SUMS`,
|
|
383
|
-
* `SHA256SUMS.sig`, `index.json` — and object storage cannot swap them
|
|
384
|
-
* atomically. The edge cache in front of the artifacts host widens the unsafe
|
|
385
|
-
* window from "between two uploads" to "as long as any one object can still be
|
|
386
|
-
* served stale from cache", which measures the same as its `Cache-Control`
|
|
387
|
-
* max-age.
|
|
388
|
-
*
|
|
389
|
-
* Three symptoms of reading inside that window are retried: a signature under
|
|
390
|
-
* no trusted key, an index that disagrees with the sums, and (in
|
|
391
|
-
* {@link installGoldens}) a release-level file whose hash disagrees with its
|
|
392
|
-
* own SHA256SUMS row, or that the sums list but the source does not yet serve.
|
|
393
|
-
* Every retry re-reads the WHOLE set from scratch — never one freshly
|
|
394
|
-
* re-fetched object checked against another attempt's stale one. Nothing else
|
|
395
|
-
* is retried, and neither are these three once the attempts run out: the same
|
|
396
|
-
* error surfaces, with the same code, as it did before. A tarball whose hash
|
|
397
|
-
* is wrong is never retried; that is the release lying about a byte, not a
|
|
398
|
-
* half-finished upload.
|
|
399
|
-
*
|
|
400
|
-
* Only an HTTP source can be mid-publish, so a `file://` or directory source is
|
|
401
|
-
* read exactly once and refuses on the first look.
|
|
402
|
-
*/
|
|
403
|
-
async function loadRelease(source, options, emit) {
|
|
404
|
-
const attempts = source instanceof HttpSource ? RELEASE_RETRY.attempts : 1;
|
|
405
|
-
let delayMs = RELEASE_RETRY.delayMs;
|
|
406
|
-
for (let attempt = 1;; attempt++) {
|
|
407
|
-
try {
|
|
408
|
-
return await loadReleaseOnce(source, options, emit);
|
|
409
|
-
}
|
|
410
|
-
catch (err) {
|
|
411
|
-
const retryable = err instanceof ArtifactUntrustedError || err instanceof ArtifactCorruptError;
|
|
412
|
-
if (!retryable || attempt >= attempts)
|
|
413
|
-
throw err;
|
|
414
|
-
emit({
|
|
415
|
-
type: 'status',
|
|
416
|
-
message: `${String(err)} (attempt ${attempt}/${attempts}) — this is what a release being ` +
|
|
417
|
-
`published looks like from outside; retrying in ${delayMs / 1000}s`,
|
|
418
|
-
});
|
|
419
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
420
|
-
delayMs *= 2;
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
function parseIndex(bytes, where) {
|
|
425
|
-
let doc;
|
|
426
|
-
try {
|
|
427
|
-
doc = JSON.parse(bytes.toString('utf8'));
|
|
428
|
-
}
|
|
429
|
-
catch (err) {
|
|
430
|
-
throw new ArtifactCorruptError(`chtypes: index.json at ${where} is not JSON: ${errorText(err)}`, { cause: err });
|
|
431
|
-
}
|
|
432
|
-
if (typeof doc !== 'object' || doc === null)
|
|
433
|
-
throw new ArtifactCorruptError(`chtypes: index.json at ${where} is not an object`);
|
|
434
|
-
const d = doc;
|
|
435
|
-
if (d['schema'] !== INDEX_SCHEMA) {
|
|
436
|
-
throw new ArtifactCorruptError(`chtypes: index.json schema ${JSON.stringify(d['schema'])} is not ${INDEX_SCHEMA} — this SDK cannot read it`);
|
|
437
|
-
}
|
|
438
|
-
if (!Array.isArray(d['artifacts']))
|
|
439
|
-
throw new ArtifactCorruptError(`chtypes: index.json at ${where} has no artifacts list`);
|
|
440
|
-
const artifacts = d['artifacts'].map((row, i) => indexArtifact(row, i));
|
|
441
|
-
return {
|
|
442
|
-
schema: INDEX_SCHEMA,
|
|
443
|
-
...(typeof d['generated_at'] === 'string' ? { generated_at: d['generated_at'] } : {}),
|
|
444
|
-
...(typeof d['release_tag'] === 'string' ? { release_tag: d['release_tag'] } : {}),
|
|
445
|
-
...(typeof d['license'] === 'string' ? { license: d['license'] } : {}),
|
|
446
|
-
...(typeof d['license_url'] === 'string' ? { license_url: d['license_url'] } : {}),
|
|
447
|
-
artifacts,
|
|
448
|
-
};
|
|
449
|
-
}
|
|
450
|
-
function indexArtifact(row, i) {
|
|
451
|
-
if (typeof row !== 'object' || row === null)
|
|
452
|
-
throw new ArtifactCorruptError(`chtypes: index.json artifacts[${i}] is not an object`);
|
|
453
|
-
const r = row;
|
|
454
|
-
const str = (k) => {
|
|
455
|
-
const v = r[k];
|
|
456
|
-
if (typeof v !== 'string' || v === '')
|
|
457
|
-
throw new ArtifactCorruptError(`chtypes: index.json entry for ${String(r['file'] ?? i)} is missing ${k}`);
|
|
458
|
-
return v;
|
|
459
|
-
};
|
|
460
|
-
const bytes = r['bytes'];
|
|
461
|
-
if (typeof bytes !== 'number' || !Number.isSafeInteger(bytes) || bytes <= 0) {
|
|
462
|
-
throw new ArtifactCorruptError(`chtypes: index.json entry for ${String(r['file'] ?? i)} is missing bytes`);
|
|
463
|
-
}
|
|
464
|
-
const file = str('file');
|
|
465
|
-
const version = str('clickhouse_version');
|
|
466
|
-
const minor = typeof r['clickhouse_minor'] === 'string' && r['clickhouse_minor'] !== '' ? r['clickhouse_minor'] : minorOfVersion(version);
|
|
467
|
-
return {
|
|
468
|
-
os: str('os'),
|
|
469
|
-
arch: str('arch'),
|
|
470
|
-
file,
|
|
471
|
-
sha256: str('sha256').toLowerCase(),
|
|
472
|
-
bytes,
|
|
473
|
-
clickhouse_version: version,
|
|
474
|
-
clickhouse_minor: minor,
|
|
475
|
-
library: str('library'),
|
|
476
|
-
library_sha256: str('library_sha256').toLowerCase(),
|
|
477
|
-
build: buildOf(r['build'], file),
|
|
478
|
-
core_commit: typeof r['core_commit'] === 'string' ? r['core_commit'] : '',
|
|
479
|
-
...revisionOf(r['abi_revision']),
|
|
480
|
-
};
|
|
481
|
-
}
|
|
482
|
-
/**
|
|
483
|
-
* A row's `abi_revision` when it is an integer; otherwise the row is read as
|
|
484
|
-
* declaring none — the field is left out, and never matches — rather than
|
|
485
|
-
* failing the whole listing.
|
|
486
|
-
*/
|
|
487
|
-
function revisionOf(field) {
|
|
488
|
-
return typeof field === 'number' && Number.isSafeInteger(field) ? { abi_revision: field } : {};
|
|
489
|
-
}
|
|
490
|
-
/**
|
|
491
|
-
* TEST-ONLY. When `override` is set, the ABI revision fetch selects release rows
|
|
492
|
-
* at, in place of `ABI_REVISION` (docs/guides/fetch.md §2). The fetch-fixture
|
|
493
|
-
* suite sets it to the revision its fixture release carries — derived from that
|
|
494
|
-
* release's own `index.json` — so a binding whose own revision has moved ahead
|
|
495
|
-
* of the fixtures still exercises the whole chain. It changes WHICH rows are
|
|
496
|
-
* eligible and nothing else: the default registry directory stays
|
|
497
|
-
* `abi<ABI_REVISION>/`, and the loader still refuses an artifact of another
|
|
498
|
-
* revision.
|
|
499
|
-
*
|
|
500
|
-
* A mutable object for the same reason as {@link RELEASE_RETRY}: an imported
|
|
501
|
-
* binding is read-only. Not re-exported from `index.ts` — internal to this
|
|
502
|
-
* module and its test suite, never part of the public contract.
|
|
503
|
-
*/
|
|
504
|
-
export const FETCH_ABI_REVISION = { override: null };
|
|
505
|
-
/** The one ABI revision whose rows fetch may install: this binding's own. */
|
|
506
|
-
function fetchAbiRevision() {
|
|
507
|
-
return FETCH_ABI_REVISION.override ?? ABI_REVISION;
|
|
508
|
-
}
|
|
509
|
-
/**
|
|
510
|
-
* What the release DOES have for `noun`, for an unpublished message: the ABI
|
|
511
|
-
* revision(s) its rows carry, or that it has none at any revision.
|
|
512
|
-
*/
|
|
513
|
-
/**
|
|
514
|
-
* Rows that carry no `abi_revision`, named with the reason: the artifact
|
|
515
|
-
* producer records the field from the revision that introduced it onward. Every
|
|
516
|
-
* unpublished message, `ensureAll` warning and `list` note says this in these
|
|
517
|
-
* words rather than that the release serves "none".
|
|
518
|
-
*/
|
|
519
|
-
const BUILT_BEFORE = '(built before revisions were recorded)';
|
|
520
|
-
const NO_RECORDED_REVISION = `rows that record no ABI revision ${BUILT_BEFORE}`;
|
|
521
|
-
function served(rows, noun) {
|
|
522
|
-
if (rows.length === 0)
|
|
523
|
-
return `the release does not have ${noun} at any ABI revision`;
|
|
524
|
-
const revs = declaredRevisions(rows);
|
|
525
|
-
if (revs.length === 0)
|
|
526
|
-
return `the release has ${noun} only in ${NO_RECORDED_REVISION}`;
|
|
527
|
-
let said = revs.length === 1 ? `ABI revision ${revs[0]}` : `ABI revisions ${revs.join(', ')}`;
|
|
528
|
-
if (rows.some((a) => a.abi_revision === undefined))
|
|
529
|
-
said += ` and in ${NO_RECORDED_REVISION}`;
|
|
530
|
-
return `the release has ${noun} only at ${said}`;
|
|
531
|
-
}
|
|
532
|
-
/** The distinct revisions rows declare, ascending. */
|
|
533
|
-
function declaredRevisions(rows) {
|
|
534
|
-
return [...new Set(rows.flatMap((a) => (a.abi_revision === undefined ? [] : [a.abi_revision])))].sort((a, b) => a - b);
|
|
535
|
-
}
|
|
536
|
-
/**
|
|
537
|
-
* One message per line the release has for `platform` only at another ABI
|
|
538
|
-
* revision, or only in rows that record none — the lines `ensureAll` installs
|
|
539
|
-
* nothing for — oldest line first. Exported for the test suite only; not
|
|
540
|
-
* re-exported from `index.ts`.
|
|
541
|
-
*/
|
|
542
|
-
export function skippedLines(index, platform) {
|
|
543
|
-
const [os, arch] = platform.split('-');
|
|
544
|
-
const rev = fetchAbiRevision();
|
|
545
|
-
const byLine = new Map();
|
|
546
|
-
for (const a of index.artifacts) {
|
|
547
|
-
if (a.os !== os || a.arch !== arch)
|
|
548
|
-
continue;
|
|
549
|
-
byLine.set(a.clickhouse_minor, [...(byLine.get(a.clickhouse_minor) ?? []), a]);
|
|
550
|
-
}
|
|
551
|
-
return [...byLine.entries()]
|
|
552
|
-
.filter(([, rows]) => !rows.some((a) => a.abi_revision === rev))
|
|
553
|
-
.sort(([a], [b]) => compareVersions(a, b))
|
|
554
|
-
.map(([line, rows]) => `ClickHouse line ${line} on ${platform} is not installed: ${served(rows, `that line for ${platform}`)}, ` +
|
|
555
|
-
`and this SDK speaks ABI revision ${rev}`);
|
|
556
|
-
}
|
|
557
|
-
/**
|
|
558
|
-
* `list`'s one line naming the platform's rows at another ABI revision, which it
|
|
559
|
-
* does not show (docs/guides/fetch.md §6); `undefined` when none were hidden.
|
|
560
|
-
* Rows that carry no `abi_revision` are named for what they are — built before
|
|
561
|
-
* revisions were recorded — never as a revision called "none". Exported for the
|
|
562
|
-
* test suite only; not re-exported from `index.ts`.
|
|
563
|
-
*/
|
|
564
|
-
export function notShown(index, platform) {
|
|
565
|
-
const [os, arch] = platform.split('-');
|
|
566
|
-
const rev = fetchAbiRevision();
|
|
567
|
-
const hidden = index.artifacts.filter((a) => a.os === os && a.arch === arch && a.abi_revision !== rev);
|
|
568
|
-
if (hidden.length === 0)
|
|
569
|
-
return undefined;
|
|
570
|
-
const undeclared = hidden.filter((a) => a.abi_revision === undefined).length;
|
|
571
|
-
const declared = hidden.length - undeclared;
|
|
572
|
-
const parts = [];
|
|
573
|
-
if (declared > 0)
|
|
574
|
-
parts.push(`${declared} row(s) at ABI revision(s) ${declaredRevisions(hidden).join(', ')}`);
|
|
575
|
-
if (undeclared > 0)
|
|
576
|
-
parts.push(`${undeclared} row(s) that record no ABI revision ${BUILT_BEFORE}`);
|
|
577
|
-
return `${parts.join(' and ')} not shown; this SDK speaks ${rev}`;
|
|
578
|
-
}
|
|
579
|
-
/**
|
|
580
|
-
* The wrapper build number: the row's own `build` when it has one, else the
|
|
581
|
-
* `-b<N>` suffix of the file name, else 0 — an old row, published before builds
|
|
582
|
-
* existed, which is build 0 by definition.
|
|
583
|
-
*/
|
|
584
|
-
function buildOf(field, file) {
|
|
585
|
-
if (typeof field === 'number' && Number.isSafeInteger(field) && field >= 0)
|
|
586
|
-
return field;
|
|
587
|
-
const m = /-b(\d+)\.tar\.gz$/.exec(file);
|
|
588
|
-
return m ? Number(m[1]) : 0;
|
|
589
|
-
}
|
|
590
|
-
/** `sha256sum` format: `<hex> <file>` (or `<hex> *<file>`), one per line. */
|
|
591
|
-
function parseSums(bytes) {
|
|
592
|
-
const out = new Map();
|
|
593
|
-
for (const raw of bytes.toString('utf8').split(/\r?\n/)) {
|
|
594
|
-
const m = /^([0-9a-fA-F]{64})\s+\*?(\S.*)$/.exec(raw.trim());
|
|
595
|
-
if (m)
|
|
596
|
-
out.set(m[2], m[1].toLowerCase());
|
|
597
|
-
}
|
|
598
|
-
return out;
|
|
599
|
-
}
|
|
600
|
-
function minorOfVersion(version) {
|
|
601
|
-
const parts = version.split('.');
|
|
602
|
-
return parts.length < 2 ? version : `${parts[0]}.${parts[1]}`;
|
|
603
|
-
}
|
|
604
|
-
/**
|
|
605
|
-
* The one row for a request, or `CHTYPES_ARTIFACT_UNPUBLISHED` naming what the
|
|
606
|
-
* release does have. Only rows at this binding's ABI revision are considered,
|
|
607
|
-
* FIRST (docs/guides/fetch.md §2): nothing at that revision is unpublished, never
|
|
608
|
-
* another revision's row.
|
|
609
|
-
*/
|
|
610
|
-
export function selectArtifact(index, platform, req) {
|
|
611
|
-
const all = forPlatform(index, platform);
|
|
612
|
-
const rev = fetchAbiRevision();
|
|
613
|
-
const have = all.filter((a) => a.abi_revision === rev).map((a) => a.clickhouse_version).join(', ') || 'nothing';
|
|
614
|
-
const anyRevision = req.exact !== null ? all.filter((a) => patchMatches(a.clickhouse_version, req)) : all.filter((a) => a.clickhouse_minor === req.line);
|
|
615
|
-
const hit = anyRevision.filter((a) => a.abi_revision === rev);
|
|
616
|
-
if (hit.length === 0) {
|
|
617
|
-
throw new ArtifactUnpublishedError(req.exact !== null
|
|
618
|
-
? `chtypes: you asked for exactly ClickHouse ${req.exact} on ${platform} at ABI revision ${rev} (this SDK's) and this ` +
|
|
619
|
-
`release does not publish it at that revision: ${served(anyRevision, `that patch for ${platform}`)}` +
|
|
620
|
-
`; at ABI revision ${rev} the release has: ${have}. Ask for the line (${req.line}) to take what was published.`
|
|
621
|
-
: `chtypes: no artifact for ClickHouse line ${req.line} on ${platform} at ABI revision ${rev} (this SDK's): ` +
|
|
622
|
-
`${served(anyRevision, `that line for ${platform}`)}; at ABI revision ${rev} the release has: ${have}.`);
|
|
623
|
-
}
|
|
624
|
-
// A line can carry more than one row: two patches, or the same patch built
|
|
625
|
-
// twice (a release lists every build it publishes). Take the newest
|
|
626
|
-
// by version and then by BUILD — never by list order, which is what a
|
|
627
|
-
// version-only comparison degrades to once two rows compare equal, because
|
|
628
|
-
// Array.prototype.sort is stable.
|
|
629
|
-
return hit.reduce((best, a) => (newerRow(a, best) ? a : best));
|
|
630
|
-
}
|
|
631
|
-
/**
|
|
632
|
-
* Is `a` the row to prefer over `b`? Newest ClickHouse version first, and among
|
|
633
|
-
* rows of the SAME version the highest wrapper build — a rebuild is a new row
|
|
634
|
-
* beside the old one, so without the build tie-break the older build could win
|
|
635
|
-
* on nothing but its position in the index.
|
|
636
|
-
*/
|
|
637
|
-
function newerRow(a, b) {
|
|
638
|
-
const byVersion = compareVersions(a.clickhouse_version, b.clickhouse_version);
|
|
639
|
-
return byVersion !== 0 ? byVersion > 0 : a.build > b.build;
|
|
640
|
-
}
|
|
641
|
-
/**
|
|
642
|
-
* Every line the release publishes for the platform at this binding's ABI
|
|
643
|
-
* revision, newest patch (then highest build) per line, oldest line first.
|
|
644
|
-
* Nothing at that revision is `CHTYPES_ARTIFACT_UNPUBLISHED`, naming the
|
|
645
|
-
* revision(s) the release does serve (docs/guides/fetch.md §2).
|
|
646
|
-
*/
|
|
647
|
-
export function selectAll(index, platform) {
|
|
648
|
-
const all = forPlatform(index, platform);
|
|
649
|
-
const rev = fetchAbiRevision();
|
|
650
|
-
const rows = all.filter((a) => a.abi_revision === rev);
|
|
651
|
-
if (rows.length === 0) {
|
|
652
|
-
throw new ArtifactUnpublishedError(`chtypes: the release has nothing for ${platform} at ABI revision ${rev} (this SDK's): ${served(all, `rows for ${platform}`)}`);
|
|
653
|
-
}
|
|
654
|
-
const best = new Map();
|
|
655
|
-
for (const a of rows) {
|
|
656
|
-
const cur = best.get(a.clickhouse_minor);
|
|
657
|
-
if (cur === undefined || newerRow(a, cur))
|
|
658
|
-
best.set(a.clickhouse_minor, a);
|
|
659
|
-
}
|
|
660
|
-
return [...best.values()].sort((a, b) => compareVersions(a.clickhouse_minor, b.clickhouse_minor));
|
|
661
|
-
}
|
|
662
|
-
function forPlatform(index, platform) {
|
|
663
|
-
const [os, arch] = platform.split('-');
|
|
664
|
-
const rows = index.artifacts.filter((a) => a.os === os && a.arch === arch);
|
|
665
|
-
if (rows.length === 0) {
|
|
666
|
-
const have = [...new Set(index.artifacts.map((a) => `${a.os}-${a.arch}`))].sort();
|
|
667
|
-
throw new ArtifactUnpublishedError(`chtypes: the release has nothing for ${platform} (it has: ${have.length > 0 ? have.join(', ') : 'nothing'}).`);
|
|
668
|
-
}
|
|
669
|
-
return rows;
|
|
670
|
-
}
|
|
671
|
-
// -------------------------------------------------------------- ensure
|
|
672
|
-
/** Concurrent `ensure`s of one line in one process share one fetch (docs/guides/fetch.md §6). */
|
|
673
|
-
const inFlight = new Map();
|
|
674
|
-
/**
|
|
675
|
-
* Make one ClickHouse line available in a registry directory, through the
|
|
676
|
-
* verification chain, and say where it is. Idempotent: installed and
|
|
677
|
-
* verified is a no-op that downloads nothing (`force` re-downloads).
|
|
678
|
-
*
|
|
679
|
-
* @param spelling - a line (`"25.8"`) or an exact patch (`"25.8.28.1-lts"`, a hard requirement).
|
|
680
|
-
* @param options - see `EnsureOptions`; every §6 flag has a property here.
|
|
681
|
-
* @returns where the line is and what it is — `dir` is `<registry>/<minor>`.
|
|
682
|
-
* @throws {ArtifactUntrustedError} `CHTYPES_ARTIFACT_UNTRUSTED` — no signature, or none a trusted key verifies.
|
|
683
|
-
* @throws {ArtifactCorruptError} `CHTYPES_ARTIFACT_CORRUPT` — any hash mismatch, a release that disagrees with itself.
|
|
684
|
-
* @throws {ArtifactPinnedError} `CHTYPES_ARTIFACT_PINNED` — `frozen` and the lock says otherwise.
|
|
685
|
-
* @throws {ArtifactUnpublishedError} `CHTYPES_ARTIFACT_UNPUBLISHED` — nothing for this platform/line/patch.
|
|
686
|
-
* @throws {SourceUnreachableError} `CHTYPES_SOURCE_UNREACHABLE` — the source cannot be reached, or `offline` and not installed.
|
|
687
|
-
* @throws {ChtypesError} for a spelling that is not a version, a bad platform key, or conflicting options.
|
|
688
|
-
*/
|
|
689
|
-
export async function ensure(spelling, options = {}) {
|
|
690
|
-
const req = parseVersionSpelling(spelling);
|
|
691
|
-
const platform = resolvePlatform(options.platform);
|
|
692
|
-
const dest = fetchDestination(options.dest, platform);
|
|
693
|
-
const key = [dest, platform, req.exact ?? req.line, options.force ? 'force' : ''].join(' ');
|
|
694
|
-
const running = inFlight.get(key);
|
|
695
|
-
if (running !== undefined)
|
|
696
|
-
return running;
|
|
697
|
-
const p = ensureUncached(req, platform, dest, options).finally(() => {
|
|
698
|
-
inFlight.delete(key);
|
|
699
|
-
});
|
|
700
|
-
inFlight.set(key, p);
|
|
701
|
-
return p;
|
|
702
|
-
}
|
|
703
|
-
/** A synthetic line-shaped request, for the parts of `installOne` / the lock helpers that take one — `--all` never spells an exact patch. */
|
|
704
|
-
function lineRequest(line) {
|
|
705
|
-
return { spelling: line, line, exact: null, bare: null };
|
|
706
|
-
}
|
|
707
|
-
/**
|
|
708
|
-
* `fetch --all`: every line the release publishes for the platform at this
|
|
709
|
-
* binding's ABI revision, each through `ensure`'s chain — installed flat, as
|
|
710
|
-
* a line request always is. A line it has only at another revision is not
|
|
711
|
-
* installed and is named in one loud stderr line; the rest go on.
|
|
712
|
-
*
|
|
713
|
-
* Under `--frozen` (F6): every line the lock pins for this platform, at its
|
|
714
|
-
* own newest pinned patch — never a line the release has that the lock does
|
|
715
|
-
* not pin (that is not drift in anything that was pinned). Every candidate's
|
|
716
|
-
* ABI revision is checked (F2) before the release is even read.
|
|
717
|
-
*/
|
|
718
|
-
export async function ensureAll(options = {}) {
|
|
719
|
-
const platform = resolvePlatform(options.platform);
|
|
720
|
-
const dest = fetchDestination(options.dest, platform);
|
|
721
|
-
const emit = options.onProgress ?? noop;
|
|
722
|
-
if (options.offline)
|
|
723
|
-
throw new ChtypesError('chtypes: --all needs the release listing; it cannot run offline');
|
|
724
|
-
const ctx = installContext(platform, dest, options);
|
|
725
|
-
let pinnedLines = [];
|
|
726
|
-
if (ctx.frozen) {
|
|
727
|
-
pinnedLines = linesPinnedForPlatform(ctx.lock, platform);
|
|
728
|
-
for (const line of pinnedLines)
|
|
729
|
-
checkLockRevisionEarly(ctx, lineRequest(line));
|
|
730
|
-
}
|
|
731
|
-
const source = openSource(options);
|
|
732
|
-
emit({ type: 'status', message: `every published ClickHouse line, ${platform} -> ${dest}` });
|
|
733
|
-
emit({ type: 'status', message: `source ${source.description}` });
|
|
734
|
-
const release = await loadRelease(source, options, emit);
|
|
735
|
-
if (ctx.frozen) {
|
|
736
|
-
const out = [];
|
|
737
|
-
for (const line of pinnedLines) {
|
|
738
|
-
const req = lineRequest(line);
|
|
739
|
-
const candidate = selectFromLock(ctx.lock, platform, req);
|
|
740
|
-
const art = verifyFrozenCandidate(release, ctx, req, candidate);
|
|
741
|
-
out.push(await installOne(release, art, ctx, req));
|
|
742
|
-
}
|
|
743
|
-
await installGoldens(release, ctx, options);
|
|
744
|
-
return out;
|
|
745
|
-
}
|
|
746
|
-
const rows = selectAll(release.index, platform);
|
|
747
|
-
// A line the release has only at another ABI revision is not installed —
|
|
748
|
-
// and never silently: one loud line per such line, then the rest go on.
|
|
749
|
-
for (const message of skippedLines(release.index, platform))
|
|
750
|
-
warn(`chtypes: WARNING: ${message}`);
|
|
751
|
-
const out = [];
|
|
752
|
-
for (const art of rows)
|
|
753
|
-
out.push(await installOne(release, art, ctx, lineRequest(art.clickhouse_minor)));
|
|
754
|
-
await installGoldens(release, ctx, options);
|
|
755
|
-
return out;
|
|
756
|
-
}
|
|
757
|
-
/**
|
|
758
|
-
* The served golden set: a release-level file like `index.json`, and a row in
|
|
759
|
-
* the signed `SHA256SUMS` like a tarball, so it verifies through the same chain
|
|
760
|
-
* and installs beside the artifacts as `<registry>/sdk-goldens.json` — where
|
|
761
|
-
* every binding's golden test reads it offline.
|
|
762
|
-
*/
|
|
763
|
-
const GOLDENS_ASSET = 'sdk-goldens.json';
|
|
764
|
-
/**
|
|
765
|
-
* One look at the served golden set against an already-verified `sums` map.
|
|
766
|
-
* Resolves `null` when the release simply does not list it (not an error).
|
|
767
|
-
* Throws `ArtifactCorruptError`/`SourceUnreachableError` for the new window
|
|
768
|
-
* symptom — a hash mismatch, or the sums listing it while the source does not
|
|
769
|
-
* (yet) serve it — matching the codes {@link loadRelease}'s own retry
|
|
770
|
-
* recognizes, so {@link installGoldens} can reuse the same retryable check.
|
|
771
|
-
*/
|
|
772
|
-
async function readGoldensOnce(source, sums) {
|
|
773
|
-
const want = sums.get(GOLDENS_ASSET);
|
|
774
|
-
if (want === undefined)
|
|
775
|
-
return null;
|
|
776
|
-
const blob = await source.get(GOLDENS_ASSET);
|
|
777
|
-
if (blob === null) {
|
|
778
|
-
throw new ArtifactCorruptError(`chtypes: SHA256SUMS lists ${GOLDENS_ASSET} but ${source.description} does not serve it — the release disagrees with itself; not installing it`);
|
|
779
|
-
}
|
|
780
|
-
const got = createHash('sha256').update(blob).digest('hex');
|
|
781
|
-
if (got !== want) {
|
|
782
|
-
throw new ArtifactCorruptError(`chtypes: ${GOLDENS_ASSET} hashes to ${got} but the signed SHA256SUMS says ${want} — not installing it`);
|
|
783
|
-
}
|
|
784
|
-
return blob;
|
|
785
|
-
}
|
|
786
|
-
/**
|
|
787
|
-
* Install the served golden set, if this release publishes one, and never
|
|
788
|
-
* fail the fetch that called it: a release-level file here is best-effort.
|
|
789
|
-
*
|
|
790
|
-
* The first look reuses `release.sums` — already verified by {@link loadRelease}
|
|
791
|
-
* — which is cheap and right on the overwhelmingly common case that nothing is
|
|
792
|
-
* mid-publish. Only a disagreement is retried (the same publish-window symptom
|
|
793
|
-
* as the signature and index.json), and a retry re-reads the WHOLE consistent
|
|
794
|
-
* set fresh through {@link loadReleaseOnce} — SHA256SUMS, its signature,
|
|
795
|
-
* index.json AND the golden set together — never the golden set alone checked
|
|
796
|
-
* against this call's by-then possibly-stale `release.sums`.
|
|
797
|
-
*
|
|
798
|
-
* Never throws. A release with no such row simply predates the served set, and a
|
|
799
|
-
* mismatch that never heals — or a set that cannot be written — leaves the
|
|
800
|
-
* golden tests skipping loudly, which is their job when there is nothing
|
|
801
|
-
* trustworthy to read.
|
|
802
|
-
*/
|
|
803
|
-
async function installGoldens(release, ctx, options) {
|
|
804
|
-
const attempts = release.source instanceof HttpSource ? RELEASE_RETRY.attempts : 1;
|
|
805
|
-
let delayMs = RELEASE_RETRY.delayMs;
|
|
806
|
-
let blob = null;
|
|
807
|
-
for (let attempt = 1;; attempt++) {
|
|
808
|
-
try {
|
|
809
|
-
if (attempt === 1) {
|
|
810
|
-
blob = await readGoldensOnce(release.source, release.sums);
|
|
811
|
-
}
|
|
812
|
-
else {
|
|
813
|
-
// A stale first look: re-verify everything from scratch, not just the
|
|
814
|
-
// golden set against sums that may themselves have moved on.
|
|
815
|
-
const fresh = await loadReleaseOnce(release.source, options, ctx.emit);
|
|
816
|
-
blob = await readGoldensOnce(fresh.source, fresh.sums);
|
|
817
|
-
}
|
|
818
|
-
break;
|
|
819
|
-
}
|
|
820
|
-
catch (err) {
|
|
821
|
-
const retryable = err instanceof ArtifactUntrustedError || err instanceof ArtifactCorruptError;
|
|
822
|
-
if (!retryable || attempt >= attempts) {
|
|
823
|
-
ctx.emit({ type: 'status', message: `${String(err)} — the golden tests will skip` });
|
|
824
|
-
return;
|
|
825
|
-
}
|
|
826
|
-
ctx.emit({
|
|
827
|
-
type: 'status',
|
|
828
|
-
message: `${String(err)} (attempt ${attempt}/${attempts}) — this is what a release being ` +
|
|
829
|
-
`published looks like from outside; retrying in ${delayMs / 1000}s`,
|
|
830
|
-
});
|
|
831
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
832
|
-
delayMs *= 2;
|
|
833
|
-
}
|
|
834
|
-
}
|
|
835
|
-
if (blob === null) {
|
|
836
|
-
ctx.emit({ type: 'status', message: `this release does not publish ${GOLDENS_ASSET} (the SDKs' golden tests will skip until it does)` });
|
|
837
|
-
return;
|
|
838
|
-
}
|
|
839
|
-
const out = path.join(ctx.dest, GOLDENS_ASSET);
|
|
840
|
-
try {
|
|
841
|
-
await mkdir(ctx.dest, { recursive: true });
|
|
842
|
-
await writeFile(out, blob);
|
|
843
|
-
}
|
|
844
|
-
catch (err) {
|
|
845
|
-
ctx.emit({ type: 'status', message: `could not write ${GOLDENS_ASSET}: ${String(err)} — the golden tests will skip` });
|
|
846
|
-
return;
|
|
847
|
-
}
|
|
848
|
-
ctx.emit({ type: 'status', message: `golden set verified and installed: ${out}` });
|
|
849
|
-
}
|
|
850
|
-
function installContext(platform, dest, options) {
|
|
851
|
-
const frozen = options.frozen ?? false;
|
|
852
|
-
// frozen without a lock path reads ./chtypes.lock (docs/guides/fetch.md, Decisions).
|
|
853
|
-
const lockPath = options.lock !== undefined && options.lock !== '' ? path.resolve(options.lock) : frozen ? path.resolve(DEFAULT_LOCK_FILE) : undefined;
|
|
854
|
-
const lock = lockPath !== undefined ? readLock(lockPath) : null;
|
|
855
|
-
// Decision 6: offline never reads the lock at all, so a missing lock file
|
|
856
|
-
// is not refused here — `ensureUncached`'s offline branch never consults
|
|
857
|
-
// `ctx.lock` and answers from the destination alone.
|
|
858
|
-
if (frozen && lock === null && options.offline !== true) {
|
|
859
|
-
throw new ArtifactPinnedError(`chtypes: ${lockPath} does not exist, and frozen refuses anything it does not pin`);
|
|
860
|
-
}
|
|
861
|
-
if (platform !== hostPlatform()) {
|
|
862
|
-
warn(`chtypes: note — fetching ${platform} artifacts on a ${hostPlatform()} host. They are for a ${platform} process ` +
|
|
863
|
-
`(a container, usually), not this one.`);
|
|
864
|
-
}
|
|
865
|
-
return { platform, dest, lockPath, lock, frozen, force: options.force ?? false, emit: options.onProgress ?? noop };
|
|
866
|
-
}
|
|
867
|
-
/**
|
|
868
|
-
* Every lock entry (its full `<os>-<arch>/<clickhouse_version>` key) that
|
|
869
|
-
* `req` — a patch or a line spelling, for `platform` — matches under R2.
|
|
870
|
-
*/
|
|
871
|
-
function candidatesFromLock(lock, platform, req) {
|
|
872
|
-
const prefix = `${platform}/`;
|
|
873
|
-
const rows = [];
|
|
874
|
-
for (const [key, entry] of Object.entries(lock.artifacts)) {
|
|
875
|
-
if (!key.startsWith(prefix))
|
|
876
|
-
continue;
|
|
877
|
-
const version = key.slice(prefix.length);
|
|
878
|
-
const matches = req.exact !== null ? patchMatches(version, req) : minorOfVersion(version) === req.line;
|
|
879
|
-
if (matches)
|
|
880
|
-
rows.push({ key, entry, version });
|
|
881
|
-
}
|
|
882
|
-
return rows;
|
|
883
|
-
}
|
|
884
|
-
/** F1: the newest-versioned candidate the lock pins for `req` — a patch spelling's own entry, or a line's newest pinned patch. */
|
|
885
|
-
function selectFromLock(lock, platform, req) {
|
|
886
|
-
const rows = candidatesFromLock(lock, platform, req);
|
|
887
|
-
if (rows.length === 0)
|
|
888
|
-
return undefined;
|
|
889
|
-
return rows.reduce((best, cur) => (compareVersions(cur.version, best.version) > 0 ? cur : best));
|
|
890
|
-
}
|
|
891
|
-
/** Every line the lock pins at least one patch of, for `platform` — what `--all --frozen` (F6) installs. */
|
|
892
|
-
function linesPinnedForPlatform(lock, platform) {
|
|
893
|
-
const prefix = `${platform}/`;
|
|
894
|
-
const lines = new Set();
|
|
895
|
-
for (const key of Object.keys(lock.artifacts)) {
|
|
896
|
-
if (key.startsWith(prefix))
|
|
897
|
-
lines.add(minorOfVersion(key.slice(prefix.length)));
|
|
898
|
-
}
|
|
899
|
-
return [...lines].sort(compareVersions);
|
|
900
|
-
}
|
|
901
|
-
/**
|
|
902
|
-
* Fails fast, before any network access, when `frozen`'s lock already names
|
|
903
|
-
* an ABI revision for one of `req`'s candidates that is not this binding's own
|
|
904
|
-
* (docs/guides/fetch.md §5, F2). A lock made for one ABI revision does not get
|
|
905
|
-
* a second chance disguised as a drifted pin or an unpublished line once the
|
|
906
|
-
* SDK moves to another: the fix is always the same re-lock, so the message
|
|
907
|
-
* says that directly instead of waiting to see which of the two symptoms
|
|
908
|
-
* selection would have produced.
|
|
909
|
-
*/
|
|
910
|
-
function checkLockRevisionEarly(ctx, req) {
|
|
911
|
-
if (!ctx.frozen || ctx.lock === null)
|
|
912
|
-
return;
|
|
913
|
-
for (const { key, entry } of candidatesFromLock(ctx.lock, ctx.platform, req)) {
|
|
914
|
-
if (entry.abi_revision === undefined || entry.abi_revision === fetchAbiRevision())
|
|
915
|
-
continue;
|
|
916
|
-
throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins ${key} at ABI revision ${entry.abi_revision}; this SDK speaks ABI revision ` +
|
|
917
|
-
`${fetchAbiRevision()} — re-lock with: ${FETCH_COMMAND} ${req.spelling} --lock ${ctx.lockPath}`);
|
|
918
|
-
}
|
|
919
|
-
}
|
|
920
|
-
/**
|
|
921
|
-
* The one sentence appended to a PINNED or UNPUBLISHED message when, under
|
|
922
|
-
* `frozen`, one of the lock's own candidate entries for `req` exists but
|
|
923
|
-
* names no ABI revision at all — written by an SDK before this field
|
|
924
|
-
* existed. `''` when there is nothing to add.
|
|
925
|
-
*/
|
|
926
|
-
function revisionNote(ctx, req) {
|
|
927
|
-
if (!ctx.frozen || ctx.lock === null)
|
|
928
|
-
return '';
|
|
929
|
-
const entry = candidatesFromLock(ctx.lock, ctx.platform, req).find((r) => r.entry.abi_revision === undefined);
|
|
930
|
-
if (entry === undefined)
|
|
931
|
-
return '';
|
|
932
|
-
return (` ${ctx.lockPath} records no ABI revision (written by an older SDK); this SDK speaks ABI revision ` +
|
|
933
|
-
`${fetchAbiRevision()} — re-lock with: ${FETCH_COMMAND} ${req.spelling} --lock ${ctx.lockPath}`);
|
|
934
|
-
}
|
|
935
|
-
/** F5: the release's own row for a lock candidate, verified against SHA256SUMS, the pin's own hash, and the SDK's ABI revision — in that order. */
|
|
936
|
-
function verifyFrozenCandidate(release, ctx, req, candidate) {
|
|
937
|
-
const { key, entry, version } = candidate;
|
|
938
|
-
const [os, arch] = ctx.platform.split('-');
|
|
939
|
-
const row = release.index.artifacts.find((a) => a.file === entry.file && a.os === os && a.arch === arch);
|
|
940
|
-
const listedSha = release.sums.get(entry.file);
|
|
941
|
-
if (row === undefined || listedSha === undefined) {
|
|
942
|
-
throw new ArtifactUnpublishedError(`chtypes: the release at ${release.source.description} does not list ${entry.file}, which ${ctx.lockPath} pins for ${key}.${revisionNote(ctx, req)}`);
|
|
943
|
-
}
|
|
944
|
-
if (row.sha256 !== listedSha) {
|
|
945
|
-
throw new ArtifactCorruptError(`chtypes: index.json says ${entry.file} is ${row.sha256} but SHA256SUMS says ${listedSha} — the release disagrees with itself; not installing it`);
|
|
946
|
-
}
|
|
947
|
-
if (row.abi_revision !== fetchAbiRevision()) {
|
|
948
|
-
throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins ${key} to ${entry.file}, which the release at ${release.source.description} serves at ABI revision ` +
|
|
949
|
-
`${row.abi_revision ?? 'none recorded'}; this SDK speaks ABI revision ${fetchAbiRevision()} — re-lock with: ` +
|
|
950
|
-
`${FETCH_COMMAND} ${version} --lock ${ctx.lockPath}${revisionNote(ctx, req)}`);
|
|
951
|
-
}
|
|
952
|
-
if (entry.sha256.toLowerCase() !== listedSha) {
|
|
953
|
-
throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins ${key} to ${entry.file} (${entry.sha256}) but the release lists it as ${listedSha}; ` +
|
|
954
|
-
`frozen refuses it.${revisionNote(ctx, req)}`);
|
|
955
|
-
}
|
|
956
|
-
return row;
|
|
957
|
-
}
|
|
958
|
-
/** F1-F5: install exactly what the lock pins for `req`, never a newer patch or build the release also serves. */
|
|
959
|
-
async function installFrozen(release, req, ctx, options) {
|
|
960
|
-
const lock = ctx.lock; // installContext refuses `frozen` with no lock before this is ever called.
|
|
961
|
-
const candidate = selectFromLock(lock, ctx.platform, req);
|
|
962
|
-
if (candidate === undefined) {
|
|
963
|
-
throw new ArtifactPinnedError(`chtypes: ${ctx.lockPath} pins nothing for ${ctx.platform}/${req.spelling}, and frozen refuses anything it does not pin`);
|
|
964
|
-
}
|
|
965
|
-
const art = verifyFrozenCandidate(release, ctx, req, candidate);
|
|
966
|
-
const installed = await installOne(release, art, ctx, req);
|
|
967
|
-
await installGoldens(release, ctx, options);
|
|
968
|
-
return installed;
|
|
969
|
-
}
|
|
970
|
-
async function ensureUncached(req, platform, dest, options) {
|
|
971
|
-
const ctx = installContext(platform, dest, options);
|
|
972
|
-
if (options.offline) {
|
|
973
|
-
// Never touch the source, and never the lock (Decision 6): installed and
|
|
974
|
-
// hashing what its own manifest says, at either slot, is the whole test.
|
|
975
|
-
const patches = await installedPatchesForLine(dest, req.line);
|
|
976
|
-
const match = req.exact !== null
|
|
977
|
-
? (patches.find((p) => patchMatches(p.version, req)) ?? null)
|
|
978
|
-
: patches.reduce((best, cur) => (best === null || compareVersions(cur.version, best.version) > 0 ? cur : best), null);
|
|
979
|
-
if (match !== null && match.ok) {
|
|
980
|
-
ctx.emit({ type: 'status', message: `offline — already installed and verified: ${match.dir}` });
|
|
981
|
-
return {
|
|
982
|
-
dir: match.dir,
|
|
983
|
-
registry: dest,
|
|
984
|
-
line: req.line,
|
|
985
|
-
version: match.version,
|
|
986
|
-
platform,
|
|
987
|
-
file: '',
|
|
988
|
-
sha256: '',
|
|
989
|
-
library: match.library,
|
|
990
|
-
librarySha256: match.expected,
|
|
991
|
-
installed: false,
|
|
992
|
-
source: 'offline',
|
|
993
|
-
signed: false,
|
|
994
|
-
keyId: null,
|
|
995
|
-
};
|
|
996
|
-
}
|
|
997
|
-
throw new SourceUnreachableError(`chtypes: offline — ClickHouse ${req.spelling} (${platform}) is not installed and verified in ${dest}` +
|
|
998
|
-
(match !== null && !match.ok ? ` (${match.problem})` : '') +
|
|
999
|
-
', and offline forbids fetching it');
|
|
1000
|
-
}
|
|
1001
|
-
checkLockRevisionEarly(ctx, req);
|
|
1002
|
-
const source = openSource(options);
|
|
1003
|
-
ctx.emit({
|
|
1004
|
-
type: 'status',
|
|
1005
|
-
message: `ClickHouse ${req.spelling} -> line ${req.line}${req.exact !== null ? ` (exact ${req.exact})` : ''}, ${platform}`,
|
|
1006
|
-
});
|
|
1007
|
-
ctx.emit({ type: 'status', message: `source ${source.description}` });
|
|
1008
|
-
const release = await loadRelease(source, options, ctx.emit);
|
|
1009
|
-
if (ctx.frozen)
|
|
1010
|
-
return installFrozen(release, req, ctx, options);
|
|
1011
|
-
const art = selectArtifact(release.index, platform, req);
|
|
1012
|
-
const installed = await installOne(release, art, ctx, req);
|
|
1013
|
-
await installGoldens(release, ctx, options);
|
|
1014
|
-
return installed;
|
|
1015
|
-
}
|
|
1016
|
-
/**
|
|
1017
|
-
* Same-filesystem, atomic: move the current flat `<dest>/<minor>/` occupant
|
|
1018
|
-
* aside to `<dest>/patches/<minor>/<its version>/` — NEVER deleted — before a
|
|
1019
|
-
* different patch takes the flat slot (issue #284, the layout rule). A
|
|
1020
|
-
* no-op when nothing is flat yet, or when the flat occupant already IS
|
|
1021
|
-
* `incomingVersion` (the ordinary re-verify/rebuild path handles that case).
|
|
1022
|
-
*/
|
|
1023
|
-
async function demoteFlatIfDifferent(dest, minor, incomingVersion) {
|
|
1024
|
-
const flat = path.join(dest, minor);
|
|
1025
|
-
if (!existsSync(flat))
|
|
1026
|
-
return;
|
|
1027
|
-
const existing = readInstalledManifest(flat);
|
|
1028
|
-
if (existing === null || existing.version === incomingVersion)
|
|
1029
|
-
return;
|
|
1030
|
-
const patchesDir = path.join(dest, 'patches', minor);
|
|
1031
|
-
const target = path.join(patchesDir, existing.version);
|
|
1032
|
-
await mkdir(patchesDir, { recursive: true });
|
|
1033
|
-
if (existsSync(target))
|
|
1034
|
-
await rm(target, { recursive: true, force: true });
|
|
1035
|
-
await rename(flat, target);
|
|
1036
|
-
}
|
|
1037
|
-
/** Does `dir` already hold `art`, byte-verified — a manifest and a library hashing `art.library_sha256`? */
|
|
1038
|
-
async function verifiedInstall(dir, art) {
|
|
1039
|
-
const libPath = path.join(dir, art.library);
|
|
1040
|
-
if (!existsSync(path.join(dir, 'manifest.json')) || !existsSync(libPath))
|
|
1041
|
-
return false;
|
|
1042
|
-
return (await sha256File(libPath)) === art.library_sha256;
|
|
1043
|
-
}
|
|
1044
|
-
/**
|
|
1045
|
-
* One index row, from the release into its slot — steps 2 through 4. A LINE
|
|
1046
|
-
* request's pick installs flat, `<dest>/<minor>/`, demoting a different flat
|
|
1047
|
-
* occupant first (never deleting it). Any other exact patch request installs
|
|
1048
|
-
* at `<dest>/patches/<minor>/<clickhouse_version>/` — unless that exact
|
|
1049
|
-
* version already sits flat, byte-verified, in which case it counts as
|
|
1050
|
-
* installed there and no second copy is made (docs/guides/fetch.md §4).
|
|
1051
|
-
*/
|
|
1052
|
-
async function installOne(release, art, ctx, req) {
|
|
1053
|
-
const { dest, platform, emit } = ctx;
|
|
1054
|
-
const minor = art.clickhouse_minor;
|
|
1055
|
-
const version = art.clickhouse_version;
|
|
1056
|
-
const wantsFlat = req.exact === null;
|
|
1057
|
-
const flat = path.join(dest, minor);
|
|
1058
|
-
const nested = path.join(dest, 'patches', minor, version);
|
|
1059
|
-
const install = wantsFlat ? flat : nested;
|
|
1060
|
-
const lockKey = `${platform}/${version}`;
|
|
1061
|
-
emit({ type: 'status', message: `${art.file} (${art.bytes} bytes, ClickHouse ${art.clickhouse_version}, library ${art.library})` });
|
|
1062
|
-
// Step 2: the authentic SHA256SUMS must agree with index.json, byte for byte.
|
|
1063
|
-
const listed = release.sums.get(art.file);
|
|
1064
|
-
if (listed === undefined)
|
|
1065
|
-
throw new ArtifactCorruptError(`chtypes: SHA256SUMS has no line for ${art.file}`);
|
|
1066
|
-
if (listed !== art.sha256) {
|
|
1067
|
-
throw new ArtifactCorruptError(`chtypes: index.json says ${art.file} is ${art.sha256} but SHA256SUMS says ${listed} — the release disagrees with itself; not installing it`);
|
|
1068
|
-
}
|
|
1069
|
-
const result = (installed, dir) => ({
|
|
1070
|
-
dir,
|
|
1071
|
-
registry: dest,
|
|
1072
|
-
line: minor,
|
|
1073
|
-
version: art.clickhouse_version,
|
|
1074
|
-
platform,
|
|
1075
|
-
file: art.file,
|
|
1076
|
-
sha256: art.sha256,
|
|
1077
|
-
library: art.library,
|
|
1078
|
-
librarySha256: art.library_sha256,
|
|
1079
|
-
installed,
|
|
1080
|
-
source: release.source.description,
|
|
1081
|
-
signed: release.signed,
|
|
1082
|
-
keyId: release.keyId,
|
|
1083
|
-
});
|
|
1084
|
-
const recordLock = () => {
|
|
1085
|
-
if (ctx.lockPath !== undefined && !ctx.frozen) {
|
|
1086
|
-
writeLockEntry(ctx.lockPath, lockKey, { file: art.file, sha256: art.sha256, abi_revision: fetchAbiRevision() });
|
|
1087
|
-
}
|
|
1088
|
-
};
|
|
1089
|
-
// Already installed and intact? The same hash the install path ends with,
|
|
1090
|
-
// so "already there" is a verified claim, not an inference from a directory.
|
|
1091
|
-
if (!ctx.force) {
|
|
1092
|
-
if (await verifiedInstall(install, art)) {
|
|
1093
|
-
emit({ type: 'status', message: `already installed and verified: ${path.join(install, art.library)}` });
|
|
1094
|
-
recordLock();
|
|
1095
|
-
return result(false, install);
|
|
1096
|
-
}
|
|
1097
|
-
// §4: a patch request whose exact version already sits flat (put there by
|
|
1098
|
-
// an earlier line fetch) is already installed — no second, nested copy.
|
|
1099
|
-
if (!wantsFlat && (await verifiedInstall(flat, art))) {
|
|
1100
|
-
emit({ type: 'status', message: `already installed and verified (flat): ${path.join(flat, art.library)}` });
|
|
1101
|
-
recordLock();
|
|
1102
|
-
return result(false, flat);
|
|
1103
|
-
}
|
|
1104
|
-
}
|
|
1105
|
-
// A line request whose pick differs from the current flat occupant demotes
|
|
1106
|
-
// that occupant to patches/ BEFORE the incoming patch takes the flat slot.
|
|
1107
|
-
if (wantsFlat)
|
|
1108
|
-
await demoteFlatIfDifferent(dest, minor, version);
|
|
1109
|
-
await mkdir(dest, { recursive: true });
|
|
1110
|
-
await mkdir(path.dirname(install), { recursive: true });
|
|
1111
|
-
const tag = `${process.pid}.${randomBytes(4).toString('hex')}`;
|
|
1112
|
-
// Hidden siblings: the loader skips dot-directories, so a half-written
|
|
1113
|
-
// install is never scanned as a version.
|
|
1114
|
-
const tarball = path.join(dest, `.${minor}.download.${tag}`);
|
|
1115
|
-
const incoming = path.join(dest, `.${minor}.incoming.${tag}`);
|
|
1116
|
-
try {
|
|
1117
|
-
emit({ type: 'status', message: `downloading ${art.file}` });
|
|
1118
|
-
const got = await release.source.download(art.file, tarball, (received) => emit({ type: 'download', file: art.file, received, total: art.bytes }));
|
|
1119
|
-
if (!got)
|
|
1120
|
-
throw new SourceUnreachableError(`chtypes: could not download ${art.file} from ${release.source.description}`);
|
|
1121
|
-
// Step 3: size, then hash, before a byte is unpacked.
|
|
1122
|
-
const size = statSync(tarball).size;
|
|
1123
|
-
if (size !== art.bytes)
|
|
1124
|
-
throw new ArtifactCorruptError(`chtypes: ${art.file} is ${size} bytes, index.json says ${art.bytes}`);
|
|
1125
|
-
const sha = await sha256File(tarball);
|
|
1126
|
-
if (sha !== art.sha256) {
|
|
1127
|
-
throw new ArtifactCorruptError(`chtypes: ${art.file} FAILED its sha256: got ${sha}, want ${art.sha256} — not unpacking it`);
|
|
1128
|
-
}
|
|
1129
|
-
emit({ type: 'status', message: `sha256 verified before unpacking: ${sha}` });
|
|
1130
|
-
await mkdir(incoming);
|
|
1131
|
-
await extractTarGz(tarball, incoming);
|
|
1132
|
-
const manifest = readInstalledManifest(incoming);
|
|
1133
|
-
if (manifest === null)
|
|
1134
|
-
throw new ArtifactCorruptError(`chtypes: ${art.file} contains no usable manifest.json at its root`);
|
|
1135
|
-
// The index is a convenience; the manifest is the artifact's own claim
|
|
1136
|
-
// about itself. They must agree, or the index was built from a different artifact.
|
|
1137
|
-
if (manifest.library !== art.library ||
|
|
1138
|
-
manifest.version !== art.clickhouse_version ||
|
|
1139
|
-
manifest.minor !== minor ||
|
|
1140
|
-
manifest.librarySha256 !== art.library_sha256) {
|
|
1141
|
-
throw new ArtifactCorruptError(`chtypes: manifest.json inside ${art.file} disagrees with index.json ` +
|
|
1142
|
-
`(${manifest.library}/${manifest.version}/${manifest.minor}/${manifest.librarySha256} vs ` +
|
|
1143
|
-
`${art.library}/${art.clickhouse_version}/${minor}/${art.library_sha256})`);
|
|
1144
|
-
}
|
|
1145
|
-
if (!existsSync(path.join(incoming, manifest.library))) {
|
|
1146
|
-
throw new ArtifactCorruptError(`chtypes: ${art.file} names library ${manifest.library} but does not contain it`);
|
|
1147
|
-
}
|
|
1148
|
-
// Into place through a sibling: an interrupted install never leaves a
|
|
1149
|
-
// half-populated slot for the loader to dlopen.
|
|
1150
|
-
const replaced = path.join(dest, `.${minor}.replaced.${tag}`);
|
|
1151
|
-
if (existsSync(install))
|
|
1152
|
-
await rename(install, replaced);
|
|
1153
|
-
await rename(incoming, install);
|
|
1154
|
-
await rm(replaced, { recursive: true, force: true });
|
|
1155
|
-
// Step 4: re-hash the installed file, in place. Everything up to here
|
|
1156
|
-
// proved the bytes were right somewhere else.
|
|
1157
|
-
const installedLib = path.join(install, manifest.library);
|
|
1158
|
-
const finalSha = await sha256File(installedLib);
|
|
1159
|
-
if (finalSha !== manifest.librarySha256) {
|
|
1160
|
-
const aside = path.join(dest, `.${minor}.corrupt.${tag}`);
|
|
1161
|
-
await rename(install, aside);
|
|
1162
|
-
throw new ArtifactCorruptError(`chtypes: installed ${installedLib} hashes ${finalSha}, manifest says ${manifest.librarySha256} — the install is bad; moved aside to ${aside}`);
|
|
1163
|
-
}
|
|
1164
|
-
emit({ type: 'status', message: `installed and verified: ${install} (${manifest.library} sha256 ${finalSha})` });
|
|
1165
|
-
recordLock();
|
|
1166
|
-
return result(true, install);
|
|
1167
|
-
}
|
|
1168
|
-
finally {
|
|
1169
|
-
await rm(tarball, { force: true });
|
|
1170
|
-
await rm(incoming, { recursive: true, force: true });
|
|
1171
|
-
}
|
|
1172
|
-
}
|
|
1173
|
-
// ------------------------------------------------------ verify and list
|
|
1174
|
-
/**
|
|
1175
|
-
* `chtypes verify`: re-hash every installed PATCH in a registry directory —
|
|
1176
|
-
* the flat slot and every `patches/<minor>/<version>/` sibling (issue #284)
|
|
1177
|
-
* — against its own manifest. A directory whose manifest cannot be read is
|
|
1178
|
-
* not a patch (a scratch directory) and is skipped, as the loader skips it.
|
|
1179
|
-
*/
|
|
1180
|
-
export async function verifyInstalled(dest, platform) {
|
|
1181
|
-
const dir = fetchDestination(dest, resolvePlatform(platform));
|
|
1182
|
-
const out = [];
|
|
1183
|
-
for (const sub of allInstalledDirs(dir)) {
|
|
1184
|
-
const state = await installedPatch(sub);
|
|
1185
|
-
if (state !== null)
|
|
1186
|
-
out.push(state);
|
|
1187
|
-
}
|
|
1188
|
-
return out.sort((a, b) => compareVersions(a.version, b.version));
|
|
1189
|
-
}
|
|
1190
|
-
/** `chtypes list`: what is installed in the registry directory (one record per patch), and what the release offers for the platform. */
|
|
1191
|
-
export async function listArtifacts(options = {}) {
|
|
1192
|
-
const platform = resolvePlatform(options.platform);
|
|
1193
|
-
const dir = fetchDestination(options.dest, platform);
|
|
1194
|
-
const installed = [];
|
|
1195
|
-
for (const sub of allInstalledDirs(dir)) {
|
|
1196
|
-
const m = readInstalledManifest(sub);
|
|
1197
|
-
if (m === null)
|
|
1198
|
-
continue;
|
|
1199
|
-
const present = existsSync(path.join(sub, m.library));
|
|
1200
|
-
installed.push({
|
|
1201
|
-
line: m.minor,
|
|
1202
|
-
dir: sub,
|
|
1203
|
-
version: m.version,
|
|
1204
|
-
library: m.library,
|
|
1205
|
-
expected: m.librarySha256,
|
|
1206
|
-
actual: null,
|
|
1207
|
-
ok: present,
|
|
1208
|
-
problem: present ? null : `${m.library} is missing`,
|
|
1209
|
-
});
|
|
1210
|
-
}
|
|
1211
|
-
installed.sort((a, b) => compareVersions(a.version, b.version));
|
|
1212
|
-
if (options.offline)
|
|
1213
|
-
return { registry: dir, platform, installed, offered: null, source: null };
|
|
1214
|
-
const source = openSource(options);
|
|
1215
|
-
const release = await loadRelease(source, options, options.onProgress ?? noop);
|
|
1216
|
-
let offered;
|
|
1217
|
-
try {
|
|
1218
|
-
offered = selectAll(release.index, platform);
|
|
1219
|
-
}
|
|
1220
|
-
catch (err) {
|
|
1221
|
-
if (!(err instanceof ArtifactUnpublishedError))
|
|
1222
|
-
throw err;
|
|
1223
|
-
offered = [];
|
|
1224
|
-
}
|
|
1225
|
-
const note = notShown(release.index, platform);
|
|
1226
|
-
return { registry: dir, platform, installed, offered, source: source.description, ...(note !== undefined ? { notShown: note } : {}) };
|
|
1227
|
-
}
|
|
1228
|
-
// ------------------------------------------------------------ the lock
|
|
1229
|
-
/** `chtypes-<version>-<os>-<arch>[-b<N>].tar.gz` (docs/guides/artifacts.md), version captured whole (it may itself contain dots and a channel). */
|
|
1230
|
-
const ASSET_FILE_NAME = /^chtypes-(.+)-(linux|darwin)-(arm64|amd64)(?:-b\d+)?\.tar\.gz$/;
|
|
1231
|
-
/** The ClickHouse version a served asset file name claims, or `null` when it does not parse. */
|
|
1232
|
-
function versionFromAssetFile(file) {
|
|
1233
|
-
const m = ASSET_FILE_NAME.exec(file);
|
|
1234
|
-
return m === null ? null : m[1];
|
|
1235
|
-
}
|
|
1236
|
-
/** True for a PATCH spelling (four or more numeric components) — what schema 2's keys must name, never a bare line. */
|
|
1237
|
-
function isPatchVersion(version) {
|
|
1238
|
-
try {
|
|
1239
|
-
return parseVersionSpelling(version).exact !== null;
|
|
1240
|
-
}
|
|
1241
|
-
catch {
|
|
1242
|
-
return false;
|
|
1243
|
-
}
|
|
1244
|
-
}
|
|
1245
|
-
/**
|
|
1246
|
-
* Read a lock file; `null` when it does not exist. Accepts schema 1 and
|
|
1247
|
-
* schema 2 (docs/guides/fetch.md §5); `.schema` reports the number that was
|
|
1248
|
-
* actually on disk (so a caller can tell which it read), while `.artifacts`
|
|
1249
|
-
* is ALWAYS schema-2-shaped: a schema-1 entry `<os>-<arch>/<minor>` is
|
|
1250
|
-
* converted to the schema-2 entry `<os>-<arch>/<v>`, `<v>` being the
|
|
1251
|
-
* ClickHouse version its own `file` names — the one schema-1 could not
|
|
1252
|
-
* record, because it never had two patches of a line to distinguish. Every
|
|
1253
|
-
* other reader in this module treats `.artifacts` as schema-2-keyed
|
|
1254
|
-
* unconditionally; only this doc comment and `.schema` itself ever mention
|
|
1255
|
-
* schema 1 again. Any other schema number is refused, naming both 1 and
|
|
1256
|
-
* `LOCK_SCHEMA`. An entry that does not parse, under either schema — including
|
|
1257
|
-
* a schema-2 key that names a bare line rather than an exact patch — makes the
|
|
1258
|
-
* whole lock unreadable and the error names it.
|
|
1259
|
-
*/
|
|
1260
|
-
export function readLock(file) {
|
|
1261
|
-
let text;
|
|
1262
|
-
try {
|
|
1263
|
-
text = readFileSync(file, 'utf8');
|
|
1264
|
-
}
|
|
1265
|
-
catch (err) {
|
|
1266
|
-
if (isNoEntry(err))
|
|
1267
|
-
return null;
|
|
1268
|
-
throw new ChtypesError(`chtypes: cannot read lock file ${file}: ${errorText(err)}`, { cause: err });
|
|
1269
|
-
}
|
|
1270
|
-
let doc;
|
|
1271
|
-
try {
|
|
1272
|
-
doc = JSON.parse(text);
|
|
1273
|
-
}
|
|
1274
|
-
catch (err) {
|
|
1275
|
-
throw new ChtypesError(`chtypes: lock file ${file} is not JSON: ${errorText(err)}`, { cause: err });
|
|
1276
|
-
}
|
|
1277
|
-
if (typeof doc !== 'object' || doc === null) {
|
|
1278
|
-
throw new ChtypesError(`chtypes: lock file ${file} is not schema ${LOCK_SCHEMA_LEGACY} or ${LOCK_SCHEMA}`);
|
|
1279
|
-
}
|
|
1280
|
-
const schema = doc['schema'];
|
|
1281
|
-
if (schema !== LOCK_SCHEMA_LEGACY && schema !== LOCK_SCHEMA) {
|
|
1282
|
-
throw new ChtypesError(`chtypes: lock file ${file} is schema ${JSON.stringify(schema)}, not ${LOCK_SCHEMA_LEGACY} or ${LOCK_SCHEMA}`);
|
|
1283
|
-
}
|
|
1284
|
-
const raw = doc['artifacts'];
|
|
1285
|
-
const artifacts = {};
|
|
1286
|
-
if (typeof raw === 'object' && raw !== null) {
|
|
1287
|
-
for (const [k, v] of Object.entries(raw)) {
|
|
1288
|
-
if (typeof v !== 'object' || v === null)
|
|
1289
|
-
continue;
|
|
1290
|
-
const e = v;
|
|
1291
|
-
if (typeof e['file'] !== 'string' || typeof e['sha256'] !== 'string')
|
|
1292
|
-
continue;
|
|
1293
|
-
// abi_revision is optional and additive (docs/guides/fetch.md §5): an
|
|
1294
|
-
// entry written before this SDK recorded it simply has none, read the
|
|
1295
|
-
// same way an index row that carries none is — never a typed number
|
|
1296
|
-
// (TS cannot tell 5.0 from 5, so anything not a safe integer is absent).
|
|
1297
|
-
const entry = { file: e['file'], sha256: e['sha256'], ...revisionOf(e['abi_revision']) };
|
|
1298
|
-
const slash = k.indexOf('/');
|
|
1299
|
-
if (slash < 0) {
|
|
1300
|
-
throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)} is not <os>-<arch>/<version>`);
|
|
1301
|
-
}
|
|
1302
|
-
const platformKey = k.slice(0, slash);
|
|
1303
|
-
const versionOrMinor = k.slice(slash + 1);
|
|
1304
|
-
if (schema === LOCK_SCHEMA_LEGACY) {
|
|
1305
|
-
const version = versionFromAssetFile(entry.file);
|
|
1306
|
-
if (version === null) {
|
|
1307
|
-
throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)}'s file ${JSON.stringify(entry.file)} does not name a ` +
|
|
1308
|
-
'ClickHouse version — cannot convert this schema 1 lock to schema 2');
|
|
1309
|
-
}
|
|
1310
|
-
if (minorOfVersion(version) !== versionOrMinor) {
|
|
1311
|
-
throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)} names line ${versionOrMinor} but its file ` +
|
|
1312
|
-
`${JSON.stringify(entry.file)} is ${version} — cannot convert this schema 1 lock to schema 2`);
|
|
1313
|
-
}
|
|
1314
|
-
artifacts[`${platformKey}/${version}`] = entry;
|
|
1315
|
-
}
|
|
1316
|
-
else {
|
|
1317
|
-
if (!isPatchVersion(versionOrMinor)) {
|
|
1318
|
-
throw new ChtypesError(`chtypes: lock file ${file}: entry ${JSON.stringify(k)} names a line, not an exact patch — schema ${LOCK_SCHEMA} keys are <os>-<arch>/<clickhouse_version>`);
|
|
1319
|
-
}
|
|
1320
|
-
artifacts[k] = entry;
|
|
1321
|
-
}
|
|
1322
|
-
}
|
|
1323
|
-
}
|
|
1324
|
-
return { schema, artifacts };
|
|
1325
|
-
}
|
|
1326
|
-
function writeLockEntry(file, key, entry) {
|
|
1327
|
-
const current = readLock(file) ?? { schema: LOCK_SCHEMA, artifacts: {} };
|
|
1328
|
-
const merged = { ...current.artifacts, [key]: entry };
|
|
1329
|
-
const sorted = Object.fromEntries(Object.keys(merged).sort().map((k) => [k, merged[k]]));
|
|
1330
|
-
writeFileSync(file, `${JSON.stringify({ schema: LOCK_SCHEMA, artifacts: sorted }, null, 2)}\n`);
|
|
1331
|
-
}
|
|
1332
|
-
// -------------------------------------------------------------- helpers
|
|
1333
|
-
/** The platform a fetch is for: the option, else `CHTYPES_TARGET`, else this host. */
|
|
1334
|
-
export function resolvePlatform(explicit) {
|
|
1335
|
-
const env = process.env['CHTYPES_TARGET'];
|
|
1336
|
-
const key = explicit !== undefined && explicit !== '' ? explicit : env !== undefined && env !== '' ? env : hostPlatform();
|
|
1337
|
-
if (!isPlatformKey(key))
|
|
1338
|
-
throw new ChtypesError(`chtypes: not a known platform key: ${key} ((linux|darwin)-(arm64|amd64))`);
|
|
1339
|
-
return key;
|
|
1340
|
-
}
|
|
1341
|
-
function readInstalledManifest(dir) {
|
|
1342
|
-
try {
|
|
1343
|
-
const m = JSON.parse(readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
|
|
1344
|
-
if (typeof m['library'] !== 'string' || m['library'] === '')
|
|
1345
|
-
return null;
|
|
1346
|
-
if (typeof m['library_sha256'] !== 'string' || m['library_sha256'] === '')
|
|
1347
|
-
return null;
|
|
1348
|
-
const version = typeof m['clickhouse_version'] === 'string' ? m['clickhouse_version'] : '';
|
|
1349
|
-
// clickhouse_minor is absent from the first Linux artifacts; deriving it
|
|
1350
|
-
// from clickhouse_version is what the loader does, so both agree.
|
|
1351
|
-
const minor = typeof m['clickhouse_minor'] === 'string' && m['clickhouse_minor'] !== '' ? m['clickhouse_minor'] : minorOfVersion(version);
|
|
1352
|
-
return { library: m['library'], version, minor, librarySha256: m['library_sha256'].toLowerCase() };
|
|
1353
|
-
}
|
|
1354
|
-
catch {
|
|
1355
|
-
return null;
|
|
1356
|
-
}
|
|
1357
|
-
}
|
|
1358
|
-
/**
|
|
1359
|
-
* The state of one installed patch directory (flat or `patches/<minor>/<version>/`):
|
|
1360
|
-
* null when it is not one (no usable manifest). `line` comes from the
|
|
1361
|
-
* manifest's own claim, never from the directory name — a nested directory is
|
|
1362
|
-
* named after the VERSION, not the line.
|
|
1363
|
-
*/
|
|
1364
|
-
async function installedPatch(dir) {
|
|
1365
|
-
const m = readInstalledManifest(dir);
|
|
1366
|
-
if (m === null)
|
|
1367
|
-
return null;
|
|
1368
|
-
const libPath = path.join(dir, m.library);
|
|
1369
|
-
if (!existsSync(libPath)) {
|
|
1370
|
-
return { line: m.minor, dir, version: m.version, library: m.library, expected: m.librarySha256, actual: null, ok: false, problem: `${m.library} is missing` };
|
|
1371
|
-
}
|
|
1372
|
-
const actual = await sha256File(libPath);
|
|
1373
|
-
const ok = actual === m.librarySha256;
|
|
1374
|
-
return {
|
|
1375
|
-
line: m.minor,
|
|
1376
|
-
dir,
|
|
1377
|
-
version: m.version,
|
|
1378
|
-
library: m.library,
|
|
1379
|
-
expected: m.librarySha256,
|
|
1380
|
-
actual,
|
|
1381
|
-
ok,
|
|
1382
|
-
problem: ok ? null : `${m.library} hashes ${actual}, manifest says ${m.librarySha256}`,
|
|
1383
|
-
};
|
|
1384
|
-
}
|
|
1385
|
-
/** Every installed-patch directory in a registry root: the flat slot per line, plus every `patches/<minor>/<version>/`. */
|
|
1386
|
-
function allInstalledDirs(dir) {
|
|
1387
|
-
const out = [];
|
|
1388
|
-
let entries;
|
|
1389
|
-
try {
|
|
1390
|
-
entries = readdirSync(dir);
|
|
1391
|
-
}
|
|
1392
|
-
catch {
|
|
1393
|
-
return out;
|
|
1394
|
-
}
|
|
1395
|
-
for (const entry of entries.sort()) {
|
|
1396
|
-
if (entry.startsWith('.') || entry === 'patches')
|
|
1397
|
-
continue;
|
|
1398
|
-
if (existsSync(path.join(dir, entry, 'manifest.json')))
|
|
1399
|
-
out.push(path.join(dir, entry));
|
|
1400
|
-
}
|
|
1401
|
-
const patchesRoot = path.join(dir, 'patches');
|
|
1402
|
-
let minorEntries;
|
|
1403
|
-
try {
|
|
1404
|
-
minorEntries = readdirSync(patchesRoot);
|
|
1405
|
-
}
|
|
1406
|
-
catch {
|
|
1407
|
-
return out;
|
|
1408
|
-
}
|
|
1409
|
-
for (const minorEntry of minorEntries.sort()) {
|
|
1410
|
-
if (minorEntry.startsWith('.'))
|
|
1411
|
-
continue;
|
|
1412
|
-
const minorDir = path.join(patchesRoot, minorEntry);
|
|
1413
|
-
let versionEntries;
|
|
1414
|
-
try {
|
|
1415
|
-
versionEntries = readdirSync(minorDir);
|
|
1416
|
-
}
|
|
1417
|
-
catch {
|
|
1418
|
-
continue;
|
|
1419
|
-
}
|
|
1420
|
-
for (const versionEntry of versionEntries.sort()) {
|
|
1421
|
-
if (versionEntry.startsWith('.'))
|
|
1422
|
-
continue;
|
|
1423
|
-
const sub = path.join(minorDir, versionEntry);
|
|
1424
|
-
if (existsSync(path.join(sub, 'manifest.json')))
|
|
1425
|
-
out.push(sub);
|
|
1426
|
-
}
|
|
1427
|
-
}
|
|
1428
|
-
return out;
|
|
1429
|
-
}
|
|
1430
|
-
/**
|
|
1431
|
-
* Every installed patch of `minor` in `dest` — the flat slot and every
|
|
1432
|
-
* `patches/<minor>/<version>/` sibling — byte-verified against its own
|
|
1433
|
-
* manifest. Used offline, where the source and the lock are both off limits
|
|
1434
|
-
* (Decision 6): "installed" there means hashed and matching, not merely present.
|
|
1435
|
-
*/
|
|
1436
|
-
async function installedPatchesForLine(dest, minor) {
|
|
1437
|
-
const flat = path.join(dest, minor);
|
|
1438
|
-
const dirs = existsSync(path.join(flat, 'manifest.json')) ? [flat] : [];
|
|
1439
|
-
const patchesDir = path.join(dest, 'patches', minor);
|
|
1440
|
-
let versionEntries = [];
|
|
1441
|
-
try {
|
|
1442
|
-
versionEntries = readdirSync(patchesDir);
|
|
1443
|
-
}
|
|
1444
|
-
catch {
|
|
1445
|
-
versionEntries = [];
|
|
1446
|
-
}
|
|
1447
|
-
for (const versionEntry of versionEntries.sort()) {
|
|
1448
|
-
if (versionEntry.startsWith('.'))
|
|
1449
|
-
continue;
|
|
1450
|
-
const sub = path.join(patchesDir, versionEntry);
|
|
1451
|
-
if (existsSync(path.join(sub, 'manifest.json')))
|
|
1452
|
-
dirs.push(sub);
|
|
1453
|
-
}
|
|
1454
|
-
const out = [];
|
|
1455
|
-
for (const d of dirs) {
|
|
1456
|
-
const state = await installedPatch(d);
|
|
1457
|
-
if (state !== null)
|
|
1458
|
-
out.push(state);
|
|
1459
|
-
}
|
|
1460
|
-
return out;
|
|
1461
|
-
}
|
|
1462
|
-
/** sha256 of a file, streamed — a 300 MB library never sits in memory whole. */
|
|
1463
|
-
export async function sha256File(file) {
|
|
1464
|
-
const hash = createHash('sha256');
|
|
1465
|
-
await pipeline(createReadStream(file), async function* (chunks) {
|
|
1466
|
-
for await (const chunk of chunks)
|
|
1467
|
-
hash.update(chunk);
|
|
1468
|
-
});
|
|
1469
|
-
return hash.digest('hex');
|
|
1470
|
-
}
|
|
1471
|
-
function progressTap(onReceived) {
|
|
1472
|
-
let received = 0;
|
|
1473
|
-
return new Transform({
|
|
1474
|
-
transform(chunk, _enc, cb) {
|
|
1475
|
-
received += chunk.length;
|
|
1476
|
-
onReceived(received);
|
|
1477
|
-
cb(null, chunk);
|
|
1478
|
-
},
|
|
1479
|
-
});
|
|
1480
|
-
}
|
|
1481
|
-
function envFlag(name) {
|
|
1482
|
-
const v = process.env[name];
|
|
1483
|
-
return v !== undefined && ['1', 'true', 'yes', 'on'].includes(v.trim().toLowerCase());
|
|
1484
|
-
}
|
|
1485
|
-
function isNoEntry(err) {
|
|
1486
|
-
return typeof err === 'object' && err !== null && err.code === 'ENOENT';
|
|
1487
|
-
}
|
|
1488
|
-
function errorText(err) {
|
|
1489
|
-
if (!(err instanceof Error))
|
|
1490
|
-
return String(err);
|
|
1491
|
-
const cause = err.cause;
|
|
1492
|
-
return cause instanceof Error && cause.message !== err.message ? `${err.message}: ${cause.message}` : err.message;
|
|
1493
|
-
}
|
|
1494
|
-
function warn(message) {
|
|
1495
|
-
process.stderr.write(`${message}\n`);
|
|
1496
|
-
}
|
|
1497
|
-
const noop = () => { };
|
|
1498
|
-
//# sourceMappingURL=fetch.js.map
|