@indigoai-us/hq-cli 5.33.0 → 5.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/__fixtures__/make-tar.d.ts +46 -0
- package/dist/commands/__fixtures__/make-tar.js +105 -0
- package/dist/commands/master-sync.d.ts +11 -0
- package/dist/commands/master-sync.js +15 -0
- package/dist/commands/pack-install.d.ts +187 -1
- package/dist/commands/pack-install.js +405 -16
- package/dist/commands/packs.js +9 -4
- package/dist/commands/publish.d.ts +186 -0
- package/dist/commands/publish.js +375 -0
- package/dist/commands/rescue.d.ts +33 -0
- package/dist/commands/rescue.js +161 -0
- package/dist/commands/safe-extract.d.ts +154 -0
- package/dist/commands/safe-extract.js +347 -0
- package/dist/index.js +17 -2
- package/dist/lib/local-tree-diff.d.ts +21 -0
- package/dist/lib/local-tree-diff.js +18 -3
- package/dist/types.d.ts +22 -0
- package/dist/utils/vault-api.d.ts +11 -0
- package/dist/utils/vault-api.js +39 -2
- package/package.json +2 -2
- package/src/commands/__fixtures__/make-tar.ts +126 -0
- package/src/commands/artifact-verify.test.ts +177 -0
- package/src/commands/marketplace-install.test.ts +414 -0
- package/src/commands/marketplace-security.test.ts +646 -0
- package/src/commands/master-sync.ts +23 -0
- package/src/commands/pack-install.test.ts +209 -1
- package/src/commands/pack-install.ts +617 -15
- package/src/commands/packs.ts +8 -1
- package/src/commands/publish.test.ts +538 -0
- package/src/commands/publish.ts +517 -0
- package/src/commands/rescue.test.ts +39 -0
- package/src/commands/rescue.ts +210 -0
- package/src/commands/safe-extract.test.ts +459 -0
- package/src/commands/safe-extract.ts +444 -0
- package/src/index.ts +18 -0
- package/src/lib/local-tree-diff.test.ts +19 -0
- package/src/lib/local-tree-diff.ts +17 -1
- package/src/types.ts +23 -0
- package/src/utils/vault-api.ts +41 -0
|
@@ -39,6 +39,12 @@ import * as os from 'os';
|
|
|
39
39
|
import * as path from 'path';
|
|
40
40
|
import * as readline from 'readline';
|
|
41
41
|
import * as yaml from 'js-yaml';
|
|
42
|
+
import {
|
|
43
|
+
createHash,
|
|
44
|
+
createPublicKey,
|
|
45
|
+
verify as cryptoVerify,
|
|
46
|
+
type KeyObject,
|
|
47
|
+
} from 'node:crypto';
|
|
42
48
|
import { execFileSync, spawnSync } from 'child_process';
|
|
43
49
|
import { Command } from 'commander';
|
|
44
50
|
import chalk from 'chalk';
|
|
@@ -47,15 +53,26 @@ import semverValid from 'semver/functions/valid.js';
|
|
|
47
53
|
import semverValidRange from 'semver/ranges/valid.js';
|
|
48
54
|
import semverGt from 'semver/functions/gt.js';
|
|
49
55
|
import { findHqRoot } from '../utils/manifest.js';
|
|
56
|
+
import { safeExtractTarball } from './safe-extract.js';
|
|
57
|
+
import { vaultApiFetchPublic } from '../utils/vault-api.js';
|
|
50
58
|
import type { PackManifest, PackContributeKey } from '../types.js';
|
|
51
59
|
|
|
52
60
|
// ---------------------------------------------------------------------------
|
|
53
61
|
// Source classification
|
|
54
62
|
// ---------------------------------------------------------------------------
|
|
55
63
|
|
|
56
|
-
export type Transport = 'npm' | 'git' | 'local';
|
|
64
|
+
export type Transport = 'npm' | 'git' | 'local' | 'marketplace';
|
|
65
|
+
|
|
66
|
+
/** Prefix that routes a source through the HQ marketplace transport (US-006). */
|
|
67
|
+
export const MARKETPLACE_PREFIX = 'marketplace:';
|
|
57
68
|
|
|
58
69
|
export function classify(source: string): Transport {
|
|
70
|
+
// US-006: marketplace:<slug>[@version] is recognized BEFORE the legacy
|
|
71
|
+
// registry fallback and ahead of every other transport. The slug is resolved
|
|
72
|
+
// against the public listings API, the presigned tarball is downloaded,
|
|
73
|
+
// verified (US-021), and safe-extracted (US-020) into the existing local
|
|
74
|
+
// install + symlink-wiring path.
|
|
75
|
+
if (source.startsWith(MARKETPLACE_PREFIX)) return 'marketplace';
|
|
59
76
|
if (source.startsWith('@')) return 'npm';
|
|
60
77
|
if (
|
|
61
78
|
source.startsWith('http://') ||
|
|
@@ -154,7 +171,25 @@ interface FetchResult {
|
|
|
154
171
|
resolvedSha?: string; // git transport only
|
|
155
172
|
}
|
|
156
173
|
|
|
157
|
-
|
|
174
|
+
/**
|
|
175
|
+
* Optional marketplace integrity metadata for a tarball-based install (US-021).
|
|
176
|
+
* When present, `verifyArtifact` runs over the downloaded tarball bytes BEFORE
|
|
177
|
+
* `safeExtractTarball` — a hash/signature mismatch aborts the install before
|
|
178
|
+
* any byte is unpacked. US-006 populates this from the listing detail; the npm
|
|
179
|
+
* fetch path also honors it so a registry-fetched tarball can be verified.
|
|
180
|
+
*/
|
|
181
|
+
export interface ArtifactIntegrity {
|
|
182
|
+
expectedHash: string;
|
|
183
|
+
signature?: string;
|
|
184
|
+
publicKey?: string;
|
|
185
|
+
requireSignature?: boolean;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function fetchNpm(
|
|
189
|
+
source: string,
|
|
190
|
+
tmpDir: string,
|
|
191
|
+
integrity?: ArtifactIntegrity,
|
|
192
|
+
): FetchResult {
|
|
158
193
|
// Use `npm pack` to grab the tarball without actually installing anything.
|
|
159
194
|
// Capture output to get the produced filename. Arguments are passed as an
|
|
160
195
|
// argv array — never interpolated into a shell string — so `source` cannot
|
|
@@ -170,10 +205,28 @@ function fetchNpm(source: string, tmpDir: string): FetchResult {
|
|
|
170
205
|
}
|
|
171
206
|
const tarballPath = path.join(tmpDir, tarball);
|
|
172
207
|
const extractDir = path.join(tmpDir, 'extracted');
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
208
|
+
// SECURITY (US-021): verify integrity + authenticity over the EXACT
|
|
209
|
+
// downloaded bytes BEFORE we extract. A hash/signature mismatch throws
|
|
210
|
+
// `ArtifactVerificationError` here, so a tampered artifact is never unpacked
|
|
211
|
+
// or wired. This MUST precede `safeExtractTarball` (US-020). Only runs when
|
|
212
|
+
// the caller supplied a pinned hash (i.e. a marketplace listing did); the
|
|
213
|
+
// plain `npm pack` path with no listing metadata is unaffected.
|
|
214
|
+
if (integrity) {
|
|
215
|
+
const tarballBytes = fs.readFileSync(tarballPath);
|
|
216
|
+
verifyArtifact({
|
|
217
|
+
tarballBytes,
|
|
218
|
+
expectedHash: integrity.expectedHash,
|
|
219
|
+
signature: integrity.signature,
|
|
220
|
+
publicKey: integrity.publicKey,
|
|
221
|
+
requireSignature: integrity.requireSignature,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
// SECURITY (US-020): the tarball is untrusted. Extract through the hardened
|
|
225
|
+
// path — zip-slip / link-escape containment + decompression-bomb caps +
|
|
226
|
+
// staged-then-atomic commit with full rollback — instead of a blind
|
|
227
|
+
// `tar -xzf`. safeExtractTarball requires the destination not to pre-exist,
|
|
228
|
+
// so we do NOT mkdir extractDir first; it is created atomically on success.
|
|
229
|
+
safeExtractTarball(tarballPath, extractDir);
|
|
177
230
|
// npm tarballs unpack into ./package/
|
|
178
231
|
const payloadDir = path.join(extractDir, 'package');
|
|
179
232
|
if (!fs.existsSync(payloadDir)) {
|
|
@@ -194,6 +247,270 @@ function stripVersion(src: string): string {
|
|
|
194
247
|
return src;
|
|
195
248
|
}
|
|
196
249
|
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
251
|
+
// Marketplace transport (US-006)
|
|
252
|
+
//
|
|
253
|
+
// `marketplace:<slug>[@version]` resolves a slug against the PUBLIC listings
|
|
254
|
+
// API (US-005, NONE-auth), downloads the presigned tarball, verifies it
|
|
255
|
+
// (US-021) BEFORE extraction, and feeds it into the existing safe-extract +
|
|
256
|
+
// validate + symlink-wire path. The legacy registry/npm/git/local transports
|
|
257
|
+
// are untouched.
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
/** Parsed `marketplace:<slug>[@version]` source. */
|
|
261
|
+
export function parseMarketplaceSource(source: string): {
|
|
262
|
+
slug: string;
|
|
263
|
+
version?: string;
|
|
264
|
+
} {
|
|
265
|
+
if (!source.startsWith(MARKETPLACE_PREFIX)) {
|
|
266
|
+
throw new Error(`Not a marketplace source: "${source}"`);
|
|
267
|
+
}
|
|
268
|
+
const rest = source.slice(MARKETPLACE_PREFIX.length).trim();
|
|
269
|
+
if (!rest) {
|
|
270
|
+
throw new Error('marketplace: source requires a slug (marketplace:<slug>[@version]).');
|
|
271
|
+
}
|
|
272
|
+
// Split on the LAST '@' so a slug never legitimately contains one, but be
|
|
273
|
+
// defensive: only treat the suffix as a version if there's a non-empty slug.
|
|
274
|
+
const at = rest.lastIndexOf('@');
|
|
275
|
+
if (at > 0) {
|
|
276
|
+
return { slug: rest.slice(0, at), version: rest.slice(at + 1) || undefined };
|
|
277
|
+
}
|
|
278
|
+
return { slug: rest };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* A resolved marketplace listing detail — the public `GET /v1/listings/{id}`
|
|
283
|
+
* shape (US-005) reduced to the fields the installer needs. The presigned
|
|
284
|
+
* `downloadUrl` is short-lived; `listingId` lets us re-resolve a fresh one if
|
|
285
|
+
* it expires mid-download.
|
|
286
|
+
*/
|
|
287
|
+
export interface MarketplaceListing {
|
|
288
|
+
listingId: string;
|
|
289
|
+
slug: string;
|
|
290
|
+
version?: string;
|
|
291
|
+
/** Short-lived presigned S3 URL for the tarball. */
|
|
292
|
+
downloadUrl: string;
|
|
293
|
+
/** Lowercase-hex sha256 the listing pinned + approval is bound to (US-021). */
|
|
294
|
+
contentHash: string;
|
|
295
|
+
/** Hash algorithm the server reported (e.g. `sha256`), when present. */
|
|
296
|
+
contentHashAlg?: string;
|
|
297
|
+
/** Base64 Ed25519 signature over the contentHash (optional during key-defer). */
|
|
298
|
+
signature?: string;
|
|
299
|
+
/** Identifier of the signing key the server used, when present. */
|
|
300
|
+
signingKeyId?: string;
|
|
301
|
+
/** Platform public key (PEM/SPKI) to verify the signature against. */
|
|
302
|
+
publicKey?: string;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Network seams for the marketplace transport, dependency-injected so unit
|
|
307
|
+
* tests can mock resolve + download without real HTTP/S3. Production wiring
|
|
308
|
+
* lives in `defaultMarketplaceDeps`.
|
|
309
|
+
*/
|
|
310
|
+
export interface MarketplaceDeps {
|
|
311
|
+
/**
|
|
312
|
+
* Resolve `slug[@version]` to a listing detail (id + presigned URL + the
|
|
313
|
+
* approved hash/signature/publicKey). Throws if no approved listing matches.
|
|
314
|
+
*/
|
|
315
|
+
resolveListing: (slug: string, version?: string) => Promise<MarketplaceListing>;
|
|
316
|
+
/** Re-fetch the listing detail by id to mint a FRESH presigned URL. */
|
|
317
|
+
refreshListing: (listingId: string) => Promise<MarketplaceListing>;
|
|
318
|
+
/** Download the tarball bytes from a presigned URL. Returns null on 403/expired. */
|
|
319
|
+
download: (url: string) => Promise<Uint8Array | { expired: true }>;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** True when a download result signals an expired/forbidden presigned URL. */
|
|
323
|
+
function isExpired(r: Uint8Array | { expired: true }): r is { expired: true } {
|
|
324
|
+
return !(r instanceof Uint8Array) && (r as { expired?: true }).expired === true;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
interface RawListingSummary {
|
|
328
|
+
listingId?: string;
|
|
329
|
+
id?: string;
|
|
330
|
+
slug?: string;
|
|
331
|
+
version?: string;
|
|
332
|
+
latestVersion?: string;
|
|
333
|
+
status?: string;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
interface RawListingDetailFields extends RawListingSummary {
|
|
337
|
+
downloadUrl?: string;
|
|
338
|
+
url?: string;
|
|
339
|
+
contentHash?: string;
|
|
340
|
+
contentHashAlg?: string;
|
|
341
|
+
sha256?: string;
|
|
342
|
+
signature?: string;
|
|
343
|
+
signingKeyId?: string;
|
|
344
|
+
publicKey?: string;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* `GET /v1/listings/{id}` returns the listing WRAPPED in an envelope
|
|
349
|
+
* (`{ listing: { id, downloadUrl, contentHash, ... } }`). Older/back-compat
|
|
350
|
+
* responses returned the listing fields at the top level, so we accept both.
|
|
351
|
+
*/
|
|
352
|
+
interface RawListingDetail extends RawListingDetailFields {
|
|
353
|
+
listing?: RawListingDetailFields;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** Map a raw `GET /v1/listings/{id}` body into a MarketplaceListing. */
|
|
357
|
+
export function toMarketplaceListing(raw: RawListingDetail): MarketplaceListing {
|
|
358
|
+
// Unwrap the `{ listing: {...} }` envelope when present; fall back to the
|
|
359
|
+
// top level for the legacy flat shape (backwards-compat).
|
|
360
|
+
const detail: RawListingDetailFields =
|
|
361
|
+
raw.listing && typeof raw.listing === 'object' ? raw.listing : raw;
|
|
362
|
+
// The detail uses `id`; the legacy flat shape may use `listingId`.
|
|
363
|
+
const listingId = detail.id ?? detail.listingId;
|
|
364
|
+
const downloadUrl = detail.downloadUrl ?? detail.url;
|
|
365
|
+
const contentHash = detail.contentHash ?? detail.sha256;
|
|
366
|
+
if (!listingId) throw new Error('Listing detail missing an id.');
|
|
367
|
+
if (!downloadUrl) {
|
|
368
|
+
throw new Error(`Listing ${listingId} has no download URL — it may not be approved yet.`);
|
|
369
|
+
}
|
|
370
|
+
if (!contentHash) {
|
|
371
|
+
throw new Error(
|
|
372
|
+
`Listing ${listingId} has no content hash — refusing to install an unverifiable artifact.`,
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
return {
|
|
376
|
+
listingId,
|
|
377
|
+
slug: detail.slug ?? '',
|
|
378
|
+
version: detail.version ?? detail.latestVersion,
|
|
379
|
+
downloadUrl,
|
|
380
|
+
contentHash,
|
|
381
|
+
contentHashAlg: detail.contentHashAlg,
|
|
382
|
+
signature: detail.signature,
|
|
383
|
+
signingKeyId: detail.signingKeyId,
|
|
384
|
+
publicKey: detail.publicKey,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** Default production marketplace deps — public listings API + presigned S3. */
|
|
389
|
+
export function defaultMarketplaceDeps(): MarketplaceDeps {
|
|
390
|
+
const fetchDetail = async (listingId: string): Promise<MarketplaceListing> => {
|
|
391
|
+
const res = await vaultApiFetchPublic({
|
|
392
|
+
path: `/v1/listings/${encodeURIComponent(listingId)}`,
|
|
393
|
+
});
|
|
394
|
+
if (!res.ok) {
|
|
395
|
+
throw new Error(
|
|
396
|
+
`Failed to resolve listing ${listingId} (HTTP ${res.status}). The pack may be unavailable or unapproved.`,
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
const body = (await res.json().catch(() => ({}))) as RawListingDetail;
|
|
400
|
+
return toMarketplaceListing(body);
|
|
401
|
+
};
|
|
402
|
+
return {
|
|
403
|
+
resolveListing: async (slug, version) => {
|
|
404
|
+
// Search by slug (public). The API returns approved listings only.
|
|
405
|
+
const res = await vaultApiFetchPublic({
|
|
406
|
+
path: '/v1/listings',
|
|
407
|
+
query: version ? { slug, version } : { slug },
|
|
408
|
+
});
|
|
409
|
+
if (!res.ok) {
|
|
410
|
+
throw new Error(
|
|
411
|
+
`Failed to search marketplace for "${slug}" (HTTP ${res.status}).`,
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
const body = (await res.json().catch(() => ({}))) as {
|
|
415
|
+
listings?: RawListingSummary[];
|
|
416
|
+
};
|
|
417
|
+
const listings = body.listings ?? [];
|
|
418
|
+
const match = version
|
|
419
|
+
? listings.find((l) => (l.version ?? l.latestVersion) === version)
|
|
420
|
+
: listings[0];
|
|
421
|
+
if (!match) {
|
|
422
|
+
throw new Error(
|
|
423
|
+
`No approved marketplace listing found for "${slug}"${version ? `@${version}` : ''}.`,
|
|
424
|
+
);
|
|
425
|
+
}
|
|
426
|
+
const id = match.listingId ?? match.id;
|
|
427
|
+
if (!id) throw new Error(`Listing for "${slug}" has no id.`);
|
|
428
|
+
// The summary may omit the presigned URL; always fetch detail to mint it.
|
|
429
|
+
return fetchDetail(id);
|
|
430
|
+
},
|
|
431
|
+
refreshListing: fetchDetail,
|
|
432
|
+
download: async (url) => {
|
|
433
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(120_000) });
|
|
434
|
+
if (response.status === 403) return { expired: true };
|
|
435
|
+
if (!response.ok) {
|
|
436
|
+
throw new Error(`Tarball download failed (HTTP ${response.status}).`);
|
|
437
|
+
}
|
|
438
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
439
|
+
},
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Fetch + verify + safe-extract a marketplace pack. Mirrors `fetchNpm`'s
|
|
445
|
+
* contract (returns a payloadDir the caller mvs into core/packages/), but
|
|
446
|
+
* resolves the tarball from the marketplace and ALWAYS verifies it against the
|
|
447
|
+
* listing's approved hash/signature BEFORE extraction (US-021 → US-020).
|
|
448
|
+
*
|
|
449
|
+
* @param requireSignature production posture once signing keys exist; during
|
|
450
|
+
* the key-deferral window callers pass false (hash still always checked).
|
|
451
|
+
*/
|
|
452
|
+
export async function fetchMarketplace(
|
|
453
|
+
source: string,
|
|
454
|
+
tmpDir: string,
|
|
455
|
+
deps: MarketplaceDeps,
|
|
456
|
+
opts: { requireSignature?: boolean } = {},
|
|
457
|
+
): Promise<FetchResult> {
|
|
458
|
+
const { slug, version } = parseMarketplaceSource(source);
|
|
459
|
+
let listing = await deps.resolveListing(slug, version);
|
|
460
|
+
|
|
461
|
+
// Download with a single expired-URL retry: a presigned URL can expire
|
|
462
|
+
// between resolve and download, surfacing as a raw S3 403. We re-resolve the
|
|
463
|
+
// detail by id to mint a fresh URL and retry ONCE rather than leaking the 403.
|
|
464
|
+
let bytes = await deps.download(listing.downloadUrl);
|
|
465
|
+
if (isExpired(bytes)) {
|
|
466
|
+
listing = await deps.refreshListing(listing.listingId);
|
|
467
|
+
bytes = await deps.download(listing.downloadUrl);
|
|
468
|
+
if (isExpired(bytes)) {
|
|
469
|
+
throw new Error(
|
|
470
|
+
`Marketplace download URL for "${slug}" expired and re-resolving did not help. ` +
|
|
471
|
+
`Try again in a moment.`,
|
|
472
|
+
);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
const tarballBytes = bytes as Uint8Array;
|
|
476
|
+
|
|
477
|
+
// SECURITY (US-021): verify integrity (+ authenticity) over the EXACT
|
|
478
|
+
// downloaded bytes BEFORE extraction. A hash/signature mismatch throws
|
|
479
|
+
// `ArtifactVerificationError` and nothing is unpacked or wired. The hash is
|
|
480
|
+
// ALWAYS verified when present; signature is enforced when requireSignature.
|
|
481
|
+
verifyArtifact({
|
|
482
|
+
tarballBytes,
|
|
483
|
+
expectedHash: listing.contentHash,
|
|
484
|
+
signature: listing.signature,
|
|
485
|
+
publicKey: listing.publicKey,
|
|
486
|
+
requireSignature: opts.requireSignature ?? false,
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
// Write to a tmp tarball and run it through the SAME hardened extractor
|
|
490
|
+
// (US-020) as every other tarball transport.
|
|
491
|
+
const tarballPath = path.join(tmpDir, 'marketplace.tar.gz');
|
|
492
|
+
fs.writeFileSync(tarballPath, tarballBytes);
|
|
493
|
+
const extractDir = path.join(tmpDir, 'extracted');
|
|
494
|
+
safeExtractTarball(tarballPath, extractDir);
|
|
495
|
+
|
|
496
|
+
// Marketplace tarballs are packed with `tar -C payloadDir .` (publish.ts), so
|
|
497
|
+
// the manifest + contributes dirs are at the archive root — no `package/`
|
|
498
|
+
// wrapper to descend into.
|
|
499
|
+
const payloadDir = extractDir;
|
|
500
|
+
if (!fs.existsSync(path.join(payloadDir, 'package.yaml'))) {
|
|
501
|
+
throw new Error(
|
|
502
|
+
`Marketplace tarball for "${slug}" has no package.yaml at its root.`,
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
// Record the marketplace source for re-install + `hq packs update`. We stamp
|
|
507
|
+
// the resolved version so a later `update` can detect a newer listing.
|
|
508
|
+
const resolvedSource = listing.version
|
|
509
|
+
? `${MARKETPLACE_PREFIX}${slug}@${listing.version}`
|
|
510
|
+
: `${MARKETPLACE_PREFIX}${slug}`;
|
|
511
|
+
return { payloadDir, resolvedSource };
|
|
512
|
+
}
|
|
513
|
+
|
|
197
514
|
function fetchGit(
|
|
198
515
|
source: string,
|
|
199
516
|
tmpDir: string,
|
|
@@ -384,6 +701,21 @@ export function resolveLatest(
|
|
|
384
701
|
return { transport, updateAvailable: null, error: 'local source — re-run to re-sync' };
|
|
385
702
|
}
|
|
386
703
|
|
|
704
|
+
if (transport === 'marketplace') {
|
|
705
|
+
// The marketplace probe needs an async network call (listings API), which
|
|
706
|
+
// this synchronous helper cannot make. Callers that want the marketplace
|
|
707
|
+
// update check use the async `resolveLatestMarketplace`; here we report the
|
|
708
|
+
// installed version but leave availability undeterminable (null) so the
|
|
709
|
+
// sync menubar `--check-updates` path is not regressed.
|
|
710
|
+
const { version } = parseMarketplaceSource(source);
|
|
711
|
+
return {
|
|
712
|
+
transport,
|
|
713
|
+
current: installedVersion ?? version,
|
|
714
|
+
updateAvailable: null,
|
|
715
|
+
error: 'marketplace source — use async update check',
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
|
|
387
719
|
if (transport === 'npm') {
|
|
388
720
|
const pkg = stripVersion(source);
|
|
389
721
|
const current =
|
|
@@ -427,11 +759,206 @@ export function resolveLatest(
|
|
|
427
759
|
}
|
|
428
760
|
}
|
|
429
761
|
|
|
762
|
+
/**
|
|
763
|
+
* Async marketplace update probe (US-006): resolve the slug's latest approved
|
|
764
|
+
* listing version and compare it to the installed version. Reuses the same
|
|
765
|
+
* `MarketplaceDeps.resolveListing` seam as the install path (resolving with NO
|
|
766
|
+
* version pin returns the latest listing). Never throws — network/parse
|
|
767
|
+
* failures return `{ updateAvailable: null, error }` so `hq packs update`
|
|
768
|
+
* stays resilient.
|
|
769
|
+
*
|
|
770
|
+
* @param source the stamped `marketplace:<slug>[@version]` source
|
|
771
|
+
* @param installedVersion the installed pack's manifest version (semver compare)
|
|
772
|
+
* @param deps injected for tests; defaults to the public API
|
|
773
|
+
*/
|
|
774
|
+
export async function resolveLatestMarketplace(
|
|
775
|
+
source: string,
|
|
776
|
+
installedVersion?: string,
|
|
777
|
+
deps?: MarketplaceDeps,
|
|
778
|
+
): Promise<LatestResult> {
|
|
779
|
+
let slug: string;
|
|
780
|
+
let pinned: string | undefined;
|
|
781
|
+
try {
|
|
782
|
+
({ slug, version: pinned } = parseMarketplaceSource(source));
|
|
783
|
+
} catch (e) {
|
|
784
|
+
return { transport: 'marketplace', updateAvailable: null, error: (e as Error).message };
|
|
785
|
+
}
|
|
786
|
+
const current = installedVersion ?? pinned;
|
|
787
|
+
try {
|
|
788
|
+
const d = deps ?? defaultMarketplaceDeps();
|
|
789
|
+
// No version pin → latest approved listing.
|
|
790
|
+
const listing = await d.resolveListing(slug);
|
|
791
|
+
const latest = listing.version;
|
|
792
|
+
let updateAvailable: boolean | null = null;
|
|
793
|
+
if (current && latest) {
|
|
794
|
+
// Prefer semver compare; fall back to inequality if either side isn't
|
|
795
|
+
// valid semver (so a non-semver listing version still flags a change).
|
|
796
|
+
updateAvailable =
|
|
797
|
+
semverValid(latest) && semverValid(current)
|
|
798
|
+
? semverGt(latest, current)
|
|
799
|
+
: latest !== current;
|
|
800
|
+
}
|
|
801
|
+
return { transport: 'marketplace', current, latest, updateAvailable };
|
|
802
|
+
} catch (e) {
|
|
803
|
+
return {
|
|
804
|
+
transport: 'marketplace',
|
|
805
|
+
current,
|
|
806
|
+
updateAvailable: null,
|
|
807
|
+
error: `marketplace check failed: ${(e as Error).message}`,
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// ---------------------------------------------------------------------------
|
|
813
|
+
// Marketplace artifact verification (US-021, INSTALL side)
|
|
814
|
+
//
|
|
815
|
+
// Guarantees the bytes we install are EXACTLY the bytes a moderator approved:
|
|
816
|
+
// 1. recompute sha256 over the downloaded tarball bytes and compare it to the
|
|
817
|
+
// hash the listing pinned at publish (and approval bound to);
|
|
818
|
+
// 2. verify the platform Ed25519 signature over that hash against a
|
|
819
|
+
// distributable PUBLIC key.
|
|
820
|
+
//
|
|
821
|
+
// SHARED CONTRACT (must match hq-pro `src/listings/signing.ts`):
|
|
822
|
+
// • CANONICAL BYTES = the gzipped tarball object EXACTLY as uploaded to S3
|
|
823
|
+
// (i.e. exactly the bytes we downloaded — no transform).
|
|
824
|
+
// • contentHash = sha256(canonical bytes), lowercase hex.
|
|
825
|
+
// • signature = Ed25519 over the UTF-8 bytes of the lowercase-hex
|
|
826
|
+
// contentHash, base64-encoded.
|
|
827
|
+
//
|
|
828
|
+
// This runs BEFORE safe-extract (US-020): a mismatch must abort the install
|
|
829
|
+
// before any bytes are unpacked or wired. US-006 wires the marketplace
|
|
830
|
+
// transport that surfaces the listing's {hash, signature, publicKey}; this
|
|
831
|
+
// function is the reusable verifier it calls.
|
|
832
|
+
// ---------------------------------------------------------------------------
|
|
833
|
+
|
|
834
|
+
/** Thrown when a downloaded artifact fails hash or signature verification. */
|
|
835
|
+
export class ArtifactVerificationError extends Error {
|
|
836
|
+
constructor(message: string) {
|
|
837
|
+
super(message);
|
|
838
|
+
this.name = 'ArtifactVerificationError';
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Hash algorithm for the content hash — pinned to sha256 (matches publish). */
|
|
843
|
+
export const ARTIFACT_HASH_ALG = 'sha256' as const;
|
|
844
|
+
|
|
845
|
+
export interface VerifyArtifactInput {
|
|
846
|
+
/** The downloaded gzipped tarball bytes (canonical bytes). */
|
|
847
|
+
tarballBytes: Uint8Array;
|
|
848
|
+
/** Lowercase-hex sha256 the listing pinned at publish (approval bound to it). */
|
|
849
|
+
expectedHash: string;
|
|
850
|
+
/**
|
|
851
|
+
* Base64 Ed25519 signature over the lowercase-hex `expectedHash`. Optional:
|
|
852
|
+
* when the listing carries no signature (e.g. published before signing-key
|
|
853
|
+
* provisioning), hash verification still runs but signature verification is
|
|
854
|
+
* skipped UNLESS `requireSignature` is set.
|
|
855
|
+
*/
|
|
856
|
+
signature?: string;
|
|
857
|
+
/**
|
|
858
|
+
* The platform PUBLIC key (Ed25519) as a PEM/SPKI string or a pre-parsed
|
|
859
|
+
* KeyObject. Required to verify a signature. The public key is distributable
|
|
860
|
+
* (embedded in / fetched by the installer).
|
|
861
|
+
*/
|
|
862
|
+
publicKey?: string | KeyObject;
|
|
863
|
+
/**
|
|
864
|
+
* When true, a missing signature OR missing public key is a hard failure
|
|
865
|
+
* (production posture once signing keys are provisioned). Default false so an
|
|
866
|
+
* unsigned listing still hash-verifies during the deferred-key window.
|
|
867
|
+
*/
|
|
868
|
+
requireSignature?: boolean;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/** Lowercase-hex sha256 of the given bytes — the install-side content hash. */
|
|
872
|
+
export function computeArtifactHash(tarballBytes: Uint8Array): string {
|
|
873
|
+
return createHash(ARTIFACT_HASH_ALG).update(tarballBytes).digest('hex');
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Verify a downloaded marketplace artifact's integrity (hash) and authenticity
|
|
878
|
+
* (signature). Throws `ArtifactVerificationError` on ANY mismatch — the caller
|
|
879
|
+
* MUST let it propagate so the install aborts before extraction.
|
|
880
|
+
*
|
|
881
|
+
* Constant-time-ish hash comparison: hashes are fixed-length lowercase hex, and
|
|
882
|
+
* the early-exit risk is negligible (the hash is public), but we still compare
|
|
883
|
+
* full strings rather than prefixes.
|
|
884
|
+
*/
|
|
885
|
+
export function verifyArtifact(input: VerifyArtifactInput): void {
|
|
886
|
+
const {
|
|
887
|
+
tarballBytes,
|
|
888
|
+
expectedHash,
|
|
889
|
+
signature,
|
|
890
|
+
publicKey,
|
|
891
|
+
requireSignature = false,
|
|
892
|
+
} = input;
|
|
893
|
+
|
|
894
|
+
if (!expectedHash || !/^[0-9a-f]{64}$/.test(expectedHash)) {
|
|
895
|
+
throw new ArtifactVerificationError(
|
|
896
|
+
`Refusing to install: listing did not provide a valid ${ARTIFACT_HASH_ALG} content hash.`,
|
|
897
|
+
);
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// 1. Integrity — recompute the hash over the EXACT downloaded bytes.
|
|
901
|
+
const actualHash = computeArtifactHash(tarballBytes);
|
|
902
|
+
if (actualHash !== expectedHash) {
|
|
903
|
+
throw new ArtifactVerificationError(
|
|
904
|
+
`Artifact hash mismatch — refusing to install. ` +
|
|
905
|
+
`expected ${ARTIFACT_HASH_ALG}=${expectedHash}, got ${actualHash}. ` +
|
|
906
|
+
`The downloaded bytes are NOT the bytes that were approved.`,
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// 2. Authenticity — verify the Ed25519 signature over the hash.
|
|
911
|
+
if (!signature || !publicKey) {
|
|
912
|
+
if (requireSignature) {
|
|
913
|
+
throw new ArtifactVerificationError(
|
|
914
|
+
signature
|
|
915
|
+
? 'Artifact signature present but no public key available to verify it — refusing to install.'
|
|
916
|
+
: 'Artifact is unsigned and a signature is required — refusing to install.',
|
|
917
|
+
);
|
|
918
|
+
}
|
|
919
|
+
// Deferred-key window: hash verified, no signature to check. Caller opted to
|
|
920
|
+
// allow unsigned (requireSignature=false).
|
|
921
|
+
return;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
let key: KeyObject;
|
|
925
|
+
try {
|
|
926
|
+
key = typeof publicKey === 'string' ? createPublicKey(publicKey) : publicKey;
|
|
927
|
+
} catch (e) {
|
|
928
|
+
throw new ArtifactVerificationError(
|
|
929
|
+
`Invalid artifact public key — refusing to install: ${(e as Error).message}`,
|
|
930
|
+
);
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
let ok = false;
|
|
934
|
+
try {
|
|
935
|
+
// Ed25519: digest algorithm is null (the scheme hashes internally). The
|
|
936
|
+
// signed payload is the UTF-8 bytes of the lowercase-hex hash — identical to
|
|
937
|
+
// what hq-pro signed at publish.
|
|
938
|
+
ok = cryptoVerify(
|
|
939
|
+
null,
|
|
940
|
+
Buffer.from(expectedHash, 'utf-8'),
|
|
941
|
+
key,
|
|
942
|
+
Buffer.from(signature, 'base64'),
|
|
943
|
+
);
|
|
944
|
+
} catch (e) {
|
|
945
|
+
throw new ArtifactVerificationError(
|
|
946
|
+
`Artifact signature verification errored — refusing to install: ${(e as Error).message}`,
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
if (!ok) {
|
|
950
|
+
throw new ArtifactVerificationError(
|
|
951
|
+
'Artifact signature is invalid — refusing to install. ' +
|
|
952
|
+
'The artifact was not signed by the platform key (possible tampering).',
|
|
953
|
+
);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
430
957
|
// ---------------------------------------------------------------------------
|
|
431
958
|
// Manifest validation (spec §Validation, 10 checks)
|
|
432
959
|
// ---------------------------------------------------------------------------
|
|
433
960
|
|
|
434
|
-
function validateManifest(
|
|
961
|
+
export function validateManifest(
|
|
435
962
|
payloadDir: string,
|
|
436
963
|
hqVersion: string | null
|
|
437
964
|
): PackManifest {
|
|
@@ -506,6 +1033,28 @@ function validateManifest(
|
|
|
506
1033
|
}
|
|
507
1034
|
}
|
|
508
1035
|
}
|
|
1036
|
+
// author + capabilities (US-001) — both OPTIONAL and backwards-compatible.
|
|
1037
|
+
// Absent → fine (legacy packs). Present → must be well-shaped so a malformed
|
|
1038
|
+
// attribution can't masquerade as a valid one.
|
|
1039
|
+
if (m.author !== undefined) {
|
|
1040
|
+
const a = m.author as unknown as Record<string, unknown>;
|
|
1041
|
+
if (!a || typeof a !== 'object' || Array.isArray(a)) {
|
|
1042
|
+
throw new Error('author must be a mapping with uid, handle, displayName');
|
|
1043
|
+
}
|
|
1044
|
+
for (const field of ['uid', 'handle', 'displayName'] as const) {
|
|
1045
|
+
if (typeof a[field] !== 'string' || (a[field] as string).trim() === '') {
|
|
1046
|
+
throw new Error(`author.${field} must be a non-empty string`);
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
if (m.capabilities !== undefined) {
|
|
1051
|
+
if (
|
|
1052
|
+
!Array.isArray(m.capabilities) ||
|
|
1053
|
+
!m.capabilities.every((c) => typeof c === 'string')
|
|
1054
|
+
) {
|
|
1055
|
+
throw new Error('capabilities must be a list of strings');
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
509
1058
|
return m as PackManifest;
|
|
510
1059
|
}
|
|
511
1060
|
|
|
@@ -629,13 +1178,32 @@ export function installToPackages(
|
|
|
629
1178
|
const packagesDir = path.join(hqRoot, 'core', 'packages');
|
|
630
1179
|
fs.mkdirSync(packagesDir, { recursive: true });
|
|
631
1180
|
const destDir = path.join(packagesDir, pkg.name);
|
|
632
|
-
|
|
633
|
-
|
|
1181
|
+
|
|
1182
|
+
// SECURITY/ATOMICITY (US-020): wire atomically. rsync into a sibling staging
|
|
1183
|
+
// dir on the SAME filesystem first; only swap it into the final destination
|
|
1184
|
+
// once the copy fully succeeds. A failed/partial rsync therefore never
|
|
1185
|
+
// leaves a half-wired pack dir in place (which scan-packages.sh would then
|
|
1186
|
+
// wire into host paths). On any error the staging dir is rolled back.
|
|
1187
|
+
const stagingDir = fs.mkdtempSync(path.join(packagesDir, `.${pkg.name}.staging-`));
|
|
1188
|
+
try {
|
|
1189
|
+
// rsync preserves modes/symlinks; argv form — no shell.
|
|
1190
|
+
const srcSlashed = payloadDir.endsWith('/') ? payloadDir : `${payloadDir}/`;
|
|
1191
|
+
const stagingSlashed = stagingDir.endsWith('/') ? stagingDir : `${stagingDir}/`;
|
|
1192
|
+
execFileSync('rsync', ['-a', srcSlashed, stagingSlashed], { stdio: 'inherit' });
|
|
1193
|
+
|
|
1194
|
+
// Swap into place: remove any prior install, then atomic rename. The brief
|
|
1195
|
+
// window between rm and rename is unavoidable with a same-name dir, but the
|
|
1196
|
+
// staged copy guarantees the NEW contents are fully present before we touch
|
|
1197
|
+
// the live location.
|
|
1198
|
+
if (fs.existsSync(destDir)) {
|
|
1199
|
+
fs.rmSync(destDir, { recursive: true, force: true });
|
|
1200
|
+
}
|
|
1201
|
+
fs.renameSync(stagingDir, destDir);
|
|
1202
|
+
} finally {
|
|
1203
|
+
if (fs.existsSync(stagingDir)) {
|
|
1204
|
+
fs.rmSync(stagingDir, { recursive: true, force: true });
|
|
1205
|
+
}
|
|
634
1206
|
}
|
|
635
|
-
// rsync preserves modes/symlinks; tar also works. argv form — no shell.
|
|
636
|
-
const srcSlashed = payloadDir.endsWith('/') ? payloadDir : `${payloadDir}/`;
|
|
637
|
-
const destSlashed = destDir.endsWith('/') ? destDir : `${destDir}/`;
|
|
638
|
-
execFileSync('rsync', ['-a', srcSlashed, destSlashed], { stdio: 'inherit' });
|
|
639
1207
|
return destDir;
|
|
640
1208
|
}
|
|
641
1209
|
|
|
@@ -745,6 +1313,25 @@ export interface InstallPackOptions {
|
|
|
745
1313
|
* `rsync -a`, `git clone` progress -> stderr), so this is sufficient.
|
|
746
1314
|
*/
|
|
747
1315
|
quiet?: boolean;
|
|
1316
|
+
/**
|
|
1317
|
+
* US-021 marketplace artifact integrity. When provided (a marketplace listing
|
|
1318
|
+
* carries a pinned content hash + signature + public key), the downloaded
|
|
1319
|
+
* tarball is verified BEFORE extraction; a mismatch aborts the install. US-006
|
|
1320
|
+
* populates this from the listing detail response.
|
|
1321
|
+
*/
|
|
1322
|
+
integrity?: ArtifactIntegrity;
|
|
1323
|
+
/**
|
|
1324
|
+
* US-006 marketplace transport seams (resolve/refresh/download), injected for
|
|
1325
|
+
* tests. When omitted, `defaultMarketplaceDeps()` (public listings API +
|
|
1326
|
+
* presigned S3) is used. Only consulted when `source` is `marketplace:...`.
|
|
1327
|
+
*/
|
|
1328
|
+
marketplaceDeps?: MarketplaceDeps;
|
|
1329
|
+
/**
|
|
1330
|
+
* US-006: require a valid signature for marketplace artifacts. Default false
|
|
1331
|
+
* during the signing-key-deferral window (the content hash is still ALWAYS
|
|
1332
|
+
* verified); set true once platform signing keys are provisioned.
|
|
1333
|
+
*/
|
|
1334
|
+
requireSignature?: boolean;
|
|
748
1335
|
}
|
|
749
1336
|
|
|
750
1337
|
export async function installPack(
|
|
@@ -764,7 +1351,7 @@ export async function installPack(
|
|
|
764
1351
|
let fetched: FetchResult;
|
|
765
1352
|
switch (transport) {
|
|
766
1353
|
case 'npm':
|
|
767
|
-
fetched = fetchNpm(source, tmpDir);
|
|
1354
|
+
fetched = fetchNpm(source, tmpDir, opts.integrity);
|
|
768
1355
|
break;
|
|
769
1356
|
case 'git':
|
|
770
1357
|
fetched = fetchGit(source, tmpDir, opts.followBranch ?? false);
|
|
@@ -772,6 +1359,14 @@ export async function installPack(
|
|
|
772
1359
|
case 'local':
|
|
773
1360
|
fetched = fetchLocal(source, tmpDir);
|
|
774
1361
|
break;
|
|
1362
|
+
case 'marketplace':
|
|
1363
|
+
fetched = await fetchMarketplace(
|
|
1364
|
+
source,
|
|
1365
|
+
tmpDir,
|
|
1366
|
+
opts.marketplaceDeps ?? defaultMarketplaceDeps(),
|
|
1367
|
+
{ requireSignature: opts.requireSignature },
|
|
1368
|
+
);
|
|
1369
|
+
break;
|
|
775
1370
|
}
|
|
776
1371
|
|
|
777
1372
|
const pkg = validateManifest(fetched.payloadDir, hqVersion);
|
|
@@ -812,7 +1407,14 @@ export async function installPack(
|
|
|
812
1407
|
// setup.sh can dedup against `core/core.yaml:recommended_packages` on
|
|
813
1408
|
// re-runs. Stamping the literal input (not the resolved SHA/version)
|
|
814
1409
|
// matches the verbatim equality check in setup.sh.
|
|
815
|
-
|
|
1410
|
+
//
|
|
1411
|
+
// US-006 EXCEPTION: for the marketplace transport we stamp the RESOLVED
|
|
1412
|
+
// source (`marketplace:<slug>@<version>`) rather than the literal input.
|
|
1413
|
+
// The version is what lets `hq packs update` detect a newer listing — the
|
|
1414
|
+
// bare `marketplace:<slug>` input carries no version to compare against.
|
|
1415
|
+
const stampedSource =
|
|
1416
|
+
transport === 'marketplace' ? fetched.resolvedSource : source;
|
|
1417
|
+
stampInstallSource(destDir, stampedSource);
|
|
816
1418
|
runScanPackages(hqRoot, { quiet: opts.quiet });
|
|
817
1419
|
|
|
818
1420
|
say(
|