@rightkit/release 0.6.6 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -9
- package/admission-files.mjs +3 -2
- package/advisory-db.json +4 -0
- package/advisory-db.mjs +124 -0
- package/advisory-exceptions.mjs +13 -10
- package/app-check.mjs +257 -38
- package/bin/darwin/rightkit-swap +0 -0
- package/bin/rightkit-agent-hook.mjs +9 -0
- package/bin/rightkit-gate.mjs +495 -344
- package/build-release.mjs +13 -6
- package/cli/right-release.mjs +29 -6
- package/compile-inputs.mjs +66 -0
- package/consumer-contracts.json +1044 -0
- package/consumer-contracts.mjs +627 -0
- package/consumer-input-contracts.mjs +422 -0
- package/consumer-policy.mjs +588 -0
- package/consumer-source-contracts.mjs +469 -0
- package/deny-default.toml +42 -0
- package/deterministic-scans.mjs +240 -29
- package/disk-command.mjs +270 -30
- package/disk-policy.json +50 -10
- package/disk-policy.mjs +482 -35
- package/disk-report.mjs +129 -0
- package/door-policy.mjs +228 -0
- package/gate-executor.mjs +30 -0
- package/gate-stages.mjs +110 -0
- package/github-updater-release.mjs +48 -8
- package/init-command.mjs +41 -0
- package/legion-semantic-contracts.mjs +428 -0
- package/ownership-class-rules.mjs +277 -0
- package/ownership-key-authority.mjs +3 -1
- package/ownership-mandatory-set.mjs +3 -2
- package/package.json +10 -3
- package/publish-cargo.mjs +5 -2
- package/publish-update.mjs +14 -11
- package/release-config.mjs +51 -6
- package/rightkit-versions.json +172 -167
- package/shell-contract-bridge.json +251 -0
- package/shell-contract.mjs +1069 -0
- package/sign-updater.mjs +24 -4
- package/source-closure-rules.mjs +204 -0
- package/stage-sidecar-command.mjs +2 -1
- package/stage-sidecar.mjs +437 -114
- package/suite-api.generated.mjs +96 -6
- package/suite-trust-roots.mjs +198 -0
package/stage-sidecar.mjs
CHANGED
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
//
|
|
7
7
|
// sidecars: [
|
|
8
8
|
// { kind: "cargo-bin", package: "mailright-engine", bin?, name?, profile?: "release" | { dev, release }, features?, manifestPath?, sha256?, dest?, clean?, platforms? },
|
|
9
|
-
// { kind: "ffmpeg", variant?, tools?, notices: { dir, sourceOfferContact }, catalog, stage?, bundleDir? },
|
|
10
|
-
// { kind: "onnxruntime", version, catalog, resourceDir?: "src-tauri/resources/runtime", as?, licenseDir?, cacheDir? },
|
|
9
|
+
// { kind: "ffmpeg", variant?, variants?, tools?, notices: { dir, sourceOfferContact }, catalog, stage?, bundleDir?, resourceDir?, sources?, probe? },
|
|
10
|
+
// { kind: "onnxruntime", version, catalog, resourceDir?: "src-tauri/resources/runtime", as?, licenseDir?, cacheDir?, companions? }, // the alias of native-library { library: "onnxruntime" }
|
|
11
|
+
// { kind: "native-library", library: "onnxruntime" | "directml" | "pdfium" | "whisper-cpp" | "ggml" | "sherpa-onnx", version, catalog, resourceDir?, licenseDir?, cacheDir?, as?, files?, companions?, source?: { dir }, platforms? },
|
|
11
12
|
// { kind: "resource", from, to, sha256?, machO?, clean?, platforms? }, // `from` may be a directory (copied recursively, every file hashed)
|
|
12
13
|
// { kind: "command", name, cmd, args?, cwd?, output, externalBin: true | (false with dest), dest?, clean?, platforms? },
|
|
13
14
|
// { kind: "remote", name, url (https), sha256, dest | to, size?, extract?, memberSha256?, executable?, clean?, platforms? },
|
|
@@ -22,11 +23,13 @@
|
|
|
22
23
|
// `catalog` is a relative path, an absolute path, { crate: "rightkit-ffmpeg", file? } (resolved through `rustright cargo
|
|
23
24
|
// metadata`) or { npm: "<pkg>/<file>" }. The wrapper form { sidecars, profiles: { dev, release } } sets the default cargo
|
|
24
25
|
// profile pair; RIGHTKIT_BUILD_MODE (dev|release, default release) selects. The target `universal-apple-darwin` builds
|
|
25
|
-
// both slices of a cargo-bin and lipo-creates `<name>-universal-apple-darwin`. `ffmpeg` and `
|
|
26
|
-
// per-arch catalog rows (each slice staged under .right-release/slices/<triple>, then lipo-merged;
|
|
26
|
+
// both slices of a cargo-bin and lipo-creates `<name>-universal-apple-darwin`. `ffmpeg`, `onnxruntime` and `native-library` do the same from their
|
|
27
|
+
// per-arch catalog rows (each slice staged under .right-release/slices/<triple>, then lipo-merged; a `universal` macOS row is staged once, lipo -archs checked, no merge), a `cmake` entry builds with
|
|
27
28
|
// CMAKE_OSX_ARCHITECTURES="arm64;x86_64". A `prebuilt` entry does it from per-arch host-built inputs (`from: { aarch64, x86_64 }`, or the
|
|
28
29
|
// staged `<name>-<slice triple>` files), each verified against its own manifest entry and then lipo-merged. An `ffmpeg` source may be a
|
|
29
|
-
// local path (`sources.ffmpeg: { path: "../vendor/ffmpeg-lgpl" }`) instead of a pinned https archive.
|
|
30
|
+
// local path (`sources.ffmpeg: { path: "../vendor/ffmpeg-lgpl" }`) instead of a pinned https archive. An `ffmpeg` entry with `resourceDir`
|
|
31
|
+
// stages `<resourceDir>/<tool>[.exe]` (no target triple, not an externalBin) from the same rows, with the same lipo merge, notices, digest
|
|
32
|
+
// check, sources, probe and receipt; `bundleDir` belongs to the externalBin layout, so the two keys are refused together.
|
|
30
33
|
//
|
|
31
34
|
// It never signs (release.mjs signs staged files) and never calls bare cargo. Every effect goes through `deps`.
|
|
32
35
|
import { createHash } from "node:crypto";
|
|
@@ -38,7 +41,7 @@ import { resolveCommand } from "./development.mjs";
|
|
|
38
41
|
import { managedHost } from "./managed-cargo.mjs";
|
|
39
42
|
import { APPROVAL_MACHO_STRIP_LINKEDIT_V1, APPROVAL_RAW_SHA256, approvalDigest } from "./macho-approval-hash.mjs";
|
|
40
43
|
import {
|
|
41
|
-
planMediaSidecar, sourceOfferText, stagedFileName, validateMediaSidecarConfig, verifyStagedSidecars, writeMediaSidecarNotices,
|
|
44
|
+
bundledFileName, planMediaSidecar, sourceOfferText, stagedFileName, validateMediaSidecarConfig, verifyStagedSidecars, writeMediaSidecarNotices,
|
|
42
45
|
} from "./media-sidecar.mjs";
|
|
43
46
|
import { ContentDigestError, contentIdentity, digestFor } from "./content-digest.mjs";
|
|
44
47
|
|
|
@@ -60,7 +63,7 @@ export class StageSidecarError extends Error {
|
|
|
60
63
|
const fail = (code, message, details) => { throw new StageSidecarError(code, message, details); };
|
|
61
64
|
const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
62
65
|
const SHA256 = /^[0-9a-f]{64}$/;
|
|
63
|
-
const KINDS = new Set(["cargo-bin", "ffmpeg", "onnxruntime", "resource", "command", "remote", "cmake", "prebuilt"]);
|
|
66
|
+
const KINDS = new Set(["cargo-bin", "ffmpeg", "onnxruntime", "native-library", "resource", "command", "remote", "cmake", "prebuilt"]);
|
|
64
67
|
export const NESTED_BROKER_CODE = "STAGE_SIDECAR_NESTED_BROKER";
|
|
65
68
|
export const REMOTE_CACHE_DIR = ".right-release/cache/remote";
|
|
66
69
|
/** Per-arch staging for a universal ffmpeg or onnxruntime entry: `<SLICE_ROOT>/<triple>/...`, merged with lipo afterwards. */
|
|
@@ -95,10 +98,17 @@ export function normalizeArchiveMember(member) {
|
|
|
95
98
|
if (segments.length > 1 && !segments.every((segment) => MEMBER_SEGMENT.test(segment))) return { error: "has a segment with characters outside A-Z a-z 0-9 _ . + @ ~ -" };
|
|
96
99
|
return { value, byPath: segments.length > 1 };
|
|
97
100
|
}
|
|
101
|
+
/** Native-catalog libraries a `native-library` entry stages from rows (dynamic libraries); the others are served by another route. */
|
|
102
|
+
const NATIVE_LIBRARIES = Object.freeze(["onnxruntime", "directml", "pdfium", "whisper-cpp", "ggml", "sherpa-onnx"]);
|
|
103
|
+
const STATIC_CRATE_ROUTE = "is a static crate: depend on it through Cargo (its catalog row pins the crate checksum); there is no file to stage";
|
|
104
|
+
const NATIVE_OTHER_ROUTE = Object.freeze({ ffmpeg: "has its own kind: use { kind: \"ffmpeg\", ... }", "sqlite-vec": STATIC_CRATE_ROUTE, libgit2: STATIC_CRATE_ROUTE });
|
|
105
|
+
/** A catalog version token (1.24.4, chromium/8021): only compared with the rows, never used as a path. */
|
|
106
|
+
const NATIVE_VERSION = /^[A-Za-z0-9][A-Za-z0-9._+/-]{0,63}$/;
|
|
98
107
|
const KEYS = Object.freeze({
|
|
99
108
|
"cargo-bin": ["kind", "package", "bin", "name", "profile", "features", "manifestPath", "sha256", "dest", "clean", "platforms"],
|
|
100
|
-
ffmpeg: ["kind", "variant", "variants", "tools", "notices", "catalog", "stage", "bundleDir", "sources", "probe", "platforms"],
|
|
101
|
-
onnxruntime: ["kind", "version", "catalog", "resourceDir", "as", "licenseDir", "cacheDir", "platforms"],
|
|
109
|
+
ffmpeg: ["kind", "variant", "variants", "tools", "notices", "catalog", "stage", "bundleDir", "resourceDir", "sources", "probe", "platforms"],
|
|
110
|
+
onnxruntime: ["kind", "version", "catalog", "resourceDir", "as", "licenseDir", "cacheDir", "companions", "platforms"],
|
|
111
|
+
"native-library": ["kind", "library", "version", "catalog", "resourceDir", "licenseDir", "cacheDir", "as", "files", "companions", "source", "platforms"],
|
|
102
112
|
resource: ["kind", "from", "to", "sha256", "machO", "clean", "platforms"],
|
|
103
113
|
command: ["kind", "name", "cmd", "args", "cwd", "output", "externalBin", "dest", "clean", "platforms"],
|
|
104
114
|
remote: ["kind", "name", "url", "sha256", "dest", "to", "size", "extract", "memberSha256", "executable", "clean", "platforms"],
|
|
@@ -147,22 +157,27 @@ function relativePath(value, label, errors, { allowRoot = false } = {}) {
|
|
|
147
157
|
|
|
148
158
|
/** A relative output path in comparison form: no "." or empty segments, lower case (Windows and macOS are case-insensitive). */
|
|
149
159
|
const canonicalOutput = (value) => String(value).replaceAll("\\", "/").split("/").filter((segment) => segment !== "" && segment !== ".").join("/").toLowerCase();
|
|
160
|
+
/** The claim in `claimed` that `key` equals, contains or sits inside (outputs may not be nested inside each other). */
|
|
161
|
+
const findClaimClash = (claimed, key) => claimed.find((other) => other.key === key || other.key.startsWith(`${key}/`) || key.startsWith(`${other.key}/`));
|
|
150
162
|
/** `output` equals `protectedPath` or is an ancestor of it (so writing or cleaning `output` would reach it). */
|
|
151
163
|
const reaches = (output, protectedPath) => output === protectedPath || protectedPath.startsWith(`${output}/`);
|
|
152
164
|
|
|
153
165
|
/**
|
|
154
166
|
* Paths stage-sidecar never writes to or cleans, app-relative: the stage directory, the receipt, the remote cache and the
|
|
155
|
-
* `.right-release` state, onnxruntime cache directories, and the app's source and dependency roots. A destination that equals
|
|
167
|
+
* `.right-release` state, onnxruntime and native-library cache directories, and the app's source and dependency roots. A destination that equals
|
|
156
168
|
* or contains one of these is refused (validation, and again just before any delete).
|
|
157
169
|
*/
|
|
158
170
|
function protectedOutputPaths({ stageDir, receiptPath, entries = [] }) {
|
|
159
|
-
const cacheDirs = entries.filter((entry) => entry.kind === "onnxruntime" && entry.cacheDir).map((entry) => entry.cacheDir);
|
|
171
|
+
const cacheDirs = entries.filter((entry) => (entry.kind === "onnxruntime" || entry.kind === "native-library") && entry.cacheDir).map((entry) => entry.cacheDir);
|
|
160
172
|
return [stageDir, receiptPath, ".right-release", REMOTE_CACHE_DIR, ".git", "src", "src-tauri", "node_modules", "package.json", ...cacheDirs].filter(Boolean).map(canonicalOutput);
|
|
161
173
|
}
|
|
162
174
|
|
|
163
175
|
/** `<name>-<triple>[.exe]`, with `<name>-universal-apple-darwin` for the universal target. */
|
|
164
176
|
const stagedName = (name, t) => (t.arch === "universal" ? `${name}-${UNIVERSAL}` : stagedFileName(name, t.os, t.arch));
|
|
165
177
|
|
|
178
|
+
/** An ffmpeg tool's app-relative output: `<resourceDir>/<tool>[.exe]` for a resourceDir entry (no triple, not an externalBin), else `<stageDir>/<tool>-<triple>[.exe]`. */
|
|
179
|
+
const ffmpegOutput = (entry, stageDir, tool, t) => (entry.resourceDir ? `${entry.resourceDir}/${bundledFileName(tool, t.os)}` : `${stageDir}/${stagedName(tool, t)}`);
|
|
180
|
+
|
|
166
181
|
/** RIGHTKIT_BUILD_MODE: dev | release (default release). Anything else is refused. */
|
|
167
182
|
export function resolveBuildMode(env = process.env) {
|
|
168
183
|
const value = env?.RIGHTKIT_BUILD_MODE;
|
|
@@ -183,7 +198,7 @@ function parseProfile(value, label, errors) {
|
|
|
183
198
|
return { dev: value.dev, release: value.release };
|
|
184
199
|
}
|
|
185
200
|
|
|
186
|
-
const CATALOG_DEFAULT_FILE = Object.freeze({ ffmpeg: "catalog/artifacts.json", onnxruntime: "catalog/native-libraries.json" });
|
|
201
|
+
const CATALOG_DEFAULT_FILE = Object.freeze({ ffmpeg: "catalog/artifacts.json", onnxruntime: "catalog/native-libraries.json", "native-library": "catalog/native-libraries.json" });
|
|
187
202
|
|
|
188
203
|
/**
|
|
189
204
|
* A catalog reference: a relative path (inside the app), an absolute path, { crate, file? } or { npm: "<pkg>/<file>" }.
|
|
@@ -246,6 +261,19 @@ function parseLocalSource(value, label, errors) {
|
|
|
246
261
|
return { path: normalized.replace(/\/+$/, ""), ...(typeof value.member === "string" ? { member: value.member } : {}) };
|
|
247
262
|
}
|
|
248
263
|
|
|
264
|
+
/**
|
|
265
|
+
* A `native-library` `source.dir`: the directory holding the shipped file under its catalog artifactFileName (rows with no archive,
|
|
266
|
+
* measured from a retained build). Relative to the app root and MAY step out of it, or absolute. Nothing is downloaded from it.
|
|
267
|
+
*/
|
|
268
|
+
function parseSourceDir(raw, label, errors) {
|
|
269
|
+
const normalized = typeof raw === "string" ? raw.replaceAll("\\", "/") : "";
|
|
270
|
+
if (typeof raw !== "string" || !raw.trim()) { errors.push(`${label} must be a non-empty directory path (relative to the app root, or absolute)`); return undefined; }
|
|
271
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(normalized)) { errors.push(`${label} must be a directory path, not a URL (a source dir is never downloaded)`); return undefined; }
|
|
272
|
+
if (path.isAbsolute(raw) || /^[A-Za-z]:[\\/]/.test(raw)) return raw;
|
|
273
|
+
if (/^[A-Za-z]:/.test(raw) || !normalized.split("/").some((segment) => segment !== "" && segment !== "." && segment !== "..")) { errors.push(`${label} must name a directory (relative to the app root, or absolute), not the app root, a bare parent or a drive-relative path: ${raw}`); return undefined; }
|
|
274
|
+
return normalized.replace(/\/+$/, "");
|
|
275
|
+
}
|
|
276
|
+
|
|
249
277
|
const parseFfmpegSource = (value, label, errors) => (isObject(value) && "path" in value ? parseLocalSource(value, label, errors) : parseArchiveSource(value, label, errors));
|
|
250
278
|
|
|
251
279
|
/**
|
|
@@ -310,7 +338,7 @@ export function validateSidecarManifest(raw, { target, mode = "release" } = {})
|
|
|
310
338
|
const claimed = [];
|
|
311
339
|
const claim = (output, label) => {
|
|
312
340
|
const key = canonicalOutput(output);
|
|
313
|
-
const clash = claimed
|
|
341
|
+
const clash = findClaimClash(claimed, key);
|
|
314
342
|
if (clash?.key === key) errors.push(`${label} writes ${output}, which ${clash.label} already writes`);
|
|
315
343
|
else if (clash) errors.push(`${label} writes ${output}, which overlaps ${clash.output} that ${clash.label} writes (outputs may not be nested inside each other)`);
|
|
316
344
|
else claimed.push({ key, output, label });
|
|
@@ -353,7 +381,7 @@ export function validateSidecarManifest(raw, { target, mode = "release" } = {})
|
|
|
353
381
|
if (!skipped) { if (dest) claim(dest, where); else if (typeof name === "string" && stageDir) claim(stagedClaim(name), where); }
|
|
354
382
|
Object.assign(entry, { package: item.package, bin, name, profile, features: [...features], ...(manifestPath ? { manifestPath } : {}), ...(item.sha256 ? { sha256: item.sha256 } : {}), ...(dest ? { dest } : {}), ...(clean ? { clean } : {}) });
|
|
355
383
|
} else if (item.kind === "ffmpeg") {
|
|
356
|
-
const { kind: _kind, platforms: _platforms, catalog: rawCatalog, sources: rawSources, probe: rawProbe, ...rest } = item;
|
|
384
|
+
const { kind: _kind, platforms: _platforms, catalog: rawCatalog, sources: rawSources, probe: rawProbe, resourceDir: rawResourceDir, ...rest } = item;
|
|
357
385
|
if (rest.stage !== undefined && isObject(rest.stage) && rest.stage.cmd === "cargo") errors.push(`${where}: stage.cmd must not be bare cargo; use { cmd: "rustright", args: ["cargo", ...] }`);
|
|
358
386
|
if (rest.stage !== undefined && isObject(rest.stage) && rest.stage.cmd === "rightkit") errors.push(`${where}: stage.cmd "rightkit" is the retired name of the RustRight client (rustright#91); use { cmd: "rustright", args: ["cargo", ...] }`);
|
|
359
387
|
const catalogSpec = parseCatalogSpec(rawCatalog, `${where}: catalog`, errors, "ffmpeg");
|
|
@@ -362,6 +390,10 @@ export function validateSidecarManifest(raw, { target, mode = "release" } = {})
|
|
|
362
390
|
if (rawProbe === true) entry.probe = true;
|
|
363
391
|
const sources = parseFfmpegSources(rawSources, where, errors, { universal: t.arch === "universal" });
|
|
364
392
|
if (sources) entry.sources = sources;
|
|
393
|
+
// `resourceDir` stages flat files (<resourceDir>/ffmpeg[.exe]); `bundleDir` is the key of the externalBin layout, so the two never combine.
|
|
394
|
+
const resourceDir = rawResourceDir === undefined ? undefined : relativePath(rawResourceDir, `${where}: resourceDir`, errors);
|
|
395
|
+
if (rawResourceDir !== undefined && item.bundleDir !== undefined) errors.push(`${where}: bundleDir and resourceDir cannot be combined: bundleDir places an externalBin in the app bundle, resourceDir stages flat files (drop bundleDir)`);
|
|
396
|
+
if (resourceDir) entry.resourceDir = resourceDir;
|
|
365
397
|
if (!skipped) {
|
|
366
398
|
const mediaErrors = (error) => errors.push(...String(error.message).split("\n").filter((line) => line.startsWith("- ")).map((line) => `${where}: ${line.slice(2)}`).concat(error.message.startsWith("invalid") ? [] : [`${where}: ${error.message}`]));
|
|
367
399
|
if (t.arch === "universal") {
|
|
@@ -372,24 +404,53 @@ export function validateSidecarManifest(raw, { target, mode = "release" } = {})
|
|
|
372
404
|
return { triple, arch: slice.arch, media: validateMediaSidecarConfig({ ...rest, arch: slice.arch, stageDir: `${SLICE_ROOT}/${triple}` }, { platform: t.platform, hostArch: slice.arch }) };
|
|
373
405
|
});
|
|
374
406
|
entry.media = entry.slices[0].media;
|
|
375
|
-
for (const tool of entry.media.tools) claim(
|
|
407
|
+
for (const tool of entry.media.tools) claim(ffmpegOutput(entry, stageDir, tool, t), where);
|
|
376
408
|
} catch (error) { mediaErrors(error); }
|
|
377
409
|
} else {
|
|
378
410
|
try {
|
|
379
411
|
const config = validateMediaSidecarConfig({ ...rest, arch: t.arch, stageDir }, { platform: t.platform, hostArch: t.arch });
|
|
380
|
-
for (const tool of config.tools) claim(
|
|
412
|
+
for (const tool of config.tools) claim(ffmpegOutput(entry, stageDir, tool, t), where);
|
|
381
413
|
entry.media = config;
|
|
382
414
|
} catch (error) { mediaErrors(error); }
|
|
383
415
|
}
|
|
384
416
|
}
|
|
385
|
-
} else if (item.kind === "onnxruntime") {
|
|
386
|
-
|
|
417
|
+
} else if (item.kind === "onnxruntime" || item.kind === "native-library") {
|
|
418
|
+
// `onnxruntime` is the alias of `native-library` with library onnxruntime: one code path, the alias keeps its own keys and receipt kind.
|
|
419
|
+
const native = item.kind === "native-library";
|
|
420
|
+
const library = native ? item.library : "onnxruntime";
|
|
421
|
+
if (native) {
|
|
422
|
+
if (typeof library !== "string" || !library) errors.push(`${where}: library is required (one of ${NATIVE_LIBRARIES.join(", ")})`);
|
|
423
|
+
else if (Object.hasOwn(NATIVE_OTHER_ROUTE, library)) errors.push(`${where}: library ${library} ${NATIVE_OTHER_ROUTE[library]}`);
|
|
424
|
+
else if (!NATIVE_LIBRARIES.includes(library)) errors.push(`${where}: unknown library ${JSON.stringify(library)} (one of ${NATIVE_LIBRARIES.join(", ")})`);
|
|
425
|
+
if (typeof item.version !== "string" || !NATIVE_VERSION.test(item.version)) errors.push(`${where}: version must be a catalog version token (letters, digits and . _ + / -), such as 1.24.4 or chromium/8021`);
|
|
426
|
+
} else if (typeof item.version !== "string" || !/^\d+\.\d+\.\d+$/.test(item.version)) errors.push(`${where}: version must be x.y.z`);
|
|
387
427
|
const resourceDir = relativePath(item.resourceDir ?? DEFAULT_ORT_RESOURCE_DIR, `${where}: resourceDir`, errors);
|
|
388
428
|
const licenseDir = relativePath(item.licenseDir ?? (resourceDir ? `${resourceDir}/LICENSES` : "x"), `${where}: licenseDir`, errors);
|
|
389
|
-
const cacheDir = relativePath(item.cacheDir ?? ".cache/onnxruntime", `${where}: cacheDir`, errors);
|
|
390
|
-
const catalogSpec = parseCatalogSpec(item.catalog, `${where}: catalog`, errors,
|
|
429
|
+
const cacheDir = relativePath(item.cacheDir ?? (native ? ".cache/native-library" : ".cache/onnxruntime"), `${where}: cacheDir`, errors);
|
|
430
|
+
const catalogSpec = parseCatalogSpec(item.catalog, `${where}: catalog`, errors, item.kind);
|
|
391
431
|
if (item.as !== undefined && (typeof item.as !== "string" || !NAME.test(item.as))) errors.push(`${where}: as must be a bare file name`);
|
|
392
|
-
|
|
432
|
+
if (item.companions !== undefined && typeof item.companions !== "boolean") errors.push(`${where}: companions must be a boolean`);
|
|
433
|
+
let files;
|
|
434
|
+
let source;
|
|
435
|
+
if (native) {
|
|
436
|
+
if (item.files !== undefined) {
|
|
437
|
+
if (!Array.isArray(item.files) || !item.files.length || item.files.some((file) => typeof file !== "string" || !NAME.test(file))) errors.push(`${where}: files must be a non-empty list of bare file names (catalog artifactFileName values)`);
|
|
438
|
+
else if (new Set(item.files).size !== item.files.length) errors.push(`${where}: files names a file twice`);
|
|
439
|
+
else files = [...item.files];
|
|
440
|
+
if (item.as !== undefined && Array.isArray(item.files) && item.files.length !== 1) errors.push(`${where}: as renames one file, so files must list exactly one (it lists ${item.files.length})`);
|
|
441
|
+
}
|
|
442
|
+
if (item.source !== undefined) {
|
|
443
|
+
if (!isObject(item.source)) errors.push(`${where}: source must be { dir }`);
|
|
444
|
+
else {
|
|
445
|
+
for (const key of Object.keys(item.source)) if (key !== "dir") errors.push(`${where}: source: unknown key ${JSON.stringify(key)} (allowed: dir)`);
|
|
446
|
+
source = parseSourceDir(item.source.dir, `${where}: source.dir`, errors);
|
|
447
|
+
if (source) source = { dir: source };
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
// Only the names known now are claimed; a catalog-driven entry (no files, no as) writes whatever its rows name.
|
|
451
|
+
if (!skipped && resourceDir) for (const name of typeof item.as === "string" ? [item.as] : (files ?? [])) claim(`${resourceDir}/${name}`, where);
|
|
452
|
+
}
|
|
453
|
+
Object.assign(entry, { library, version: item.version, resourceDir, licenseDir, cacheDir, companions: item.companions !== false, ...(catalogSpec ? { catalogSpec } : {}), ...(item.as ? { as: item.as } : {}), ...(files ? { files } : {}), ...(source ? { source } : {}) });
|
|
393
454
|
} else if (item.kind === "command") {
|
|
394
455
|
const name = item.name;
|
|
395
456
|
if (typeof name !== "string" || !NAME.test(name) || /\.exe$/i.test(name) || name !== path.basename(name)) errors.push(`${where}: name must be a bare sidecar name (no path, no .exe, no target triple)`);
|
|
@@ -496,7 +557,7 @@ export function validateSidecarManifest(raw, { target, mode = "release" } = {})
|
|
|
496
557
|
if (hit) errors.push(`${label} writes ${output}, which is or contains ${hit}; stage-sidecar never overwrites or cleans the app's stage directory, receipt, cache, .git, src, src-tauri, node_modules or package.json`);
|
|
497
558
|
}
|
|
498
559
|
if (errors.length) fail("STAGE_SIDECAR_CONFIG_INVALID", `invalid sidecars configuration:\n${errors.map((line) => `- ${line}`).join("\n")}`, { errors });
|
|
499
|
-
return { target: t, stageDir, receiptPath, wipeStageDir: wrapper.wipeStageDir === true, entries };
|
|
560
|
+
return { target: t, stageDir, receiptPath, wipeStageDir: wrapper.wipeStageDir === true, entries, claims: claimed };
|
|
500
561
|
}
|
|
501
562
|
|
|
502
563
|
const sha256Of = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
@@ -792,7 +853,11 @@ async function fetchFfmpegSources(entry, ctx, plan, arch) {
|
|
|
792
853
|
for (const file of plan.files) {
|
|
793
854
|
const spec = ffmpegSourceFor(entry.sources, file.tool, arch);
|
|
794
855
|
if (!spec) continue;
|
|
795
|
-
if (fs.existsSync(file.stagedPath) && catalogContentDigest(fs.readFileSync(file.stagedPath), arch) === file.contentSha256) {
|
|
856
|
+
if (fs.existsSync(file.stagedPath) && catalogContentDigest(fs.readFileSync(file.stagedPath), arch) === file.contentSha256) {
|
|
857
|
+
ctx.log(`stage-sidecar: ${rel(ctx, file.stagedPath)} already staged`);
|
|
858
|
+
if (!ctx.windows) fs.chmodSync(file.stagedPath, 0o755);
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
796
861
|
if (spec.path) {
|
|
797
862
|
const local = resolveLocalFfmpegSource(ctx, spec, file.tool);
|
|
798
863
|
const actual = catalogContentDigest(fs.readFileSync(local), arch);
|
|
@@ -841,11 +906,13 @@ function probeFfmpegBinary(ctx, { file, tool, target }) {
|
|
|
841
906
|
/**
|
|
842
907
|
* Stage one architecture of an ffmpeg entry: plan the catalog rows, fetch declared sources, run the optional stage command,
|
|
843
908
|
* write the notices (unless a universal merge writes them once for every slice), verify each staged file against its row.
|
|
844
|
-
*
|
|
909
|
+
* `flat` (a thin target of a `resourceDir` entry) moves each file to `<resourceDir>/<tool>[.exe]`; a universal slice stays
|
|
910
|
+
* `<SLICE_ROOT>/<triple>/<tool>-<triple>[.exe]` and only its merge result is flat. Returns { plan, rows }; each row is a receipt row.
|
|
845
911
|
*/
|
|
846
|
-
async function stageFfmpegSlice(entry, ctx, resolved, { config, t, writeNotices }) {
|
|
912
|
+
async function stageFfmpegSlice(entry, ctx, resolved, { config, t, writeNotices, flat = false }) {
|
|
847
913
|
const { appRoot, run, env, fs } = ctx;
|
|
848
|
-
const
|
|
914
|
+
const planned = planMediaSidecar({ config, catalog: resolved.catalog, crateDir: resolved.crateDir, crateVersion: resolved.crateVersion, catalogBytes: resolved.catalogBytes, root: appRoot });
|
|
915
|
+
const plan = flat ? { ...planned, files: planned.files.map((file) => ({ ...file, stagedPath: path.resolve(appRoot, entry.resourceDir, file.bundledName) })) } : planned;
|
|
849
916
|
await fetchFfmpegSources(entry, ctx, plan, t.arch);
|
|
850
917
|
if (config.stage) {
|
|
851
918
|
const universal = ctx.target.arch === "universal";
|
|
@@ -863,12 +930,12 @@ async function stageFfmpegSlice(entry, ctx, resolved, { config, t, writeNotices
|
|
|
863
930
|
const contentDigest = catalogContentDigest(fs.readFileSync(file.stagedPath), t.arch);
|
|
864
931
|
if (contentDigest !== file.contentSha256) fail("STAGE_SIDECAR_HASH_MISMATCH", `${file.tool} staged with content digest ${contentDigest}, the approved catalog row ${file.artifactId} says ${file.contentSha256}`, { expected: file.contentSha256, actual: contentDigest });
|
|
865
932
|
const probe = entry.probe ? probeFfmpegBinary(ctx, { file: file.stagedPath, tool: file.tool, target: t }) : undefined;
|
|
866
|
-
return { kind: "ffmpeg", name: file.tool, path: rel(ctx, file.stagedPath), sha256: file.preSignSha256, sizeBytes: file.preSignSizeBytes, changed: null, catalogRow: file.artifactId, version: file.version, catalogContentSha256: file.contentSha256, ...(file.variant ? { variant: file.variant } : {}), externalBin:
|
|
933
|
+
return { kind: "ffmpeg", name: file.tool, path: rel(ctx, file.stagedPath), sha256: file.preSignSha256, sizeBytes: file.preSignSizeBytes, changed: null, catalogRow: file.artifactId, version: file.version, catalogContentSha256: file.contentSha256, ...(file.variant ? { variant: file.variant } : {}), externalBin: !flat, license: "LGPL-2.1-or-later", notices, ...(probe ? { probe } : {}) };
|
|
867
934
|
});
|
|
868
935
|
return { plan, rows };
|
|
869
936
|
}
|
|
870
937
|
|
|
871
|
-
/** universal-apple-darwin: stage each arch slice under SLICE_ROOT, then lipo-merge every tool into <stageDir>/<tool>-universal-apple-darwin. */
|
|
938
|
+
/** universal-apple-darwin: stage each arch slice under SLICE_ROOT, then lipo-merge every tool into <stageDir>/<tool>-universal-apple-darwin (<resourceDir>/<tool> for a resourceDir entry). */
|
|
872
939
|
async function stageFfmpegUniversal(entry, ctx, resolved) {
|
|
873
940
|
const { appRoot, fs } = ctx;
|
|
874
941
|
const results = [];
|
|
@@ -889,12 +956,12 @@ async function stageFfmpegUniversal(entry, ctx, resolved) {
|
|
|
889
956
|
const notices = noticeFiles.map((file) => ({ name: path.basename(file), path: rel(ctx, file), sha256: sha256Of(fs.readFileSync(file)) }));
|
|
890
957
|
return entry.media.tools.map((tool) => {
|
|
891
958
|
const parts = results.map(({ slice, rows }) => ({ slice, row: rows.find((row) => row.name === tool) }));
|
|
892
|
-
const dest = path.resolve(appRoot, ctx.stageDir,
|
|
959
|
+
const dest = path.resolve(appRoot, ffmpegOutput(entry, ctx.stageDir, tool, ctx.target));
|
|
893
960
|
const merged = lipoMergeInto(ctx, parts.map(({ row }) => path.resolve(appRoot, row.path)), dest, `ffmpeg sidecar ${tool}`, { executable: true });
|
|
894
961
|
const probe = entry.probe ? probeFfmpegBinary(ctx, { file: dest, tool, target: ctx.target }) : undefined;
|
|
895
962
|
return {
|
|
896
963
|
kind: "ffmpeg", name: tool, path: rel(ctx, dest), ...merged, sourceSha256: merged.sha256, catalogRow: parts.map(({ row }) => row.catalogRow).join("+"), version: parts[0].row.version,
|
|
897
|
-
externalBin:
|
|
964
|
+
externalBin: !entry.resourceDir, license: "LGPL-2.1-or-later", notices, ...(probe ? { probe } : {}),
|
|
898
965
|
slices: parts.map(({ slice, row }) => ({ triple: slice.triple, sha256: row.sha256, catalogRow: row.catalogRow, catalogContentSha256: row.catalogContentSha256 })),
|
|
899
966
|
};
|
|
900
967
|
});
|
|
@@ -910,7 +977,7 @@ async function stageFfmpeg(entry, ctx) {
|
|
|
910
977
|
resolved = readJsonCatalog(resolveCatalogFile(entry.catalogSpec, ctx, "ffmpeg"), appRoot, fs, "rightkit-ffmpeg");
|
|
911
978
|
}
|
|
912
979
|
if (target.arch === "universal") return await stageFfmpegUniversal(entry, ctx, resolved);
|
|
913
|
-
return (await stageFfmpegSlice(entry, ctx, resolved, { config, t: target, writeNotices: true })).rows;
|
|
980
|
+
return (await stageFfmpegSlice(entry, ctx, resolved, { config, t: target, writeNotices: true, flat: Boolean(entry.resourceDir) })).rows;
|
|
914
981
|
} catch (error) {
|
|
915
982
|
if (error instanceof StageSidecarError || error instanceof ContentDigestError) throw error;
|
|
916
983
|
return fail("STAGE_SIDECAR_FFMPEG_FAILED", `ffmpeg sidecar (${target.triple}): ${error.message}`, { cause: error, causeCode: error.code });
|
|
@@ -920,124 +987,369 @@ async function stageFfmpeg(entry, ctx) {
|
|
|
920
987
|
/** Catalog statuses that may serve a staged library; `stale-unverified` and `planned` rows never do. */
|
|
921
988
|
const SERVING_STATUS = new Set(["approved", "pinned"]);
|
|
922
989
|
|
|
990
|
+
/** An aarch64 or x86_64 macOS target: a `universal` row serves it when no exact row does. */
|
|
991
|
+
const appleThin = (t) => t.os === "macos" && (t.arch === "aarch64" || t.arch === "x86_64");
|
|
992
|
+
/** Does `row` serve target `t`: the exact (os, arch), or a macOS `universal` row for an Apple thin target. */
|
|
993
|
+
const rowServes = (row, t) => row.os === t.os && (row.arch === t.arch || (appleThin(t) && row.arch === "universal"));
|
|
994
|
+
|
|
923
995
|
/**
|
|
924
|
-
* The
|
|
925
|
-
* exact-(os, arch)
|
|
926
|
-
* target. A stale-unverified or planned row is never selected. With nothing serving, the typed error names the exact-arch row's
|
|
927
|
-
*
|
|
996
|
+
* The catalog rows of the entry's library and version that serve target `t` (mirrors rightkit-onnx provision.rs select()): the approved or
|
|
997
|
+
* pinned exact-(os, arch) rows win; only when none exists do the macOS `universal` rows (both Apple slices) serve an aarch64 or x86_64
|
|
998
|
+
* target. A stale-unverified or planned row is never selected. With nothing serving, the typed error names the exact-arch row's status
|
|
999
|
+
* when there is one, else the missing row. The onnxruntime alias stages one file, so it takes exactly one row.
|
|
928
1000
|
*/
|
|
929
|
-
function
|
|
930
|
-
const candidates = (loaded.catalog?.libraries ?? []).filter((row) => row.library ===
|
|
1001
|
+
function selectNativeRows(loaded, entry, t, where) {
|
|
1002
|
+
const candidates = (loaded.catalog?.libraries ?? []).filter((row) => row.library === entry.library && row.version === entry.version && row.os === t.os);
|
|
931
1003
|
const serving = candidates.filter((row) => SERVING_STATUS.has(row.status));
|
|
932
1004
|
const exact = serving.filter((row) => row.arch === t.arch);
|
|
933
|
-
const
|
|
934
|
-
|
|
935
|
-
if (hits.length
|
|
936
|
-
if (hits.length > 1) return fail("STAGE_SIDECAR_NOT_APPROVED", `the native catalog has ${hits.length} ${where} rows`);
|
|
1005
|
+
const hits = exact.length ? exact : appleThin(t) ? serving.filter((row) => row.arch === "universal") : [];
|
|
1006
|
+
if (hits.length > 1 && entry.kind === "onnxruntime") return fail("STAGE_SIDECAR_NOT_APPROVED", `the native catalog has ${hits.length} ${where} rows`);
|
|
1007
|
+
if (hits.length) return hits;
|
|
937
1008
|
const unserved = candidates.filter((row) => row.arch === t.arch);
|
|
938
1009
|
if (unserved.length === 0) return fail("STAGE_SIDECAR_NOT_APPROVED", `the native catalog has no ${where} row`);
|
|
939
|
-
if (unserved.length
|
|
940
|
-
return fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where}
|
|
1010
|
+
if (unserved.length === 1) return fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} row ${unserved[0].id} is ${unserved[0].status}, not approved or pinned`);
|
|
1011
|
+
return fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} rows ${unserved.map((row) => `${row.id} (${row.status})`).join(", ")} are not approved or pinned`);
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/** The rows a row's `companions` name (staged beside it): each must exist, serve the same os and arch, and be approved or pinned. */
|
|
1015
|
+
function nativeCompanionRows(loaded, parent, t) {
|
|
1016
|
+
if (parent.companions === undefined) return [];
|
|
1017
|
+
if (!Array.isArray(parent.companions) || parent.companions.some((id) => typeof id !== "string" || !id)) return fail("STAGE_SIDECAR_NOT_APPROVED", `the row ${parent.id} lists companions that are not catalog row ids`);
|
|
1018
|
+
return parent.companions.map((id) => {
|
|
1019
|
+
const found = (loaded.catalog?.libraries ?? []).filter((row) => row.id === id);
|
|
1020
|
+
if (found.length !== 1) return fail("STAGE_SIDECAR_NOT_APPROVED", `the companion ${id} of ${parent.id} matches ${found.length} native catalog rows, expected exactly one`);
|
|
1021
|
+
const [row] = found;
|
|
1022
|
+
if (!rowServes(row, t)) return fail("STAGE_SIDECAR_NOT_APPROVED", `the companion ${id} of ${parent.id} is a ${row.os}/${row.arch} row, it does not serve ${t.os}/${t.arch}`);
|
|
1023
|
+
if (!SERVING_STATUS.has(row.status)) return fail("STAGE_SIDECAR_NOT_APPROVED", `the companion ${id} of ${parent.id} is ${row.status}, not approved or pinned`);
|
|
1024
|
+
if (Array.isArray(row.companions) && row.companions.length) return fail("STAGE_SIDECAR_NOT_APPROVED", `the companion ${id} of ${parent.id} lists companions of its own, which are not staged`);
|
|
1025
|
+
return row;
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
/**
|
|
1030
|
+
* A catalog row that cannot be staged is refused before anything is downloaded or written. The onnxruntime alias checks the whole-file
|
|
1031
|
+
* sha256 only (its rows must carry one); a `native-library` row may pin sha256, contentSha256 or contentSha256ByArch (universal rows) and
|
|
1032
|
+
* needs at least one, so a row with no digest never serves. hostProof is only "never-run", and never on an approved row.
|
|
1033
|
+
*/
|
|
1034
|
+
function checkNativeRow(entry, row, where) {
|
|
1035
|
+
const bad = (message) => fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} row ${row.id} ${message}`);
|
|
1036
|
+
const native = entry.kind === "native-library";
|
|
1037
|
+
if (row.hostProof !== undefined && (row.hostProof !== "never-run" || row.status === "approved")) bad(`carries hostProof ${JSON.stringify(row.hostProof)}; only "never-run" is defined, and never on an approved row`);
|
|
1038
|
+
if (!native && row.sha256 === undefined) bad("has no sha256");
|
|
1039
|
+
if (row.artifactFileName === undefined) bad("has no artifactFileName");
|
|
1040
|
+
if (typeof row.artifactFileName !== "string" || !NAME.test(row.artifactFileName)) bad(`has an artifactFileName that is not a bare file name: ${JSON.stringify(row.artifactFileName)}`);
|
|
1041
|
+
const digest = (value, name) => { if (value !== undefined && !(typeof value === "string" && SHA256.test(value))) bad(`${name} is not a lowercase 64-hex SHA-256`); };
|
|
1042
|
+
digest(row.sha256, "sha256");
|
|
1043
|
+
if (native) {
|
|
1044
|
+
digest(row.contentSha256, "contentSha256");
|
|
1045
|
+
if (row.contentSha256ByArch !== undefined) {
|
|
1046
|
+
if (!isObject(row.contentSha256ByArch) || Object.keys(row.contentSha256ByArch).sort().join() !== SOURCE_ARCHES.join()) bad("contentSha256ByArch must be exactly { aarch64, x86_64 }");
|
|
1047
|
+
for (const arch of SOURCE_ARCHES) digest(row.contentSha256ByArch[arch], `contentSha256ByArch.${arch}`);
|
|
1048
|
+
}
|
|
1049
|
+
if (row.arch === "universal" && row.contentSha256 !== undefined) bad("is a universal row: it pins contentSha256ByArch, not contentSha256");
|
|
1050
|
+
if (row.arch !== "universal" && row.contentSha256ByArch !== undefined) bad("pins contentSha256ByArch, which only a universal row carries");
|
|
1051
|
+
if (row.sha256 === undefined && row.contentSha256 === undefined && row.contentSha256ByArch === undefined) bad("pins no digest (sha256, contentSha256 or contentSha256ByArch), so it never serves");
|
|
1052
|
+
}
|
|
1053
|
+
// A native-library entry may stage from source.dir, so an archive is optional there (and checked when present); the alias always downloads.
|
|
1054
|
+
if (row.archive !== undefined || !native) {
|
|
1055
|
+
const { archive } = row;
|
|
1056
|
+
if (!isObject(archive) || typeof archive.url !== "string" || !/^https:\/\/[^\s/]+\/\S*$/.test(archive.url) || typeof archive.sha256 !== "string" || !SHA256.test(archive.sha256)) bad("has no valid archive (an https url and a sha256 digest)");
|
|
1057
|
+
if (!archiveFileName(archive.url)) bad("has an archive url that names no file");
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
/** The cached archive's file name: the last segment of the URL path ("" when it has none). */
|
|
1062
|
+
function archiveFileName(url) {
|
|
1063
|
+
try { return path.posix.basename(new URL(url).pathname); } catch { return ""; }
|
|
941
1064
|
}
|
|
942
1065
|
|
|
943
1066
|
/**
|
|
944
1067
|
* The archive member of a catalog row: archive.member when present; else, for a macOS `.zip` archive, `<archive name without
|
|
945
1068
|
* .zip>/lib/<artifactFileName>` (as provision.rs derives it); else the bare artifactFileName.
|
|
946
1069
|
*/
|
|
947
|
-
function
|
|
1070
|
+
function nativeArchiveMember(row) {
|
|
948
1071
|
if (row.archive.member !== undefined) return row.archive.member;
|
|
949
|
-
const archiveName =
|
|
1072
|
+
const archiveName = archiveFileName(row.archive.url);
|
|
950
1073
|
if (row.os === "macos" && archiveName.endsWith(".zip")) return `${archiveName.slice(0, -".zip".length)}/lib/${row.artifactFileName}`;
|
|
951
1074
|
return row.artifactFileName;
|
|
952
1075
|
}
|
|
953
1076
|
|
|
954
1077
|
/**
|
|
955
|
-
*
|
|
956
|
-
*
|
|
1078
|
+
* Plan one target's files without any effect: the rows to stage (every serving row of the library, or the `files` named, each exactly
|
|
1079
|
+
* once), then the companion rows of each. Returns [{ row, outName, where, companionOf? }]; every row has passed checkNativeRow.
|
|
1080
|
+
* `as` renames the one file the library stages (a companion keeps its catalog name: it is loaded by that name).
|
|
1081
|
+
*/
|
|
1082
|
+
function planNativeSet(entry, ctx, loaded, t) {
|
|
1083
|
+
const where = `${entry.library} ${entry.version} ${t.os}/${t.arch}`;
|
|
1084
|
+
const hits = selectNativeRows(loaded, entry, t, where);
|
|
1085
|
+
let primary = hits;
|
|
1086
|
+
if (!entry.files) {
|
|
1087
|
+
// Every serving row is staged; a row of the same library and arch that does not serve is left out, and said so (an incomplete set is not silent).
|
|
1088
|
+
for (const row of loaded.catalog?.libraries ?? []) {
|
|
1089
|
+
if (row.library === entry.library && row.version === entry.version && row.os === t.os && row.arch === t.arch && !SERVING_STATUS.has(row.status)) ctx.warn(`stage-sidecar: not staging ${row.id} (${row.status}); only approved or pinned rows are staged`);
|
|
1090
|
+
}
|
|
1091
|
+
} else {
|
|
1092
|
+
primary = entry.files.map((name) => {
|
|
1093
|
+
const named = hits.filter((row) => row.artifactFileName === name);
|
|
1094
|
+
if (named.length === 1) return named[0];
|
|
1095
|
+
if (named.length > 1) return fail("STAGE_SIDECAR_NOT_APPROVED", `the native catalog has ${named.length} ${where} rows for ${name}`);
|
|
1096
|
+
const unserved = (loaded.catalog?.libraries ?? []).find((row) => row.library === entry.library && row.version === entry.version && row.os === t.os && row.arch === t.arch && row.artifactFileName === name);
|
|
1097
|
+
return fail("STAGE_SIDECAR_NOT_APPROVED", unserved ? `the ${where} row ${unserved.id} for ${name} is ${unserved.status}, not approved or pinned` : `the native catalog has no ${where} row for ${name}`);
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
for (const row of primary) {
|
|
1101
|
+
const same = primary.filter((other) => (other.artifactFileName ?? other.id) === (row.artifactFileName ?? row.id));
|
|
1102
|
+
if (same.length > 1) return fail("STAGE_SIDECAR_NOT_APPROVED", `the native catalog has ${same.length} ${where} rows for ${row.artifactFileName}`);
|
|
1103
|
+
}
|
|
1104
|
+
if (entry.as && primary.length > 1) fail("STAGE_SIDECAR_CONFIG_INVALID", `as ${JSON.stringify(entry.as)} renames one file, but the ${where} rows stage ${primary.length} (${primary.map((row) => row.artifactFileName).join(", ")}); drop as, or name one file in files`);
|
|
1105
|
+
const items = primary.map((row) => ({ row, outName: entry.as ?? row.artifactFileName, where }));
|
|
1106
|
+
if (entry.companions) {
|
|
1107
|
+
for (const parent of primary) {
|
|
1108
|
+
for (const row of nativeCompanionRows(loaded, parent, t)) {
|
|
1109
|
+
if (!items.some((item) => item.row.id === row.id)) items.push({ row, outName: row.artifactFileName, where: `${row.library} ${row.version} ${t.os}/${t.arch}`, companionOf: parent.id });
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
for (const item of items) checkNativeRow(entry, item.row, item.where);
|
|
1114
|
+
for (const item of items) {
|
|
1115
|
+
const clash = items.find((other) => other !== item && other.outName.toLowerCase() === item.outName.toLowerCase());
|
|
1116
|
+
if (clash) fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} rows ${item.row.id} and ${clash.row.id} both stage ${item.outName}`);
|
|
1117
|
+
}
|
|
1118
|
+
return items;
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
/**
|
|
1122
|
+
* The digest `bytes` must match for `row`, or the first disagreement { what, expected, actual }. `fromSource` is a copy read from
|
|
1123
|
+
* source.dir (not byte-identical to the pinned archive by construction, and possibly re-signed): it is held to the content digest when
|
|
1124
|
+
* the row pins one, else to the whole-file sha256. A file extracted from the pinned archive must equal the whole-file sha256 the row
|
|
1125
|
+
* has AND its content digest. The onnxruntime alias keeps the whole-file check only. A malformed recognized format is a mismatch.
|
|
957
1126
|
*/
|
|
958
|
-
|
|
1127
|
+
function nativeDigestMismatch(entry, row, bytes, { fromSource }) {
|
|
1128
|
+
const checkContent = entry.kind === "native-library" && (row.contentSha256 !== undefined || row.contentSha256ByArch !== undefined);
|
|
1129
|
+
if (row.sha256 !== undefined && !(fromSource && checkContent)) {
|
|
1130
|
+
const actual = sha256Of(bytes);
|
|
1131
|
+
if (actual !== row.sha256) return { what: "sha256", expected: row.sha256, actual };
|
|
1132
|
+
}
|
|
1133
|
+
if (!checkContent) return undefined;
|
|
1134
|
+
try {
|
|
1135
|
+
if (row.contentSha256ByArch) {
|
|
1136
|
+
const identity = contentIdentity(bytes);
|
|
1137
|
+
for (const arch of SOURCE_ARCHES) {
|
|
1138
|
+
const actual = digestFor(identity, arch);
|
|
1139
|
+
if (actual !== row.contentSha256ByArch[arch]) return { what: `${arch} content digest`, expected: row.contentSha256ByArch[arch], actual };
|
|
1140
|
+
}
|
|
1141
|
+
return undefined;
|
|
1142
|
+
}
|
|
1143
|
+
// A thin row pins one slice of one thin file: a universal Mach-O (a good slice beside a foreign one) is not that file.
|
|
1144
|
+
const identity = contentIdentity(bytes);
|
|
1145
|
+
if (identity.format === "macho-universal" || identity.slices.length !== 1) {
|
|
1146
|
+
return { what: "file identity", expected: `a thin ${row.arch} file with content digest ${row.contentSha256}`, actual: `a ${identity.format === "macho-universal" ? "universal Mach-O" : "file"} with ${identity.slices.length} slices (${identity.slices.map((slice) => slice.arch ?? "none").join(", ")}); a thin row accepts exactly one slice` };
|
|
1147
|
+
}
|
|
1148
|
+
const actual = digestFor(identity, row.arch) ?? identity.slices[0].sha256;
|
|
1149
|
+
return actual === row.contentSha256 ? undefined : { what: "content digest", expected: row.contentSha256, actual };
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
if (!(error instanceof ContentDigestError)) throw error;
|
|
1152
|
+
return { what: "content digest", expected: row.contentSha256 ?? SOURCE_ARCHES.map((arch) => row.contentSha256ByArch[arch]).join("+"), reason: error.message };
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
const failNativeMismatch = (row, label, mismatch) => fail("STAGE_SIDECAR_HASH_MISMATCH",
|
|
1157
|
+
`${label} has ${mismatch.what} ${mismatch.actual ?? `unreadable (${mismatch.reason ?? "no matching slice"})`}, the catalog row ${row.id} pins ${mismatch.expected}; nothing staged`, { expected: mismatch.expected, actual: mismatch.actual ?? null });
|
|
1158
|
+
|
|
1159
|
+
/**
|
|
1160
|
+
* Verify one planned file and find its bytes; nothing is written except the verified archive cache. A file already staged is kept only
|
|
1161
|
+
* when it passes the same digest check as a fresh one (never trusted because it exists). Otherwise the bytes come from `source.dir`
|
|
1162
|
+
* (read and checked against the row before anything is copied), or from the row's pinned archive (digest checked before use, one member
|
|
1163
|
+
* unpacked into a scratch directory the caller removes). Returns the item with its notice copies, destination and `from` (undefined when already staged).
|
|
1164
|
+
*/
|
|
1165
|
+
async function prepareNativeFile(entry, ctx, loaded, item, { destDir, scratchDirs }) {
|
|
959
1166
|
const { appRoot, fs } = ctx;
|
|
960
|
-
const
|
|
961
|
-
const row = selectOnnxRuntimeRow(loaded, entry, t, where);
|
|
962
|
-
for (const key of ["sha256", "artifactFileName"]) if (typeof row[key] !== "string" || !row[key]) fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} row ${row.id} has no ${key}`);
|
|
963
|
-
if (!SHA256.test(row.sha256) || !SHA256.test(row.archive?.sha256 ?? "") || typeof row.archive?.url !== "string") fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} row ${row.id} lacks a valid library digest or archive url and digest`);
|
|
1167
|
+
const { row, outName, where } = item;
|
|
964
1168
|
if (!Array.isArray(row.noticeFiles) || !row.noticeFiles.length) fail("STAGE_SIDECAR_NOTICE_MISSING", `the ${where} row ${row.id} lists no notice files; the licence must ship with the library`);
|
|
965
|
-
const crateDir = loaded.crateDir;
|
|
966
1169
|
const notices = row.noticeFiles.map((file) => {
|
|
967
1170
|
const normalized = String(file).replaceAll("\\", "/");
|
|
968
1171
|
if (path.isAbsolute(file) || normalized.split("/").includes("..")) fail("STAGE_SIDECAR_NOTICE_MISSING", `notice file path escapes the catalog crate: ${file}`);
|
|
969
|
-
return { source: path.join(crateDir, ...normalized.split("/")), dest: path.resolve(appRoot, entry.licenseDir, path.posix.basename(normalized)) };
|
|
1172
|
+
return { source: path.join(loaded.crateDir, ...normalized.split("/")), dest: path.resolve(appRoot, entry.licenseDir, path.posix.basename(normalized)) };
|
|
970
1173
|
});
|
|
971
1174
|
for (const { source } of notices) if (!fs.existsSync(source)) fail("STAGE_SIDECAR_NOTICE_MISSING", `notice file not found: ${source}`);
|
|
972
1175
|
|
|
973
1176
|
// The archive member is the catalog's archive.member (an archive path) when present, else the library file name; the output name never comes from it.
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1177
|
+
let member;
|
|
1178
|
+
if (!entry.source && row.archive) {
|
|
1179
|
+
member = nativeArchiveMember(row);
|
|
1180
|
+
const memberCheck = normalizeArchiveMember(member);
|
|
1181
|
+
if (memberCheck.error) fail("STAGE_SIDECAR_NOT_APPROVED", `the ${where} row ${row.id} has an invalid archive member ${JSON.stringify(member)}: ${memberCheck.error}`);
|
|
1182
|
+
}
|
|
978
1183
|
const dest = path.resolve(appRoot, destDir ?? entry.resourceDir, outName);
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
ctx.removeDir(scratch);
|
|
998
|
-
}
|
|
1184
|
+
const prepared = { ...item, dest, notices };
|
|
1185
|
+
if (fs.existsSync(dest) && !nativeDigestMismatch(entry, row, fs.readFileSync(dest), { fromSource: Boolean(entry.source) })) return prepared;
|
|
1186
|
+
|
|
1187
|
+
if (entry.source) {
|
|
1188
|
+
const file = path.resolve(appRoot, entry.source.dir, row.artifactFileName);
|
|
1189
|
+
if (!(fs.existsSync(file) && fs.statSync(file).isFile())) fail("STAGE_SIDECAR_BINARY_NOT_FOUND", `the ${where} row ${row.id} is staged from source.dir ${entry.source.dir}, which has no ${row.artifactFileName} (looked at ${file}); nothing is downloaded for a source dir`, { file });
|
|
1190
|
+
const mismatch = nativeDigestMismatch(entry, row, fs.readFileSync(file), { fromSource: true });
|
|
1191
|
+
if (mismatch) failNativeMismatch(row, file, mismatch);
|
|
1192
|
+
return { ...prepared, from: file };
|
|
1193
|
+
}
|
|
1194
|
+
if (!row.archive) fail("STAGE_SIDECAR_CONFIG_INVALID", `the ${where} row ${row.id} has no archive, so it is staged from a local build: set source: { dir } on this entry to the directory holding ${row.artifactFileName}`);
|
|
1195
|
+
const archive = path.resolve(appRoot, entry.cacheDir, archiveFileName(row.archive.url));
|
|
1196
|
+
if (!(fs.existsSync(archive) && sha256Of(fs.readFileSync(archive)) === row.archive.sha256)) {
|
|
1197
|
+
const bytes = await ctx.fetchBytes(row.archive.url);
|
|
1198
|
+
const actual = sha256Of(bytes);
|
|
1199
|
+
if (actual !== row.archive.sha256) fail("STAGE_SIDECAR_HASH_MISMATCH", `${row.archive.url} downloaded as ${actual}, the catalog pins ${row.archive.sha256}; nothing staged`, { expected: row.archive.sha256, actual });
|
|
1200
|
+
fs.mkdirSync(path.dirname(archive), { recursive: true });
|
|
1201
|
+
fs.writeFileSync(archive, bytes);
|
|
999
1202
|
}
|
|
1000
|
-
const
|
|
1203
|
+
const scratch = ctx.makeTempDir();
|
|
1204
|
+
scratchDirs.push(scratch);
|
|
1205
|
+
const library = await ctx.extractMember({ archive, fileName: member, destDir: scratch });
|
|
1206
|
+
const mismatch = nativeDigestMismatch(entry, row, fs.readFileSync(library), { fromSource: false });
|
|
1207
|
+
if (mismatch) failNativeMismatch(row, member, mismatch);
|
|
1208
|
+
return { ...prepared, from: library };
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
/** Write one verified file (unless it is already staged) and its notices; the staged bytes are re-verified against the row and re-read for the receipt. */
|
|
1212
|
+
function commitNativeFile(entry, ctx, prepared) {
|
|
1213
|
+
const { fs } = ctx;
|
|
1214
|
+
const { row, dest, from } = prepared;
|
|
1215
|
+
if (from) {
|
|
1216
|
+
copyInto(ctx, from, dest, { executable: false });
|
|
1217
|
+
// The source was read again for the copy: hold the staged bytes to the row too, so a file that changed after it was verified is never left staged.
|
|
1218
|
+
const mismatch = nativeDigestMismatch(entry, row, fs.readFileSync(dest), { fromSource: Boolean(entry.source) });
|
|
1219
|
+
if (mismatch) {
|
|
1220
|
+
fs.rmSync(dest, { force: true });
|
|
1221
|
+
failNativeMismatch(row, `${rel(ctx, dest)} (copied from ${from}, which changed after it was verified)`, mismatch);
|
|
1222
|
+
}
|
|
1223
|
+
} else ctx.log(`stage-sidecar: ${rel(ctx, dest)} already staged`);
|
|
1224
|
+
if (row.hostProof === "never-run") ctx.warn(`stage-sidecar: warning: ${row.id} has hostProof never-run: its bytes were pinned but never loaded or run on a ${row.os} host, so ${path.basename(dest)} is staged but not host-proven`);
|
|
1225
|
+
const staged = fs.readFileSync(dest);
|
|
1226
|
+
const notices = prepared.notices.map(({ source, dest: out }) => {
|
|
1001
1227
|
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
1002
1228
|
fs.copyFileSync(source, out);
|
|
1003
1229
|
return { name: path.basename(out), path: rel(ctx, out), sha256: sha256Of(fs.readFileSync(out)) };
|
|
1004
1230
|
});
|
|
1005
|
-
return {
|
|
1231
|
+
return { ...prepared, sha256: sha256Of(staged), sizeBytes: staged.length, notices };
|
|
1006
1232
|
}
|
|
1007
1233
|
|
|
1008
|
-
|
|
1234
|
+
/**
|
|
1235
|
+
* Stage one planned set (one target or one universal slice) in two phases: every file is verified first (`beforeWrite` may veto a
|
|
1236
|
+
* verified file), then everything is written, so a bad file leaves nothing of the set behind. Scratch directories are always removed.
|
|
1237
|
+
*/
|
|
1238
|
+
async function stageNativeSet(entry, ctx, loaded, items, { destDir, beforeWrite } = {}) {
|
|
1239
|
+
const scratchDirs = [];
|
|
1240
|
+
try {
|
|
1241
|
+
const prepared = [];
|
|
1242
|
+
for (const item of items) prepared.push(await prepareNativeFile(entry, ctx, loaded, item, { destDir, scratchDirs }));
|
|
1243
|
+
if (beforeWrite) for (const file of prepared) beforeWrite(file);
|
|
1244
|
+
return prepared.map((file) => commitNativeFile(entry, ctx, file));
|
|
1245
|
+
} finally {
|
|
1246
|
+
for (const dir of scratchDirs) ctx.removeDir(dir);
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
/** Is this file's receipt row evidence from the pinned archive (not source.dir)? */
|
|
1251
|
+
const fromArchive = (entry, row) => !entry.source && row.archive !== undefined;
|
|
1252
|
+
|
|
1253
|
+
/** The digests a `native-library` row pins, for its receipt row. */
|
|
1254
|
+
const pinnedContent = (row) => ({ ...(row.contentSha256 ? { contentSha256: row.contentSha256 } : {}), ...(row.contentSha256ByArch ? { contentSha256ByArch: { ...row.contentSha256ByArch } } : {}) });
|
|
1255
|
+
|
|
1256
|
+
/** The receipt row of one staged file (a thin target, or a universal row staged once). The alias keeps its original row shape. */
|
|
1257
|
+
function nativeReceiptRow(entry, ctx, file, extra = {}) {
|
|
1258
|
+
const { row } = file;
|
|
1259
|
+
const alias = entry.kind === "onnxruntime";
|
|
1260
|
+
return {
|
|
1261
|
+
kind: entry.kind, ...(alias ? {} : { library: row.library }), name: file.outName, path: rel(ctx, file.dest), sha256: file.sha256, sizeBytes: file.sizeBytes, changed: null,
|
|
1262
|
+
catalogRow: row.id, version: row.version, ...(fromArchive(entry, row) ? { archiveSha256: row.archive.sha256 } : {}), ...(alias ? {} : pinnedContent(row)),
|
|
1263
|
+
license: row.licenseSpdx ?? null, notices: file.notices, externalBin: false,
|
|
1264
|
+
...(row.hostProof ? { hostProof: row.hostProof } : {}), ...(file.companionOf ? { companionOf: file.companionOf } : {}), ...extra,
|
|
1265
|
+
};
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
/** The native catalog of an entry, loaded once per run (the claim pre-pass and the staging share it). */
|
|
1269
|
+
async function loadNativeCatalogOf(entry, ctx) {
|
|
1270
|
+
if (ctx.nativeCatalogs.has(entry)) return ctx.nativeCatalogs.get(entry);
|
|
1271
|
+
const { appRoot, fs } = ctx;
|
|
1272
|
+
const loaded = await (ctx.loadNativeCatalog({ entry, appRoot }) ?? (entry.catalogSpec ? readJsonCatalog(resolveCatalogFile(entry.catalogSpec, ctx, entry.kind), appRoot, fs, "native-libraries") : undefined));
|
|
1273
|
+
if (!loaded) fail("STAGE_SIDECAR_CATALOG_UNREADABLE", `${entry.kind} needs a catalog path (the rightkit-ffmpeg native-libraries.json) or deps.loadNativeCatalog`);
|
|
1274
|
+
ctx.nativeCatalogs.set(entry, loaded);
|
|
1275
|
+
return loaded;
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
/**
|
|
1279
|
+
* Validation claims only the file names a config states, but a native-library or onnxruntime entry writes what its catalog rows name (and
|
|
1280
|
+
* the alias stages the DirectML companion beside the library by default), so another entry can write the same file. Before anything is
|
|
1281
|
+
* written, plan each such entry without effects (a catalog or row problem is skipped here and raised in order by the staging itself), claim
|
|
1282
|
+
* the planned file names beside the validation claims, and refuse a file two entries write, naming both and the remedy.
|
|
1283
|
+
*/
|
|
1284
|
+
async function claimNativeOutputs(plan, ctx) {
|
|
1285
|
+
const natives = plan.entries.filter((entry) => !entry.skipped && (entry.kind === "onnxruntime" || entry.kind === "native-library"));
|
|
1286
|
+
if (!natives.length) return;
|
|
1287
|
+
const claimed = [...plan.claims];
|
|
1288
|
+
const quiet = { ...ctx, log: () => {}, warn: () => {} };
|
|
1289
|
+
const errors = [];
|
|
1290
|
+
for (const entry of natives) {
|
|
1291
|
+
const label = `sidecars[${entry.index}] (${entry.kind})`;
|
|
1292
|
+
let items;
|
|
1293
|
+
try {
|
|
1294
|
+
const loaded = await loadNativeCatalogOf(entry, ctx);
|
|
1295
|
+
items = (ctx.target.arch === "universal" ? SLICES.map((triple) => parseTargetTriple(triple)) : [ctx.target]).flatMap((t) => planNativeSet(entry, quiet, loaded, t));
|
|
1296
|
+
} catch { continue; }
|
|
1297
|
+
for (const item of items) {
|
|
1298
|
+
const output = `${entry.resourceDir}/${item.outName}`;
|
|
1299
|
+
const key = canonicalOutput(output);
|
|
1300
|
+
const clash = findClaimClash(claimed, key);
|
|
1301
|
+
if (!clash) { claimed.push({ key, output, label, ...(item.companionOf ? { companion: true } : {}) }); continue; }
|
|
1302
|
+
if (clash.label === label) continue;
|
|
1303
|
+
// The alias and native-library stage companions by default: whichever side is one is told how to stand down.
|
|
1304
|
+
const companionLabel = item.companionOf ? label : clash.companion ? clash.label : undefined;
|
|
1305
|
+
const remedy = companionLabel ? `; drop the other entry's copy, or set companions: false on ${companionLabel} to stage the library alone` : "";
|
|
1306
|
+
const what = item.companionOf ? `the ${item.row.library} companion ${item.outName} of ${item.companionOf}` : `${item.outName} (catalog row ${item.row.id})`;
|
|
1307
|
+
errors.push(`${label} stages ${what} into ${entry.resourceDir}, which ${clash.label} also writes (${clash.output})${remedy}`);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
if (errors.length) fail("STAGE_SIDECAR_CONFIG_INVALID", `invalid sidecars configuration:\n${[...new Set(errors)].map((line) => `- ${line}`).join("\n")}`, { errors });
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
async function stageNativeLibrary(entry, ctx) {
|
|
1009
1314
|
const { appRoot, fs, target } = ctx;
|
|
1010
|
-
const loaded = await (
|
|
1011
|
-
|
|
1315
|
+
const loaded = await loadNativeCatalogOf(entry, ctx);
|
|
1316
|
+
const rowOf = (file, extra) => nativeReceiptRow(entry, ctx, file, extra);
|
|
1012
1317
|
if (target.arch !== "universal") {
|
|
1013
|
-
const
|
|
1014
|
-
return
|
|
1318
|
+
const staged = await stageNativeSet(entry, ctx, loaded, planNativeSet(entry, ctx, loaded, target));
|
|
1319
|
+
return staged.map((file) => rowOf(file));
|
|
1015
1320
|
}
|
|
1016
|
-
|
|
1017
|
-
const
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1321
|
+
const plans = SLICES.map((triple) => ({ triple, items: planNativeSet(entry, ctx, loaded, parseTargetTriple(triple)) }));
|
|
1322
|
+
const rows = plans.flatMap((plan) => plan.items.map((item) => item.row));
|
|
1323
|
+
// universal-apple-darwin: `universal` rows serving both slices are staged once, in place, with no lipo merge (nothing mixes with exact-arch rows).
|
|
1324
|
+
if (rows.some((row) => row.arch === "universal")) {
|
|
1325
|
+
if (!rows.every((row) => row.arch === "universal") || plans[0].items.map((item) => item.row.id).join() !== plans[1].items.map((item) => item.row.id).join()) {
|
|
1326
|
+
fail("STAGE_SIDECAR_NOT_APPROVED", `the native catalog mixes a universal row with exact-arch rows (${[...new Set(rows.map((row) => row.id))].join(", ")}); universal-apple-darwin takes either one universal row or one row per arch`);
|
|
1327
|
+
}
|
|
1328
|
+
const label = (file) => `${file.row.library} ${file.row.version} ${file.row.id}`;
|
|
1329
|
+
const staged = await stageNativeSet(entry, ctx, loaded, plans[0].items, { beforeWrite: (file) => requireUniversalFile(ctx, file.from ?? file.dest, label(file)) });
|
|
1330
|
+
return staged.map((file) => rowOf(file, {
|
|
1331
|
+
sourceSha256: file.sha256,
|
|
1332
|
+
slices: SLICES.map((triple, index) => ({ triple, sha256: file.sha256, catalogRow: file.row.id, ...(fromArchive(entry, file.row) ? { archiveSha256: file.row.archive.sha256 } : {}), ...(entry.kind === "native-library" && file.row.contentSha256ByArch ? { contentSha256: file.row.contentSha256ByArch[SOURCE_ARCHES[index]] } : {}) })),
|
|
1333
|
+
}));
|
|
1027
1334
|
}
|
|
1028
|
-
// Otherwise each arch
|
|
1335
|
+
// Otherwise each arch's files are staged thin under SLICE_ROOT, then merged file by file into the entry's resource directory.
|
|
1029
1336
|
const slices = [];
|
|
1030
|
-
for (const
|
|
1031
|
-
const
|
|
1032
|
-
if (
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1337
|
+
for (const plan of plans) slices.push({ triple: plan.triple, files: await stageNativeSet(entry, ctx, loaded, plan.items, { destDir: `${SLICE_ROOT}/${plan.triple}` }) });
|
|
1338
|
+
const [first, second] = slices.map((slice) => slice.files.map((file) => file.outName));
|
|
1339
|
+
if ([...first].sort().join("\n") !== [...second].sort().join("\n")) fail("STAGE_SIDECAR_NOT_APPROVED", `the arch rows name different library files (aarch64: ${first.join(", ")}; x86_64: ${second.join(", ")}); each file needs one name on both arches (a single file can be renamed with "as")`);
|
|
1340
|
+
return first.map((name) => {
|
|
1341
|
+
const parts = slices.map((slice) => ({ triple: slice.triple, file: slice.files.find((file) => file.outName === name) }));
|
|
1342
|
+
const dest = path.resolve(appRoot, entry.resourceDir, name);
|
|
1343
|
+
const merged = lipoMergeInto(ctx, parts.map(({ file }) => file.dest), dest, `${parts[0].file.row.library} ${parts[0].file.row.version}`);
|
|
1344
|
+
const hostProof = parts.find(({ file }) => file.row.hostProof)?.file.row.hostProof;
|
|
1345
|
+
const alias = entry.kind === "onnxruntime";
|
|
1346
|
+
return {
|
|
1347
|
+
kind: entry.kind, ...(alias ? {} : { library: parts[0].file.row.library }), name, path: rel(ctx, dest), ...merged, sourceSha256: merged.sha256, catalogRow: parts.map(({ file }) => file.row.id).join("+"), version: parts[0].file.row.version,
|
|
1348
|
+
license: parts[0].file.row.licenseSpdx ?? null, notices: [...new Map(parts.flatMap(({ file }) => file.notices).map((notice) => [notice.path, notice])).values()], externalBin: false,
|
|
1349
|
+
...(hostProof ? { hostProof } : {}), ...(parts[0].file.companionOf ? { companionOf: parts.map(({ file }) => file.companionOf).join("+") } : {}),
|
|
1350
|
+
slices: parts.map(({ triple, file }) => ({ triple, sha256: file.sha256, catalogRow: file.row.id, ...(fromArchive(entry, file.row) ? { archiveSha256: file.row.archive.sha256 } : {}), ...(!alias && file.row.contentSha256 ? { contentSha256: file.row.contentSha256 } : {}) })),
|
|
1351
|
+
};
|
|
1352
|
+
});
|
|
1041
1353
|
}
|
|
1042
1354
|
|
|
1043
1355
|
/** Every file under `dir`, sorted, as paths relative to it (forward slashes). */
|
|
@@ -1345,7 +1657,8 @@ async function stagePrebuilt(entry, ctx) {
|
|
|
1345
1657
|
/**
|
|
1346
1658
|
* `wipeStageDir`: remove top-level files in stageDir named exactly `<name>-<current triple>[.exe]` for a name this manifest
|
|
1347
1659
|
* declares (a cargo-bin, command or ffmpeg tool) that this run did not produce, such as an entry now skipped on this platform
|
|
1348
|
-
* or now placed at `dest`. Never recursive; other triples, notices, directories, temporaries and unrelated files
|
|
1660
|
+
* or now placed at `dest` or an ffmpeg `resourceDir`. Never recursive; other triples, notices, directories, temporaries and unrelated files
|
|
1661
|
+
* are left alone, and it only ever lists stageDir, so a `resourceDir` file is never a candidate.
|
|
1349
1662
|
*/
|
|
1350
1663
|
function wipeStaleStageDir(ctx, plan, produced) {
|
|
1351
1664
|
const dir = path.resolve(ctx.appRoot, ctx.stageDir);
|
|
@@ -1399,14 +1712,16 @@ export function planSidecarStaging({ manifest, target, env = process.env } = {})
|
|
|
1399
1712
|
if (["cargo-bin", "command", "cmake", "prebuilt"].includes(entry.kind)) return [entry.dest ?? `${plan.stageDir}/${stagedName(entry.name, t)}`];
|
|
1400
1713
|
if (entry.kind === "remote") return [entry.dest];
|
|
1401
1714
|
if (entry.kind === "resource") return [entry.to];
|
|
1402
|
-
|
|
1403
|
-
return (entry.
|
|
1715
|
+
// Without a catalog at check time: the files named, else the one name `as` gives, else what the catalog row will say.
|
|
1716
|
+
if (entry.kind === "onnxruntime" || entry.kind === "native-library") return (entry.files && !entry.as ? entry.files : [entry.as ?? "<catalog artifactFileName>"]).map((name) => `${entry.resourceDir}/${name}`);
|
|
1717
|
+
return (entry.media?.tools ?? []).map((tool) => ffmpegOutput(entry, plan.stageDir, tool, t));
|
|
1404
1718
|
};
|
|
1405
1719
|
const entries = plan.entries.map((entry) => ({
|
|
1406
1720
|
index: entry.index, kind: entry.kind, skipped: entry.skipped, ...(entry.platforms ? { platforms: entry.platforms } : {}),
|
|
1407
|
-
...(entry.skipped ? {} : { outputs: outputsOf(entry), externalBin: ["cargo-bin", "command", "cmake", "prebuilt"].includes(entry.kind) ? !entry.dest : entry.kind === "ffmpeg" }),
|
|
1721
|
+
...(entry.skipped ? {} : { outputs: outputsOf(entry), externalBin: ["cargo-bin", "command", "cmake", "prebuilt"].includes(entry.kind) ? !entry.dest : entry.kind === "ffmpeg" && !entry.resourceDir }),
|
|
1408
1722
|
...(entry.kind === "cargo-bin" ? { package: entry.package, bin: entry.bin, profile: entry.profile } : {}),
|
|
1409
1723
|
...(entry.kind === "remote" ? { url: entry.url } : {}),
|
|
1724
|
+
...(entry.kind === "native-library" ? { library: entry.library, version: entry.version } : {}),
|
|
1410
1725
|
...(entry.kind === "cmake" ? { repo: entry.repo, commit: entry.commit } : {}),
|
|
1411
1726
|
...(entry.clean ? { clean: true } : {}),
|
|
1412
1727
|
}));
|
|
@@ -1416,7 +1731,8 @@ export function planSidecarStaging({ manifest, target, env = process.env } = {})
|
|
|
1416
1731
|
/**
|
|
1417
1732
|
* Stage every configured sidecar for `target` (a Rust triple) and write the staging receipt.
|
|
1418
1733
|
* Returns { receipt, receiptPath, files, externalBinNames }. `deps` (all optional) injects every effect:
|
|
1419
|
-
* fs (existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, renameSync, chmodSync, statSync, readdirSync, rmSync), run, fetch (https downloads of `remote`), fetchBytes, extractMember, loadMediaCatalog, loadNativeCatalog, makeTempDir, removeDir, now, pid, log, env.
|
|
1734
|
+
* fs (existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, renameSync, chmodSync, statSync, readdirSync, rmSync), run, fetch (https downloads of `remote`), fetchBytes, extractMember, loadMediaCatalog, loadNativeCatalog, makeTempDir, removeDir, now, pid, log (progress), warn (warnings; defaults to log), env.
|
|
1735
|
+
* The API is quiet by default; the `right-release stage-sidecar` command passes `warn` so the CLI prints warnings on stderr.
|
|
1420
1736
|
*/
|
|
1421
1737
|
export async function stageSidecars({ appRoot, manifest, target, deps = {} } = {}) {
|
|
1422
1738
|
if (typeof appRoot !== "string" || !path.isAbsolute(appRoot)) fail("STAGE_SIDECAR_CONFIG_INVALID", "appRoot must be an absolute path");
|
|
@@ -1432,6 +1748,9 @@ export async function stageSidecars({ appRoot, manifest, target, deps = {} } = {
|
|
|
1432
1748
|
host: deps.host ?? { os: { darwin: "macos", win32: "windows", linux: "linux" }[process.platform] ?? process.platform, arch: { arm64: "aarch64", x64: "x86_64" }[process.arch] ?? process.arch },
|
|
1433
1749
|
approvalDigest: deps.approvalDigest ?? ((file, algorithm) => approvalDigest(file, algorithm)),
|
|
1434
1750
|
log: deps.log ?? (() => {}),
|
|
1751
|
+
// Warnings (a never-run row, a row left out) go to deps.warn; without one they join the progress log, so a caller that sets only `log` still sees them.
|
|
1752
|
+
warn: deps.warn ?? deps.log ?? (() => {}),
|
|
1753
|
+
nativeCatalogs: new Map(),
|
|
1435
1754
|
fetchBytes: deps.fetchBytes ?? defaultFetchBytes,
|
|
1436
1755
|
fetch: deps.fetch ?? ((...args) => globalThis.fetch(...args)),
|
|
1437
1756
|
extractMember: deps.extractMember ?? ((args) => defaultExtractMember({ ...args, run })),
|
|
@@ -1440,13 +1759,14 @@ export async function stageSidecars({ appRoot, manifest, target, deps = {} } = {
|
|
|
1440
1759
|
makeTempDir: deps.makeTempDir ?? (() => nodeFs.mkdtempSync(path.join(os.tmpdir(), "rightkit-stage-"))),
|
|
1441
1760
|
removeDir: deps.removeDir ?? ((dir) => nodeFs.rmSync(dir, { recursive: true, force: true })),
|
|
1442
1761
|
};
|
|
1762
|
+
await claimNativeOutputs(plan, ctx);
|
|
1443
1763
|
const files = [];
|
|
1444
1764
|
const skipped = [];
|
|
1445
1765
|
for (const entry of plan.entries) {
|
|
1446
1766
|
if (entry.skipped) { skipped.push({ kind: entry.kind, index: entry.index, platforms: entry.platforms }); continue; }
|
|
1447
1767
|
const staged = entry.kind === "cargo-bin" ? stageCargoBin(entry, ctx)
|
|
1448
1768
|
: entry.kind === "ffmpeg" ? await stageFfmpeg(entry, ctx)
|
|
1449
|
-
: entry.kind === "onnxruntime" ? await
|
|
1769
|
+
: entry.kind === "onnxruntime" || entry.kind === "native-library" ? await stageNativeLibrary(entry, ctx)
|
|
1450
1770
|
: entry.kind === "command" ? stageCommand(entry, ctx)
|
|
1451
1771
|
: entry.kind === "remote" ? await stageRemote(entry, ctx)
|
|
1452
1772
|
: entry.kind === "cmake" ? stageCmake(entry, ctx)
|
|
@@ -1482,6 +1802,9 @@ export function validateStageSidecarReceipt(receipt, expected = {}) {
|
|
|
1482
1802
|
if (typeof file.name !== "string" || !file.name || typeof file.path !== "string" || !file.path) bad("file without a name or path");
|
|
1483
1803
|
if (!SHA256.test(file.sha256 ?? "")) bad(`${file.name}.sha256 must be a SHA-256`);
|
|
1484
1804
|
if (!Number.isSafeInteger(file.sizeBytes) || file.sizeBytes < (file.kind === "resource" ? 0 : 1)) bad(`${file.name}.sizeBytes must be positive`);
|
|
1805
|
+
if (file.kind === "native-library" && (typeof file.library !== "string" || !file.library)) bad(`${file.name} has no library`);
|
|
1806
|
+
if (file.hostProof !== undefined && file.hostProof !== "never-run") bad(`${file.name}.hostProof must be "never-run"`);
|
|
1807
|
+
if (file.companionOf !== undefined && (typeof file.companionOf !== "string" || !file.companionOf)) bad(`${file.name}.companionOf must be a catalog row id`);
|
|
1485
1808
|
}
|
|
1486
1809
|
if (new Set(receipt.files.map((file) => file.path)).size !== receipt.files.length) bad("duplicate staged path");
|
|
1487
1810
|
if (expected.target && receipt.target !== expected.target) bad(`target ${receipt.target} is not ${expected.target}`);
|