@moldesk/runtime 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docker/index.d.ts +41 -4
- package/dist/docker/index.js +140 -14
- package/dist/docker/index.test.d.ts +1 -0
- package/dist/docker/index.test.js +108 -0
- package/dist/filesystem/companion-inputs.d.ts +34 -0
- package/dist/filesystem/companion-inputs.js +123 -0
- package/dist/filesystem/companion-inputs.test.d.ts +1 -0
- package/dist/filesystem/companion-inputs.test.js +256 -0
- package/dist/filesystem/index.d.ts +4 -0
- package/dist/filesystem/index.js +2 -0
- package/dist/filesystem/paths.d.ts +20 -0
- package/dist/filesystem/paths.js +79 -0
- package/dist/filesystem/paths.test.d.ts +1 -0
- package/dist/filesystem/paths.test.js +35 -0
- package/dist/hardware/index.d.ts +21 -0
- package/dist/hardware/index.js +62 -0
- package/dist/hardware/index.test.d.ts +1 -0
- package/dist/hardware/index.test.js +27 -0
- package/dist/index.d.ts +22 -5
- package/dist/index.js +17 -12
- package/dist/install/index.d.ts +83 -0
- package/dist/install/index.js +487 -0
- package/dist/install/index.test.d.ts +1 -0
- package/dist/install/index.test.js +628 -0
- package/dist/nvidia/index.d.ts +29 -0
- package/dist/nvidia/index.js +119 -0
- package/dist/nvidia/index.test.d.ts +1 -0
- package/dist/nvidia/index.test.js +91 -0
- package/dist/process/index.d.ts +34 -0
- package/dist/process/index.js +99 -4
- package/dist/process/index.test.d.ts +1 -0
- package/dist/process/index.test.js +36 -0
- package/dist/providers/python.d.ts +14 -0
- package/dist/providers/python.js +80 -0
- package/dist/providers/python.test.d.ts +1 -0
- package/dist/providers/python.test.js +68 -0
- package/dist/providers.d.ts +49 -0
- package/dist/providers.js +1 -0
- package/dist/providers.test.d.ts +1 -0
- package/dist/providers.test.js +10 -0
- package/dist/python/index.d.ts +14 -3
- package/dist/python/index.js +30 -12
- package/dist/redact.d.ts +6 -0
- package/dist/redact.js +16 -0
- package/dist/redact.test.d.ts +1 -0
- package/dist/redact.test.js +16 -0
- package/dist/system.d.ts +39 -0
- package/dist/system.js +42 -0
- package/dist/system.test.d.ts +1 -0
- package/dist/system.test.js +138 -0
- package/package.json +8 -3
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { AssetSpec, DockerRuntimeSpec, PythonRuntimeSpec } from "@moldesk/registry";
|
|
2
|
+
import { type MoldeskPaths } from "../filesystem/index.js";
|
|
3
|
+
import { type RunResult } from "../process/index.js";
|
|
4
|
+
export type InstallCommandRunner = (command: string, args: string[], options?: {
|
|
5
|
+
cwd?: string;
|
|
6
|
+
env?: NodeJS.ProcessEnv;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
}) => Promise<RunResult>;
|
|
9
|
+
export interface InstallProgress {
|
|
10
|
+
step: string;
|
|
11
|
+
message: string;
|
|
12
|
+
receivedBytes?: number;
|
|
13
|
+
totalBytes?: number;
|
|
14
|
+
}
|
|
15
|
+
export interface PythonInstallRequest {
|
|
16
|
+
targetDir: string;
|
|
17
|
+
repository: string;
|
|
18
|
+
revision: string;
|
|
19
|
+
runtime: PythonRuntimeSpec;
|
|
20
|
+
runner?: InstallCommandRunner;
|
|
21
|
+
fetch?: AssetFetch;
|
|
22
|
+
paths?: MoldeskPaths;
|
|
23
|
+
uvExecutable?: string;
|
|
24
|
+
onProgress?: (progress: InstallProgress) => void;
|
|
25
|
+
}
|
|
26
|
+
export interface PostInstallHookResult {
|
|
27
|
+
hook: string;
|
|
28
|
+
stdout: string;
|
|
29
|
+
stderr: string;
|
|
30
|
+
}
|
|
31
|
+
export interface PreparedPythonEnvironment {
|
|
32
|
+
executable: string;
|
|
33
|
+
lock: string[];
|
|
34
|
+
pythonVersion: string;
|
|
35
|
+
/** One entry per `runtime.postInstall` hook that ran, in declared order. Empty
|
|
36
|
+
* when the runtime declares no postInstall hooks. */
|
|
37
|
+
postInstall: PostInstallHookResult[];
|
|
38
|
+
}
|
|
39
|
+
export interface PreparedDockerImage {
|
|
40
|
+
image: string;
|
|
41
|
+
digest: string;
|
|
42
|
+
}
|
|
43
|
+
type FetchResponse = {
|
|
44
|
+
ok: boolean;
|
|
45
|
+
status: number;
|
|
46
|
+
url?: string;
|
|
47
|
+
headers: {
|
|
48
|
+
get(name: string): string | null;
|
|
49
|
+
};
|
|
50
|
+
body: {
|
|
51
|
+
getReader(): {
|
|
52
|
+
read(): Promise<{
|
|
53
|
+
done: boolean;
|
|
54
|
+
value?: Uint8Array;
|
|
55
|
+
}>;
|
|
56
|
+
};
|
|
57
|
+
} | null;
|
|
58
|
+
};
|
|
59
|
+
export type AssetFetch = (url: string, init?: {
|
|
60
|
+
headers?: Record<string, string>;
|
|
61
|
+
}) => Promise<FetchResponse>;
|
|
62
|
+
export declare const MANAGED_UV_VERSION = "0.12.8";
|
|
63
|
+
export declare const UV_RELEASES: Record<string, {
|
|
64
|
+
triple: string;
|
|
65
|
+
sha256: string;
|
|
66
|
+
sizeBytes: number;
|
|
67
|
+
}>;
|
|
68
|
+
export declare function managedUvDownloadBytes(paths?: MoldeskPaths): number;
|
|
69
|
+
export declare function ensureManagedUv(options?: {
|
|
70
|
+
paths?: MoldeskPaths;
|
|
71
|
+
fetch?: AssetFetch;
|
|
72
|
+
runner?: InstallCommandRunner;
|
|
73
|
+
onProgress?: (progress: InstallProgress) => void;
|
|
74
|
+
}): Promise<string>;
|
|
75
|
+
export declare function preparePythonEnvironment(request: PythonInstallRequest): Promise<PreparedPythonEnvironment>;
|
|
76
|
+
export declare function cacheAsset(asset: AssetSpec, paths: MoldeskPaths, options?: {
|
|
77
|
+
fetch?: AssetFetch;
|
|
78
|
+
onProgress?: (progress: InstallProgress) => void;
|
|
79
|
+
}): Promise<string>;
|
|
80
|
+
export declare function materializeAsset(asset: AssetSpec, objectPath: string, assetsDir: string): Promise<string>;
|
|
81
|
+
export declare function prepareDockerImage(runtime: DockerRuntimeSpec, runner?: InstallCommandRunner): Promise<PreparedDockerImage>;
|
|
82
|
+
export declare function temporaryInstallDir(targetDir: string): string;
|
|
83
|
+
export {};
|
|
@@ -0,0 +1,487 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { MoldeskError } from "@moldesk/registry";
|
|
5
|
+
import { getMoldeskPaths, pythonToolDir, uvToolDir } from "../filesystem/index.js";
|
|
6
|
+
import { runCommand } from "../process/index.js";
|
|
7
|
+
export const MANAGED_UV_VERSION = "0.12.8";
|
|
8
|
+
export const UV_RELEASES = {
|
|
9
|
+
"darwin-arm64": {
|
|
10
|
+
triple: "aarch64-apple-darwin",
|
|
11
|
+
sha256: "8ce083658dbff20143607ca7af8e0c1d64b6fd7bf03a5cdcb62bf3d47d991b5f",
|
|
12
|
+
sizeBytes: 16_657_135,
|
|
13
|
+
},
|
|
14
|
+
"linux-x64": {
|
|
15
|
+
triple: "x86_64-unknown-linux-gnu",
|
|
16
|
+
sha256: "2e2b37e9811e17675a9e70bed5e1a58fc8c0388be63d751d72cc735188c149ff",
|
|
17
|
+
sizeBytes: 19_428_791,
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
function installError(code, message, remediation, details) {
|
|
21
|
+
return new MoldeskError({ code, message, remediation, details });
|
|
22
|
+
}
|
|
23
|
+
async function checked(runner, command, args, options = {}) {
|
|
24
|
+
const result = await runner(command, args, { ...options, timeoutMs: options.timeoutMs ?? 30 * 60_000 });
|
|
25
|
+
if (result.code !== 0 || result.timedOut) {
|
|
26
|
+
// Surface the tail of the tool's own output so a failed install is diagnosable
|
|
27
|
+
// from the CLI message alone, without digging through structured details.
|
|
28
|
+
const tail = (result.stderr || result.stdout || "").trim().split("\n").slice(-15).join("\n");
|
|
29
|
+
throw installError("INSTALL_COMMAND_FAILED", `${command} ${args.join(" ")} failed${result.timedOut ? " (timed out)" : ` with exit code ${result.code}`}.${tail ? `\n${tail}` : ""}`, "Review command output, network access, and disk space, then run install --reinstall.", { command, args, stdout: result.stdout, stderr: result.stderr, timedOut: result.timedOut });
|
|
30
|
+
}
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
function managedEnvironment(paths, pythonVersion, options = {}) {
|
|
34
|
+
const env = {
|
|
35
|
+
PATH: process.env["PATH"],
|
|
36
|
+
TMPDIR: process.env["TMPDIR"],
|
|
37
|
+
SSL_CERT_FILE: process.env["SSL_CERT_FILE"],
|
|
38
|
+
SSL_CERT_DIR: process.env["SSL_CERT_DIR"],
|
|
39
|
+
UV_CACHE_DIR: path.join(paths.cache, "uv"),
|
|
40
|
+
UV_PYTHON_INSTALL_DIR: pythonToolDir(pythonVersion, { MOLDESK_HOME: paths.home }),
|
|
41
|
+
UV_PYTHON_PREFERENCE: "only-managed",
|
|
42
|
+
UV_NO_PROGRESS: "1",
|
|
43
|
+
};
|
|
44
|
+
// Deliberately never overrides UV_INDEX_URL (the default/primary index stays
|
|
45
|
+
// PyPI) — a runtime entry that needs a CUDA-specific extra index (e.g. PyTorch's
|
|
46
|
+
// own cu117 index) must not lose access to ordinary packages like pandas/scipy
|
|
47
|
+
// by replacing the default index outright. `extraIndexUrls` (PEP-503-style
|
|
48
|
+
// indexes) and `findLinks` (flat wheel-listing pages, a distinct pip/uv
|
|
49
|
+
// mechanism) are kept as separate fields precisely because conflating them
|
|
50
|
+
// breaks resolution — a page meant for `--find-links` is not always a valid
|
|
51
|
+
// PEP-503 index.
|
|
52
|
+
if (options.extraIndexUrls && options.extraIndexUrls.length > 0) {
|
|
53
|
+
env["UV_EXTRA_INDEX_URL"] = options.extraIndexUrls.join(" ");
|
|
54
|
+
// uv's default first-index strategy would let a manifest-declared extra index
|
|
55
|
+
// (PyTorch's cu117 index also lists e.g. `torchmetrics`) shadow an exact PyPI
|
|
56
|
+
// pin it lacks. The extra indexes are schema-validated manifest entries and
|
|
57
|
+
// every requirement is an exact `==` pin, so best-match across them is safe.
|
|
58
|
+
env["UV_INDEX_STRATEGY"] = "unsafe-best-match";
|
|
59
|
+
}
|
|
60
|
+
if (options.findLinks && options.findLinks.length > 0) {
|
|
61
|
+
env["UV_FIND_LINKS"] = options.findLinks.join(" ");
|
|
62
|
+
}
|
|
63
|
+
// Schema-validated closed enum (e.g. "cu126"); uv routes torch-ecosystem
|
|
64
|
+
// packages to the matching official PyTorch index.
|
|
65
|
+
if (options.torchBackend)
|
|
66
|
+
env["UV_TORCH_BACKEND"] = options.torchBackend;
|
|
67
|
+
for (const name of ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) {
|
|
68
|
+
if (process.env[name])
|
|
69
|
+
env[name] = process.env[name];
|
|
70
|
+
}
|
|
71
|
+
return Object.fromEntries(Object.entries(env).filter((entry) => entry[1] !== undefined));
|
|
72
|
+
}
|
|
73
|
+
function platformId() {
|
|
74
|
+
return `${process.platform}-${process.arch}`;
|
|
75
|
+
}
|
|
76
|
+
export function managedUvDownloadBytes(paths = getMoldeskPaths()) {
|
|
77
|
+
const release = UV_RELEASES[platformId()];
|
|
78
|
+
if (!release)
|
|
79
|
+
return 0;
|
|
80
|
+
const executable = path.join(uvToolDir(MANAGED_UV_VERSION, { MOLDESK_HOME: paths.home }), "uv");
|
|
81
|
+
return fs.existsSync(executable) ? 0 : release.sizeBytes;
|
|
82
|
+
}
|
|
83
|
+
export async function ensureManagedUv(options = {}) {
|
|
84
|
+
const paths = options.paths ?? getMoldeskPaths();
|
|
85
|
+
const runner = options.runner ?? runCommand;
|
|
86
|
+
const release = UV_RELEASES[platformId()];
|
|
87
|
+
if (!release) {
|
|
88
|
+
throw installError("UV_PLATFORM_UNSUPPORTED", `Managed uv is unavailable for ${platformId()}.`, "Use macOS ARM64 or Linux x64 for v0.1.");
|
|
89
|
+
}
|
|
90
|
+
const toolDir = uvToolDir(MANAGED_UV_VERSION, { MOLDESK_HOME: paths.home });
|
|
91
|
+
const executable = path.join(toolDir, "uv");
|
|
92
|
+
const receipt = path.join(toolDir, "verified.json");
|
|
93
|
+
if (fs.existsSync(executable) && fs.existsSync(receipt)) {
|
|
94
|
+
try {
|
|
95
|
+
const metadata = JSON.parse(fs.readFileSync(receipt, "utf8"));
|
|
96
|
+
if (metadata.version === MANAGED_UV_VERSION && metadata.sha256 === release.sha256) {
|
|
97
|
+
const verified = await runner(executable, ["--version"], { timeoutMs: 15_000 });
|
|
98
|
+
if (verified.code === 0 && verified.stdout.includes(MANAGED_UV_VERSION))
|
|
99
|
+
return executable;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
// Replace an incomplete or unverifiable tool installation below.
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
options.onProgress?.({ step: "uv", message: `Downloading managed uv ${MANAGED_UV_VERSION}` });
|
|
107
|
+
const archiveName = `uv-${release.triple}.tar.gz`;
|
|
108
|
+
const asset = {
|
|
109
|
+
id: `uv-${MANAGED_UV_VERSION}-${release.triple}`,
|
|
110
|
+
url: `https://github.com/astral-sh/uv/releases/download/${MANAGED_UV_VERSION}/${archiveName}`,
|
|
111
|
+
source: "Official astral-sh/uv GitHub release",
|
|
112
|
+
license: "Apache-2.0 OR MIT",
|
|
113
|
+
sha256: release.sha256,
|
|
114
|
+
target: archiveName,
|
|
115
|
+
sizeBytes: release.sizeBytes,
|
|
116
|
+
archive: "none",
|
|
117
|
+
};
|
|
118
|
+
const object = await cacheAsset(asset, paths, { fetch: options.fetch, onProgress: options.onProgress });
|
|
119
|
+
const staging = `${toolDir}.partial-${process.pid}-${randomUUID()}`;
|
|
120
|
+
const backup = `${toolDir}.backup-${randomUUID()}`;
|
|
121
|
+
let backedUp = false;
|
|
122
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
123
|
+
fs.mkdirSync(staging, { recursive: true });
|
|
124
|
+
try {
|
|
125
|
+
const listing = await checked(runner, "tar", ["-tzf", object]);
|
|
126
|
+
if (listing.stdout.split(/\r?\n/).filter(Boolean).some((entry) => entry.startsWith("/") || entry.split("/").includes(".."))) {
|
|
127
|
+
throw installError("UNSAFE_ARCHIVE_ENTRY", "Managed uv archive contains an unsafe path.", "Report the release artifact as unsafe.");
|
|
128
|
+
}
|
|
129
|
+
// Managed volumes may forbid chown even for root; archive owners are not
|
|
130
|
+
// part of the tool's provenance and should never replace the local owner.
|
|
131
|
+
await checked(runner, "tar", ["-xzf", object, "--no-same-owner", "-C", staging]);
|
|
132
|
+
const extracted = path.join(staging, `uv-${release.triple}`, "uv");
|
|
133
|
+
if (!fs.existsSync(extracted)) {
|
|
134
|
+
throw installError("UV_ARCHIVE_INVALID", `Managed uv archive did not contain ${extracted}.`, "Retry download or report the pinned artifact.");
|
|
135
|
+
}
|
|
136
|
+
fs.chmodSync(extracted, 0o755);
|
|
137
|
+
const verified = await checked(runner, extracted, ["--version"]);
|
|
138
|
+
if (!verified.stdout.includes(MANAGED_UV_VERSION)) {
|
|
139
|
+
throw installError("UV_VERSION_MISMATCH", `Expected uv ${MANAGED_UV_VERSION}, received ${verified.stdout.trim()}.`, "Report the pinned artifact.");
|
|
140
|
+
}
|
|
141
|
+
fs.writeFileSync(path.join(staging, "verified.json"), `${JSON.stringify({ version: MANAGED_UV_VERSION, sha256: release.sha256 })}\n`, { mode: 0o600 });
|
|
142
|
+
fs.copyFileSync(extracted, path.join(staging, "uv"));
|
|
143
|
+
fs.chmodSync(path.join(staging, "uv"), 0o755);
|
|
144
|
+
if (fs.existsSync(toolDir)) {
|
|
145
|
+
fs.renameSync(toolDir, backup);
|
|
146
|
+
backedUp = true;
|
|
147
|
+
}
|
|
148
|
+
fs.renameSync(staging, toolDir);
|
|
149
|
+
fs.rmSync(backup, { recursive: true, force: true });
|
|
150
|
+
return executable;
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
154
|
+
if (backedUp && fs.existsSync(backup) && !fs.existsSync(toolDir))
|
|
155
|
+
fs.renameSync(backup, toolDir);
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
function requirementSpecifier(requirement) {
|
|
160
|
+
const nameWithExtras = requirement.extras && requirement.extras.length > 0
|
|
161
|
+
? `${requirement.name}[${requirement.extras.join(",")}]`
|
|
162
|
+
: requirement.name;
|
|
163
|
+
if (requirement.source) {
|
|
164
|
+
const revision = requirement.revision ? `@${requirement.revision}` : "";
|
|
165
|
+
return `${nameWithExtras} @ git+${requirement.source}${revision}`;
|
|
166
|
+
}
|
|
167
|
+
return requirement.version ? `${nameWithExtras}==${requirement.version}` : nameWithExtras;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* One requirements-file line per requirement, with `--hash=sha256:<digest>` tokens
|
|
171
|
+
* appended when the requirement declares hashes. Hash-checking mode (uv/pip's
|
|
172
|
+
* `--require-hashes`) is only reliable via a requirements file — not as bare CLI
|
|
173
|
+
* args mixed across multiple packages — so hashed installs always go through
|
|
174
|
+
* `preparePythonEnvironment`'s `-r <file>` path, never the positional-args path.
|
|
175
|
+
*/
|
|
176
|
+
function requirementFileLine(requirement) {
|
|
177
|
+
const specifier = requirementSpecifier(requirement);
|
|
178
|
+
if (!requirement.hashes || requirement.hashes.length === 0)
|
|
179
|
+
return specifier;
|
|
180
|
+
return `${specifier} ${requirement.hashes.map((h) => `--hash=sha256:${h}`).join(" ")}`;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Fixed dispatch table for `postInstall` hook names: named, non-shell fixups. Each
|
|
184
|
+
* handler receives the venv's bin directory and runs a literal, reviewed executable
|
|
185
|
+
* name from it via the injected runner — never a shell, never an interpolated
|
|
186
|
+
* manifest string, so `postInstall` can never become a generic command escape hatch.
|
|
187
|
+
*/
|
|
188
|
+
const POST_INSTALL_HOOKS = {
|
|
189
|
+
"boltz-fix-macos-libomp": (venvBinDir, runner) => checked(runner, path.join(venvBinDir, "boltz-fix-macos-libomp"), []),
|
|
190
|
+
};
|
|
191
|
+
/**
|
|
192
|
+
* Runs the existing two-pass install (verify hash-declared requirements by
|
|
193
|
+
* artifact digest with `--no-deps`, then a normal resolving pass over every
|
|
194
|
+
* requirement in the set) for one *stage* of requirements — a stage being
|
|
195
|
+
* either the main `requirements[]` set or a `buildAfter`-deferred subset,
|
|
196
|
+
* installed via its own separate invocation so a git-built package that
|
|
197
|
+
* imports an earlier dependency at build time (e.g. openfold importing torch)
|
|
198
|
+
* only ever builds after that dependency is already fully installed.
|
|
199
|
+
* `--only-binary <name>` is appended for every requirement in `binaryOnlyNames`
|
|
200
|
+
* so an ABI-sensitive CUDA extension package can never silently fall back to
|
|
201
|
+
* building from source — install fails closed instead.
|
|
202
|
+
* `noBuildIsolation` builds in the venv itself (not uv's throwaway build env), so
|
|
203
|
+
* a build that imports an already-installed dependency (torch) can find it.
|
|
204
|
+
*/
|
|
205
|
+
async function installRequirementSet(runner, uv, python, targetDir, requirements, binaryOnlyNames, env, lockFileSuffix, noBuildIsolation = false) {
|
|
206
|
+
if (requirements.length === 0)
|
|
207
|
+
return;
|
|
208
|
+
const isolationArgs = noBuildIsolation ? ["--no-build-isolation"] : [];
|
|
209
|
+
const binaryOnlyArgs = requirements
|
|
210
|
+
.filter((r) => binaryOnlyNames.has(r.name))
|
|
211
|
+
.flatMap((r) => ["--only-binary", r.name]);
|
|
212
|
+
const hashedRequirements = requirements.filter((r) => r.hashes && r.hashes.length > 0);
|
|
213
|
+
if (hashedRequirements.length > 0) {
|
|
214
|
+
const requirementsFile = path.join(targetDir, `.requirements-lock${lockFileSuffix}.txt`);
|
|
215
|
+
const content = `${hashedRequirements.map((r) => requirementFileLine(r)).join("\n")}\n`;
|
|
216
|
+
fs.writeFileSync(requirementsFile, content, { mode: 0o600 });
|
|
217
|
+
await checked(runner, uv, ["pip", "install", "--python", python, "--require-hashes", "--no-deps", ...isolationArgs, ...binaryOnlyArgs, "-r", requirementsFile], { env });
|
|
218
|
+
}
|
|
219
|
+
await checked(runner, uv, ["pip", "install", "--python", python, ...isolationArgs, ...binaryOnlyArgs, ...requirements.map((r) => requirementSpecifier(r))], { env });
|
|
220
|
+
}
|
|
221
|
+
export async function preparePythonEnvironment(request) {
|
|
222
|
+
const runner = request.runner ?? runCommand;
|
|
223
|
+
const paths = request.paths ?? getMoldeskPaths();
|
|
224
|
+
const uv = request.uvExecutable ?? await ensureManagedUv({ paths, fetch: request.fetch, runner, onProgress: request.onProgress });
|
|
225
|
+
const env = managedEnvironment(paths, request.runtime.python, {
|
|
226
|
+
extraIndexUrls: request.runtime.extraIndexUrls,
|
|
227
|
+
findLinks: request.runtime.findLinks,
|
|
228
|
+
torchBackend: request.runtime.torchBackend,
|
|
229
|
+
});
|
|
230
|
+
const sourceDir = path.join(request.targetDir, "source");
|
|
231
|
+
const venvDir = path.join(request.targetDir, ".venv");
|
|
232
|
+
const python = process.platform === "win32"
|
|
233
|
+
? path.join(venvDir, "Scripts", "python.exe")
|
|
234
|
+
: path.join(venvDir, "bin", "python");
|
|
235
|
+
request.onProgress?.({ step: "source", message: "Fetching pinned source revision" });
|
|
236
|
+
fs.mkdirSync(request.targetDir, { recursive: true });
|
|
237
|
+
await checked(runner, "git", ["init", sourceDir]);
|
|
238
|
+
await checked(runner, "git", ["-C", sourceDir, "remote", "add", "origin", request.repository]);
|
|
239
|
+
await checked(runner, "git", ["-C", sourceDir, "fetch", "--depth", "1", "origin", request.revision]);
|
|
240
|
+
await checked(runner, "git", ["-C", sourceDir, "checkout", "--detach", "FETCH_HEAD"]);
|
|
241
|
+
const revision = await checked(runner, "git", ["-C", sourceDir, "rev-parse", "HEAD"]);
|
|
242
|
+
if (revision.stdout.trim().toLowerCase() !== request.revision.toLowerCase()) {
|
|
243
|
+
throw installError("SOURCE_REVISION_MISMATCH", `Fetched source revision ${revision.stdout.trim()} does not match ${request.revision}.`, "Retry installation. If this persists, report a registry integrity issue.");
|
|
244
|
+
}
|
|
245
|
+
request.onProgress?.({ step: "python", message: `Preparing managed Python ${request.runtime.python}` });
|
|
246
|
+
await checked(runner, uv, ["python", "install", request.runtime.python], { env });
|
|
247
|
+
// --relocatable: installs happen in a `.partial-<pid>-<uuid>` staging directory
|
|
248
|
+
// that is atomically renamed to its final target on success (see
|
|
249
|
+
// packages/core/src/installation.ts). Without this flag, console scripts uv
|
|
250
|
+
// generates (e.g. `boltz`) bake the staging path into their shebang line and
|
|
251
|
+
// break after the rename — confirmed by a real install+run on this machine
|
|
252
|
+
// (`exec: .../.partial-.../.venv/bin/python: No such file or directory`).
|
|
253
|
+
await checked(runner, uv, ["venv", "--relocatable", "--python", request.runtime.python, venvDir], { env });
|
|
254
|
+
request.onProgress?.({ step: "dependencies", message: "Installing pinned dependencies" });
|
|
255
|
+
// Stage 0: preInstall requirements, each its own separate invocation, strictly
|
|
256
|
+
// before anything else — for a package (e.g. a pinned older `setuptools`) whose
|
|
257
|
+
// own version a later requirement's source build depends on already being fully
|
|
258
|
+
// resolved, not merely present somewhere in the same combined resolver graph.
|
|
259
|
+
// preInstall resolves against PyPI only: uv's default first-index strategy would
|
|
260
|
+
// otherwise let an extra index that merely lists the package name (e.g. PyTorch's
|
|
261
|
+
// cu117 index carries a stale `setuptools`) shadow the pinned PyPI version.
|
|
262
|
+
const preInstallEnv = managedEnvironment(paths, request.runtime.python);
|
|
263
|
+
for (const req of request.runtime.preInstall ?? []) {
|
|
264
|
+
request.onProgress?.({ step: "dependencies", message: `Installing ${req.name} (preInstall)` });
|
|
265
|
+
await checked(runner, uv, ["pip", "install", "--python", python, requirementSpecifier(req)], { env: preInstallEnv });
|
|
266
|
+
}
|
|
267
|
+
// uv/pip's --require-hashes mode demands every requirement resolved by a given
|
|
268
|
+
// install invocation be hashed, including the full transitive closure — not
|
|
269
|
+
// just the manifest-declared top-level packages (confirmed directly: `uv pip
|
|
270
|
+
// install --require-hashes` refuses over an unhashed transitive dependency like
|
|
271
|
+
// `filelock`). Hand-pinning an entire transitive closure per model isn't
|
|
272
|
+
// practical or what this manifest schema is for. Instead, verify each
|
|
273
|
+
// hash-declared requirement's own artifact by digest with `--no-deps` (so only
|
|
274
|
+
// that one wheel/sdist is checked, no transitive resolution), then do a second,
|
|
275
|
+
// ordinary install pass over every requirement (hashed ones included, by their
|
|
276
|
+
// plain specifier) so normal dependency resolution installs the rest of the
|
|
277
|
+
// closure. A requirement pinned by git source+revision has no fixed artifact
|
|
278
|
+
// hash to check at all (it's built from source) — that's a distinct, equally
|
|
279
|
+
// valid provenance mechanism, not a gap. `binaryOnly` names get `--only-binary`
|
|
280
|
+
// on both passes so an ABI-sensitive package can never silently fall back to a
|
|
281
|
+
// from-source build.
|
|
282
|
+
const binaryOnlyNames = new Set(request.runtime.binaryOnly ?? []);
|
|
283
|
+
const deferredNames = new Set(request.runtime.buildAfter ?? []);
|
|
284
|
+
const immediateRequirements = request.runtime.requirements.filter((r) => !deferredNames.has(r.name));
|
|
285
|
+
const deferredRequirements = request.runtime.requirements.filter((r) => deferredNames.has(r.name));
|
|
286
|
+
await installRequirementSet(runner, uv, python, request.targetDir, immediateRequirements, binaryOnlyNames, env, "");
|
|
287
|
+
// Stage 2: buildAfter requirements, in their own separate final invocation,
|
|
288
|
+
// strictly after every other requirement (including preInstall) has finished —
|
|
289
|
+
// for a package whose own build process imports an earlier dependency (e.g. an
|
|
290
|
+
// unpackaged git build that imports torch at setup time to select CUDA
|
|
291
|
+
// architecture flags), which a single combined resolver invocation cannot
|
|
292
|
+
// reliably sequence.
|
|
293
|
+
if (deferredRequirements.length > 0) {
|
|
294
|
+
request.onProgress?.({ step: "dependencies", message: "Installing buildAfter dependencies" });
|
|
295
|
+
await installRequirementSet(runner, uv, python, request.targetDir, deferredRequirements, binaryOnlyNames, env, "-deferred", true);
|
|
296
|
+
}
|
|
297
|
+
const postInstall = [];
|
|
298
|
+
for (const hook of request.runtime.postInstall ?? []) {
|
|
299
|
+
const handler = POST_INSTALL_HOOKS[hook];
|
|
300
|
+
if (!handler) {
|
|
301
|
+
throw installError("POST_INSTALL_HOOK_UNKNOWN", `Unknown postInstall hook "${hook}".`, "This is a MoleculeDesk bug — report it; the schema should have rejected this value.", { hook });
|
|
302
|
+
}
|
|
303
|
+
request.onProgress?.({ step: "post-install", message: `Running postInstall hook ${hook}` });
|
|
304
|
+
const venvBinDir = path.dirname(python);
|
|
305
|
+
const result = await handler(venvBinDir, runner);
|
|
306
|
+
postInstall.push({ hook, stdout: result.stdout, stderr: result.stderr });
|
|
307
|
+
}
|
|
308
|
+
const freeze = await checked(runner, uv, ["pip", "freeze", "--python", python], { env });
|
|
309
|
+
const version = await checked(runner, python, ["--version"], { env });
|
|
310
|
+
return {
|
|
311
|
+
executable: python,
|
|
312
|
+
lock: freeze.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).sort(),
|
|
313
|
+
pythonVersion: `${version.stdout}\n${version.stderr}`.trim(),
|
|
314
|
+
postInstall,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
async function sha256(file) {
|
|
318
|
+
const hash = createHash("sha256");
|
|
319
|
+
for await (const chunk of fs.createReadStream(file))
|
|
320
|
+
hash.update(chunk);
|
|
321
|
+
return hash.digest("hex");
|
|
322
|
+
}
|
|
323
|
+
async function cacheAssetUnlocked(asset, paths, options = {}) {
|
|
324
|
+
fs.mkdirSync(paths.downloads, { recursive: true });
|
|
325
|
+
fs.mkdirSync(paths.objects, { recursive: true });
|
|
326
|
+
const expected = asset.sha256.toLowerCase();
|
|
327
|
+
const objectPath = path.join(paths.objects, expected);
|
|
328
|
+
if (fs.existsSync(objectPath)) {
|
|
329
|
+
if (await sha256(objectPath) === expected)
|
|
330
|
+
return objectPath;
|
|
331
|
+
fs.rmSync(objectPath, { force: true });
|
|
332
|
+
}
|
|
333
|
+
const partial = path.join(paths.downloads, `${expected}.part`);
|
|
334
|
+
if (fs.existsSync(partial) && await sha256(partial) === expected) {
|
|
335
|
+
fs.renameSync(partial, objectPath);
|
|
336
|
+
return objectPath;
|
|
337
|
+
}
|
|
338
|
+
let received = fs.existsSync(partial) ? fs.statSync(partial).size : 0;
|
|
339
|
+
const fetcher = options.fetch ?? globalThis.fetch;
|
|
340
|
+
let response = await fetcher(asset.url, received > 0 ? { headers: { Range: `bytes=${received}-` } } : undefined);
|
|
341
|
+
if (received > 0 && response.status !== 206) {
|
|
342
|
+
fs.truncateSync(partial, 0);
|
|
343
|
+
received = 0;
|
|
344
|
+
response = await fetcher(asset.url);
|
|
345
|
+
}
|
|
346
|
+
if (!response.ok || !response.body) {
|
|
347
|
+
throw installError("ASSET_DOWNLOAD_FAILED", `Could not download ${asset.id}: HTTP ${response.status}.`, "Check network access and retry; partial downloads are preserved.", { asset: asset.id, url: asset.url });
|
|
348
|
+
}
|
|
349
|
+
if (response.url && new URL(response.url).protocol !== "https:") {
|
|
350
|
+
throw installError("ASSET_REDIRECT_UNSAFE", `Download for ${asset.id} redirected to unsupported URL ${response.url}.`, "Report the registry asset as unsafe.");
|
|
351
|
+
}
|
|
352
|
+
const contentLength = Number(response.headers.get("content-length") ?? 0);
|
|
353
|
+
const totalBytes = contentLength > 0 ? received + contentLength : asset.sizeBytes;
|
|
354
|
+
const file = fs.openSync(partial, received > 0 ? "a" : "w");
|
|
355
|
+
try {
|
|
356
|
+
const reader = response.body.getReader();
|
|
357
|
+
while (true) {
|
|
358
|
+
const { done, value } = await reader.read();
|
|
359
|
+
if (done)
|
|
360
|
+
break;
|
|
361
|
+
if (!value)
|
|
362
|
+
continue;
|
|
363
|
+
fs.writeSync(file, value);
|
|
364
|
+
received += value.byteLength;
|
|
365
|
+
options.onProgress?.({ step: "asset", message: `Downloading ${asset.id}`, receivedBytes: received, totalBytes });
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
finally {
|
|
369
|
+
fs.closeSync(file);
|
|
370
|
+
}
|
|
371
|
+
if (asset.sizeBytes !== undefined && received !== asset.sizeBytes) {
|
|
372
|
+
fs.rmSync(partial, { force: true });
|
|
373
|
+
throw installError("ASSET_SIZE_MISMATCH", `Size mismatch for ${asset.id}: expected ${asset.sizeBytes} bytes, received ${received}.`, "Retry download. If mismatch persists, report a registry integrity issue.");
|
|
374
|
+
}
|
|
375
|
+
const actual = await sha256(partial);
|
|
376
|
+
if (actual !== expected) {
|
|
377
|
+
fs.rmSync(partial, { force: true });
|
|
378
|
+
throw installError("ASSET_CHECKSUM_MISMATCH", `Checksum mismatch for ${asset.id}: expected ${expected}, received ${actual}.`, "Retry download. If mismatch persists, report a registry integrity issue.");
|
|
379
|
+
}
|
|
380
|
+
try {
|
|
381
|
+
fs.renameSync(partial, objectPath);
|
|
382
|
+
}
|
|
383
|
+
catch (error) {
|
|
384
|
+
if (!fs.existsSync(objectPath))
|
|
385
|
+
throw error;
|
|
386
|
+
if (await sha256(objectPath) !== expected)
|
|
387
|
+
throw error;
|
|
388
|
+
fs.rmSync(partial, { force: true });
|
|
389
|
+
}
|
|
390
|
+
return objectPath;
|
|
391
|
+
}
|
|
392
|
+
async function acquireDownloadLock(paths, digest) {
|
|
393
|
+
fs.mkdirSync(paths.locks, { recursive: true });
|
|
394
|
+
const lock = path.join(paths.locks, `download-${digest}.lock`);
|
|
395
|
+
for (let attempt = 0; attempt < 1200; attempt += 1) {
|
|
396
|
+
try {
|
|
397
|
+
fs.mkdirSync(lock);
|
|
398
|
+
fs.writeFileSync(path.join(lock, "owner.json"), JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() }));
|
|
399
|
+
return lock;
|
|
400
|
+
}
|
|
401
|
+
catch (error) {
|
|
402
|
+
if (error.code !== "EEXIST")
|
|
403
|
+
throw error;
|
|
404
|
+
try {
|
|
405
|
+
const owner = JSON.parse(fs.readFileSync(path.join(lock, "owner.json"), "utf8"));
|
|
406
|
+
let stale = Date.now() - fs.statSync(lock).mtimeMs > 60 * 60_000;
|
|
407
|
+
if (typeof owner.pid === "number") {
|
|
408
|
+
try {
|
|
409
|
+
process.kill(owner.pid, 0);
|
|
410
|
+
}
|
|
411
|
+
catch (processError) {
|
|
412
|
+
stale ||= processError.code === "ESRCH";
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (stale) {
|
|
416
|
+
fs.rmSync(lock, { recursive: true, force: true });
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
catch {
|
|
421
|
+
fs.rmSync(lock, { recursive: true, force: true });
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
throw installError("DOWNLOAD_LOCKED", `Timed out waiting for download ${digest}.`, "Wait for the other installation to finish, then retry.");
|
|
428
|
+
}
|
|
429
|
+
export async function cacheAsset(asset, paths, options = {}) {
|
|
430
|
+
fs.mkdirSync(paths.objects, { recursive: true });
|
|
431
|
+
const expected = asset.sha256.toLowerCase();
|
|
432
|
+
const objectPath = path.join(paths.objects, expected);
|
|
433
|
+
if (fs.existsSync(objectPath) && await sha256(objectPath) === expected)
|
|
434
|
+
return objectPath;
|
|
435
|
+
const lock = await acquireDownloadLock(paths, expected);
|
|
436
|
+
try {
|
|
437
|
+
return await cacheAssetUnlocked(asset, paths, options);
|
|
438
|
+
}
|
|
439
|
+
finally {
|
|
440
|
+
fs.rmSync(lock, { recursive: true, force: true });
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
export async function materializeAsset(asset, objectPath, assetsDir) {
|
|
444
|
+
const target = path.resolve(assetsDir, asset.target);
|
|
445
|
+
const root = `${path.resolve(assetsDir)}${path.sep}`;
|
|
446
|
+
if (!target.startsWith(root)) {
|
|
447
|
+
throw installError("UNSAFE_ASSET_TARGET", `Asset target escapes installation: ${asset.target}.`, "Fix the registry manifest.");
|
|
448
|
+
}
|
|
449
|
+
if (!asset.archive || asset.archive === "none") {
|
|
450
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
451
|
+
try {
|
|
452
|
+
fs.linkSync(objectPath, target);
|
|
453
|
+
}
|
|
454
|
+
catch {
|
|
455
|
+
fs.copyFileSync(objectPath, target, fs.constants.COPYFILE_FICLONE);
|
|
456
|
+
}
|
|
457
|
+
return target;
|
|
458
|
+
}
|
|
459
|
+
fs.mkdirSync(target, { recursive: true });
|
|
460
|
+
const listing = asset.archive === "zip"
|
|
461
|
+
? await checked(runCommand, "unzip", ["-Z1", objectPath])
|
|
462
|
+
: await checked(runCommand, "tar", [asset.archive === "tar.gz" ? "-tzf" : "-tf", objectPath]);
|
|
463
|
+
const unsafeEntry = listing.stdout.split(/\r?\n/).filter(Boolean).find((entry) => {
|
|
464
|
+
const normalized = entry.replaceAll("\\", "/");
|
|
465
|
+
return normalized.startsWith("/") || normalized.split("/").some((part) => part === "..");
|
|
466
|
+
});
|
|
467
|
+
if (unsafeEntry) {
|
|
468
|
+
fs.rmSync(target, { recursive: true, force: true });
|
|
469
|
+
throw installError("UNSAFE_ARCHIVE_ENTRY", `Archive ${asset.id} contains unsafe path ${unsafeEntry}.`, "Report the registry asset as unsafe.");
|
|
470
|
+
}
|
|
471
|
+
if (asset.archive === "zip")
|
|
472
|
+
await checked(runCommand, "unzip", ["-q", objectPath, "-d", target]);
|
|
473
|
+
else
|
|
474
|
+
await checked(runCommand, "tar", [asset.archive === "tar.gz" ? "-xzf" : "-xf", objectPath, "--no-same-owner", "-C", target]);
|
|
475
|
+
return target;
|
|
476
|
+
}
|
|
477
|
+
export async function prepareDockerImage(runtime, runner = runCommand) {
|
|
478
|
+
const reference = `${runtime.image}@${runtime.digest}`;
|
|
479
|
+
const inspect = await runner("docker", ["image", "inspect", reference], { timeoutMs: 30_000 });
|
|
480
|
+
if (inspect.code !== 0)
|
|
481
|
+
await checked(runner, "docker", ["pull", reference], { timeoutMs: 60 * 60_000 });
|
|
482
|
+
await checked(runner, "docker", ["image", "inspect", reference], { timeoutMs: 30_000 });
|
|
483
|
+
return { image: reference, digest: runtime.digest };
|
|
484
|
+
}
|
|
485
|
+
export function temporaryInstallDir(targetDir) {
|
|
486
|
+
return `${targetDir}.partial-${process.pid}-${randomUUID()}`;
|
|
487
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|