@rehearsal-labs/retest 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/api/source-location.d.ts.map +1 -1
- package/dist/api/source-location.js +3 -1
- package/dist/browser/builds.d.ts +35 -6
- package/dist/browser/builds.d.ts.map +1 -1
- package/dist/browser/builds.js +115 -16
- package/dist/browser/cdp/transport.d.ts.map +1 -1
- package/dist/browser/cdp/transport.js +4 -4
- package/dist/browser/chromium-process.d.ts +23 -0
- package/dist/browser/chromium-process.d.ts.map +1 -1
- package/dist/browser/chromium-process.js +78 -13
- package/dist/browser/electron.d.ts +1 -1
- package/dist/browser/electron.d.ts.map +1 -1
- package/dist/browser/electron.js +6 -3
- package/dist/browser/executables.d.ts +16 -1
- package/dist/browser/executables.d.ts.map +1 -1
- package/dist/browser/executables.js +50 -18
- package/dist/browser/firefox/accessible-names.d.ts +5 -4
- package/dist/browser/firefox/accessible-names.d.ts.map +1 -1
- package/dist/browser/firefox/accessible-names.js +11 -6
- package/dist/browser/firefox/bridge.js +3 -2
- package/dist/browser/firefox/executable.d.ts +14 -14
- package/dist/browser/firefox/executable.d.ts.map +1 -1
- package/dist/browser/firefox/executable.js +29 -21
- package/dist/browser/firefox/launch.js +1 -1
- package/dist/browser/firefox/navigation.d.ts +5 -2
- package/dist/browser/firefox/navigation.d.ts.map +1 -1
- package/dist/browser/firefox/navigation.js +48 -14
- package/dist/browser/firefox/orphans.d.ts.map +1 -1
- package/dist/browser/firefox/orphans.js +34 -2
- package/dist/browser/firefox/page.d.ts.map +1 -1
- package/dist/browser/firefox/page.js +113 -19
- package/dist/browser/firefox/process.d.ts +20 -1
- package/dist/browser/firefox/process.d.ts.map +1 -1
- package/dist/browser/firefox/process.js +124 -22
- package/dist/browser/firefox/profile.d.ts +5 -0
- package/dist/browser/firefox/profile.d.ts.map +1 -1
- package/dist/browser/firefox/profile.js +17 -1
- package/dist/browser/firefox/storage.d.ts.map +1 -1
- package/dist/browser/firefox/storage.js +4 -2
- package/dist/browser/firefox/subscriptions.d.ts +26 -0
- package/dist/browser/firefox/subscriptions.d.ts.map +1 -0
- package/dist/browser/firefox/subscriptions.js +10 -0
- package/dist/browser/launch.d.ts +6 -0
- package/dist/browser/launch.d.ts.map +1 -1
- package/dist/browser/launch.js +14 -1
- package/dist/cli/commands/doctor.js +3 -3
- package/dist/cli/commands/inspect.js +3 -2
- package/dist/cli/commands/install.d.ts.map +1 -1
- package/dist/cli/commands/install.js +26 -13
- package/dist/cli/commands/licences.d.ts +22 -0
- package/dist/cli/commands/licences.d.ts.map +1 -1
- package/dist/cli/commands/licences.js +62 -20
- package/dist/cli/commands/run.js +2 -2
- package/dist/cli/doctor/checks.js +4 -1
- package/dist/cli/inspect/diagnostics.js +3 -3
- package/dist/cli/inspect/test-timeline.js +10 -10
- package/dist/cli/install/doctor-rows.d.ts.map +1 -1
- package/dist/cli/install/doctor-rows.js +5 -4
- package/dist/cli/install/file-changes.d.ts +44 -0
- package/dist/cli/install/file-changes.d.ts.map +1 -0
- package/dist/cli/install/file-changes.js +81 -0
- package/dist/cli/install/install-archive.d.ts +3 -1
- package/dist/cli/install/install-archive.d.ts.map +1 -1
- package/dist/cli/install/install-archive.js +80 -33
- package/dist/cli/install/lock.d.ts.map +1 -1
- package/dist/cli/install/lock.js +108 -21
- package/dist/cli/install/media-doctor.d.ts.map +1 -1
- package/dist/cli/install/media-doctor.js +2 -1
- package/dist/cli/install/media-install.d.ts.map +1 -1
- package/dist/cli/install/media-install.js +38 -19
- package/dist/cli/install/media-pins.d.ts +7 -1
- package/dist/cli/install/media-pins.d.ts.map +1 -1
- package/dist/cli/install/media-pins.js +23 -12
- package/dist/cli/install/media-record.d.ts.map +1 -1
- package/dist/cli/install/media-record.js +10 -9
- package/dist/cli/install/media-report.d.ts +3 -1
- package/dist/cli/install/media-report.d.ts.map +1 -1
- package/dist/cli/install/media-report.js +7 -5
- package/dist/cli/install/media-tools.d.ts.map +1 -1
- package/dist/cli/install/media-tools.js +6 -4
- package/dist/cli/install/process-start.d.ts +3 -2
- package/dist/cli/install/process-start.d.ts.map +1 -1
- package/dist/cli/install/process-start.js +4 -0
- package/dist/cli/install/report.d.ts.map +1 -1
- package/dist/cli/install/report.js +11 -7
- package/dist/cli/install/supervisor-install.d.ts +33 -0
- package/dist/cli/install/supervisor-install.d.ts.map +1 -0
- package/dist/cli/install/supervisor-install.js +158 -0
- package/dist/cli/install/supervisor-locate.d.ts +47 -0
- package/dist/cli/install/supervisor-locate.d.ts.map +1 -0
- package/dist/cli/install/supervisor-locate.js +113 -0
- package/dist/cli/install/supervisor-pins.d.ts +21 -0
- package/dist/cli/install/supervisor-pins.d.ts.map +1 -0
- package/dist/cli/install/supervisor-pins.js +28 -0
- package/dist/cli/install/supervisor-record.d.ts +31 -0
- package/dist/cli/install/supervisor-record.d.ts.map +1 -0
- package/dist/cli/install/supervisor-record.js +104 -0
- package/dist/cli/install/supervisor-report.d.ts +11 -0
- package/dist/cli/install/supervisor-report.d.ts.map +1 -0
- package/dist/cli/install/supervisor-report.js +33 -0
- package/dist/cli/install/supervisor-tools.d.ts +43 -0
- package/dist/cli/install/supervisor-tools.d.ts.map +1 -0
- package/dist/cli/install/supervisor-tools.js +114 -0
- package/dist/cli/install/unpack.d.ts +23 -3
- package/dist/cli/install/unpack.d.ts.map +1 -1
- package/dist/cli/install/unpack.js +124 -10
- package/dist/cli/install/zip-member.d.ts +15 -0
- package/dist/cli/install/zip-member.d.ts.map +1 -0
- package/dist/cli/install/zip-member.js +77 -0
- package/dist/cli/interrupt.d.ts +1 -1
- package/dist/cli/interrupt.d.ts.map +1 -1
- package/dist/cli/interrupt.js +15 -10
- package/dist/cli/terminal.d.ts +9 -1
- package/dist/cli/terminal.d.ts.map +1 -1
- package/dist/cli/terminal.js +14 -0
- package/dist/config/define.d.ts +2 -2
- package/dist/config/define.js +2 -2
- package/dist/diagnostics/firefox-collector.d.ts +11 -10
- package/dist/diagnostics/firefox-collector.d.ts.map +1 -1
- package/dist/diagnostics/firefox-collector.js +16 -11
- package/dist/media/client.d.ts +2 -1
- package/dist/media/client.d.ts.map +1 -1
- package/dist/media/client.js +16 -5
- package/dist/media/host.d.ts +32 -0
- package/dist/media/host.d.ts.map +1 -0
- package/dist/media/host.js +50 -0
- package/dist/media/locate.d.ts.map +1 -1
- package/dist/media/locate.js +4 -3
- package/dist/native/processes.d.ts.map +1 -1
- package/dist/native/processes.js +7 -3
- package/dist/protocol/files.d.ts +1 -1
- package/dist/protocol/files.d.ts.map +1 -1
- package/dist/protocol/files.js +10 -5
- package/dist/reporters/commands.d.ts +12 -3
- package/dist/reporters/commands.d.ts.map +1 -1
- package/dist/reporters/commands.js +29 -11
- package/dist/reporters/human-card.js +1 -1
- package/dist/reporters/human.d.ts +3 -1
- package/dist/reporters/human.d.ts.map +1 -1
- package/dist/reporters/human.js +8 -8
- package/dist/reporters/style.d.ts +21 -2
- package/dist/reporters/style.d.ts.map +1 -1
- package/dist/reporters/style.js +11 -2
- package/dist/runner/app-server.d.ts +27 -7
- package/dist/runner/app-server.d.ts.map +1 -1
- package/dist/runner/app-server.js +74 -24
- package/dist/runner/child.js +6 -2
- package/dist/runner/media-leftovers.d.ts +7 -3
- package/dist/runner/media-leftovers.d.ts.map +1 -1
- package/dist/runner/media-leftovers.js +27 -6
- package/dist/runner/resources.d.ts +8 -0
- package/dist/runner/resources.d.ts.map +1 -1
- package/dist/runner/resources.js +12 -1
- package/dist/runner/run-session.d.ts.map +1 -1
- package/dist/runner/run-session.js +9 -4
- package/dist/runner/run.d.ts.map +1 -1
- package/dist/runner/run.js +2 -1
- package/dist/runner/target-drivers.d.ts +5 -2
- package/dist/runner/target-drivers.d.ts.map +1 -1
- package/dist/runner/target-drivers.js +10 -1
- package/dist/runner/test-file-process.d.ts +8 -0
- package/dist/runner/test-file-process.d.ts.map +1 -1
- package/dist/runner/test-file-process.js +26 -3
- package/dist/runner/test-pages.d.ts.map +1 -1
- package/dist/runner/test-pages.js +5 -2
- package/dist/shared/environment.d.ts +17 -0
- package/dist/shared/environment.d.ts.map +1 -0
- package/dist/shared/environment.js +27 -0
- package/dist/shared/metadata-process-worker.js +15 -7
- package/dist/shared/metadata-process.d.ts +5 -0
- package/dist/shared/metadata-process.d.ts.map +1 -1
- package/dist/shared/metadata-process.js +36 -6
- package/dist/shared/posix-path.d.ts +12 -0
- package/dist/shared/posix-path.d.ts.map +1 -1
- package/dist/shared/posix-path.js +16 -0
- package/dist/shared/private-file.d.ts +38 -0
- package/dist/shared/private-file.d.ts.map +1 -0
- package/dist/shared/private-file.js +132 -0
- package/dist/shared/process-ownership.d.ts +52 -2
- package/dist/shared/process-ownership.d.ts.map +1 -1
- package/dist/shared/process-ownership.js +104 -11
- package/dist/shared/process-supervisor.d.ts +309 -0
- package/dist/shared/process-supervisor.d.ts.map +1 -0
- package/dist/shared/process-supervisor.js +737 -0
- package/dist/shared/process-system.d.ts +113 -0
- package/dist/shared/process-system.d.ts.map +1 -0
- package/dist/shared/process-system.js +254 -0
- package/dist/shared/regular-file.d.ts +2 -2
- package/dist/shared/regular-file.d.ts.map +1 -1
- package/dist/shared/regular-file.js +19 -2
- package/dist/store/artifacts.d.ts +5 -4
- package/dist/store/artifacts.d.ts.map +1 -1
- package/dist/store/artifacts.js +18 -5
- package/dist/store/run-store.d.ts +5 -2
- package/dist/store/run-store.d.ts.map +1 -1
- package/dist/store/run-store.js +48 -4
- package/media/Cargo.lock +16 -0
- package/media/Cargo.toml +16 -1
- package/media/src/encoder.rs +56 -12
- package/media/src/frame.rs +7 -1
- package/media/src/main.rs +20 -2
- package/media/src/place.rs +230 -11
- package/media/src/process_ownership_windows.rs +1058 -0
- package/media/src/recording.rs +15 -7
- package/media/src/replies.rs +55 -0
- package/media/src/server.rs +72 -2
- package/media/src/store.rs +71 -2
- package/package.json +9 -4
- package/src/cli/install/media-notices.txt +465 -1
- package/supervisor/Cargo.lock +25 -0
- package/supervisor/Cargo.toml +22 -0
- package/supervisor/NOTICES +464 -0
- package/supervisor/build.rs +44 -0
- package/supervisor/src/command.rs +612 -0
- package/supervisor/src/jobs.rs +1650 -0
- package/supervisor/src/main.rs +71 -0
- package/supervisor/src/reply.rs +367 -0
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { constants } from 'node:fs';
|
|
2
|
-
import { access, lstat, mkdir, readdir,
|
|
2
|
+
import { access, lstat, mkdir, readdir, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { basename, dirname, join } from 'node:path';
|
|
4
4
|
import { buildFolder, describePin, describePlatform, describeTreeProblems, fileSha256, inspectBuild, pinRefusal, readBundledLicence, readFileSha256, readLicences, recordFile, scanTree, treeFolder } from "../../browser/builds.js";
|
|
5
5
|
import { folderChecksum } from "../../native/executors.js";
|
|
6
6
|
import { processExists } from "../../native/processes.js";
|
|
7
7
|
import { isMissingFile } from "../../shared/error-code.js";
|
|
8
8
|
import { downloadFile, mirroredUrl, shownAddress } from "./download.js";
|
|
9
|
+
import { removePath, renamePath } from "./file-changes.js";
|
|
9
10
|
import { takeInstallLock } from "./lock.js";
|
|
10
11
|
import { licenceLines, mirrorProblem } from "./report.js";
|
|
11
12
|
import { unpackArchive } from "./unpack.js";
|
|
@@ -25,7 +26,9 @@ const unpackTimeoutMs = 10 * 60_000;
|
|
|
25
26
|
* bit and links that stay inside it, and where the executable, the pinned files and every licence notice the pin names
|
|
26
27
|
* must be present with their checksums. Only then is the record written and the staging folder renamed into place, so
|
|
27
28
|
* a build folder always holds a finished, checked build, and the archive is deleted. A verified archive whose
|
|
28
|
-
* unpacking failed is kept for the next attempt.
|
|
29
|
+
* unpacking failed is kept for the next attempt. A rename or removal that does not happen is never thrown: before the
|
|
30
|
+
* build is in place it ends the install with its reason, and once the build is in place the install stands and what
|
|
31
|
+
* was left behind is reported. On Windows each is tried again while a program holds a file open (`file-changes.ts`).
|
|
29
32
|
*
|
|
30
33
|
* @example await installArchive({ pin, folders, signal, tools: systemUnpackTools, platform: process.platform, version: retestVersion, report: console.log })
|
|
31
34
|
*/
|
|
@@ -63,36 +66,65 @@ async function installLocked(options, pinnedSha256) {
|
|
|
63
66
|
const { pin, folders } = options;
|
|
64
67
|
const folder = buildFolder(folders, pin);
|
|
65
68
|
const archive = archivePath(folders, pin);
|
|
69
|
+
const changes = changeOptions(options);
|
|
66
70
|
const before = await inspectBuild(pin, folders);
|
|
67
71
|
if (before.state === 'installed') {
|
|
68
72
|
// An archive a killed install left after renaming its build into place is of no further use.
|
|
69
|
-
await
|
|
73
|
+
reportLeftover(options, archive, await removePath(archive, changes));
|
|
70
74
|
return { ok: true, action: 'already_installed', inspection: before };
|
|
71
75
|
}
|
|
72
76
|
if (before.state !== 'missing')
|
|
73
77
|
return refused(`${describePin(pin)} is in ${folder}, but not as recorded: ${before.problems.join(' ')} Remove ${folder} and run the install again.`);
|
|
74
|
-
await sweepStaging(folder);
|
|
75
|
-
await sweepPartials(dirname(archive));
|
|
78
|
+
await sweepStaging(folder, options);
|
|
79
|
+
await sweepPartials(dirname(archive), options);
|
|
76
80
|
const obtained = await obtainArchive({ ...options, archive, sha256: pinnedSha256 });
|
|
77
81
|
if (!obtained.ok)
|
|
78
82
|
return obtained;
|
|
79
83
|
const staging = `${folder}${stagingMark}${process.pid}`;
|
|
80
|
-
await
|
|
84
|
+
const cleared = await removePath(staging, changes);
|
|
85
|
+
if (!cleared.ok)
|
|
86
|
+
return notChanged(`Retest could not clear its staging folder ${staging}: ${cleared.reason}. Nothing was installed.`, cleared);
|
|
87
|
+
let placed;
|
|
88
|
+
let cleanup;
|
|
81
89
|
try {
|
|
82
|
-
|
|
83
|
-
if (!placed.ok)
|
|
84
|
-
return placed;
|
|
85
|
-
await rename(staging, folder);
|
|
90
|
+
placed = await placeBuild({ options, archive, staging, obtained }, folder);
|
|
86
91
|
}
|
|
87
92
|
finally {
|
|
88
|
-
|
|
93
|
+
// A staging folder left behind is said, and never hides how the install itself ended.
|
|
94
|
+
cleanup = await removePath(staging, changes);
|
|
89
95
|
}
|
|
90
|
-
|
|
96
|
+
if (!placed.ok)
|
|
97
|
+
return cleanup.ok ? placed : { ...placed, message: `${placed.message} ${leftover(staging, cleanup)}` };
|
|
98
|
+
// The build is in place from here on: what is left to tidy is said, and the install stands.
|
|
99
|
+
reportLeftover(options, staging, cleanup);
|
|
100
|
+
reportLeftover(options, archive, await removePath(archive, changes));
|
|
91
101
|
const after = await inspectBuild(pin, folders);
|
|
92
102
|
if (after.state !== 'installed')
|
|
93
103
|
return refused(`${describePin(pin)} was unpacked into ${folder}, then did not read as installed: ${after.problems.join(' ')}`);
|
|
94
104
|
return { ok: true, action: 'installed', inspection: after };
|
|
95
105
|
}
|
|
106
|
+
// The unpacked, checked and recorded build, renamed from its staging folder to its own.
|
|
107
|
+
async function placeBuild(placement, folder) {
|
|
108
|
+
const unpacked = await unpackAndRecord(placement);
|
|
109
|
+
if (!unpacked.ok)
|
|
110
|
+
return unpacked;
|
|
111
|
+
const moved = await renamePath(placement.staging, folder, changeOptions(placement.options));
|
|
112
|
+
return moved.ok ? moved : notChanged(`Retest could not move the unpacked build into ${folder}: ${moved.reason}. Nothing was installed.`, moved);
|
|
113
|
+
}
|
|
114
|
+
function changeOptions(options) {
|
|
115
|
+
return { platform: options.platform, signal: options.signal };
|
|
116
|
+
}
|
|
117
|
+
// A change that did not happen ends the install, as stopped when the install's signal ended it.
|
|
118
|
+
function notChanged(message, settled) {
|
|
119
|
+
return { ok: false, message, stopped: settled.stopped };
|
|
120
|
+
}
|
|
121
|
+
function leftover(path, settled) {
|
|
122
|
+
return `Retest could not remove ${path}: ${settled.reason}. Remove it by hand.`;
|
|
123
|
+
}
|
|
124
|
+
function reportLeftover(options, path, settled) {
|
|
125
|
+
if (!settled.ok)
|
|
126
|
+
options.report(leftover(path, settled));
|
|
127
|
+
}
|
|
96
128
|
/**
|
|
97
129
|
* The lock that keeps two installs of one build apart on this machine: a folder of its own beside the cache's builds,
|
|
98
130
|
* which holds the lock's newest generations and stays there once made.
|
|
@@ -124,11 +156,17 @@ async function obtainArchive(options) {
|
|
|
124
156
|
const size = (await lstat(archive)).size;
|
|
125
157
|
return { ok: true, size, sha256, fetchedFrom: 'an earlier attempt' };
|
|
126
158
|
}
|
|
127
|
-
|
|
128
|
-
|
|
159
|
+
const changes = changeOptions(options);
|
|
160
|
+
if (kept.kind !== 'missing') {
|
|
161
|
+
const removed = await removePath(archive, changes);
|
|
162
|
+
if (!removed.ok)
|
|
163
|
+
return notChanged(`Retest could not remove ${archive}, which is not the pinned archive: ${removed.reason}. Nothing was installed.`, removed);
|
|
164
|
+
}
|
|
129
165
|
await mkdir(dirname(archive), { recursive: true });
|
|
130
166
|
const partial = `${archive}${partialMark}${process.pid}`;
|
|
131
|
-
await
|
|
167
|
+
const cleared = await removePath(partial, changes);
|
|
168
|
+
if (!cleared.ok)
|
|
169
|
+
return notChanged(`Retest could not clear ${partial} for the download: ${cleared.reason}. Nothing was installed.`, cleared);
|
|
132
170
|
const size = pin.archive.size === undefined ? '' : ` (${formatBytes(pin.archive.size)})`;
|
|
133
171
|
options.report(`Downloading ${describePin(pin)} for ${describePlatform(pin.platform)} from ${shownAddress(url)}${size}`);
|
|
134
172
|
const downloaded = await downloadFile({
|
|
@@ -145,18 +183,24 @@ async function obtainArchive(options) {
|
|
|
145
183
|
if (!downloaded.ok)
|
|
146
184
|
return { ok: false, message: `${downloaded.message} Nothing was installed.`, stopped: downloaded.stopped };
|
|
147
185
|
if (downloaded.sha256 !== sha256) {
|
|
148
|
-
await
|
|
149
|
-
|
|
186
|
+
const removed = await removePath(partial, changes);
|
|
187
|
+
const deleted = removed.ok ? 'It was deleted unread and nothing was installed.' : `Nothing was installed. ${leftover(partial, removed)}`;
|
|
188
|
+
return refused(`The archive from ${shownAddress(url)} has SHA-256 ${downloaded.sha256}, not the pinned ${sha256}. ${deleted}`);
|
|
189
|
+
}
|
|
190
|
+
const named = await renamePath(partial, archive, changes);
|
|
191
|
+
if (!named.ok) {
|
|
192
|
+
const removed = await removePath(partial, changes);
|
|
193
|
+
return notChanged(`Retest could not name the verified download ${archive}: ${named.reason}. Nothing was installed.${removed.ok ? '' : ` ${leftover(partial, removed)}`}`, named);
|
|
150
194
|
}
|
|
151
|
-
await rename(partial, archive);
|
|
152
195
|
options.report(`Verified: ${downloaded.size} bytes, SHA-256 ${sha256}`);
|
|
153
196
|
// A redirect's query can be a signature, such as the one on a release asset's address, and is never recorded.
|
|
154
197
|
return { ok: true, size: downloaded.size, sha256, fetchedFrom: shownAddress(downloaded.answeredBy) };
|
|
155
198
|
}
|
|
156
199
|
const partialMark = '.partial-';
|
|
157
200
|
// Partial downloads left by installs whose process is gone are deleted; one whose process still runs is left to it,
|
|
158
|
-
// and so is one of this process's, which may belong to another install it is running.
|
|
159
|
-
|
|
201
|
+
// and so is one of this process's, which may belong to another install it is running. One that cannot be deleted is
|
|
202
|
+
// said and left, since this install downloads to a name of its own.
|
|
203
|
+
async function sweepPartials(downloads, options) {
|
|
160
204
|
const names = await readdir(downloads).catch((error) => (isMissingFile(error) ? [] : Promise.reject(error)));
|
|
161
205
|
for (const name of names) {
|
|
162
206
|
const at = name.lastIndexOf(partialMark);
|
|
@@ -165,7 +209,7 @@ async function sweepPartials(downloads) {
|
|
|
165
209
|
const pid = Number(name.slice(at + partialMark.length));
|
|
166
210
|
if (!Number.isSafeInteger(pid) || pid <= 0 || pid === process.pid || mayStillRun(pid))
|
|
167
211
|
continue;
|
|
168
|
-
|
|
212
|
+
reportLeftover(options, join(downloads, name), await removePath(join(downloads, name), changeOptions(options)));
|
|
169
213
|
}
|
|
170
214
|
}
|
|
171
215
|
// Above any pid a system hands out; `process.kill` refuses even to ask about one.
|
|
@@ -186,21 +230,18 @@ async function unpackAndRecord({ options, archive, staging, obtained }) {
|
|
|
186
230
|
const { pin } = options;
|
|
187
231
|
const root = join(staging, treeFolder);
|
|
188
232
|
await mkdir(root, { recursive: true });
|
|
189
|
-
const problem = await unpackArchive({ archive, format: pin.archive, into: root, platform: options.platform, tools: options.tools, signal: options.signal, timeoutMs: unpackTimeoutMs });
|
|
233
|
+
const problem = await unpackArchive({ archive, format: pin.archive, into: root, platform: options.platform, tools: options.tools, signal: options.signal, timeoutMs: unpackTimeoutMs, report: options.report });
|
|
190
234
|
if (options.signal.aborted)
|
|
191
235
|
return { ok: false, message: `The install was stopped while the archive was unpacked. The verified archive is kept at ${archive} for the next attempt.`, stopped: true };
|
|
192
236
|
if (problem !== undefined)
|
|
193
237
|
return refused(`${problem} The verified archive is kept at ${archive} for the next attempt.`);
|
|
194
238
|
const noticesProblem = await copyBundledLicences(pin, root);
|
|
195
|
-
if (noticesProblem !== undefined)
|
|
196
|
-
|
|
197
|
-
return refused(`${noticesProblem} The archive was deleted and nothing was installed.`);
|
|
198
|
-
}
|
|
239
|
+
if (noticesProblem !== undefined)
|
|
240
|
+
return refused(`${noticesProblem} ${await deleteRefusedArchive(options, archive)}`);
|
|
199
241
|
const checked = await checkUnpacked(pin, root);
|
|
200
242
|
if (!checked.ok) {
|
|
201
|
-
await rm(archive, { force: true });
|
|
202
243
|
const notices = checked.missing === undefined ? {} : { notices: { lead: `${describePin(pin)} lacks these licence notices, so Retest deleted its archive and installed nothing:`, files: checked.missing } };
|
|
203
|
-
return { ...refused(`${checked.message}
|
|
244
|
+
return { ...refused(`${checked.message} ${await deleteRefusedArchive(options, archive)}`), ...notices };
|
|
204
245
|
}
|
|
205
246
|
const record = {
|
|
206
247
|
schemaVersion: 1,
|
|
@@ -220,8 +261,13 @@ async function unpackAndRecord({ options, archive, staging, obtained }) {
|
|
|
220
261
|
};
|
|
221
262
|
const temporary = join(staging, `${recordFile}.${process.pid}.tmp`);
|
|
222
263
|
await writeFile(temporary, `${JSON.stringify(record, null, 2)}\n`);
|
|
223
|
-
await
|
|
224
|
-
return
|
|
264
|
+
const recorded = await renamePath(temporary, join(staging, recordFile), changeOptions(options));
|
|
265
|
+
return recorded.ok ? recorded : notChanged(`Retest could not write the record of the unpacked build: ${recorded.reason}. Nothing was installed.`, recorded);
|
|
266
|
+
}
|
|
267
|
+
// An archive whose build was refused is deleted, and the sentence says whether it was.
|
|
268
|
+
async function deleteRefusedArchive(options, archive) {
|
|
269
|
+
const removed = await removePath(archive, changeOptions(options));
|
|
270
|
+
return removed.ok ? 'The archive was deleted and nothing was installed.' : `Nothing was installed. ${leftover(archive, removed)}`;
|
|
225
271
|
}
|
|
226
272
|
/** Place only checked notice bytes into the private staging tree, before its licence and tree checks. */
|
|
227
273
|
export async function copyBundledLicences(pin, root) {
|
|
@@ -303,8 +349,9 @@ function describeAbsence(reading) {
|
|
|
303
349
|
return `SHA-256 ${reading.sha256 ?? ''}, not the pinned ${reading.file.sha256 ?? ''}`;
|
|
304
350
|
return reading.state === 'missing' ? 'missing' : (reading.problem ?? 'not a file');
|
|
305
351
|
}
|
|
306
|
-
// Staging folders of installs whose process is gone are removed; one whose process still runs is left to it.
|
|
307
|
-
|
|
352
|
+
// Staging folders of installs whose process is gone are removed; one whose process still runs is left to it. One that
|
|
353
|
+
// cannot be removed is said and left, since this install stages in a folder of its own.
|
|
354
|
+
async function sweepStaging(folder, options) {
|
|
308
355
|
const parent = dirname(folder);
|
|
309
356
|
const prefix = `${basename(folder)}${stagingMark}`;
|
|
310
357
|
const names = await readdir(parent).catch((error) => (isMissingFile(error) ? [] : Promise.reject(error)));
|
|
@@ -314,7 +361,7 @@ async function sweepStaging(folder) {
|
|
|
314
361
|
const pid = Number(name.slice(prefix.length));
|
|
315
362
|
if (Number.isSafeInteger(pid) && pid > 0 && pid !== process.pid && mayStillRun(pid))
|
|
316
363
|
continue;
|
|
317
|
-
|
|
364
|
+
reportLeftover(options, join(parent, name), await removePath(join(parent, name), changeOptions(options)));
|
|
318
365
|
}
|
|
319
366
|
}
|
|
320
367
|
function refused(message) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["../../../src/cli/install/lock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;
|
|
1
|
+
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["../../../src/cli/install/lock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAG5D,OAAO,KAAK,EAAW,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAoC1F,+GAA+G;AAC/G,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,IAAI,CAAA;CAAE,CAAA;AAE9G,2FAA2F;AAC3F,KAAK,QAAQ,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;CAAE,CAAA;AAE7E,yIAAyI;AACzI,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAA;IACnC,QAAQ,CAAC,SAAS,CAAC,EAAE,WAAW,CAAA;CACjC,CAAA;AAED,wHAAwH;AACxH,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAA;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB,CAAA;AAyBD;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC,CA2B1G;AAoHD,mDAAmD;AACnD,MAAM,MAAM,OAAO,GACf;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;CAAE,GAC1D;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;CAAE,GAC7D;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;CAAE,GAC7D;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;CAAE,GAC5D;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACvF;IAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAA;AAE7D;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,SAAS,CAAA;CAAE,GAAG,OAAO,GAAG,MAAM,CAclP"}
|
package/dist/cli/install/lock.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createHash, randomBytes } from 'node:crypto';
|
|
2
|
+
import { constants } from 'node:fs';
|
|
2
3
|
import { link, lstat, mkdir, open, readdir, rm } from 'node:fs/promises';
|
|
4
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
3
5
|
import { join } from 'node:path';
|
|
4
6
|
import { processExists, systemTools } from "../../native/processes.js";
|
|
5
7
|
import { errorMessage } from "../../protocol/failures.js";
|
|
@@ -7,6 +9,7 @@ import { parse, s } from "../../protocol/schema.js";
|
|
|
7
9
|
import { errorCode } from "../../shared/error-code.js";
|
|
8
10
|
import { entryKind, readRecordText } from "../../shared/regular-file.js";
|
|
9
11
|
import { readProcessTableAsync } from "../../shared/process-ownership.js";
|
|
12
|
+
import { launchesHeldByJob, readHeldProcess } from "../../shared/process-system.js";
|
|
10
13
|
import { compareStarts, readMachine, startReader } from "./process-start.js";
|
|
11
14
|
const generationSchema = s.object({
|
|
12
15
|
version: s.literal(2),
|
|
@@ -14,7 +17,7 @@ const generationSchema = s.object({
|
|
|
14
17
|
machine: s.string(),
|
|
15
18
|
host: s.string(),
|
|
16
19
|
pid: s.number({ integer: true, min: 1 }),
|
|
17
|
-
start: s.object({ clock: s.enum(['boot-ticks', 'utc-seconds']), value: s.number({ integer: true, min: 0 }) }),
|
|
20
|
+
start: s.object({ clock: s.enum(['boot-ticks', 'utc-seconds', 'held-file']), value: s.number({ integer: true, min: 0 }) }),
|
|
18
21
|
since: s.string(),
|
|
19
22
|
command: s.string(),
|
|
20
23
|
});
|
|
@@ -40,6 +43,7 @@ export async function takeInstallLock(path, options = {}) {
|
|
|
40
43
|
if (stoppedBy(signal))
|
|
41
44
|
return stopped();
|
|
42
45
|
const platform = options.platform ?? process.platform;
|
|
46
|
+
const held = platform === 'win32';
|
|
43
47
|
const readStart = options.readStart ?? startReader({ platform, tools: options.tools ?? systemTools, signal });
|
|
44
48
|
const folder = await lockFolder(path);
|
|
45
49
|
if (folder !== undefined)
|
|
@@ -49,12 +53,12 @@ export async function takeInstallLock(path, options = {}) {
|
|
|
49
53
|
return stopped();
|
|
50
54
|
if (!machine.ok)
|
|
51
55
|
return { ok: false, message: `${machine.problem} The install lock ${path} records which machine holds it, so Retest installs nothing without it.` };
|
|
52
|
-
const own = await readStart(process.pid);
|
|
56
|
+
const own = held ? { state: 'present', start: heldFileStart } : await readStart(process.pid);
|
|
53
57
|
if (stoppedBy(signal))
|
|
54
58
|
return stopped();
|
|
55
59
|
if (own.state !== 'present')
|
|
56
60
|
return { ok: false, message: `Retest could not read when this process started (${own.state === 'unreadable' ? own.problem : 'it is not listed'}). The install lock ${path} names its holder by its start, so Retest installs nothing without it.` };
|
|
57
|
-
const context = { path, machine: machine.machine, tag: machineTag(machine.machine), ownStart: own.start, readStart };
|
|
61
|
+
const context = { path, machine: machine.machine, tag: machineTag(machine.machine), ownStart: own.start, readStart, held };
|
|
58
62
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
59
63
|
if (stoppedBy(signal))
|
|
60
64
|
return stopped();
|
|
@@ -75,6 +79,8 @@ export async function takeInstallLock(path, options = {}) {
|
|
|
75
79
|
function stoppedBy(signal) {
|
|
76
80
|
return signal?.aborted === true;
|
|
77
81
|
}
|
|
82
|
+
// What a holder on Windows records as its start: none, since its held file is what tells it alive.
|
|
83
|
+
const heldFileStart = { clock: 'held-file', value: 0 };
|
|
78
84
|
// One reading of the folder and, when the newest generation is free, one try at the next.
|
|
79
85
|
async function tryOnce(context) {
|
|
80
86
|
const { path } = context;
|
|
@@ -94,13 +100,21 @@ async function tryOnce(context) {
|
|
|
94
100
|
if (legacy.machine !== undefined && legacy.machine !== context.machine.id)
|
|
95
101
|
return { kind: 'refused', message: `The install generation ${file} belongs to another machine or boot namespace, whose process table this machine cannot confirm. Remove ${file} once no install of this build is running there.` };
|
|
96
102
|
try {
|
|
97
|
-
|
|
103
|
+
// Windows has no `ps`; there the pid is read through retest-supervisor, as every process is.
|
|
104
|
+
const present = launchesHeldByJob ? readHeldProcess(legacy.pid) !== undefined : (await readProcessTableAsync()).some(entry => entry.pid === legacy.pid);
|
|
105
|
+
if (present)
|
|
98
106
|
return { kind: 'refused', message: `Retest cannot confirm the recorded start time zone in the install generation ${file}; pid ${legacy.pid} is present. Remove ${file} once no install of this build is running.` };
|
|
99
107
|
}
|
|
100
108
|
catch (error) {
|
|
101
109
|
return { kind: 'refused', message: `Retest cannot confirm absence for install generation ${file}: ${errorMessage(error)}. Remove ${file} once no install of this build is running.` };
|
|
102
110
|
}
|
|
103
111
|
}
|
|
112
|
+
else if (context.held) {
|
|
113
|
+
const generation = reading.generation;
|
|
114
|
+
const decision = (await isReleased(path, newest, generation.token)) ? 'free' : await decideHeld(path, newest, generation, context);
|
|
115
|
+
if (decision !== 'free')
|
|
116
|
+
return { kind: 'refused', message: describeBlocker(path, newest, decision) };
|
|
117
|
+
}
|
|
104
118
|
else {
|
|
105
119
|
const generation = reading.generation;
|
|
106
120
|
const released = await isReleased(path, newest, generation.token);
|
|
@@ -136,11 +150,59 @@ async function tryOnce(context) {
|
|
|
136
150
|
// that is still the newest holds the lock.
|
|
137
151
|
const after = await listGenerations(context);
|
|
138
152
|
if (!after.ok || after.newest !== newest + 1) {
|
|
139
|
-
await release(path, newest + 1, made.token);
|
|
153
|
+
await release(path, newest + 1, made.token, made.hold);
|
|
140
154
|
return after.ok ? { kind: 'again' } : { kind: 'refused', message: after.message };
|
|
141
155
|
}
|
|
142
156
|
await removeOldGenerations(path, newest + 1);
|
|
143
|
-
return { kind: 'held', lock: heldLock(path, newest + 1, made.token) };
|
|
157
|
+
return { kind: 'held', lock: heldLock(path, newest + 1, made.token, made.hold) };
|
|
158
|
+
}
|
|
159
|
+
// Held files this process holds open, by path, so its own generation reads as its own rather than as another holder's.
|
|
160
|
+
const holding = new Set();
|
|
161
|
+
// libuv's UV_FS_O_EXLOCK (uv/win.h): the file is opened sharing nothing, so every other open of it fails while this one
|
|
162
|
+
// lasts. Node hands numeric flags to libuv as they are; only Windows reads this one.
|
|
163
|
+
const openedAlone = 0x10000000;
|
|
164
|
+
function holdName(number, token) {
|
|
165
|
+
return `${number}.${token}.held`;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Whether a generation on Windows leaves the lock free: its holder's held file opens, so nothing holds it any more. One
|
|
169
|
+
* that does not open is held, by this process or another; one that is missing, or cannot be read, proves nothing.
|
|
170
|
+
*/
|
|
171
|
+
async function decideHeld(path, number, generation, context) {
|
|
172
|
+
if (generation.machine !== context.machine.id)
|
|
173
|
+
return { kind: 'foreign', generation };
|
|
174
|
+
const file = join(path, holdName(number, generation.token));
|
|
175
|
+
if (holding.has(file))
|
|
176
|
+
return { kind: 'self', generation };
|
|
177
|
+
const probe = await probeHeldFile(file);
|
|
178
|
+
if (probe === 'free')
|
|
179
|
+
return 'free';
|
|
180
|
+
if (probe === 'held')
|
|
181
|
+
return { kind: 'running', generation };
|
|
182
|
+
return { kind: 'unknown', generation, problem: probe === 'missing' ? `its held file ${file} is missing` : probe.problem };
|
|
183
|
+
}
|
|
184
|
+
// A held file that opens alone is held by no one. A virus scanner may hold it for a moment, so one that will not open is
|
|
185
|
+
// asked again a few times before it counts as held.
|
|
186
|
+
async function probeHeldFile(file) {
|
|
187
|
+
for (let attempt = 1;; attempt += 1) {
|
|
188
|
+
let handle;
|
|
189
|
+
try {
|
|
190
|
+
handle = await open(file, constants.O_RDONLY | openedAlone);
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
const code = errorCode(error);
|
|
194
|
+
if (code === 'ENOENT')
|
|
195
|
+
return 'missing';
|
|
196
|
+
if (code !== 'EBUSY')
|
|
197
|
+
return { problem: `${file} cannot be opened: ${errorMessage(error)}` };
|
|
198
|
+
if (attempt >= 3)
|
|
199
|
+
return 'held';
|
|
200
|
+
await sleep(20);
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
await handle.close();
|
|
204
|
+
return 'free';
|
|
205
|
+
}
|
|
144
206
|
}
|
|
145
207
|
/**
|
|
146
208
|
* Whether a generation leaves the lock free, as a taker on `machine` reads it: let go; or held by this machine's
|
|
@@ -258,13 +320,19 @@ async function makeGeneration(context, number) {
|
|
|
258
320
|
const token = randomBytes(16).toString('hex');
|
|
259
321
|
const generation = { version: 2, token, machine: context.machine.id, host: context.machine.host, pid: process.pid, start: context.ownStart, since: new Date().toISOString(), command: holderCommand() };
|
|
260
322
|
const temporary = join(path, `.${context.tag}-${process.pid}-${randomBytes(8).toString('hex')}.tmp`);
|
|
323
|
+
let hold;
|
|
261
324
|
try {
|
|
325
|
+
// The held file is open before the generation that names it exists, so no taker ever reads it as let go.
|
|
326
|
+
if (context.held)
|
|
327
|
+
hold = await holdFile(join(path, holdName(number, token)));
|
|
262
328
|
await writeDurably(temporary, `${JSON.stringify(generation)}\n`);
|
|
263
329
|
await link(temporary, join(path, `${number}.json`));
|
|
264
330
|
await syncFolder(path);
|
|
265
|
-
return { kind: 'made', token };
|
|
331
|
+
return { kind: 'made', token, hold };
|
|
266
332
|
}
|
|
267
333
|
catch (error) {
|
|
334
|
+
if (hold !== undefined)
|
|
335
|
+
await letGo(hold, { remove: true });
|
|
268
336
|
if (errorCode(error) === 'EEXIST')
|
|
269
337
|
return { kind: 'taken' };
|
|
270
338
|
return { kind: 'failed', message: `Retest could not take the install lock ${path}: ${errorMessage(error)}.` };
|
|
@@ -273,25 +341,44 @@ async function makeGeneration(context, number) {
|
|
|
273
341
|
await rm(temporary, { force: true }).catch(() => undefined);
|
|
274
342
|
}
|
|
275
343
|
}
|
|
276
|
-
function
|
|
344
|
+
async function holdFile(file) {
|
|
345
|
+
const handle = await open(file, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | openedAlone);
|
|
346
|
+
holding.add(file);
|
|
347
|
+
return { file, handle };
|
|
348
|
+
}
|
|
349
|
+
// Closing the file is what tells other processes this holder let go; removing it is only tidying.
|
|
350
|
+
async function letGo(hold, { remove }) {
|
|
351
|
+
holding.delete(hold.file);
|
|
352
|
+
await hold.handle.close().catch(() => undefined);
|
|
353
|
+
if (remove)
|
|
354
|
+
await rm(hold.file, { force: true }).catch(() => undefined);
|
|
355
|
+
}
|
|
356
|
+
function heldLock(path, number, token, hold) {
|
|
277
357
|
let releasing;
|
|
278
|
-
return { ok: true, release: () => (releasing ??= release(path, number, token)) };
|
|
358
|
+
return { ok: true, release: () => (releasing ??= release(path, number, token, hold)) };
|
|
279
359
|
}
|
|
280
360
|
// Lets go of generation `number` only while it is still this holder's, by its token: a folder removed and made again
|
|
281
|
-
// meanwhile holds another process's generation under that number, which this release must not free.
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
if (reading.kind !== 'found' || reading.generation.token !== token)
|
|
285
|
-
return `The install lock ${path} no longer holds this process's generation ${number}, so nothing there was let go.`;
|
|
361
|
+
// meanwhile holds another process's generation under that number, which this release must not free. A held file is
|
|
362
|
+
// closed after the released mark is written, and whatever the release met, since a holder that let go holds nothing.
|
|
363
|
+
async function release(path, number, token, hold) {
|
|
286
364
|
try {
|
|
287
|
-
await
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
365
|
+
const reading = await readGeneration(path, number);
|
|
366
|
+
if (reading.kind !== 'found' || reading.generation.token !== token)
|
|
367
|
+
return `The install lock ${path} no longer holds this process's generation ${number}, so nothing there was let go.`;
|
|
368
|
+
try {
|
|
369
|
+
await writeDurably(join(path, releasedName(number, token)), `${token}\n`);
|
|
370
|
+
await syncFolder(path);
|
|
293
371
|
return undefined;
|
|
294
|
-
|
|
372
|
+
}
|
|
373
|
+
catch (error) {
|
|
374
|
+
if (errorCode(error) === 'EEXIST')
|
|
375
|
+
return undefined;
|
|
376
|
+
return `The install lock ${path} was not let go: ${errorMessage(error)}. The next install passes it over once this process has ended.`;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
finally {
|
|
380
|
+
if (hold !== undefined)
|
|
381
|
+
await letGo(hold, { remove: false });
|
|
295
382
|
}
|
|
296
383
|
}
|
|
297
384
|
// Keeps the newest generation and the one before it, which a taker may still be reading, and removes the rest. A file
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"media-doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/install/media-doctor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"media-doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/install/media-doctor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAMxD,sGAAsG;AACtG,wBAAsB,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAGnH;AAED,wBAAsB,eAAe,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAkClG;AAED,qGAAqG;AACrG,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAMrD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMxF"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { notAnEncoder } from "../../media/host.js";
|
|
1
2
|
import { locateFfmpeg, ffmpegFix, locateMedia } from "../../media/locate.js";
|
|
2
3
|
import { mediaTarget, minimumRust } from "./media-pins.js";
|
|
3
4
|
import { findMediaTool, probeMediaBinary, rustAtLeast, runMediaTool, rustFix } from "./media-tools.js";
|
|
@@ -11,7 +12,7 @@ export async function mediaDoctorRows(env, signal) {
|
|
|
11
12
|
const checks = [];
|
|
12
13
|
if (signal.aborted)
|
|
13
14
|
return [{ group: 'media', subject: 'retest-media', ok: false, text: 'The media check was stopped.', fix: rustFix }];
|
|
14
|
-
const found = await locateMedia({ env, ffmpeg:
|
|
15
|
+
const found = await locateMedia({ env, ffmpeg: notAnEncoder, signal });
|
|
15
16
|
checks.push(found.ok
|
|
16
17
|
? { group: 'media', subject: 'retest-media', ok: true, text: `retest-media ${found.hello.version}, protocol ${found.hello.protocol}, from ${found.source}; source builds need Rust 1.88 or later, edition 2024 needs Rust 1.85 or later${found.inspection?.record === undefined ? '; no cache record for this explicit binary' : `; SHA-256 ${found.inspection.record.binarySha256} matches its record`}`, detail: found.executable, fix: rustFix }
|
|
17
18
|
: { group: 'media', subject: 'retest-media', ok: false, text: found.message, fix: 'Run npx retest install media. For an explicit release binary set RETEST_MEDIA_BINARY.' });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"media-install.d.ts","sourceRoot":"","sources":["../../../src/cli/install/media-install.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"media-install.d.ts","sourceRoot":"","sources":["../../../src/cli/install/media-install.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACzE,OAAO,KAAK,EAAE,eAAe,EAAe,MAAM,mBAAmB,CAAA;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAkBpE,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/H,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,eAAe,CAAA;CAC1I,CAAA;AACD,MAAM,MAAM,kBAAkB,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,mBAAmB,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAA;CAAE,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAAA;AAE9N,yHAAyH;AACzH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA6C5F"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import { chmod, mkdir,
|
|
2
|
+
import { chmod, mkdir, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { MEDIA_PROTOCOL_VERSION } from "../../media/protocol.js";
|
|
5
5
|
import { describeCommand } from "../../native/processes.js";
|
|
@@ -7,9 +7,11 @@ import { errorMessage } from "../../protocol/failures.js";
|
|
|
7
7
|
import { Deadline } from "../../protocol/deadline.js";
|
|
8
8
|
import { openRegularFile, readFileSha256, readRecordText } from "../../shared/regular-file.js";
|
|
9
9
|
import { downloadFile, mirroredUrl } from "./download.js";
|
|
10
|
+
import { removePath, renamePath } from "./file-changes.js";
|
|
10
11
|
import { takeInstallLock } from "./lock.js";
|
|
12
|
+
import { mediaBuildFolder, notAnEncoder } from "../../media/host.js";
|
|
11
13
|
import { checkMediaSource, inspectMedia, mediaFolder, mediaLockPath, mediaNoticePins, mediaPackageRoot } from "./media-record.js";
|
|
12
|
-
import { maximumMediaBinaryBytes, mediaPrebuiltPins, mediaSourceDigest, mediaSourceFiles, mediaTarget, mediaVersion } from "./media-pins.js";
|
|
14
|
+
import { maximumMediaBinaryBytes, mediaExecutableName, mediaPrebuiltPins, mediaSourceDigest, mediaSourceFiles, mediaTarget, mediaVersion } from "./media-pins.js";
|
|
13
15
|
import { probeMediaBinary, readRustTools, runMediaTool } from "./media-tools.js";
|
|
14
16
|
import { mirrorProblem } from "./report.js";
|
|
15
17
|
/** Explicit source build or exactly pinned prebuilt. Uses the current install lock and never repairs a damaged cache. */
|
|
@@ -17,7 +19,7 @@ export async function installMedia(options) {
|
|
|
17
19
|
const refuse = (message) => ({ ok: false, message, stopped: options.signal.aborted });
|
|
18
20
|
const target = options.target ?? mediaTarget();
|
|
19
21
|
if (target === undefined)
|
|
20
|
-
return refuse(`
|
|
22
|
+
return refuse(`Retest builds retest-media for macOS arm64, Linux x64 and Windows x64, and this machine is ${process.platform} ${process.arch}.`);
|
|
21
23
|
const pins = options.pins ?? mediaPrebuiltPins;
|
|
22
24
|
const prebuilt = options.binary !== undefined || options.mirror !== undefined;
|
|
23
25
|
const pin = prebuilt ? pins.find((candidate) => candidate.target === target && candidate.version === mediaVersion && candidate.protocol === MEDIA_PROTOCOL_VERSION && candidate.sourceDigest === mediaSourceDigest) : undefined;
|
|
@@ -43,18 +45,24 @@ export async function installMedia(options) {
|
|
|
43
45
|
let result;
|
|
44
46
|
let cleanupProblem;
|
|
45
47
|
const staging = `${folder}.staging-${process.pid}-${randomUUID()}`;
|
|
48
|
+
const build = mediaBuildFolder(staging);
|
|
46
49
|
try {
|
|
47
|
-
result = await installLocked(options, target, folder, root, staging, pin);
|
|
50
|
+
result = await installLocked(options, target, folder, root, staging, build, pin);
|
|
48
51
|
}
|
|
49
52
|
catch (error) {
|
|
50
53
|
result = refuse(`Media install failed: ${errorMessage(error)} Nothing was installed.`);
|
|
51
54
|
}
|
|
52
55
|
finally {
|
|
53
56
|
try {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
// A removal settles without throwing, so the staging folder is removed even when the build folder could not be.
|
|
58
|
+
const problems = [];
|
|
59
|
+
for (const path of [build, staging]) {
|
|
60
|
+
const removed = await removePath(path, changeOptions(options));
|
|
61
|
+
if (!removed.ok)
|
|
62
|
+
problems.push(`Media staging cleanup failed at ${path}: ${removed.reason}.`);
|
|
63
|
+
}
|
|
64
|
+
if (problems.length > 0)
|
|
65
|
+
cleanupProblem = problems.join(' ');
|
|
58
66
|
}
|
|
59
67
|
finally {
|
|
60
68
|
const problem = await lock.release();
|
|
@@ -68,7 +76,7 @@ export async function installMedia(options) {
|
|
|
68
76
|
: { ...result, message: `${result.message} ${cleanupProblem}` };
|
|
69
77
|
return result;
|
|
70
78
|
}
|
|
71
|
-
async function installLocked(options, target, folder, root, staging, pin) {
|
|
79
|
+
async function installLocked(options, target, folder, root, staging, build, pin) {
|
|
72
80
|
const refuse = (message) => ({ ok: false, message, stopped: options.signal.aborted });
|
|
73
81
|
const before = await inspectMedia(options.env, { target, pins: options.pins ?? mediaPrebuiltPins, verify: true, signal: options.signal });
|
|
74
82
|
if (before?.state === 'installed')
|
|
@@ -79,7 +87,7 @@ async function installLocked(options, target, folder, root, staging, pin) {
|
|
|
79
87
|
return refuse('Media install was stopped before it built or copied anything.');
|
|
80
88
|
let rustcVersion = null;
|
|
81
89
|
await mkdir(staging, { recursive: true });
|
|
82
|
-
const executable = join(staging,
|
|
90
|
+
const executable = join(staging, mediaExecutableName(target));
|
|
83
91
|
if (pin === undefined) {
|
|
84
92
|
const run = options.run ?? runMediaTool;
|
|
85
93
|
const tools = await readRustTools(options.env, options.signal, run);
|
|
@@ -101,20 +109,24 @@ async function installLocked(options, target, folder, root, staging, pin) {
|
|
|
101
109
|
const copiedProblem = await checkMediaSource(source, { signal: options.signal });
|
|
102
110
|
if (copiedProblem !== undefined)
|
|
103
111
|
return refuse(copiedProblem);
|
|
104
|
-
const
|
|
105
|
-
const args = ['build', '--release', '--locked', ...(options.offline === true ? ['--offline'] : []), '--manifest-path', join(source, 'media', 'Cargo.toml'), '--target', target, '--target-dir', targetDir];
|
|
112
|
+
const args = ['build', '--release', '--locked', ...(options.offline === true ? ['--offline'] : []), '--manifest-path', join(source, 'media', 'Cargo.toml'), '--target', target, '--target-dir', build];
|
|
106
113
|
options.report(`Building retest-media ${mediaVersion} with ${tools.rustcVersion}; Cargo.lock is pinned. ffmpeg is a separate host prerequisite.`);
|
|
107
|
-
const built = await run(tools.cargo, args, { env: options.env, signal: options.signal, cwd: source, timeoutMs: 30 * 60_000, environment: { RUSTC: tools.rustc, CARGO_TARGET_DIR:
|
|
114
|
+
const built = await run(tools.cargo, args, { env: options.env, signal: options.signal, cwd: source, timeoutMs: 30 * 60_000, environment: { RUSTC: tools.rustc, CARGO_TARGET_DIR: build } });
|
|
108
115
|
const afterProblem = await checkMediaSource(source, { signal: options.signal });
|
|
109
116
|
if (afterProblem !== undefined)
|
|
110
117
|
return refuse(`The build changed the pinned source or Cargo.lock. ${afterProblem}`);
|
|
118
|
+
// Cargo's last line only says the build failed; its first error says why.
|
|
119
|
+
const cause = built.stderr.split('\n').find((line) => line.startsWith('error')) ?? '';
|
|
111
120
|
if (built.code !== 0 || built.stopped || built.timedOut || built.cleanupProblems.length > 0)
|
|
112
|
-
return refuse(`${describeCommand('cargo build --release --locked', built)}. Nothing was installed.`);
|
|
113
|
-
const placed = await copyRegularBinary(join(
|
|
121
|
+
return refuse(`${describeCommand('cargo build --release --locked', built)}.${cause === '' ? '' : ` ${cause.slice(0, 300)}`} Nothing was installed.`);
|
|
122
|
+
const placed = await copyRegularBinary(join(build, target, 'release', mediaExecutableName(target)), executable);
|
|
114
123
|
if (placed !== undefined)
|
|
115
124
|
return refuse(placed);
|
|
116
|
-
|
|
117
|
-
|
|
125
|
+
for (const path of [source, build]) {
|
|
126
|
+
const removed = await removePath(path, changeOptions(options));
|
|
127
|
+
if (!removed.ok)
|
|
128
|
+
return refuse(`Retest could not remove ${path} after the build: ${removed.reason}. Nothing was installed.`);
|
|
129
|
+
}
|
|
118
130
|
}
|
|
119
131
|
else {
|
|
120
132
|
if (options.binary !== undefined) {
|
|
@@ -136,7 +148,7 @@ async function installLocked(options, target, folder, root, staging, pin) {
|
|
|
136
148
|
if (options.signal.aborted)
|
|
137
149
|
return refuse('Media install was stopped before the binary probe. Nothing was installed.');
|
|
138
150
|
await chmod(executable, 0o755);
|
|
139
|
-
const probe = await probeMediaBinary(executable,
|
|
151
|
+
const probe = await probeMediaBinary(executable, notAnEncoder, target);
|
|
140
152
|
if (!probe.ok)
|
|
141
153
|
return refuse(probe.message);
|
|
142
154
|
if (options.signal.aborted)
|
|
@@ -157,10 +169,17 @@ async function installLocked(options, target, folder, root, staging, pin) {
|
|
|
157
169
|
return refuse(sourceProblem);
|
|
158
170
|
const record = { schemaVersion: 1, version: mediaVersion, protocol: MEDIA_PROTOCOL_VERSION, sourceDigest: mediaSourceDigest, target, method: pin === undefined ? 'source' : 'prebuilt', rustcVersion, binarySha256: hash.sha256, notices: mediaNoticePins(), installedAt: new Date().toISOString(), installedBy: options.version };
|
|
159
171
|
await writeFile(join(staging, 'build.json'), `${JSON.stringify(record, null, 2)}\n`, { flag: 'wx' });
|
|
160
|
-
await
|
|
172
|
+
const moved = await renamePath(staging, folder, changeOptions(options));
|
|
173
|
+
if (!moved.ok)
|
|
174
|
+
return refuse(`Retest could not move the built media folder into ${folder}: ${moved.reason}. Nothing was installed.`);
|
|
161
175
|
const after = await inspectMedia(options.env, { target, pins: options.pins ?? mediaPrebuiltPins, verify: true, signal: options.signal });
|
|
162
176
|
return after?.state === 'installed' ? { ok: true, action: 'installed', inspection: after } : refuse(`The media folder did not read as installed: ${after?.problems.join(' ') ?? 'no cache'}.`);
|
|
163
177
|
}
|
|
178
|
+
// Windows refuses to rename or remove a file a virus scanner holds open, as it does a freshly built retest-media.exe, so
|
|
179
|
+
// each change the install makes in the cache is tried again there for a while.
|
|
180
|
+
function changeOptions(options) {
|
|
181
|
+
return { platform: process.platform, signal: options.signal };
|
|
182
|
+
}
|
|
164
183
|
async function copyRegularBinary(from, to, expectedSize) {
|
|
165
184
|
const opened = await openRegularFile(from);
|
|
166
185
|
if (opened.kind !== 'opened')
|