@hominis/fireforge 0.46.2 → 0.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/bin/fireforge.js +6 -0
- package/dist/build-info.json +4 -4
- package/dist/src/commands/config.js +1 -2
- package/dist/src/commands/furnace/deploy-fragment-includers.d.ts +59 -0
- package/dist/src/commands/furnace/deploy-fragment-includers.js +120 -0
- package/dist/src/commands/furnace/deploy.js +36 -3
- package/dist/src/commands/furnace/validation-output.d.ts +5 -0
- package/dist/src/commands/furnace/validation-output.js +35 -9
- package/dist/src/commands/patch/delete-gap-notice.d.ts +24 -0
- package/dist/src/commands/patch/delete-gap-notice.js +46 -0
- package/dist/src/commands/patch/delete.js +11 -0
- package/dist/src/commands/re-export-lint.d.ts +26 -1
- package/dist/src/commands/re-export-lint.js +60 -20
- package/dist/src/commands/re-export-scan.d.ts +8 -0
- package/dist/src/commands/re-export-scan.js +12 -2
- package/dist/src/commands/re-export-single.js +24 -5
- package/dist/src/commands/run.js +42 -17
- package/dist/src/commands/test-harness-env.d.ts +24 -0
- package/dist/src/commands/test-harness-env.js +61 -0
- package/dist/src/commands/test-harness-teardown.d.ts +62 -0
- package/dist/src/commands/test-harness-teardown.js +128 -0
- package/dist/src/commands/test-register.js +8 -3
- package/dist/src/commands/test-run.d.ts +2 -0
- package/dist/src/commands/test-run.js +36 -7
- package/dist/src/commands/test-verdict.d.ts +21 -4
- package/dist/src/commands/test-verdict.js +37 -4
- package/dist/src/commands/test.js +27 -26
- package/dist/src/core/build-jar-escalation.d.ts +1 -4
- package/dist/src/core/build-jar-escalation.js +1 -4
- package/dist/src/core/build-prepare.js +72 -55
- package/dist/src/core/config-paths.d.ts +1 -1
- package/dist/src/core/config-paths.js +1 -0
- package/dist/src/core/config-validate-test-toolchains.js +7 -0
- package/dist/src/core/file-lock.js +16 -9
- package/dist/src/core/firefox-cache.js +1 -2
- package/dist/src/core/furnace-apply-output.d.ts +10 -0
- package/dist/src/core/furnace-apply-output.js +19 -4
- package/dist/src/core/furnace-css-fragments.d.ts +7 -0
- package/dist/src/core/furnace-css-fragments.js +12 -3
- package/dist/src/core/furnace-fragment-includers.d.ts +56 -0
- package/dist/src/core/furnace-fragment-includers.js +122 -0
- package/dist/src/core/git-diff-new-file.js +1 -2
- package/dist/src/core/git-file-ops.js +1 -2
- package/dist/src/core/git.js +1 -2
- package/dist/src/core/harness-helper-pids.d.ts +98 -0
- package/dist/src/core/harness-helper-pids.js +240 -0
- package/dist/src/core/harness-orphans.d.ts +53 -8
- package/dist/src/core/harness-orphans.js +125 -44
- package/dist/src/core/mach-build-artifacts.js +1 -2
- package/dist/src/core/mach.d.ts +16 -0
- package/dist/src/core/mach.js +6 -1
- package/dist/src/core/marionette-preflight.js +9 -3
- package/dist/src/core/mochitest-server-port.d.ts +43 -4
- package/dist/src/core/mochitest-server-port.js +64 -11
- package/dist/src/core/parent-exit-watchdog.d.ts +62 -0
- package/dist/src/core/parent-exit-watchdog.js +69 -0
- package/dist/src/core/patch-lint-imports.js +4 -4
- package/dist/src/core/smoke-patterns.d.ts +24 -0
- package/dist/src/core/smoke-patterns.js +50 -5
- package/dist/src/types/commands/options.d.ts +17 -1
- package/dist/src/types/config.d.ts +7 -0
- package/dist/src/utils/process-group.d.ts +5 -1
- package/dist/src/utils/process-group.js +7 -3
- package/dist/src/utils/process.d.ts +18 -0
- package/dist/src/utils/process.js +14 -2
- package/docs/configuration.md +15 -1
- package/docs/exit-codes.md +19 -19
- package/docs/lifecycle-invariants.md +17 -1
- package/docs/machine-output.md +44 -19
- package/docs/testing.md +80 -4
- package/package.json +5 -5
|
@@ -242,8 +242,7 @@ async function removeArchivePartFiles(archive, cacheDir) {
|
|
|
242
242
|
.filter((name) => name.startsWith(partPrefix))
|
|
243
243
|
.map((name) => removeFile(join(cacheDir, name))));
|
|
244
244
|
}
|
|
245
|
-
catch
|
|
246
|
-
void error;
|
|
245
|
+
catch {
|
|
247
246
|
// Cache dir may not exist yet. That's fine.
|
|
248
247
|
}
|
|
249
248
|
}
|
|
@@ -13,4 +13,14 @@ type ApplyResultWithActions = ApplyResult & {
|
|
|
13
13
|
* @param isDryRun - Whether apply was invoked with dryRun=true
|
|
14
14
|
*/
|
|
15
15
|
export declare function logApplyResult(result: ApplyResultWithActions, isDryRun: boolean): void;
|
|
16
|
+
/**
|
|
17
|
+
* Prints the operator-facing apply warnings (patch-owned overwrites) one
|
|
18
|
+
* per line and returns how many were printed. Shared by `furnace apply`
|
|
19
|
+
* and the pre-build source → engine sync, so a warning the apply computed
|
|
20
|
+
* is never dropped on one path and printed on the other.
|
|
21
|
+
*
|
|
22
|
+
* @param warnings - `result.warnings` from applyAllComponents
|
|
23
|
+
* @returns Number of warning lines printed
|
|
24
|
+
*/
|
|
25
|
+
export declare function logApplyWarnings(warnings: readonly string[] | undefined): number;
|
|
16
26
|
export {};
|
|
@@ -55,12 +55,27 @@ export function logApplyResult(result, isDryRun) {
|
|
|
55
55
|
// outcome, including the rolled-back branch, where the overwrite
|
|
56
56
|
// happened before the rollback restored it and the operator still needs
|
|
57
57
|
// to know the deployed copy was momentarily replaced.
|
|
58
|
-
if (!isDryRun
|
|
59
|
-
|
|
60
|
-
warn(line);
|
|
61
|
-
}
|
|
58
|
+
if (!isDryRun) {
|
|
59
|
+
logApplyWarnings(result.warnings);
|
|
62
60
|
}
|
|
63
61
|
for (const err of result.errors) {
|
|
64
62
|
error(`${err.name} — ${err.error}`);
|
|
65
63
|
}
|
|
66
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Prints the operator-facing apply warnings (patch-owned overwrites) one
|
|
67
|
+
* per line and returns how many were printed. Shared by `furnace apply`
|
|
68
|
+
* and the pre-build source → engine sync, so a warning the apply computed
|
|
69
|
+
* is never dropped on one path and printed on the other.
|
|
70
|
+
*
|
|
71
|
+
* @param warnings - `result.warnings` from applyAllComponents
|
|
72
|
+
* @returns Number of warning lines printed
|
|
73
|
+
*/
|
|
74
|
+
export function logApplyWarnings(warnings) {
|
|
75
|
+
if (warnings === undefined)
|
|
76
|
+
return 0;
|
|
77
|
+
for (const line of warnings) {
|
|
78
|
+
warn(line);
|
|
79
|
+
}
|
|
80
|
+
return warnings.length;
|
|
81
|
+
}
|
|
@@ -20,6 +20,13 @@
|
|
|
20
20
|
*/
|
|
21
21
|
import type { ValidationIssue } from '../types/furnace.js';
|
|
22
22
|
export { SHARED_FRAGMENTS_DIR } from './furnace-config.js';
|
|
23
|
+
/**
|
|
24
|
+
* Reads a shared fragment as the exact body an expansion inlines: the
|
|
25
|
+
* trailing newline is dropped so the fenced block ends on the end marker.
|
|
26
|
+
* One definition, so every consumer (expansion, validation, the stale
|
|
27
|
+
* includer probe) compares the same bytes.
|
|
28
|
+
*/
|
|
29
|
+
export declare function readFragmentSource(sharedDir: string, name: string): Promise<string>;
|
|
23
30
|
/** Returns the fragment names referenced by `@fireforge-include` directives. */
|
|
24
31
|
export declare function listFragmentIncludes(css: string): string[];
|
|
25
32
|
/**
|
|
@@ -29,6 +29,15 @@ export { SHARED_FRAGMENTS_DIR } from './furnace-config.js';
|
|
|
29
29
|
import { SHARED_FRAGMENTS_DIR } from './furnace-config.js';
|
|
30
30
|
const INCLUDE_PATTERN = /^\s*\/\*\s*@fireforge-include\s+([\w./-]+)\s*\*\/\s*$/;
|
|
31
31
|
const END_INCLUDE_PATTERN = /^\s*\/\*\s*@fireforge-end-include\s+([\w./-]+)\s*\*\/\s*$/;
|
|
32
|
+
/**
|
|
33
|
+
* Reads a shared fragment as the exact body an expansion inlines: the
|
|
34
|
+
* trailing newline is dropped so the fenced block ends on the end marker.
|
|
35
|
+
* One definition, so every consumer (expansion, validation, the stale
|
|
36
|
+
* includer probe) compares the same bytes.
|
|
37
|
+
*/
|
|
38
|
+
export async function readFragmentSource(sharedDir, name) {
|
|
39
|
+
return (await readText(join(sharedDir, name))).replace(/\n$/, '');
|
|
40
|
+
}
|
|
32
41
|
/** Returns the fragment names referenced by `@fireforge-include` directives. */
|
|
33
42
|
export function listFragmentIncludes(css) {
|
|
34
43
|
const names = [];
|
|
@@ -93,12 +102,12 @@ export async function expandCssFragments(css, sharedDir) {
|
|
|
93
102
|
throw new FurnaceError(`CSS fragment "${name}" not found in components/${SHARED_FRAGMENTS_DIR}/. ` +
|
|
94
103
|
'Create the fragment file or remove the @fireforge-include directive.');
|
|
95
104
|
}
|
|
96
|
-
const content = await
|
|
105
|
+
const content = await readFragmentSource(sharedDir, name);
|
|
97
106
|
if (listFragmentIncludes(content).length > 0) {
|
|
98
107
|
throw new FurnaceError(`CSS fragment "${name}" contains an @fireforge-include directive of its own; ` +
|
|
99
108
|
'nested fragment includes are not supported.');
|
|
100
109
|
}
|
|
101
|
-
fragments.set(name, content
|
|
110
|
+
fragments.set(name, content);
|
|
102
111
|
}
|
|
103
112
|
const out = [];
|
|
104
113
|
for (const line of stripped.split('\n')) {
|
|
@@ -226,7 +235,7 @@ export async function validateCssFragments(componentDir, tagName, sharedDir, eng
|
|
|
226
235
|
}
|
|
227
236
|
if (deployedBodies === null)
|
|
228
237
|
continue;
|
|
229
|
-
const fragmentContent =
|
|
238
|
+
const fragmentContent = await readFragmentSource(sharedDir, include);
|
|
230
239
|
const deployed = deployedBodies.get(include);
|
|
231
240
|
if (deployed === undefined || deployed !== fragmentContent) {
|
|
232
241
|
issues.push({
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reverse index from shared CSS fragments to the custom components that
|
|
3
|
+
* include them, plus the staleness probe a targeted `furnace deploy <tag>`
|
|
4
|
+
* runs over the OTHER includers of the fragments it just refreshed.
|
|
5
|
+
*
|
|
6
|
+
* Fragment staleness was only ever a validator finding
|
|
7
|
+
* (`stale-fragment-expansion` in {@link validateCssFragments}), printed per
|
|
8
|
+
* includer by `furnace validate` and `deploy --dry-run`. A targeted deploy
|
|
9
|
+
* validates only the component it was asked for, so the one command that
|
|
10
|
+
* changes a fragment's deployed state for SOME includers is the one that
|
|
11
|
+
* never mentioned the rest: after a fragment edit, `deploy <a> <b>` left
|
|
12
|
+
* every other consumer carrying the old expansion, and nothing said so
|
|
13
|
+
* until a `grep` over the engine found it.
|
|
14
|
+
*
|
|
15
|
+
* Only custom components are indexed. Override components are copied
|
|
16
|
+
* byte-for-byte and never expand fragments.
|
|
17
|
+
*/
|
|
18
|
+
import type { CustomComponentConfig } from '../types/furnace.js';
|
|
19
|
+
import type { FurnacePaths } from './furnace-config.js';
|
|
20
|
+
/** One includer whose deployed sheets carry a stale expansion. */
|
|
21
|
+
export interface StaleFragmentIncluder {
|
|
22
|
+
/** Custom component tag. */
|
|
23
|
+
tag: string;
|
|
24
|
+
/** Fragments (relative to `components/shared/`) whose expansion is stale. */
|
|
25
|
+
fragments: string[];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Maps every fragment name to the custom tags whose workspace sheets
|
|
29
|
+
* include it.
|
|
30
|
+
*
|
|
31
|
+
* @param customDir - `components/custom/` directory
|
|
32
|
+
* @param custom - The `custom` map from furnace.json
|
|
33
|
+
*/
|
|
34
|
+
export declare function collectFragmentIncluders(customDir: string, custom: Readonly<Record<string, CustomComponentConfig>>): Promise<Map<string, Set<string>>>;
|
|
35
|
+
/**
|
|
36
|
+
* Finds the other DEPLOYED includers of the fragments `targetTag` includes
|
|
37
|
+
* whose engine sheets do not carry the current fragment source.
|
|
38
|
+
*
|
|
39
|
+
* Only fragments the target includes are considered: a fragment unrelated
|
|
40
|
+
* to this deploy is not its business. An includer whose engine sheet does
|
|
41
|
+
* not exist is not deployed and is skipped. A deployed sheet with no
|
|
42
|
+
* expansion at all counts as stale, the same rule `validateCssFragments`
|
|
43
|
+
* applies.
|
|
44
|
+
*
|
|
45
|
+
* @returns The fragments the target includes, and the stale includers
|
|
46
|
+
* (sorted by tag), each naming the fragments that are stale for it
|
|
47
|
+
*/
|
|
48
|
+
export declare function findStaleIncludersOfTargetFragments(args: {
|
|
49
|
+
targetTag: string;
|
|
50
|
+
custom: Readonly<Record<string, CustomComponentConfig>>;
|
|
51
|
+
furnacePaths: FurnacePaths;
|
|
52
|
+
engineDir: string;
|
|
53
|
+
}): Promise<{
|
|
54
|
+
fragments: string[];
|
|
55
|
+
stale: StaleFragmentIncluder[];
|
|
56
|
+
}>;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// SPDX-License-Identifier: EUPL-1.2
|
|
2
|
+
/**
|
|
3
|
+
* Reverse index from shared CSS fragments to the custom components that
|
|
4
|
+
* include them, plus the staleness probe a targeted `furnace deploy <tag>`
|
|
5
|
+
* runs over the OTHER includers of the fragments it just refreshed.
|
|
6
|
+
*
|
|
7
|
+
* Fragment staleness was only ever a validator finding
|
|
8
|
+
* (`stale-fragment-expansion` in {@link validateCssFragments}), printed per
|
|
9
|
+
* includer by `furnace validate` and `deploy --dry-run`. A targeted deploy
|
|
10
|
+
* validates only the component it was asked for, so the one command that
|
|
11
|
+
* changes a fragment's deployed state for SOME includers is the one that
|
|
12
|
+
* never mentioned the rest: after a fragment edit, `deploy <a> <b>` left
|
|
13
|
+
* every other consumer carrying the old expansion, and nothing said so
|
|
14
|
+
* until a `grep` over the engine found it.
|
|
15
|
+
*
|
|
16
|
+
* Only custom components are indexed. Override components are copied
|
|
17
|
+
* byte-for-byte and never expand fragments.
|
|
18
|
+
*/
|
|
19
|
+
import { readdir } from 'node:fs/promises';
|
|
20
|
+
import { join } from 'node:path';
|
|
21
|
+
import { pathExists, readText } from '../utils/fs.js';
|
|
22
|
+
import { extractExpandedFragmentBodies, listFragmentIncludes, readFragmentSource, } from './furnace-css-fragments.js';
|
|
23
|
+
/** Workspace `.css` sheets of one custom component, with their includes. */
|
|
24
|
+
async function listComponentSheetIncludes(componentDir) {
|
|
25
|
+
if (!(await pathExists(componentDir)))
|
|
26
|
+
return [];
|
|
27
|
+
let entries;
|
|
28
|
+
try {
|
|
29
|
+
entries = await readdir(componentDir);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// An unreadable component directory includes nothing. Same degradation
|
|
33
|
+
// as validateCssFragments: the index must never crash a deploy.
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
const sheets = [];
|
|
37
|
+
for (const fileName of entries.sort()) {
|
|
38
|
+
if (!fileName.endsWith('.css'))
|
|
39
|
+
continue;
|
|
40
|
+
const includes = listFragmentIncludes(await readText(join(componentDir, fileName)));
|
|
41
|
+
if (includes.length > 0)
|
|
42
|
+
sheets.push({ fileName, includes });
|
|
43
|
+
}
|
|
44
|
+
return sheets;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Maps every fragment name to the custom tags whose workspace sheets
|
|
48
|
+
* include it.
|
|
49
|
+
*
|
|
50
|
+
* @param customDir - `components/custom/` directory
|
|
51
|
+
* @param custom - The `custom` map from furnace.json
|
|
52
|
+
*/
|
|
53
|
+
export async function collectFragmentIncluders(customDir, custom) {
|
|
54
|
+
const index = new Map();
|
|
55
|
+
for (const tag of Object.keys(custom).sort()) {
|
|
56
|
+
for (const sheet of await listComponentSheetIncludes(join(customDir, tag))) {
|
|
57
|
+
for (const fragment of sheet.includes) {
|
|
58
|
+
const tags = index.get(fragment) ?? new Set();
|
|
59
|
+
tags.add(tag);
|
|
60
|
+
index.set(fragment, tags);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return index;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Finds the other DEPLOYED includers of the fragments `targetTag` includes
|
|
68
|
+
* whose engine sheets do not carry the current fragment source.
|
|
69
|
+
*
|
|
70
|
+
* Only fragments the target includes are considered: a fragment unrelated
|
|
71
|
+
* to this deploy is not its business. An includer whose engine sheet does
|
|
72
|
+
* not exist is not deployed and is skipped. A deployed sheet with no
|
|
73
|
+
* expansion at all counts as stale, the same rule `validateCssFragments`
|
|
74
|
+
* applies.
|
|
75
|
+
*
|
|
76
|
+
* @returns The fragments the target includes, and the stale includers
|
|
77
|
+
* (sorted by tag), each naming the fragments that are stale for it
|
|
78
|
+
*/
|
|
79
|
+
export async function findStaleIncludersOfTargetFragments(args) {
|
|
80
|
+
const { targetTag, custom, furnacePaths, engineDir } = args;
|
|
81
|
+
const index = await collectFragmentIncluders(furnacePaths.customDir, custom);
|
|
82
|
+
const fragments = [...index]
|
|
83
|
+
.filter(([, tags]) => tags.has(targetTag))
|
|
84
|
+
.map(([fragment]) => fragment)
|
|
85
|
+
.sort();
|
|
86
|
+
if (fragments.length === 0)
|
|
87
|
+
return { fragments, stale: [] };
|
|
88
|
+
const targetFragments = new Set(fragments);
|
|
89
|
+
const sources = new Map();
|
|
90
|
+
const stale = [];
|
|
91
|
+
for (const tag of Object.keys(custom).sort()) {
|
|
92
|
+
if (tag === targetTag)
|
|
93
|
+
continue;
|
|
94
|
+
const componentConfig = custom[tag];
|
|
95
|
+
if (!componentConfig)
|
|
96
|
+
continue;
|
|
97
|
+
const engineTargetDir = join(engineDir, componentConfig.targetPath);
|
|
98
|
+
const staleFragments = new Set();
|
|
99
|
+
for (const sheet of await listComponentSheetIncludes(join(furnacePaths.customDir, tag))) {
|
|
100
|
+
const relevant = sheet.includes.filter((f) => targetFragments.has(f));
|
|
101
|
+
if (relevant.length === 0)
|
|
102
|
+
continue;
|
|
103
|
+
const destPath = join(engineTargetDir, sheet.fileName);
|
|
104
|
+
if (!(await pathExists(destPath)))
|
|
105
|
+
continue;
|
|
106
|
+
const bodies = extractExpandedFragmentBodies(await readText(destPath));
|
|
107
|
+
for (const fragment of relevant) {
|
|
108
|
+
let source = sources.get(fragment);
|
|
109
|
+
if (source === undefined) {
|
|
110
|
+
source = await readFragmentSource(furnacePaths.sharedDir, fragment);
|
|
111
|
+
sources.set(fragment, source);
|
|
112
|
+
}
|
|
113
|
+
if (bodies.get(fragment) !== source)
|
|
114
|
+
staleFragments.add(fragment);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (staleFragments.size > 0) {
|
|
118
|
+
stale.push({ tag, fragments: [...staleFragments].sort() });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return { fragments, stale };
|
|
122
|
+
}
|
package/dist/src/core/git.js
CHANGED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Teardown reaper for the helper processes one mach test dispatch launched.
|
|
3
|
+
*
|
|
4
|
+
* The mochitest harness starts its httpd (`xpcshell … server.js`), the
|
|
5
|
+
* websocket server, ssltunnel and the websocket/process bridge as plain
|
|
6
|
+
* children, and the browser through mozprocess, which puts it in a process
|
|
7
|
+
* group of its own. It logs each launch as `runtests.py | <what> pid: <n>`.
|
|
8
|
+
* When mach dies before its own cleanup runs (a forwarded SIGTERM, a
|
|
9
|
+
* no-output timeout, a harness crash, a supervisor kill), the helpers do not
|
|
10
|
+
* necessarily die with it: the 2026-09-13 incident left the httpd reparented
|
|
11
|
+
* to launchd at 100% CPU for hours after a gate killed the run at its bound.
|
|
12
|
+
*
|
|
13
|
+
* The exec layer already sweeps the dispatch's process group after close.
|
|
14
|
+
* That reaches every helper that stayed in the group, and nothing else: the
|
|
15
|
+
* browser is in its own group, and a helper that re-grouped itself is not.
|
|
16
|
+
* This module closes that gap in two passes:
|
|
17
|
+
*
|
|
18
|
+
* - {@link createHelperPidTracker} collects the announced pids from the
|
|
19
|
+
* streamed output. {@link reapTrackedHarnessPids} runs after mach has
|
|
20
|
+
* exited; a tracked pid still alive at that point is by definition a
|
|
21
|
+
* survivor (its parent is gone), so no age heuristic is needed. Before
|
|
22
|
+
* signalling, each pid is re-read from `ps` and must still be anchored to
|
|
23
|
+
* the objdir and still look like the thing the harness launched: a pid
|
|
24
|
+
* recycled onto an unrelated process between the launch line and
|
|
25
|
+
* teardown is skipped.
|
|
26
|
+
* - Not every helper is announced. mozserve starts `moz-http2` with
|
|
27
|
+
* `start_new_session=True` and logs no pid, so it is in neither the
|
|
28
|
+
* group nor the tracker, and it was the other survivor of the incident.
|
|
29
|
+
* {@link snapshotObjdirHelpers} records which same-objdir helpers existed
|
|
30
|
+
* before the dispatch spawned; after mach exits, a helper-shaped process
|
|
31
|
+
* under THIS checkout's absolute objdir that is not in that baseline and
|
|
32
|
+
* is reparented (its parent, the harness, is gone) is one this dispatch
|
|
33
|
+
* launched, and is reaped too. The absolute-path anchor keeps a sibling
|
|
34
|
+
* checkout's live harness out of it; the baseline keeps a survivor of an
|
|
35
|
+
* earlier run out of it, so the preflight census's report-only posture
|
|
36
|
+
* still decides that one's fate.
|
|
37
|
+
*/
|
|
38
|
+
/**
|
|
39
|
+
* What the harness said it launched, as it names it in the log line, or
|
|
40
|
+
* `unannounced` for a survivor found by the post-close objdir scan.
|
|
41
|
+
*/
|
|
42
|
+
export type TrackedHarnessKind = 'Server' | 'Websocket server' | 'SSL tunnel' | 'websocket/process bridge' | 'Application' | 'unannounced';
|
|
43
|
+
/** One helper the harness launched. */
|
|
44
|
+
export interface TrackedHarnessPid {
|
|
45
|
+
pid: number;
|
|
46
|
+
kind: TrackedHarnessKind;
|
|
47
|
+
/** Command line, when the entry came from `ps` rather than a log line. */
|
|
48
|
+
command?: string;
|
|
49
|
+
}
|
|
50
|
+
/** Which child stream a chunk came from. */
|
|
51
|
+
export type OutputStreamName = 'stdout' | 'stderr';
|
|
52
|
+
/** Collects harness launch announcements from streamed output chunks. */
|
|
53
|
+
export interface HelperPidTracker {
|
|
54
|
+
/** Feed one chunk. Chunk boundaries need not align with lines. */
|
|
55
|
+
feed(stream: OutputStreamName, chunk: string): void;
|
|
56
|
+
/** Every pid announced so far, in announcement order, without duplicates. */
|
|
57
|
+
tracked(): TrackedHarnessPid[];
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Creates a tracker for one dispatch. The two streams are buffered apart so
|
|
61
|
+
* a half-line on one cannot be glued to a half-line on the other.
|
|
62
|
+
*/
|
|
63
|
+
export declare function createHelperPidTracker(): HelperPidTracker;
|
|
64
|
+
/**
|
|
65
|
+
* Decides whether a `ps` row still describes the process the harness
|
|
66
|
+
* announced. Exported for fixture tests.
|
|
67
|
+
*
|
|
68
|
+
* The helpers must match the census's helper pattern and the objdir anchor.
|
|
69
|
+
* The browser has no helper-shaped name, so the anchor alone attributes it:
|
|
70
|
+
* its binary lives under the objdir (`<objdir>/dist/<App>.app/…`).
|
|
71
|
+
*/
|
|
72
|
+
export declare function isStillTrackedProcess(tracked: TrackedHarnessPid, command: string, objDir: string | undefined): boolean;
|
|
73
|
+
/**
|
|
74
|
+
* Records the same-objdir helpers alive before a dispatch spawns anything,
|
|
75
|
+
* so the post-close pass can tell this dispatch's unannounced helpers from
|
|
76
|
+
* an earlier run's survivors. Undefined when `ps` is unusable or no objdir
|
|
77
|
+
* is known; the post-close pass then reaps no unannounced process at all.
|
|
78
|
+
*
|
|
79
|
+
* @param objDir - This project's absolute objdir
|
|
80
|
+
* @returns The pids to leave alone at teardown, or undefined for "no baseline"
|
|
81
|
+
*/
|
|
82
|
+
export declare function snapshotObjdirHelpers(objDir: string | undefined): Promise<Set<number> | undefined>;
|
|
83
|
+
/**
|
|
84
|
+
* Terminates every process this dispatch launched that is still alive after
|
|
85
|
+
* mach exited: the announced pids still attributable to the dispatch, and
|
|
86
|
+
* the unannounced same-objdir survivors that appeared since the baseline.
|
|
87
|
+
*
|
|
88
|
+
* Best-effort like the census: a host without `ps` logs at verbose and
|
|
89
|
+
* reaps nothing rather than signalling on the strength of a pid alone.
|
|
90
|
+
*
|
|
91
|
+
* @param tracked - The pids the harness announced during the dispatch
|
|
92
|
+
* @param objDir - This project's absolute objdir, for the anchor rule
|
|
93
|
+
* @param baseline - Same-objdir helpers alive before the dispatch spawned
|
|
94
|
+
* ({@link snapshotObjdirHelpers}); never touched here. Undefined disables
|
|
95
|
+
* the unannounced pass entirely
|
|
96
|
+
* @returns How many survivors are confirmed gone afterwards
|
|
97
|
+
*/
|
|
98
|
+
export declare function reapTrackedHarnessPids(tracked: readonly TrackedHarnessPid[], objDir: string | undefined, baseline?: ReadonlySet<number>): Promise<number>;
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
// SPDX-License-Identifier: EUPL-1.2
|
|
2
|
+
/**
|
|
3
|
+
* Teardown reaper for the helper processes one mach test dispatch launched.
|
|
4
|
+
*
|
|
5
|
+
* The mochitest harness starts its httpd (`xpcshell … server.js`), the
|
|
6
|
+
* websocket server, ssltunnel and the websocket/process bridge as plain
|
|
7
|
+
* children, and the browser through mozprocess, which puts it in a process
|
|
8
|
+
* group of its own. It logs each launch as `runtests.py | <what> pid: <n>`.
|
|
9
|
+
* When mach dies before its own cleanup runs (a forwarded SIGTERM, a
|
|
10
|
+
* no-output timeout, a harness crash, a supervisor kill), the helpers do not
|
|
11
|
+
* necessarily die with it: the 2026-09-13 incident left the httpd reparented
|
|
12
|
+
* to launchd at 100% CPU for hours after a gate killed the run at its bound.
|
|
13
|
+
*
|
|
14
|
+
* The exec layer already sweeps the dispatch's process group after close.
|
|
15
|
+
* That reaches every helper that stayed in the group, and nothing else: the
|
|
16
|
+
* browser is in its own group, and a helper that re-grouped itself is not.
|
|
17
|
+
* This module closes that gap in two passes:
|
|
18
|
+
*
|
|
19
|
+
* - {@link createHelperPidTracker} collects the announced pids from the
|
|
20
|
+
* streamed output. {@link reapTrackedHarnessPids} runs after mach has
|
|
21
|
+
* exited; a tracked pid still alive at that point is by definition a
|
|
22
|
+
* survivor (its parent is gone), so no age heuristic is needed. Before
|
|
23
|
+
* signalling, each pid is re-read from `ps` and must still be anchored to
|
|
24
|
+
* the objdir and still look like the thing the harness launched: a pid
|
|
25
|
+
* recycled onto an unrelated process between the launch line and
|
|
26
|
+
* teardown is skipped.
|
|
27
|
+
* - Not every helper is announced. mozserve starts `moz-http2` with
|
|
28
|
+
* `start_new_session=True` and logs no pid, so it is in neither the
|
|
29
|
+
* group nor the tracker, and it was the other survivor of the incident.
|
|
30
|
+
* {@link snapshotObjdirHelpers} records which same-objdir helpers existed
|
|
31
|
+
* before the dispatch spawned; after mach exits, a helper-shaped process
|
|
32
|
+
* under THIS checkout's absolute objdir that is not in that baseline and
|
|
33
|
+
* is reparented (its parent, the harness, is gone) is one this dispatch
|
|
34
|
+
* launched, and is reaped too. The absolute-path anchor keeps a sibling
|
|
35
|
+
* checkout's live harness out of it; the baseline keeps a survivor of an
|
|
36
|
+
* earlier run out of it, so the preflight census's report-only posture
|
|
37
|
+
* still decides that one's fate.
|
|
38
|
+
*/
|
|
39
|
+
import { toError } from '../utils/errors.js';
|
|
40
|
+
import { dispatchCompleteLines } from '../utils/line-dispatch.js';
|
|
41
|
+
import { info, verbose } from '../utils/logger.js';
|
|
42
|
+
import { exec } from '../utils/process.js';
|
|
43
|
+
import { isObjdirAnchored, isPidAlive, matchesHarnessHelper, terminateHarnessProcesses, } from './harness-orphans.js';
|
|
44
|
+
/**
|
|
45
|
+
* The launch announcements in `testing/mochitest/runtests.py`. mozlog puts a
|
|
46
|
+
* timestamp and level in front, so the match is unanchored.
|
|
47
|
+
*/
|
|
48
|
+
const HARNESS_PID_LINE = /runtests\.py \| (Server|Websocket server|SSL tunnel|websocket\/process bridge|Application) pid: (\d+)\b/;
|
|
49
|
+
/**
|
|
50
|
+
* Creates a tracker for one dispatch. The two streams are buffered apart so
|
|
51
|
+
* a half-line on one cannot be glued to a half-line on the other.
|
|
52
|
+
*/
|
|
53
|
+
export function createHelperPidTracker() {
|
|
54
|
+
const seen = new Map();
|
|
55
|
+
const partial = { stdout: '', stderr: '' };
|
|
56
|
+
const onLine = (line) => {
|
|
57
|
+
const match = HARNESS_PID_LINE.exec(line);
|
|
58
|
+
if (!match)
|
|
59
|
+
return;
|
|
60
|
+
const pid = Number(match[2]);
|
|
61
|
+
if (!seen.has(pid)) {
|
|
62
|
+
seen.set(pid, { pid, kind: match[1] });
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
feed(stream, chunk) {
|
|
67
|
+
partial[stream] = dispatchCompleteLines(partial[stream] + chunk, onLine);
|
|
68
|
+
},
|
|
69
|
+
tracked() {
|
|
70
|
+
return [...seen.values()];
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** Log prefix for every line this reaper writes. */
|
|
75
|
+
const TEARDOWN_LABEL = 'harness teardown';
|
|
76
|
+
/**
|
|
77
|
+
* Decides whether a `ps` row still describes the process the harness
|
|
78
|
+
* announced. Exported for fixture tests.
|
|
79
|
+
*
|
|
80
|
+
* The helpers must match the census's helper pattern and the objdir anchor.
|
|
81
|
+
* The browser has no helper-shaped name, so the anchor alone attributes it:
|
|
82
|
+
* its binary lives under the objdir (`<objdir>/dist/<App>.app/…`).
|
|
83
|
+
*/
|
|
84
|
+
export function isStillTrackedProcess(tracked, command, objDir) {
|
|
85
|
+
if (!isObjdirAnchored(command, objDir))
|
|
86
|
+
return false;
|
|
87
|
+
return tracked.kind === 'Application' || matchesHarnessHelper(command);
|
|
88
|
+
}
|
|
89
|
+
/** Lists every process. Throws when `ps` is unusable, like the census. */
|
|
90
|
+
async function listAllProcesses() {
|
|
91
|
+
const result = await exec('ps', ['-axo', 'pid=,ppid=,command='], { timeout: 10000 });
|
|
92
|
+
if (result.exitCode !== 0)
|
|
93
|
+
throw new Error(`ps exited ${result.exitCode}`);
|
|
94
|
+
const rows = [];
|
|
95
|
+
for (const line of result.stdout.split('\n')) {
|
|
96
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(.+)$/.exec(line);
|
|
97
|
+
if (match) {
|
|
98
|
+
rows.push({
|
|
99
|
+
pid: Number(match[1]),
|
|
100
|
+
ppid: Number(match[2]),
|
|
101
|
+
command: (match[3] ?? '').trim(),
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return rows;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* A helper-shaped process under this checkout's objdir. The anchor here is
|
|
109
|
+
* the absolute objdir path alone, on purpose: the census's generic `obj-…`
|
|
110
|
+
* fallback would also match a sibling checkout's live harness.
|
|
111
|
+
*/
|
|
112
|
+
function isThisObjdirHelper(command, objDir) {
|
|
113
|
+
return command.includes(objDir) && matchesHarnessHelper(command);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Records the same-objdir helpers alive before a dispatch spawns anything,
|
|
117
|
+
* so the post-close pass can tell this dispatch's unannounced helpers from
|
|
118
|
+
* an earlier run's survivors. Undefined when `ps` is unusable or no objdir
|
|
119
|
+
* is known; the post-close pass then reaps no unannounced process at all.
|
|
120
|
+
*
|
|
121
|
+
* @param objDir - This project's absolute objdir
|
|
122
|
+
* @returns The pids to leave alone at teardown, or undefined for "no baseline"
|
|
123
|
+
*/
|
|
124
|
+
export async function snapshotObjdirHelpers(objDir) {
|
|
125
|
+
if (process.platform === 'win32' || objDir === undefined)
|
|
126
|
+
return undefined;
|
|
127
|
+
try {
|
|
128
|
+
const rows = await listAllProcesses();
|
|
129
|
+
return new Set(rows.filter((r) => isThisObjdirHelper(r.command, objDir)).map((r) => r.pid));
|
|
130
|
+
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
verbose(`${TEARDOWN_LABEL}: could not snapshot helpers (${toError(error).message}).`);
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Helpers this dispatch launched without announcing them: helper-shaped,
|
|
138
|
+
* under this checkout's objdir, not in the pre-dispatch baseline, and
|
|
139
|
+
* reparented to pid 1 because the harness that started them is gone.
|
|
140
|
+
* Returns nothing when `ps` is unusable rather than guessing. Without a
|
|
141
|
+
* baseline (a snapshot that failed) it also returns nothing: an earlier
|
|
142
|
+
* run's survivor must stay the census's call.
|
|
143
|
+
*/
|
|
144
|
+
async function findUnannouncedSurvivors(objDir, baseline, exclude) {
|
|
145
|
+
if (baseline === undefined)
|
|
146
|
+
return [];
|
|
147
|
+
let rows;
|
|
148
|
+
try {
|
|
149
|
+
rows = await listAllProcesses();
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
verbose(`${TEARDOWN_LABEL}: could not scan for unannounced helpers (${toError(error).message}).`);
|
|
153
|
+
return [];
|
|
154
|
+
}
|
|
155
|
+
return rows
|
|
156
|
+
.filter((r) => r.ppid === 1 &&
|
|
157
|
+
r.pid !== process.pid &&
|
|
158
|
+
!baseline.has(r.pid) &&
|
|
159
|
+
!exclude.has(r.pid) &&
|
|
160
|
+
isThisObjdirHelper(r.command, objDir))
|
|
161
|
+
.map((r) => ({ pid: r.pid, kind: 'unannounced', command: r.command }));
|
|
162
|
+
}
|
|
163
|
+
/** Reads `pid command` rows for the given pids. Missing pids simply produce no row. */
|
|
164
|
+
async function listCommands(pids) {
|
|
165
|
+
const result = await exec('ps', ['-o', 'pid=,command=', '-p', pids.join(',')], {
|
|
166
|
+
timeout: 10000,
|
|
167
|
+
});
|
|
168
|
+
const rows = new Map();
|
|
169
|
+
// `ps -p` exits 1 when any listed pid is gone, while still printing the
|
|
170
|
+
// rest, so the exit code is not a failure signal here.
|
|
171
|
+
for (const line of result.stdout.split('\n')) {
|
|
172
|
+
const match = /^\s*(\d+)\s+(.+)$/.exec(line);
|
|
173
|
+
if (match)
|
|
174
|
+
rows.set(Number(match[1]), (match[2] ?? '').trim());
|
|
175
|
+
}
|
|
176
|
+
return rows;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Re-reads the announced pids that are still alive and keeps the ones still
|
|
180
|
+
* attributable to this dispatch. Empty when `ps` is unusable.
|
|
181
|
+
*/
|
|
182
|
+
async function survivingTrackedPids(tracked, objDir) {
|
|
183
|
+
const alive = tracked.filter((entry) => isPidAlive(entry.pid));
|
|
184
|
+
if (alive.length === 0)
|
|
185
|
+
return [];
|
|
186
|
+
let commands;
|
|
187
|
+
try {
|
|
188
|
+
commands = await listCommands(alive.map((entry) => entry.pid));
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
verbose(`${TEARDOWN_LABEL}: could not read ps for tracked helpers (${toError(error).message}); skipping.`);
|
|
192
|
+
return [];
|
|
193
|
+
}
|
|
194
|
+
const targets = [];
|
|
195
|
+
for (const entry of alive) {
|
|
196
|
+
const command = commands.get(entry.pid);
|
|
197
|
+
if (command === undefined)
|
|
198
|
+
continue; // exited between the probe and ps
|
|
199
|
+
if (!isStillTrackedProcess(entry, command, objDir)) {
|
|
200
|
+
verbose(`${TEARDOWN_LABEL}: PID ${entry.pid} (announced as ${entry.kind}) is now "${command.slice(0, 120)}", ` +
|
|
201
|
+
'not attributable to this run; left alone.');
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
targets.push(entry);
|
|
205
|
+
}
|
|
206
|
+
return targets;
|
|
207
|
+
}
|
|
208
|
+
function describeTarget(target) {
|
|
209
|
+
const excerpt = target.command === undefined ? '' : ` "${target.command.slice(0, 80)}"`;
|
|
210
|
+
return `${target.kind} ${target.pid}${excerpt}`;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Terminates every process this dispatch launched that is still alive after
|
|
214
|
+
* mach exited: the announced pids still attributable to the dispatch, and
|
|
215
|
+
* the unannounced same-objdir survivors that appeared since the baseline.
|
|
216
|
+
*
|
|
217
|
+
* Best-effort like the census: a host without `ps` logs at verbose and
|
|
218
|
+
* reaps nothing rather than signalling on the strength of a pid alone.
|
|
219
|
+
*
|
|
220
|
+
* @param tracked - The pids the harness announced during the dispatch
|
|
221
|
+
* @param objDir - This project's absolute objdir, for the anchor rule
|
|
222
|
+
* @param baseline - Same-objdir helpers alive before the dispatch spawned
|
|
223
|
+
* ({@link snapshotObjdirHelpers}); never touched here. Undefined disables
|
|
224
|
+
* the unannounced pass entirely
|
|
225
|
+
* @returns How many survivors are confirmed gone afterwards
|
|
226
|
+
*/
|
|
227
|
+
export async function reapTrackedHarnessPids(tracked, objDir, baseline) {
|
|
228
|
+
if (process.platform === 'win32')
|
|
229
|
+
return 0;
|
|
230
|
+
const targets = await survivingTrackedPids(tracked, objDir);
|
|
231
|
+
if (objDir !== undefined) {
|
|
232
|
+
const known = new Set(tracked.map((t) => t.pid));
|
|
233
|
+
targets.push(...(await findUnannouncedSurvivors(objDir, baseline, known)));
|
|
234
|
+
}
|
|
235
|
+
if (targets.length === 0)
|
|
236
|
+
return 0;
|
|
237
|
+
info(`${TEARDOWN_LABEL}: ${targets.length} harness process(es) outlived mach ` +
|
|
238
|
+
`(${targets.map(describeTarget).join(', ')}); terminating.`);
|
|
239
|
+
return terminateHarnessProcesses(targets, TEARDOWN_LABEL);
|
|
240
|
+
}
|