@link-assistant/hive-mind 2.11.1 → 2.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/src/fix.ci-cd-issue.lib.mjs +18 -5
- package/src/fix.ci-cd.lib.mjs +79 -8
- package/src/fix.mjs +1 -1
- package/src/use-m-bootstrap.lib.mjs +23 -4
- package/src/use-m-single-flight.lib.mjs +350 -0
- package/src/use-with-retry.lib.mjs +8 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.11.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 82053e1: Stop concurrent use-m installs from corrupting the global npm alias (issue #2113). `use()` performs one `npm install -g <alias>@npm:<package>@<version>` per call with no in-flight deduplication, and Node evaluates sibling top-level-await subgraphs concurrently, so a cold container running `fix` or `task` launched six simultaneous global installs of the same directory; npm does not lock the global prefix, so those installs deleted and re-extracted each other's trees, surfacing as `ENOTEMPTY` or as `ERR_MODULE_NOT_FOUND` for an arbitrary internal file. `src/use-m-single-flight.lib.mjs` now wraps `use()` inside `ensureUseM()` with per-specifier single flight, an in-process per-alias mutex, and a cross-process advisory lock over the alias directory (Node built-ins only, `HIVE_MIND_USE_M_LOCK_DIR` to relocate it); measured on a cold prefix, 24 concurrent loads went from 24/24 failures in 54.8s to 0/24 in 3.3s. Dependency loading is also traced under `--verbose` as well as `HIVE_MIND_USE_M_DEBUG=1`, because both reported failures were captured with `--verbose` and contained no loader output at all. Reported upstream as link-foundation/use-m#70 and fixed there in `use-m@8.15.0` (cross-process alias install lock plus a post-install marker); the pinned CDN bootstrap fallback moves from 8.14.4 to 8.15.0, verified with the standalone reproduction — 22/24 concurrent loads fail on 8.14.4, 0/24 on 8.15.0.
|
|
8
|
+
|
|
9
|
+
## 2.11.2
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 2777cf5: Stop `/task --ci-cd` and `/fix --ci-cd` from listing the same workflow many times in the generated issue (issue #2125). When the latest default-branch commit has no workflow runs — typical for release commits — the collector falls back to the recent runs of the default branch, which span many commits; every one of them became a table row, so `link-assistant/agent#287` listed two workflows twenty times and reported "20 (9 not passing)". `dedupeRunsByWorkflow()` now keeps only the most recent run of each workflow (by `workflow_id`, then `path`/`name`, resolved with `created_at`/`run_attempt`/`id`), the failure summary counts the same deduplicated set, and the branch-fallback table gained a Commit column because its rows can come from different commits. The fallback fetches 100 runs instead of 20 so a rarely-run workflow still appears after collapsing, and `prepareCiCdIssue()` logs how many runs it collapsed.
|
|
14
|
+
|
|
3
15
|
## 2.11.1
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { spawn } from 'child_process';
|
|
7
|
-
import { CI_CD_ISSUE_LABELS, CI_CD_ISSUE_TYPE, buildCiCdIssueBody, buildCiCdIssueTitle } from './fix.ci-cd.lib.mjs';
|
|
7
|
+
import { CI_CD_ISSUE_LABELS, CI_CD_ISSUE_TYPE, buildCiCdIssueBody, buildCiCdIssueTitle, dedupeRunsByWorkflow } from './fix.ci-cd.lib.mjs';
|
|
8
8
|
import { createTaskIssue } from './task.issue-creation.lib.mjs';
|
|
9
9
|
|
|
10
10
|
function runCommand(command, args, options = {}) {
|
|
@@ -70,7 +70,9 @@ async function getLatestCommit(repository, branch, run, warn) {
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
// `workflow_id`, `created_at` and `run_attempt` are what let
|
|
74
|
+
// `dedupeRunsByWorkflow` keep the latest run of each workflow (issue #2125).
|
|
75
|
+
const RUNS_JQ = '[.workflow_runs[] | {id: .id, name: .name, workflow_id: .workflow_id, path: .path, status: .status, conclusion: .conclusion, html_url: .html_url, head_sha: .head_sha, created_at: .created_at, run_attempt: .run_attempt}]';
|
|
74
76
|
|
|
75
77
|
async function getRunsForCommit(repository, sha, run, warn) {
|
|
76
78
|
if (!sha) return [];
|
|
@@ -87,7 +89,7 @@ async function getRunsForCommit(repository, sha, run, warn) {
|
|
|
87
89
|
async function getRecentBranchRuns(repository, branch, run, warn) {
|
|
88
90
|
if (!branch) return [];
|
|
89
91
|
try {
|
|
90
|
-
const json = await commandOutput(run, 'gh', ['api', `repos/${repository.fullName}/actions/runs?branch=${encodeURIComponent(branch)}&per_page=
|
|
92
|
+
const json = await commandOutput(run, 'gh', ['api', `repos/${repository.fullName}/actions/runs?branch=${encodeURIComponent(branch)}&per_page=100`, '--jq', RUNS_JQ]);
|
|
91
93
|
const parsed = JSON.parse(json);
|
|
92
94
|
return Array.isArray(parsed) ? parsed : [];
|
|
93
95
|
} catch (error) {
|
|
@@ -96,7 +98,7 @@ async function getRecentBranchRuns(repository, branch, run, warn) {
|
|
|
96
98
|
}
|
|
97
99
|
}
|
|
98
100
|
|
|
99
|
-
export async function prepareCiCdIssue({ repository, run = runCommand, warn = message => console.warn(message) }) {
|
|
101
|
+
export async function prepareCiCdIssue({ repository, run = runCommand, warn = message => console.warn(message), log = null }) {
|
|
100
102
|
const [languages, defaultBranch] = await Promise.all([detectLanguages(repository, run, warn), getDefaultBranch(repository, run, warn)]);
|
|
101
103
|
const commit = await getLatestCommit(repository, defaultBranch, run, warn);
|
|
102
104
|
let runs = await getRunsForCommit(repository, commit?.sha, run, warn);
|
|
@@ -110,11 +112,22 @@ export async function prepareCiCdIssue({ repository, run = runCommand, warn = me
|
|
|
110
112
|
}
|
|
111
113
|
}
|
|
112
114
|
|
|
115
|
+
// The branch fallback returns every run of every workflow across many
|
|
116
|
+
// commits; the issue must list one row per workflow (issue #2125).
|
|
117
|
+
const fetchedRuns = runs.length;
|
|
118
|
+
runs = dedupeRunsByWorkflow(runs);
|
|
119
|
+
const duplicates = fetchedRuns - runs.length;
|
|
120
|
+
if (duplicates > 0 && typeof log === 'function') {
|
|
121
|
+
log(`ℹ️ Collapsed ${duplicates} older CI/CD run(s) — keeping the latest run of each workflow (${runs.length} workflow(s), source: ${runsSource}).`);
|
|
122
|
+
}
|
|
123
|
+
|
|
113
124
|
return {
|
|
114
125
|
repository,
|
|
115
126
|
defaultBranch,
|
|
116
127
|
commit,
|
|
117
128
|
runs,
|
|
129
|
+
fetchedRuns,
|
|
130
|
+
duplicateRuns: duplicates,
|
|
118
131
|
languages,
|
|
119
132
|
runsSource,
|
|
120
133
|
title: buildCiCdIssueTitle(),
|
|
@@ -123,7 +136,7 @@ export async function prepareCiCdIssue({ repository, run = runCommand, warn = me
|
|
|
123
136
|
}
|
|
124
137
|
|
|
125
138
|
export async function createCiCdIssue({ repository, prepared = null, run = runCommand, log = null, warn = message => console.warn(message) }) {
|
|
126
|
-
const issueDraft = prepared || (await prepareCiCdIssue({ repository, run, warn }));
|
|
139
|
+
const issueDraft = prepared || (await prepareCiCdIssue({ repository, run, warn, log }));
|
|
127
140
|
const issue = await createTaskIssue({
|
|
128
141
|
repository,
|
|
129
142
|
title: issueDraft.title,
|
package/src/fix.ci-cd.lib.mjs
CHANGED
|
@@ -249,27 +249,95 @@ export function buildTemplatesSection(languages) {
|
|
|
249
249
|
return lines.join('\n');
|
|
250
250
|
}
|
|
251
251
|
|
|
252
|
-
/**
|
|
253
|
-
|
|
252
|
+
/**
|
|
253
|
+
* Stable identity of the workflow a run belongs to (issue #2125).
|
|
254
|
+
*
|
|
255
|
+
* `workflow_id` is the authoritative key: two workflow files may share the same
|
|
256
|
+
* display `name`, and one workflow file may be renamed between runs. The name
|
|
257
|
+
* (and `path`) are only fallbacks for payloads that omit the id.
|
|
258
|
+
*/
|
|
259
|
+
export function runWorkflowKey(run) {
|
|
260
|
+
const workflowId = run?.workflow_id ?? run?.workflowId;
|
|
261
|
+
if (workflowId !== undefined && workflowId !== null && workflowId !== '') return `id:${workflowId}`;
|
|
262
|
+
if (run?.path) return `path:${run.path}`;
|
|
263
|
+
const name = run?.name || run?.workflowName;
|
|
264
|
+
// A run with no identity at all cannot be proven to be a duplicate.
|
|
265
|
+
return name ? `name:${String(name).toLowerCase()}` : null;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Recency of a run: newest first, using created_at, then attempt, then id. */
|
|
269
|
+
function compareRunRecency(a, b) {
|
|
270
|
+
const timeA = Date.parse(a?.created_at || a?.run_started_at || '') || 0;
|
|
271
|
+
const timeB = Date.parse(b?.created_at || b?.run_started_at || '') || 0;
|
|
272
|
+
if (timeA !== timeB) return timeB - timeA;
|
|
273
|
+
const attemptA = Number(a?.run_attempt) || 0;
|
|
274
|
+
const attemptB = Number(b?.run_attempt) || 0;
|
|
275
|
+
if (attemptA !== attemptB) return attemptB - attemptA;
|
|
276
|
+
return (Number(b?.id) || 0) - (Number(a?.id) || 0);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Keep only the most recent run per workflow (issue #2125).
|
|
281
|
+
*
|
|
282
|
+
* When `/fix --ci-cd` falls back to "recent runs on the default branch" the
|
|
283
|
+
* GitHub API returns every run of every workflow across many commits, so the
|
|
284
|
+
* generated issue listed the same two workflows twenty times. One row per
|
|
285
|
+
* workflow — its latest run — is what makes the table actionable.
|
|
286
|
+
*
|
|
287
|
+
* Order of the surviving rows follows the input (the API returns newest first).
|
|
288
|
+
*/
|
|
289
|
+
export function dedupeRunsByWorkflow(runs) {
|
|
290
|
+
const list = Array.isArray(runs) ? runs : [];
|
|
291
|
+
const bestByWorkflow = new Map(); // key -> { run, index }
|
|
292
|
+
list.forEach((run, index) => {
|
|
293
|
+
const key = runWorkflowKey(run) ?? `index:${index}`;
|
|
294
|
+
const existing = bestByWorkflow.get(key);
|
|
295
|
+
if (!existing || compareRunRecency(run, existing.run) < 0) {
|
|
296
|
+
bestByWorkflow.set(key, { run, index: existing ? existing.index : index });
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
return [...bestByWorkflow.values()].sort((a, b) => a.index - b.index).map(entry => entry.run);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** How many rows `dedupeRunsByWorkflow` would drop (for verbose logging). */
|
|
303
|
+
export function countDuplicateRuns(runs) {
|
|
254
304
|
const list = Array.isArray(runs) ? runs : [];
|
|
305
|
+
return list.length - dedupeRunsByWorkflow(list).length;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Render the CI/CD runs section from the GitHub Actions API payload.
|
|
310
|
+
*
|
|
311
|
+
* Runs are deduplicated per workflow (issue #2125). Pass `includeCommit: true`
|
|
312
|
+
* when the rows may come from different commits (the default-branch fallback)
|
|
313
|
+
* so it stays visible which commit each run belongs to.
|
|
314
|
+
*/
|
|
315
|
+
export function buildRunsSection(runs, { emptyMessage, includeCommit = false } = {}) {
|
|
316
|
+
const list = dedupeRunsByWorkflow(runs);
|
|
255
317
|
if (list.length === 0) {
|
|
256
318
|
return emptyMessage || 'No CI/CD runs were found for the latest default-branch commit.';
|
|
257
319
|
}
|
|
258
|
-
const header = '| Workflow | Status | Conclusion | Run |\n| --- | --- | --- | --- |';
|
|
320
|
+
const header = includeCommit ? '| Workflow | Status | Conclusion | Commit | Run |\n| --- | --- | --- | --- | --- |' : '| Workflow | Status | Conclusion | Run |\n| --- | --- | --- | --- |';
|
|
259
321
|
const rows = list.map(run => {
|
|
260
322
|
const name = run.name || run.workflowName || 'unknown';
|
|
261
323
|
const status = run.status || 'unknown';
|
|
262
324
|
const conclusion = run.conclusion || (status === 'completed' ? 'unknown' : 'in_progress');
|
|
263
325
|
const url = run.html_url || run.url || '';
|
|
264
326
|
const runLabel = url ? `[run](${url})` : '—';
|
|
265
|
-
return `| ${name} | ${status} | ${conclusion} | ${runLabel} |`;
|
|
327
|
+
if (!includeCommit) return `| ${name} | ${status} | ${conclusion} | ${runLabel} |`;
|
|
328
|
+
const sha = shortSha(run.head_sha);
|
|
329
|
+
return `| ${name} | ${status} | ${conclusion} | ${sha ? `\`${sha}\`` : '—'} | ${runLabel} |`;
|
|
266
330
|
});
|
|
267
331
|
return [header, ...rows].join('\n');
|
|
268
332
|
}
|
|
269
333
|
|
|
270
|
-
/**
|
|
334
|
+
/**
|
|
335
|
+
* Count the runs that did not pass (failure/cancelled/timed_out/etc.).
|
|
336
|
+
* Counts one run per workflow so the summary matches the rendered table
|
|
337
|
+
* (issue #2125).
|
|
338
|
+
*/
|
|
271
339
|
export function summarizeRunFailures(runs) {
|
|
272
|
-
const list =
|
|
340
|
+
const list = dedupeRunsByWorkflow(runs);
|
|
273
341
|
const passing = new Set(['success', 'neutral', 'skipped']);
|
|
274
342
|
const failing = list.filter(run => {
|
|
275
343
|
const conclusion = (run.conclusion || '').toLowerCase();
|
|
@@ -375,7 +443,10 @@ export const CI_CD_ISSUE_LABELS = Object.freeze(['bug']);
|
|
|
375
443
|
*/
|
|
376
444
|
export function buildCiCdIssueBody({ repository, defaultBranch, commit, runs, languages, runsSource = 'commit', omittedOptions = FIX_FORWARDED_SOLVE_OPTIONS }) {
|
|
377
445
|
const { sortedTemplates } = mapLanguagesToTemplates(languages);
|
|
378
|
-
|
|
446
|
+
// One row per workflow: the branch fallback returns every run of every
|
|
447
|
+
// workflow across many commits (issue #2125).
|
|
448
|
+
const uniqueRuns = dedupeRunsByWorkflow(runs);
|
|
449
|
+
const { total, failing } = summarizeRunFailures(uniqueRuns);
|
|
379
450
|
|
|
380
451
|
const commitLine = commit?.sha ? `\`${shortSha(commit.sha)}\`${commit.url ? ` ([commit](${commit.url}))` : ''}${commit.message ? ` — ${String(commit.message).split('\n')[0]}` : ''}` : 'unknown';
|
|
381
452
|
|
|
@@ -385,7 +456,7 @@ export function buildCiCdIssueBody({ repository, defaultBranch, commit, runs, la
|
|
|
385
456
|
const runsHeading = runsSource === 'branch' ? `Recent CI/CD runs on \`${defaultBranch || 'default branch'}\`` : 'Latest default-branch CI/CD runs';
|
|
386
457
|
const runsEmptyMessage = runsSource === 'branch' ? `No recent CI/CD runs were found on \`${defaultBranch || 'the default branch'}\`.` : 'No CI/CD runs were found for the latest default-branch commit.';
|
|
387
458
|
|
|
388
|
-
const sections = [`### ${runsHeading}`, '', buildRunsSection(
|
|
459
|
+
const sections = [`### ${runsHeading}`, '', buildRunsSection(uniqueRuns, { emptyMessage: runsEmptyMessage, includeCommit: runsSource === 'branch' }), '', buildStandardPrompt({ templatesSorted: sortedTemplates, omittedOptions }), '', '---', '', '<details>', '<summary>Context collected by <code>/fix --ci-cd</code></summary>', '', `- **Repository:** [${repository?.fullName}](${repository?.url})`, `- **Default branch:** \`${defaultBranch || 'unknown'}\``, `- **Latest commit:** ${commitLine}`, `- **CI/CD runs found:** ${total} (${failing} not passing)`, '', '**Detected languages**', '', buildLanguagesSection(languages), '', '**Recommended CI/CD templates**', '', buildTemplatesSection(languages), '', '</details>'];
|
|
389
460
|
|
|
390
461
|
return sections.join('\n');
|
|
391
462
|
}
|
package/src/fix.mjs
CHANGED
|
@@ -83,7 +83,7 @@ async function main() {
|
|
|
83
83
|
const repository = parsed.repository;
|
|
84
84
|
console.log(`🔧 /fix --ci-cd for ${repository.fullName}`);
|
|
85
85
|
|
|
86
|
-
const prepared = await prepareCiCdIssue({ repository });
|
|
86
|
+
const prepared = await prepareCiCdIssue({ repository, log: message => console.log(` ${message}`) });
|
|
87
87
|
const { defaultBranch, commit, runs, runsSource, title, body } = prepared;
|
|
88
88
|
|
|
89
89
|
const { total, failing } = summarizeRunFailures(runs);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
import { wrapUseWithRetry } from './use-with-retry.lib.mjs';
|
|
4
|
+
import { wrapUseWithSingleFlight } from './use-m-single-flight.lib.mjs';
|
|
4
5
|
|
|
5
6
|
export const USE_M_BOOTSTRAP_URL = 'https://unpkg.com/use-m/use.js';
|
|
6
7
|
// Issue #2113: the fallback is only reached when unpkg cannot serve the `latest`
|
|
@@ -9,8 +10,13 @@ export const USE_M_BOOTSTRAP_URL = 'https://unpkg.com/use-m/use.js';
|
|
|
9
10
|
// dependency import to the least resilient loader available. 8.14.4 is the first
|
|
10
11
|
// release that both repairs corrupt aliases (8.14.3, use-m #66/#67) and removes
|
|
11
12
|
// them with a retry budget (8.14.4, use-m #68), so the degraded path now keeps
|
|
12
|
-
// upstream recovery instead of losing it.
|
|
13
|
-
|
|
13
|
+
// upstream recovery instead of losing it. 8.15.0 (use-m #70, the report filed
|
|
14
|
+
// from this issue) additionally serialises installs of one alias across
|
|
15
|
+
// processes with its own `.use-m/<alias>.lock` plus a post-install marker, so
|
|
16
|
+
// the pinned fallback now carries upstream prevention too — verified with the
|
|
17
|
+
// standalone reproduction: 8.14.4 fails 22/24 concurrent loads, 8.15.0 fails
|
|
18
|
+
// 0/24 (docs/case-studies/issue-2113/raw/experiment-upstream-use-m-8.15.0-fixed.log).
|
|
19
|
+
export const USE_M_BOOTSTRAP_FALLBACK_URL = 'https://unpkg.com/use-m@8.15.0/use.js';
|
|
14
20
|
|
|
15
21
|
const isMissingUseMBundle = code => /^Not found: \/use-m@[^/]+\/use\.js\s*$/.test(code.trim());
|
|
16
22
|
|
|
@@ -62,9 +68,22 @@ export const ensureUseM = async (options = {}) => {
|
|
|
62
68
|
// Only a few call sites used useWithRetry explicitly; wrapping here means
|
|
63
69
|
// every `await use(...)` in the codebase recovers by deleting the corrupt
|
|
64
70
|
// install directory and re-fetching.
|
|
65
|
-
|
|
71
|
+
//
|
|
72
|
+
// Issue #2113: retrying alone is not enough. use-m runs one
|
|
73
|
+
// `npm install -g <alias>@npm:<pkg>@<version>` per `use()` call with no
|
|
74
|
+
// in-flight dedup, and 38 modules under src/ load command-stream through
|
|
75
|
+
// use(), 31 of them with a top-level
|
|
76
|
+
// `await use('command-stream')`. Node evaluates sibling top-level-await
|
|
77
|
+
// subgraphs concurrently, so a cold container fires dozens of simultaneous
|
|
78
|
+
// global installs of the *same* alias directory; they delete and re-extract
|
|
79
|
+
// each other's trees, producing the ENOTEMPTY and half-extracted-package
|
|
80
|
+
// failures recorded in the issue. Every retry re-enters the same race, so
|
|
81
|
+
// the single-flight layer wraps the retry layer: identical loads collapse
|
|
82
|
+
// into one install, and installs of the same alias are serialised within
|
|
83
|
+
// and across processes.
|
|
84
|
+
globalThis.use = wrapUseWithSingleFlight(wrapUseWithRetry(rawUse));
|
|
66
85
|
} else {
|
|
67
|
-
globalThis.use = wrapUseWithRetry(globalThis.use);
|
|
86
|
+
globalThis.use = wrapUseWithSingleFlight(wrapUseWithRetry(globalThis.use));
|
|
68
87
|
}
|
|
69
88
|
return globalThis.use;
|
|
70
89
|
};
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Single-flight layer for `use-m` package loading (issue #2113).
|
|
5
|
+
*
|
|
6
|
+
* Root cause this file addresses
|
|
7
|
+
* -----------------------------
|
|
8
|
+
* `use-m` installs every package it resolves with a *global* npm install:
|
|
9
|
+
*
|
|
10
|
+
* npm install -g <pkg>-v-<version>@npm:<pkg>@<version>
|
|
11
|
+
*
|
|
12
|
+
* and it has no in-flight deduplication — every `use(specifier)` call runs the
|
|
13
|
+
* full `ensurePackageInstalled` → `installPackage` path. Hive Mind has 36
|
|
14
|
+
* modules under `src/` whose module body starts with a top-level
|
|
15
|
+
* `await use('command-stream')`, and Node evaluates sibling top-level-await
|
|
16
|
+
* subgraphs *concurrently*. On a cold container that means dozens of
|
|
17
|
+
* simultaneous `npm install -g command-stream-v-latest@npm:command-stream@latest`
|
|
18
|
+
* processes writing into the same global `node_modules` directory.
|
|
19
|
+
*
|
|
20
|
+
* npm has no cross-process locking for the global prefix, so those installs
|
|
21
|
+
* delete and re-extract each other's trees. The two symptoms recorded in the
|
|
22
|
+
* issue are exactly what that race produces (both reproduced in
|
|
23
|
+
* `experiments/issue-2113/reproduce-concurrent-install-race.mjs`):
|
|
24
|
+
*
|
|
25
|
+
* * `npm error ENOTEMPTY: directory not empty, rmdir
|
|
26
|
+
* '<...>/command-stream-v-latest/examples'` — one npm is removing the alias
|
|
27
|
+
* while another is extracting into it, so the directory it just emptied is
|
|
28
|
+
* repopulated before the `rmdir`;
|
|
29
|
+
* * a half-extracted tree that imports fine at the entry point but throws
|
|
30
|
+
* `ERR_MODULE_NOT_FOUND` for an arbitrary internal file
|
|
31
|
+
* (`shell-parser.mjs`, `terminal-capture.mjs`, `$.trace.mjs`).
|
|
32
|
+
*
|
|
33
|
+
* Retrying cannot fix this, because every retry re-enters the same race with
|
|
34
|
+
* the same 30-odd competitors — which is why use-m's own 3 install attempts and
|
|
35
|
+
* `useWithRetry`'s backoff both failed in the logs attached to the issue.
|
|
36
|
+
*
|
|
37
|
+
* The fix
|
|
38
|
+
* -------
|
|
39
|
+
* Make the install happen **once**:
|
|
40
|
+
*
|
|
41
|
+
* 1. in-process memoisation per specifier — the 36 concurrent
|
|
42
|
+
* `use('command-stream')` calls collapse into one load (this also removes
|
|
43
|
+
* 35 redundant `npm show command-stream version` network round-trips);
|
|
44
|
+
* 2. an in-process mutex per npm *alias* — different specifiers that map to
|
|
45
|
+
* the same alias (`yargs@17.7.2` and `yargs@17.7.2/helpers`) are
|
|
46
|
+
* serialised, because they install the same directory;
|
|
47
|
+
* 3. a cross-process advisory lock per alias — two Hive Mind processes
|
|
48
|
+
* started at the same time (worker + monitor, CI matrix jobs) share one
|
|
49
|
+
* global `node_modules`, so the lock has to outlive a single process.
|
|
50
|
+
*
|
|
51
|
+
* The lock is deliberately *advisory and self-healing*: it is an atomic
|
|
52
|
+
* `mkdir`, refreshed by a heartbeat, stolen when stale, and abandoned (with a
|
|
53
|
+
* diagnostic) after a timeout. A stuck lock therefore degrades to today's
|
|
54
|
+
* behaviour instead of hanging Hive Mind.
|
|
55
|
+
*/
|
|
56
|
+
|
|
57
|
+
import os from 'node:os';
|
|
58
|
+
import path from 'node:path';
|
|
59
|
+
import { isBuiltin } from 'node:module';
|
|
60
|
+
import { USE_RETRY_WRAPPED } from './use-with-retry.lib.mjs';
|
|
61
|
+
|
|
62
|
+
export const DEFAULT_HEARTBEAT_MS = 1000;
|
|
63
|
+
export const DEFAULT_STALE_MS = 15000;
|
|
64
|
+
export const DEFAULT_POLL_MS = 100;
|
|
65
|
+
export const DEFAULT_TIMEOUT_MS = 300000;
|
|
66
|
+
|
|
67
|
+
const USE_SINGLE_FLIGHT_WRAPPED = Symbol.for('hive-mind.use-m-single-flight.wrapped');
|
|
68
|
+
|
|
69
|
+
// Mirrors use-m's own parser (`parseModuleSpecifier`) so the alias computed
|
|
70
|
+
// here is byte-identical to the directory npm will create.
|
|
71
|
+
const SPECIFIER_PATTERN = /^(?<packageName>(@[^@/]+\/)?[^@/]+)?(?:@(?<version>[^/]*))?(?<modulePath>(?:\/[^@]+)*)?$/;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* @param {string} specifier
|
|
75
|
+
* @returns {{ packageName: string, version: string, modulePath: string } | null}
|
|
76
|
+
* `null` for anything that use-m will not install from npm (builtins,
|
|
77
|
+
* relative/absolute paths, unparseable input).
|
|
78
|
+
*/
|
|
79
|
+
export const parseSpecifier = specifier => {
|
|
80
|
+
if (typeof specifier !== 'string' || specifier.trim() === '') return null;
|
|
81
|
+
if (specifier.startsWith('.') || specifier.startsWith('/') || specifier.startsWith('node:')) return null;
|
|
82
|
+
const match = specifier.match(SPECIFIER_PATTERN);
|
|
83
|
+
const packageName = match?.groups?.packageName;
|
|
84
|
+
if (typeof packageName !== 'string' || packageName.trim() === '') return null;
|
|
85
|
+
const version = typeof match.groups.version === 'string' && match.groups.version.trim() !== '' ? match.groups.version : 'latest';
|
|
86
|
+
const modulePath = typeof match.groups.modulePath === 'string' ? match.groups.modulePath : '';
|
|
87
|
+
return { packageName, version, modulePath };
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The global `node_modules` directory name use-m installs into, e.g.
|
|
92
|
+
* `use('command-stream')` → `command-stream-v-latest`.
|
|
93
|
+
*
|
|
94
|
+
* @param {string} specifier
|
|
95
|
+
* @returns {string | null}
|
|
96
|
+
*/
|
|
97
|
+
export const aliasForSpecifier = specifier => {
|
|
98
|
+
const parsed = parseSpecifier(specifier);
|
|
99
|
+
if (!parsed) return null;
|
|
100
|
+
return `${parsed.packageName.replace('@', '').replace('/', '-')}-v-${parsed.version}`;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Does loading this specifier run `npm install -g`?
|
|
105
|
+
*
|
|
106
|
+
* `use('fs')`, `use('path')` and `use('os')` account for 57 of Hive Mind's 128
|
|
107
|
+
* `use()` call sites; use-m answers them from its built-in resolver without
|
|
108
|
+
* touching npm, so they must not pay for (or wait on) an install lock. They are
|
|
109
|
+
* still memoised — 26 identical `use('fs')` calls should resolve one promise.
|
|
110
|
+
*
|
|
111
|
+
* @param {string} specifier
|
|
112
|
+
* @returns {boolean}
|
|
113
|
+
*/
|
|
114
|
+
export const installsFromNpm = specifier => {
|
|
115
|
+
const parsed = parseSpecifier(specifier);
|
|
116
|
+
if (!parsed) return false;
|
|
117
|
+
return !isBuiltin(`${parsed.packageName}${parsed.modulePath}`);
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
export const defaultLockRoot = () => process.env.HIVE_MIND_USE_M_LOCK_DIR || path.join(os.tmpdir(), 'hive-mind-use-m-locks');
|
|
121
|
+
|
|
122
|
+
const defaultSleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
123
|
+
|
|
124
|
+
const defaultLog = message => {
|
|
125
|
+
if (process.env.HIVE_MIND_USE_M_DEBUG || process.argv.includes('--verbose')) {
|
|
126
|
+
console.error(`[use-m] ${message}`);
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
// `/` and `@` never survive alias generation, but a caller may lock on an
|
|
131
|
+
// arbitrary key in tests — keep the lock directory name filesystem-safe.
|
|
132
|
+
const lockDirectoryFor = (lockRoot, key) => path.join(lockRoot, `${key.replace(/[^\w.@-]+/g, '_')}.lock`);
|
|
133
|
+
|
|
134
|
+
const noopRelease = async () => {};
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Acquire a cross-process advisory lock for one npm alias.
|
|
138
|
+
*
|
|
139
|
+
* The lock is a directory: `mkdir` is atomic on every filesystem Hive Mind runs
|
|
140
|
+
* on (ext4, overlayfs, fuse-overlayfs in the DinD image, tmpfs, APFS), unlike
|
|
141
|
+
* `writeFile` with `flag: 'wx'` on network filesystems.
|
|
142
|
+
*
|
|
143
|
+
* @param {string} key - alias name.
|
|
144
|
+
* @param {object} [options]
|
|
145
|
+
* @param {string} [options.lockRoot]
|
|
146
|
+
* @param {number} [options.heartbeatMs] - how often the owner refreshes mtime.
|
|
147
|
+
* @param {number} [options.staleMs] - age after which a lock may be stolen.
|
|
148
|
+
* @param {number} [options.pollMs] - wait between acquisition attempts.
|
|
149
|
+
* @param {number} [options.timeoutMs] - give up (and proceed unlocked) after this.
|
|
150
|
+
* @param {object} [options.fs] - injectable `node:fs/promises`.
|
|
151
|
+
* @param {(ms: number) => Promise<void>} [options.sleep]
|
|
152
|
+
* @param {() => number} [options.now]
|
|
153
|
+
* @param {(message: string) => void} [options.log]
|
|
154
|
+
* @returns {Promise<{ acquired: boolean, path: string, release: () => Promise<void> }>}
|
|
155
|
+
*/
|
|
156
|
+
export const acquireAliasLock = async (key, options = {}) => {
|
|
157
|
+
const fs = options.fs ?? (await import('node:fs/promises'));
|
|
158
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
159
|
+
const now = options.now ?? Date.now;
|
|
160
|
+
const log = options.log ?? defaultLog;
|
|
161
|
+
const lockRoot = options.lockRoot ?? defaultLockRoot();
|
|
162
|
+
const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
|
|
163
|
+
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
164
|
+
const pollMs = options.pollMs ?? DEFAULT_POLL_MS;
|
|
165
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
166
|
+
const lockPath = lockDirectoryFor(lockRoot, key);
|
|
167
|
+
const startedAt = now();
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
await fs.mkdir(lockRoot, { recursive: true });
|
|
171
|
+
} catch (error) {
|
|
172
|
+
// A lock root we cannot create means no cross-process protection; the
|
|
173
|
+
// in-process layers still dedupe, so continue instead of failing the load.
|
|
174
|
+
log(`lock root ${lockRoot} is unusable (${error?.message}); continuing without a cross-process lock`);
|
|
175
|
+
return { acquired: false, path: lockPath, release: noopRelease };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
for (;;) {
|
|
179
|
+
try {
|
|
180
|
+
await fs.mkdir(lockPath);
|
|
181
|
+
// Best-effort ownership breadcrumb: it makes a stuck lock diagnosable
|
|
182
|
+
// (`cat /tmp/hive-mind-use-m-locks/<alias>.lock/owner.json`) but nothing
|
|
183
|
+
// depends on it being readable.
|
|
184
|
+
await fs.writeFile(path.join(lockPath, 'owner.json'), `${JSON.stringify({ pid: process.pid, hostname: os.hostname(), key, startedAt: new Date(startedAt).toISOString() }, null, 2)}\n`).catch(() => {});
|
|
185
|
+
log(`acquired install lock for '${key}' at ${lockPath}`);
|
|
186
|
+
|
|
187
|
+
// Keep the mtime fresh so other processes do not mistake a slow install
|
|
188
|
+
// (a cold `npm install -g` can take a minute) for a crashed owner.
|
|
189
|
+
const heartbeat = setInterval(() => {
|
|
190
|
+
const stamp = new Date(now());
|
|
191
|
+
Promise.resolve(fs.utimes(lockPath, stamp, stamp)).catch(() => {});
|
|
192
|
+
}, heartbeatMs);
|
|
193
|
+
heartbeat.unref?.();
|
|
194
|
+
|
|
195
|
+
let released = false;
|
|
196
|
+
const release = async () => {
|
|
197
|
+
if (released) return;
|
|
198
|
+
released = true;
|
|
199
|
+
clearInterval(heartbeat);
|
|
200
|
+
try {
|
|
201
|
+
await fs.rm(lockPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
202
|
+
} catch (error) {
|
|
203
|
+
log(`failed to release install lock ${lockPath}: ${error?.message}`);
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
return { acquired: true, path: lockPath, release };
|
|
207
|
+
} catch (error) {
|
|
208
|
+
if (error?.code !== 'EEXIST') {
|
|
209
|
+
log(`could not create install lock ${lockPath} (${error?.message}); continuing without a cross-process lock`);
|
|
210
|
+
return { acquired: false, path: lockPath, release: noopRelease };
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const stats = await fs.stat(lockPath).catch(() => null);
|
|
215
|
+
if (!stats) continue; // owner released between mkdir and stat — retry immediately.
|
|
216
|
+
|
|
217
|
+
const age = now() - stats.mtimeMs;
|
|
218
|
+
if (age > staleMs) {
|
|
219
|
+
log(`stealing stale install lock ${lockPath} (idle for ${Math.round(age)}ms)`);
|
|
220
|
+
await fs.rm(lockPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }).catch(() => {});
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (now() - startedAt > timeoutMs) {
|
|
225
|
+
log(`timed out after ${timeoutMs}ms waiting for install lock ${lockPath}; proceeding without it`);
|
|
226
|
+
return { acquired: false, path: lockPath, release: noopRelease };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
await sleep(pollMs);
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Serialise `fn` against every other caller holding the same alias, in this
|
|
235
|
+
* process and across processes.
|
|
236
|
+
*
|
|
237
|
+
* @param {string} key
|
|
238
|
+
* @param {() => Promise<T>} fn
|
|
239
|
+
* @param {object} [options] - forwarded to {@link acquireAliasLock}.
|
|
240
|
+
* @returns {Promise<T>}
|
|
241
|
+
* @template T
|
|
242
|
+
*/
|
|
243
|
+
export const withAliasLock = async (key, fn, options = {}) => {
|
|
244
|
+
if (options.disabled) return fn();
|
|
245
|
+
const lock = await acquireAliasLock(key, options);
|
|
246
|
+
try {
|
|
247
|
+
return await fn();
|
|
248
|
+
} finally {
|
|
249
|
+
await lock.release();
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
const createState = () => ({ inflight: new Map(), chains: new Map() });
|
|
254
|
+
|
|
255
|
+
let sharedState = createState();
|
|
256
|
+
|
|
257
|
+
/** Drop memoised loads and alias chains (tests only). */
|
|
258
|
+
export const resetSingleFlightState = () => {
|
|
259
|
+
sharedState = createState();
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
const runOnAliasChain = (state, alias, fn) => {
|
|
263
|
+
const previous = state.chains.get(alias) ?? Promise.resolve();
|
|
264
|
+
// `.then(fn, fn)` so a failed predecessor does not strand the queue.
|
|
265
|
+
const result = previous.then(fn, fn);
|
|
266
|
+
const tail = result.then(
|
|
267
|
+
() => {},
|
|
268
|
+
() => {}
|
|
269
|
+
);
|
|
270
|
+
state.chains.set(alias, tail);
|
|
271
|
+
tail.then(() => {
|
|
272
|
+
if (state.chains.get(alias) === tail) state.chains.delete(alias);
|
|
273
|
+
});
|
|
274
|
+
return result;
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Wrap a `use` function so concurrent loads of the same package collapse into a
|
|
279
|
+
* single npm install.
|
|
280
|
+
*
|
|
281
|
+
* Composition order matters: single-flight must sit **outside**
|
|
282
|
+
* `wrapUseWithRetry`, so that the retry/repair logic (which deletes and
|
|
283
|
+
* reinstalls the alias directory) also runs under the lock. The wrapper carries
|
|
284
|
+
* both wrapper symbols, which keeps `ensureUseM()` idempotent — re-wrapping an
|
|
285
|
+
* already-protected `globalThis.use` returns it unchanged instead of nesting
|
|
286
|
+
* retries inside locks inside retries.
|
|
287
|
+
*
|
|
288
|
+
* @param {Function} use
|
|
289
|
+
* @param {object} [options]
|
|
290
|
+
* @param {boolean} [options.disabled] - skip the cross-process lock only.
|
|
291
|
+
* @param {object} [options.state] - injectable memo/chain state (tests).
|
|
292
|
+
* @returns {Function}
|
|
293
|
+
*/
|
|
294
|
+
export const wrapUseWithSingleFlight = (use, options = {}) => {
|
|
295
|
+
if (typeof use !== 'function' || use[USE_SINGLE_FLIGHT_WRAPPED]) return use;
|
|
296
|
+
const log = options.log ?? defaultLog;
|
|
297
|
+
const disabled = options.disabled ?? Boolean(process.env.HIVE_MIND_USE_M_NO_LOCK);
|
|
298
|
+
|
|
299
|
+
const wrapped = (specifier, ...args) => {
|
|
300
|
+
const state = options.state ?? sharedState;
|
|
301
|
+
const alias = aliasForSpecifier(specifier);
|
|
302
|
+
// Relative imports resolve against the *caller's* directory, so neither
|
|
303
|
+
// memoising nor serialising them is safe — pass them straight through.
|
|
304
|
+
if (!alias) return use(specifier, ...args);
|
|
305
|
+
|
|
306
|
+
// Issue #2113: both failing runs were started with `--verbose` and the log
|
|
307
|
+
// showed only the final crash. Tracing every load (specifier, alias,
|
|
308
|
+
// duration) is what makes the next incident diagnosable from the log alone.
|
|
309
|
+
const call = async () => {
|
|
310
|
+
const startedAt = Date.now();
|
|
311
|
+
log(`use('${specifier}') loading (alias ${alias})`);
|
|
312
|
+
try {
|
|
313
|
+
const module = await use(specifier, ...args);
|
|
314
|
+
log(`use('${specifier}') loaded in ${Date.now() - startedAt}ms`);
|
|
315
|
+
return module;
|
|
316
|
+
} catch (error) {
|
|
317
|
+
log(`use('${specifier}') failed after ${Date.now() - startedAt}ms: ${error?.message}`);
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
// Only npm-backed specifiers need the alias mutex and the file lock; a
|
|
322
|
+
// built-in has no install step to protect.
|
|
323
|
+
const start = installsFromNpm(specifier) ? () => runOnAliasChain(state, alias, () => withAliasLock(alias, call, { ...options, disabled, log })) : call;
|
|
324
|
+
|
|
325
|
+
// Extra arguments select a different resolver/context, so results are not
|
|
326
|
+
// interchangeable; those calls skip the memo but still take the lock.
|
|
327
|
+
if (args.length > 0) return start();
|
|
328
|
+
|
|
329
|
+
const pending = state.inflight.get(specifier);
|
|
330
|
+
if (pending) {
|
|
331
|
+
log(`use('${specifier}') joined an in-flight load (alias ${alias})`);
|
|
332
|
+
return pending;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const promise = start();
|
|
336
|
+
state.inflight.set(specifier, promise);
|
|
337
|
+
// Successful loads stay memoised for the process lifetime (Node caches the
|
|
338
|
+
// module anyway); failures are evicted so a later call can retry.
|
|
339
|
+
promise.catch(() => {
|
|
340
|
+
if (state.inflight.get(specifier) === promise) state.inflight.delete(specifier);
|
|
341
|
+
});
|
|
342
|
+
return promise;
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
Object.defineProperty(wrapped, USE_SINGLE_FLIGHT_WRAPPED, { value: true });
|
|
346
|
+
// Claim the retry symbol too: `wrapUseWithRetry` is always applied first
|
|
347
|
+
// (see ensureUseM), so an outer re-wrap would invert the intended order.
|
|
348
|
+
Object.defineProperty(wrapped, USE_RETRY_WRAPPED, { value: true });
|
|
349
|
+
return wrapped;
|
|
350
|
+
};
|
|
@@ -223,11 +223,17 @@ const defaultSleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
|
223
223
|
|
|
224
224
|
// Off by default so normal runs stay quiet; issue #2092 showed that when the
|
|
225
225
|
// loader dies there is no trace of which specifier or attempt failed.
|
|
226
|
+
// Issue #2113: both failing runs attached to the issue were started with
|
|
227
|
+
// `--verbose` and still produced zero loader diagnostics, so the log showed the
|
|
228
|
+
// final crash without a single line about which specifier, attempt or alias was
|
|
229
|
+
// involved. `--verbose` now opts into the same trace as HIVE_MIND_USE_M_DEBUG.
|
|
226
230
|
const defaultLog = message => {
|
|
227
|
-
if (process.env.HIVE_MIND_USE_M_DEBUG
|
|
231
|
+
if (process.env.HIVE_MIND_USE_M_DEBUG || process.argv.includes('--verbose')) {
|
|
232
|
+
console.error(`[use-m] ${message}`);
|
|
233
|
+
}
|
|
228
234
|
};
|
|
229
235
|
|
|
230
|
-
const USE_RETRY_WRAPPED = Symbol.for('hive-mind.use-with-retry.wrapped');
|
|
236
|
+
export const USE_RETRY_WRAPPED = Symbol.for('hive-mind.use-with-retry.wrapped');
|
|
231
237
|
|
|
232
238
|
/**
|
|
233
239
|
* Wrap a raw use-m `use` function so that *every* call site inherits the
|