@gjsify/cli 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.gjs.mjs +192 -165
- package/lib/commands/gettext.d.ts +2 -1
- package/lib/commands/gettext.js +105 -47
- package/lib/commands/onboard.d.ts +7 -0
- package/lib/commands/onboard.js +349 -96
- package/lib/commands/ship.d.ts +3 -0
- package/lib/commands/ship.js +833 -54
- package/lib/commands/trust.d.ts +6 -0
- package/lib/commands/trust.js +1 -1
- package/lib/types/config-data.d.ts +63 -1
- package/lib/utils/msgfmt-merge.d.ts +35 -0
- package/lib/utils/msgfmt-merge.js +102 -0
- package/lib/utils/npm-otp.d.ts +27 -2
- package/lib/utils/npm-otp.js +82 -7
- package/lib/utils/onboard-discovery.d.ts +68 -0
- package/lib/utils/onboard-discovery.js +178 -0
- package/lib/utils/onboard-probe.d.ts +118 -1
- package/lib/utils/onboard-probe.js +153 -4
- package/lib/utils/prompt-output.d.ts +12 -0
- package/lib/utils/prompt-output.js +54 -0
- package/lib/utils/prompt.js +22 -2
- package/lib/utils/publishable-packages.d.ts +1 -1
- package/lib/utils/publishable-packages.js +1 -0
- package/lib/utils/resolve-npm-package.d.ts +56 -0
- package/lib/utils/resolve-npm-package.js +69 -0
- package/lib/utils/ship/app-runtime.d.ts +214 -0
- package/lib/utils/ship/app-runtime.js +429 -0
- package/lib/utils/ship/depends.d.ts +50 -0
- package/lib/utils/ship/depends.js +117 -3
- package/lib/utils/ship/discover.js +18 -1
- package/lib/utils/ship/dmg.d.ts +71 -0
- package/lib/utils/ship/dmg.js +181 -0
- package/lib/utils/ship/flatpak.js +1 -1
- package/lib/utils/ship/formats.d.ts +85 -7
- package/lib/utils/ship/formats.js +734 -15
- package/lib/utils/ship/launcher.d.ts +13 -3
- package/lib/utils/ship/launcher.js +362 -17
- package/lib/utils/ship/layout.d.ts +268 -0
- package/lib/utils/ship/layout.js +351 -0
- package/lib/utils/ship/localize-metadata.d.ts +22 -0
- package/lib/utils/ship/localize-metadata.js +126 -0
- package/lib/utils/ship/mime.js +1 -9
- package/lib/utils/ship/msi.d.ts +126 -0
- package/lib/utils/ship/msi.js +417 -0
- package/lib/utils/ship/node-runtime.d.ts +71 -0
- package/lib/utils/ship/node-runtime.js +122 -0
- package/lib/utils/ship/payload.d.ts +156 -5
- package/lib/utils/ship/payload.js +580 -14
- package/lib/utils/ship/plan.d.ts +21 -0
- package/lib/utils/ship/plan.js +20 -7
- package/lib/utils/ship/plist.d.ts +58 -0
- package/lib/utils/ship/plist.js +198 -0
- package/lib/utils/ship/rpm.js +9 -3
- package/lib/utils/ship/schemas.d.ts +61 -0
- package/lib/utils/ship/schemas.js +118 -0
- package/lib/utils/ship/scripts.js +7 -3
- package/lib/utils/ship/settings.d.ts +19 -1
- package/lib/utils/ship/settings.js +12 -1
- package/lib/utils/ship/share-dirs.d.ts +23 -0
- package/lib/utils/ship/share-dirs.js +46 -0
- package/lib/utils/ship/signing.d.ts +195 -0
- package/lib/utils/ship/signing.js +317 -0
- package/lib/utils/ship/stage-manifest.d.ts +40 -12
- package/lib/utils/ship/stage-manifest.js +99 -13
- package/lib/utils/ship/stage-writer.d.ts +48 -0
- package/lib/utils/ship/stage-writer.js +71 -1
- package/lib/utils/ship/types.d.ts +158 -13
- package/lib/utils/ship/xml.d.ts +7 -0
- package/lib/utils/ship/xml.js +25 -0
- package/lib/utils/ship/zip.d.ts +69 -0
- package/lib/utils/ship/zip.js +245 -0
- package/lib/utils/trust-registry.js +5 -0
- package/package.json +19 -19
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// Package-set discovery for `gjsify onboard`, and the ONE thing that decides
|
|
2
|
+
// whether the command is a gjsify-repo chore or a monorepo tool.
|
|
3
|
+
//
|
|
4
|
+
// Onboard used to enumerate exactly one way: `discoverWorkspaces` over the root
|
|
5
|
+
// manifest's `workspaces` globs, plus this repo's two carved-out publishable
|
|
6
|
+
// directories. That set is correct HERE and empty in the repo the sweep is most
|
|
7
|
+
// needed for — `gjsify/types` publishes 703 `@girs/*` packages from a root whose
|
|
8
|
+
// only tracked file is `.gitignore`. There is no manifest, so there are no
|
|
9
|
+
// globs, so `discoverWorkspaces` throws before anything is enumerated.
|
|
10
|
+
//
|
|
11
|
+
// So the package set is now built from up to three SOURCES, and every one of
|
|
12
|
+
// them is REPORTED with its count. That reporting is not decoration: a sweep
|
|
13
|
+
// that writes to npm derives its whole blast radius from this list, and the two
|
|
14
|
+
// ways it goes wrong — enumerating nothing, and enumerating the wrong tree —
|
|
15
|
+
// look identical in a summary that only prints a total. `--packages` matching
|
|
16
|
+
// no directory is a HARD ERROR for the same reason `gjsify foreach --include`
|
|
17
|
+
// is: a filter that selects nothing is a typo or a shell-quoting bug, never an
|
|
18
|
+
// intent (and onboard's own history has the sharper version of this — it once
|
|
19
|
+
// reported "127 already done" while `@gjsify/napi` was simply absent from the
|
|
20
|
+
// list it had built).
|
|
21
|
+
import { existsSync } from 'node:fs';
|
|
22
|
+
import { spawnSync } from 'node:child_process';
|
|
23
|
+
import { discoverWorkspaces } from '@gjsify/workspace';
|
|
24
|
+
import { join } from 'node:path';
|
|
25
|
+
import { mergePublishables } from './publishable-packages.js';
|
|
26
|
+
import { parseRepoFromGitRemote } from './trust-registry.js';
|
|
27
|
+
import { findWorkspaceRoot } from './workspace-root.js';
|
|
28
|
+
/**
|
|
29
|
+
* The directory the globs and the workspace manifest are resolved against.
|
|
30
|
+
*
|
|
31
|
+
* A workspace root wins, because that is what `discoverWorkspaces` needs. When
|
|
32
|
+
* there is none, the GIT top level is the answer and the cwd is only the last
|
|
33
|
+
* resort: `--packages '*'` run from anywhere inside a non-workspace monorepo
|
|
34
|
+
* must mean the same set every time, and resolving it against the cwd would
|
|
35
|
+
* quietly mean "the packages below wherever I happen to stand".
|
|
36
|
+
*/
|
|
37
|
+
export function resolveRepoRoot(cwd) {
|
|
38
|
+
const wsRoot = findWorkspaceRoot(cwd);
|
|
39
|
+
if (wsRoot)
|
|
40
|
+
return wsRoot;
|
|
41
|
+
const git = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd, encoding: 'utf8' });
|
|
42
|
+
if (git.status === 0) {
|
|
43
|
+
const top = git.stdout.trim();
|
|
44
|
+
if (top && existsSync(top))
|
|
45
|
+
return top;
|
|
46
|
+
}
|
|
47
|
+
return cwd;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Every `--packages` glob must match at least one directory holding a
|
|
51
|
+
* `package.json` with a name. Throws with the unmatched patterns otherwise.
|
|
52
|
+
*
|
|
53
|
+
* Mirrors `gjsify foreach`'s `--include` rule, and matters more here: foreach's
|
|
54
|
+
* empty selection runs no scripts, while onboard's empty selection is a sweep
|
|
55
|
+
* that reports itself complete having examined nothing.
|
|
56
|
+
*/
|
|
57
|
+
export function assertEveryPatternMatches(root, patterns) {
|
|
58
|
+
const unmatched = patterns.filter((pattern) => {
|
|
59
|
+
try {
|
|
60
|
+
return discoverWorkspaces(root, { patterns: [pattern] }).length === 0;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
if (unmatched.length === 0)
|
|
67
|
+
return;
|
|
68
|
+
const lines = [
|
|
69
|
+
`gjsify onboard: --packages matched no package directory (${unmatched.length} of ${patterns.length} pattern(s)) under ${root}:`,
|
|
70
|
+
];
|
|
71
|
+
for (const pattern of unmatched)
|
|
72
|
+
lines.push(` ${JSON.stringify(pattern)}`);
|
|
73
|
+
lines.push(' Patterns are DIRECTORY globs resolved against the repo root, and only `*` is supported', ' (one path segment). A matched directory needs a package.json with a "name".');
|
|
74
|
+
throw new Error(lines.join('\n'));
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Collect the publishable package set from every source that applies.
|
|
78
|
+
*
|
|
79
|
+
* Order is precedence: a real workspace beats a `--packages` hit beats a scanned
|
|
80
|
+
* directory, deduped by package NAME, so the manifest a build would use is the
|
|
81
|
+
* manifest the sweep publishes.
|
|
82
|
+
*/
|
|
83
|
+
export function collectOnboardPackages(root, cwd, options = {}) {
|
|
84
|
+
const sources = [];
|
|
85
|
+
const packages = [];
|
|
86
|
+
const seen = new Set();
|
|
87
|
+
const take = (kind, found, patterns) => {
|
|
88
|
+
let added = 0;
|
|
89
|
+
for (const ws of found) {
|
|
90
|
+
if (!ws.name || seen.has(ws.name))
|
|
91
|
+
continue;
|
|
92
|
+
seen.add(ws.name);
|
|
93
|
+
packages.push(ws);
|
|
94
|
+
added++;
|
|
95
|
+
}
|
|
96
|
+
sources.push({ kind, patterns, count: added });
|
|
97
|
+
};
|
|
98
|
+
// 1. The root manifest's own `workspaces` globs, when the repo has them.
|
|
99
|
+
if (existsSync(join(root, 'package.json'))) {
|
|
100
|
+
try {
|
|
101
|
+
take('workspaces', discoverWorkspaces(root, { includeRoot: true }));
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
// A root package.json without `workspaces` is not an error here —
|
|
105
|
+
// `--packages` or the scanned dirs may still carry the whole set.
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// 2. Explicit `--packages` globs — the path for a monorepo that is not an
|
|
109
|
+
// npm/yarn workspace. Callers run `assertEveryPatternMatches` first, so
|
|
110
|
+
// an empty result here can only mean "everything was already covered".
|
|
111
|
+
if (options.patterns && options.patterns.length > 0) {
|
|
112
|
+
take('packages', discoverWorkspaces(root, { patterns: [...options.patterns] }), options.patterns);
|
|
113
|
+
}
|
|
114
|
+
// 3. This repo's publishable-but-not-workspace directories. A no-op in any
|
|
115
|
+
// repo that does not have them; see publishable-packages.ts for the
|
|
116
|
+
// incident that put it here.
|
|
117
|
+
take('scanned-dirs', mergePublishables([], cwd));
|
|
118
|
+
return { packages, sources };
|
|
119
|
+
}
|
|
120
|
+
/** One-line rendering of where the package set came from. */
|
|
121
|
+
export function describeSources(sources) {
|
|
122
|
+
return sources
|
|
123
|
+
.filter((s) => s.count > 0)
|
|
124
|
+
.map((s) => (s.patterns ? `${s.kind}(${s.patterns.join(',')})=${s.count}` : `${s.kind}=${s.count}`))
|
|
125
|
+
.join(' + ');
|
|
126
|
+
}
|
|
127
|
+
/** The `owner/repo` a package's own manifest claims, or null if it claims none. */
|
|
128
|
+
export function declaredRepository(ws) {
|
|
129
|
+
const repo = ws.manifest.repository;
|
|
130
|
+
const url = typeof repo === 'string' ? repo : repo?.url;
|
|
131
|
+
if (typeof url !== 'string' || url.length === 0)
|
|
132
|
+
return null;
|
|
133
|
+
return parseRepoFromGitRemote(url);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Refuse to configure a Trusted Publisher for a package whose OWN manifest says
|
|
137
|
+
* it lives in a different repository.
|
|
138
|
+
*
|
|
139
|
+
* This replaces a hardcoded `exclude: ['@girs/*']`, and it is worth more than
|
|
140
|
+
* the thing it replaces. Measured: `gjsify onboard` in `gjsify/ts-for-gir`
|
|
141
|
+
* selects 716 packages, ~703 of them the generated `@girs/*` under
|
|
142
|
+
* `types-dev/*` — which are workspaces there, but are PUBLISHED FROM
|
|
143
|
+
* `gjsify/types`. Trusting them for `gjsify/ts-for-gir`'s `release.yml` would
|
|
144
|
+
* point their OIDC exchange at a workflow that never publishes them, i.e. break
|
|
145
|
+
* the release the sweep exists to protect.
|
|
146
|
+
*
|
|
147
|
+
* A name pattern only ever covered the one case somebody had already been bitten
|
|
148
|
+
* by. The manifest's `repository` is the package's own claim about where it
|
|
149
|
+
* comes from, so this covers every monorepo that has another project's packages
|
|
150
|
+
* inside it — vendored, submoduled, or generated.
|
|
151
|
+
*
|
|
152
|
+
* Silence is the only safe default in the other direction: a package that
|
|
153
|
+
* declares NO repository is not evidence of a mismatch, so it passes.
|
|
154
|
+
*/
|
|
155
|
+
export function assertRepositoryAgreement(selected, repository) {
|
|
156
|
+
const target = repository.toLowerCase();
|
|
157
|
+
const foreign = selected
|
|
158
|
+
.map((ws) => ({ ws, declared: declaredRepository(ws) }))
|
|
159
|
+
.filter((e) => e.declared !== null && e.declared.toLowerCase() !== target);
|
|
160
|
+
if (foreign.length === 0)
|
|
161
|
+
return;
|
|
162
|
+
const byRepo = new Map();
|
|
163
|
+
for (const { ws, declared } of foreign) {
|
|
164
|
+
const list = byRepo.get(declared) ?? [];
|
|
165
|
+
list.push(ws.name);
|
|
166
|
+
byRepo.set(declared, list);
|
|
167
|
+
}
|
|
168
|
+
const lines = [
|
|
169
|
+
`gjsify onboard: ${foreign.length} selected package(s) declare a DIFFERENT repository than ${repository}:`,
|
|
170
|
+
];
|
|
171
|
+
for (const [repo, names] of byRepo) {
|
|
172
|
+
const shown = names.slice(0, 5).join(', ');
|
|
173
|
+
const more = names.length > 5 ? `, … (+${names.length - 5} more)` : '';
|
|
174
|
+
lines.push(` ${repo}: ${names.length} package(s) — ${shown}${more}`);
|
|
175
|
+
}
|
|
176
|
+
lines.push(" A Trusted Publisher scoped to the wrong repository points that package's OIDC exchange", ' at a workflow that does not publish it, which breaks the release this sweep protects.', ' Either exclude them (--exclude), select only what belongs here (--include), or point', ' --repository at the repo that actually publishes them.');
|
|
177
|
+
throw new Error(lines.join('\n'));
|
|
178
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type NpmrcConfig } from '@gjsify/npm-registry';
|
|
2
2
|
import type { Workspace } from '@gjsify/workspace';
|
|
3
|
-
import type { TrustRequester } from '../commands/trust.js';
|
|
3
|
+
import type { TrustRequester, TrustRequestResult } from '../commands/trust.js';
|
|
4
4
|
import { type TrustState } from './trust-registry.js';
|
|
5
5
|
/** What a package needs, computed from its published + trust state. */
|
|
6
6
|
export type PkgAction = 'skip' | 'trust' | 'publish-and-trust' | 'blocked';
|
|
@@ -24,6 +24,38 @@ export interface ProbeOptions {
|
|
|
24
24
|
retryOn401?: boolean;
|
|
25
25
|
/** Backoff before the single 401 retry. Default {@link PROBE_RETRY_DELAY_MS}. */
|
|
26
26
|
retryDelayMs?: number;
|
|
27
|
+
/** Total time to spend waiting out HTTP 429s. Default {@link RATE_LIMIT_MAX_WAIT_MS}. */
|
|
28
|
+
maxRateLimitWaitMs?: number;
|
|
29
|
+
/** First backoff for a 429; doubles per attempt. Default {@link RATE_LIMIT_BASE_DELAY_MS}. */
|
|
30
|
+
rateLimitBaseDelayMs?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Cool-down shared across the sweep. Pass ONE instance to every call so a
|
|
33
|
+
* 429 anywhere holds back everywhere; omit for an isolated request.
|
|
34
|
+
*/
|
|
35
|
+
gate?: RateLimitGate;
|
|
36
|
+
/**
|
|
37
|
+
* Called on every 429 with what npm actually sent and how long we will wait.
|
|
38
|
+
* The caller decides how loud to be; reporting the FIRST one is enough to
|
|
39
|
+
* turn "we got throttled" into "npm asked for N seconds" (or into "npm said
|
|
40
|
+
* nothing, so this delay is ours").
|
|
41
|
+
*/
|
|
42
|
+
onRateLimit?: (info: {
|
|
43
|
+
url: string;
|
|
44
|
+
headers: string;
|
|
45
|
+
waitMs: number;
|
|
46
|
+
fromRetryAfter: boolean;
|
|
47
|
+
}) => void;
|
|
48
|
+
/**
|
|
49
|
+
* Called after each state read with what is known SO FAR.
|
|
50
|
+
*
|
|
51
|
+
* The probe phase is the silent half of a sweep: 703 packages take minutes
|
|
52
|
+
* and, until this existed, printed nothing at all between the header and the
|
|
53
|
+
* plan. The only output in that window was the occasional 2FA prompt, which
|
|
54
|
+
* makes a working sweep and a wedged one look identical — and the prompt is
|
|
55
|
+
* the moment a user most needs to know the previous code accomplished
|
|
56
|
+
* something.
|
|
57
|
+
*/
|
|
58
|
+
onProgress?: (done: number, total: number, plan: PkgPlan) => void;
|
|
27
59
|
/** Injected delay (tests pass a no-op). Default: real `setTimeout`. */
|
|
28
60
|
sleep?: (ms: number) => Promise<void>;
|
|
29
61
|
/**
|
|
@@ -33,6 +65,50 @@ export interface ProbeOptions {
|
|
|
33
65
|
exists?: (registry: string, name: string) => Promise<boolean | null>;
|
|
34
66
|
}
|
|
35
67
|
export declare const PROBE_RETRY_DELAY_MS = 400;
|
|
68
|
+
/**
|
|
69
|
+
* A 429 is npm asking to be asked later — never an answer about the package.
|
|
70
|
+
*
|
|
71
|
+
* Measured on the real 703-package sweep of `gjsify/types` at the default
|
|
72
|
+
* concurrency: the first 662 reads answered and the LAST 41 came back `429`, so
|
|
73
|
+
* the plan said `41 unreadable` for packages whose state nobody had actually
|
|
74
|
+
* failed to determine. The tail of an alphabetical list is exactly where a
|
|
75
|
+
* cumulative rate limit lands, which makes the damage look like a property of
|
|
76
|
+
* those packages.
|
|
77
|
+
*
|
|
78
|
+
* Waiting is the whole fix, and it has to be bounded: an unbounded wait turns a
|
|
79
|
+
* throttled sweep into a hang with no output.
|
|
80
|
+
*/
|
|
81
|
+
/**
|
|
82
|
+
* A TIME budget, not an attempt count — and generous, because npm tells us
|
|
83
|
+
* nothing.
|
|
84
|
+
*
|
|
85
|
+
* The attempt count was the wrong unit and the real sweep proved it: 4 attempts
|
|
86
|
+
* with a doubling 2s base is 30 seconds of patience, npm's window is longer than
|
|
87
|
+
* that, so every in-flight request spent its budget inside one cooldown and
|
|
88
|
+
* `Done: 502 trusted … 73 failed` — all 73 at the tail, all 429, none of them a
|
|
89
|
+
* fact about those packages.
|
|
90
|
+
*
|
|
91
|
+
* With no `Retry-After` and no rate-limit headers to read (measured), waiting is
|
|
92
|
+
* the only instrument there is. Five minutes per request is worth it: the
|
|
93
|
+
* alternative is failing packages that then need another sweep, which is itself
|
|
94
|
+
* throttled. Bounded all the same, so a throttled sweep still ENDS.
|
|
95
|
+
*/
|
|
96
|
+
export declare const RATE_LIMIT_MAX_WAIT_MS = 300000;
|
|
97
|
+
export declare const RATE_LIMIT_BASE_DELAY_MS = 2000;
|
|
98
|
+
/** Ceiling for a single backoff step, so escalation stays legible. */
|
|
99
|
+
export declare const RATE_LIMIT_MAX_STEP_MS = 60000;
|
|
100
|
+
/**
|
|
101
|
+
* What npm told us about pacing on a 429 — verbatim, so the answer to "how fast
|
|
102
|
+
* may we go" comes from the registry rather than from a guess.
|
|
103
|
+
*
|
|
104
|
+
* Measured 2026-08-28: npm serves NO `X-RateLimit-*` headers on ordinary
|
|
105
|
+
* responses, so there is no budget to read ahead of time. `Retry-After` on the
|
|
106
|
+
* 429 itself is the only channel, and whether npm populates it on the trust
|
|
107
|
+
* endpoint is exactly what this reports the first time it happens.
|
|
108
|
+
*/
|
|
109
|
+
export declare function describeRateLimitHeaders(headers: Headers | undefined): string;
|
|
110
|
+
/** Seconds to wait per npm's `Retry-After`, or null when it said nothing usable. */
|
|
111
|
+
export declare function retryAfterMs(headers: Headers | undefined): number | null;
|
|
36
112
|
/**
|
|
37
113
|
* Does the registry serve a packument for this name?
|
|
38
114
|
*
|
|
@@ -55,6 +131,47 @@ export declare function packumentExists(registry: string, name: string): Promise
|
|
|
55
131
|
export declare const DEFAULT_PROBE_CONCURRENCY = 4;
|
|
56
132
|
/** Map a classified trust state to the action the onboard sweep should take. */
|
|
57
133
|
export declare function actionForState(state: TrustState): PkgAction;
|
|
134
|
+
/**
|
|
135
|
+
* A cool-down shared by every in-flight request of a sweep.
|
|
136
|
+
*
|
|
137
|
+
* Retrying ONE throttled request in isolation is not enough, and the measured
|
|
138
|
+
* run shows why: at serial pace `gjsify/types` was already being 429'd, so every
|
|
139
|
+
* other worker kept hammering the registry through the very window the retry was
|
|
140
|
+
* waiting out — and the retry budget then expired against a limit nobody had
|
|
141
|
+
* stopped provoking. `trust failed (HTTP 429)` was the result.
|
|
142
|
+
*
|
|
143
|
+
* A 429 anywhere therefore pauses EVERYWHERE. That also makes concurrency safe
|
|
144
|
+
* to raise: the sweep self-paces down to whatever npm is willing to serve
|
|
145
|
+
* instead of the operator guessing a number.
|
|
146
|
+
*/
|
|
147
|
+
export declare class RateLimitGate {
|
|
148
|
+
private until;
|
|
149
|
+
private readonly now;
|
|
150
|
+
private readonly sleeper;
|
|
151
|
+
constructor(opts?: {
|
|
152
|
+
now?: () => number;
|
|
153
|
+
sleep?: (ms: number) => Promise<void>;
|
|
154
|
+
});
|
|
155
|
+
/** Block while a cool-down is in effect. */
|
|
156
|
+
wait(): Promise<void>;
|
|
157
|
+
/** Hold every request back for at least `ms` from now. */
|
|
158
|
+
penalize(ms: number): void;
|
|
159
|
+
/** Milliseconds still owed, for tests and for reporting. */
|
|
160
|
+
remainingMs(): number;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Issue a request, waiting out an HTTP 429 rather than reporting it as a state.
|
|
164
|
+
*
|
|
165
|
+
* Used by BOTH the probe GET and the act-phase trust POST, because a sweep long
|
|
166
|
+
* enough to be throttled on reads is long enough to be throttled on writes, and
|
|
167
|
+
* a throttled POST would otherwise be recorded as `trust failed (HTTP 429)` —
|
|
168
|
+
* a failure attributed to the package instead of to the pacing.
|
|
169
|
+
*
|
|
170
|
+
* Bounded on purpose: it returns the last 429 once the budget is spent, so the
|
|
171
|
+
* caller reports "unreadable" rather than hanging. `Retry-After` wins over the
|
|
172
|
+
* exponential backoff whenever npm sends a usable one.
|
|
173
|
+
*/
|
|
174
|
+
export declare function requestWaitingOutRateLimit(request: TrustRequester, method: string, url: string, body?: unknown, opts?: ProbeOptions): Promise<TrustRequestResult>;
|
|
58
175
|
/**
|
|
59
176
|
* Read one package's published + trust state through the SHARED trust requester.
|
|
60
177
|
* A surviving `auth-required` means the 401 was not an answerable OTP challenge —
|
|
@@ -11,6 +11,69 @@
|
|
|
11
11
|
import { DEFAULT_REGISTRY, registryFor } from '@gjsify/npm-registry';
|
|
12
12
|
import { classifyTrustList, trustUrl } from './trust-registry.js';
|
|
13
13
|
export const PROBE_RETRY_DELAY_MS = 400;
|
|
14
|
+
/**
|
|
15
|
+
* A 429 is npm asking to be asked later — never an answer about the package.
|
|
16
|
+
*
|
|
17
|
+
* Measured on the real 703-package sweep of `gjsify/types` at the default
|
|
18
|
+
* concurrency: the first 662 reads answered and the LAST 41 came back `429`, so
|
|
19
|
+
* the plan said `41 unreadable` for packages whose state nobody had actually
|
|
20
|
+
* failed to determine. The tail of an alphabetical list is exactly where a
|
|
21
|
+
* cumulative rate limit lands, which makes the damage look like a property of
|
|
22
|
+
* those packages.
|
|
23
|
+
*
|
|
24
|
+
* Waiting is the whole fix, and it has to be bounded: an unbounded wait turns a
|
|
25
|
+
* throttled sweep into a hang with no output.
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* A TIME budget, not an attempt count — and generous, because npm tells us
|
|
29
|
+
* nothing.
|
|
30
|
+
*
|
|
31
|
+
* The attempt count was the wrong unit and the real sweep proved it: 4 attempts
|
|
32
|
+
* with a doubling 2s base is 30 seconds of patience, npm's window is longer than
|
|
33
|
+
* that, so every in-flight request spent its budget inside one cooldown and
|
|
34
|
+
* `Done: 502 trusted … 73 failed` — all 73 at the tail, all 429, none of them a
|
|
35
|
+
* fact about those packages.
|
|
36
|
+
*
|
|
37
|
+
* With no `Retry-After` and no rate-limit headers to read (measured), waiting is
|
|
38
|
+
* the only instrument there is. Five minutes per request is worth it: the
|
|
39
|
+
* alternative is failing packages that then need another sweep, which is itself
|
|
40
|
+
* throttled. Bounded all the same, so a throttled sweep still ENDS.
|
|
41
|
+
*/
|
|
42
|
+
export const RATE_LIMIT_MAX_WAIT_MS = 300_000;
|
|
43
|
+
export const RATE_LIMIT_BASE_DELAY_MS = 2000;
|
|
44
|
+
/** Ceiling for a single backoff step, so escalation stays legible. */
|
|
45
|
+
export const RATE_LIMIT_MAX_STEP_MS = 60_000;
|
|
46
|
+
/**
|
|
47
|
+
* What npm told us about pacing on a 429 — verbatim, so the answer to "how fast
|
|
48
|
+
* may we go" comes from the registry rather than from a guess.
|
|
49
|
+
*
|
|
50
|
+
* Measured 2026-08-28: npm serves NO `X-RateLimit-*` headers on ordinary
|
|
51
|
+
* responses, so there is no budget to read ahead of time. `Retry-After` on the
|
|
52
|
+
* 429 itself is the only channel, and whether npm populates it on the trust
|
|
53
|
+
* endpoint is exactly what this reports the first time it happens.
|
|
54
|
+
*/
|
|
55
|
+
export function describeRateLimitHeaders(headers) {
|
|
56
|
+
if (!headers)
|
|
57
|
+
return 'no headers captured';
|
|
58
|
+
const seen = [];
|
|
59
|
+
headers.forEach((value, key) => {
|
|
60
|
+
if (/^retry-after$/i.test(key) || /ratelimit/i.test(key))
|
|
61
|
+
seen.push(`${key}: ${value}`);
|
|
62
|
+
});
|
|
63
|
+
return seen.length > 0 ? seen.join(', ') : 'npm sent no Retry-After and no rate-limit headers';
|
|
64
|
+
}
|
|
65
|
+
/** Seconds to wait per npm's `Retry-After`, or null when it said nothing usable. */
|
|
66
|
+
export function retryAfterMs(headers) {
|
|
67
|
+
const raw = headers?.get('retry-after');
|
|
68
|
+
if (!raw)
|
|
69
|
+
return null;
|
|
70
|
+
const secs = Number.parseInt(raw.trim(), 10);
|
|
71
|
+
// Only the delta-seconds form. The HTTP-date form is legal and npm does not
|
|
72
|
+
// send it; guessing at a date here would be a second clock to get wrong.
|
|
73
|
+
if (!Number.isFinite(secs) || secs < 0)
|
|
74
|
+
return null;
|
|
75
|
+
return secs * 1000;
|
|
76
|
+
}
|
|
14
77
|
/**
|
|
15
78
|
* Does the registry serve a packument for this name?
|
|
16
79
|
*
|
|
@@ -67,6 +130,84 @@ export function actionForState(state) {
|
|
|
67
130
|
function defaultSleep(ms) {
|
|
68
131
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
69
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* A cool-down shared by every in-flight request of a sweep.
|
|
135
|
+
*
|
|
136
|
+
* Retrying ONE throttled request in isolation is not enough, and the measured
|
|
137
|
+
* run shows why: at serial pace `gjsify/types` was already being 429'd, so every
|
|
138
|
+
* other worker kept hammering the registry through the very window the retry was
|
|
139
|
+
* waiting out — and the retry budget then expired against a limit nobody had
|
|
140
|
+
* stopped provoking. `trust failed (HTTP 429)` was the result.
|
|
141
|
+
*
|
|
142
|
+
* A 429 anywhere therefore pauses EVERYWHERE. That also makes concurrency safe
|
|
143
|
+
* to raise: the sweep self-paces down to whatever npm is willing to serve
|
|
144
|
+
* instead of the operator guessing a number.
|
|
145
|
+
*/
|
|
146
|
+
export class RateLimitGate {
|
|
147
|
+
until = 0;
|
|
148
|
+
now;
|
|
149
|
+
sleeper;
|
|
150
|
+
constructor(opts = {}) {
|
|
151
|
+
this.now = opts.now ?? (() => Date.now());
|
|
152
|
+
this.sleeper = opts.sleep ?? defaultSleep;
|
|
153
|
+
}
|
|
154
|
+
/** Block while a cool-down is in effect. */
|
|
155
|
+
async wait() {
|
|
156
|
+
const remaining = this.until - this.now();
|
|
157
|
+
if (remaining > 0)
|
|
158
|
+
await this.sleeper(remaining);
|
|
159
|
+
}
|
|
160
|
+
/** Hold every request back for at least `ms` from now. */
|
|
161
|
+
penalize(ms) {
|
|
162
|
+
this.until = Math.max(this.until, this.now() + ms);
|
|
163
|
+
}
|
|
164
|
+
/** Milliseconds still owed, for tests and for reporting. */
|
|
165
|
+
remainingMs() {
|
|
166
|
+
return Math.max(0, this.until - this.now());
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Issue a request, waiting out an HTTP 429 rather than reporting it as a state.
|
|
171
|
+
*
|
|
172
|
+
* Used by BOTH the probe GET and the act-phase trust POST, because a sweep long
|
|
173
|
+
* enough to be throttled on reads is long enough to be throttled on writes, and
|
|
174
|
+
* a throttled POST would otherwise be recorded as `trust failed (HTTP 429)` —
|
|
175
|
+
* a failure attributed to the package instead of to the pacing.
|
|
176
|
+
*
|
|
177
|
+
* Bounded on purpose: it returns the last 429 once the budget is spent, so the
|
|
178
|
+
* caller reports "unreadable" rather than hanging. `Retry-After` wins over the
|
|
179
|
+
* exponential backoff whenever npm sends a usable one.
|
|
180
|
+
*/
|
|
181
|
+
export async function requestWaitingOutRateLimit(request, method, url, body, opts = {}) {
|
|
182
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
183
|
+
const budgetMs = opts.maxRateLimitWaitMs ?? RATE_LIMIT_MAX_WAIT_MS;
|
|
184
|
+
const baseDelay = opts.rateLimitBaseDelayMs ?? RATE_LIMIT_BASE_DELAY_MS;
|
|
185
|
+
const gate = opts.gate;
|
|
186
|
+
// Respect a cool-down somebody else is already serving BEFORE spending an
|
|
187
|
+
// attempt. Without this the sweep keeps provoking the limit it is waiting on.
|
|
188
|
+
if (gate)
|
|
189
|
+
await gate.wait();
|
|
190
|
+
let res = await request(method, url, body);
|
|
191
|
+
let waited = 0;
|
|
192
|
+
for (let attempt = 0; res.status === 429 && waited < budgetMs; attempt++) {
|
|
193
|
+
const advised = retryAfterMs(res.headers);
|
|
194
|
+
const step = Math.min(advised ?? baseDelay * 2 ** attempt, RATE_LIMIT_MAX_STEP_MS);
|
|
195
|
+
// Never overshoot the budget: the last wait is whatever is left of it.
|
|
196
|
+
const delay = Math.min(step, budgetMs - waited);
|
|
197
|
+
opts.onRateLimit?.({
|
|
198
|
+
url,
|
|
199
|
+
headers: describeRateLimitHeaders(res.headers),
|
|
200
|
+
waitMs: delay,
|
|
201
|
+
fromRetryAfter: advised !== null,
|
|
202
|
+
});
|
|
203
|
+
// Tell everyone, not just this call.
|
|
204
|
+
gate?.penalize(delay);
|
|
205
|
+
await (gate ? gate.wait() : sleep(delay));
|
|
206
|
+
waited += delay;
|
|
207
|
+
res = await request(method, url, body);
|
|
208
|
+
}
|
|
209
|
+
return res;
|
|
210
|
+
}
|
|
70
211
|
/**
|
|
71
212
|
* Read one package's published + trust state through the SHARED trust requester.
|
|
72
213
|
* A surviving `auth-required` means the 401 was not an answerable OTP challenge —
|
|
@@ -77,10 +218,11 @@ export async function probeTrustState(request, ws, ctx, opts = {}) {
|
|
|
77
218
|
const registry = ctx.registryOverride ?? registryFor(ws.name, ctx.npmrc) ?? DEFAULT_REGISTRY;
|
|
78
219
|
const url = trustUrl(registry, ws.name);
|
|
79
220
|
const classifyOpts = { repository: ctx.repository, workflow: ctx.workflow };
|
|
80
|
-
|
|
221
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
222
|
+
// A 429 is not an answer — wait it out BEFORE classifying anything.
|
|
223
|
+
let res = await requestWaitingOutRateLimit(request, 'GET', url, undefined, opts);
|
|
81
224
|
let state = classifyTrustList(res.status, res.json, classifyOpts);
|
|
82
225
|
if (state === 'auth-required' && opts.retryOn401 !== false) {
|
|
83
|
-
const sleep = opts.sleep ?? defaultSleep;
|
|
84
226
|
await sleep(opts.retryDelayMs ?? PROBE_RETRY_DELAY_MS);
|
|
85
227
|
res = await request('GET', url);
|
|
86
228
|
state = classifyTrustList(res.status, res.json, classifyOpts);
|
|
@@ -123,8 +265,15 @@ export async function probeTrustState(request, ws, ctx, opts = {}) {
|
|
|
123
265
|
export async function probeAllTrustStates(request, selected, concurrency, ctx, opts = {}) {
|
|
124
266
|
if (selected.length === 0)
|
|
125
267
|
return [];
|
|
126
|
-
const
|
|
127
|
-
|
|
268
|
+
const total = selected.length;
|
|
269
|
+
let done = 0;
|
|
270
|
+
const report = (plan) => {
|
|
271
|
+
done++;
|
|
272
|
+
opts.onProgress?.(done, total, plan);
|
|
273
|
+
return plan;
|
|
274
|
+
};
|
|
275
|
+
const first = report(await probeTrustState(request, selected[0], ctx, opts));
|
|
276
|
+
const rest = await mapWithConcurrency(selected.slice(1), Math.max(1, concurrency), async (ws) => report(await probeTrustState(request, ws, ctx, opts)));
|
|
128
277
|
return [first, ...rest];
|
|
129
278
|
}
|
|
130
279
|
/** Map with bounded concurrency, preserving input order in the result. */
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
type Writer = (text: string) => void;
|
|
2
|
+
/** How many prompts are currently on screen. Exposed for tests. */
|
|
3
|
+
export declare function promptDepth(): number;
|
|
4
|
+
/** Mark a prompt as open; every `writeAroundPrompt` is held until it closes. */
|
|
5
|
+
export declare function beginPrompt(): void;
|
|
6
|
+
/** Close a prompt and flush anything held while it was open. */
|
|
7
|
+
export declare function endPrompt(write?: Writer): void;
|
|
8
|
+
/** Write now, or hold until the open prompt closes. */
|
|
9
|
+
export declare function writeAroundPrompt(text: string, write?: Writer): void;
|
|
10
|
+
/** Test seam: forget any held output and reset the depth. */
|
|
11
|
+
export declare function resetPromptOutput(): void;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Stdout arbitration between a live prompt and everything else writing to it.
|
|
2
|
+
//
|
|
3
|
+
// A prompt owns the last line of the terminal. On a TTY `prompt.ts` runs in RAW
|
|
4
|
+
// mode and echoes each keystroke by hand, so the cursor sits mid-line with no
|
|
5
|
+
// newline pending — and any other writer lands INSIDE what the user is typing:
|
|
6
|
+
//
|
|
7
|
+
// Enter a NEW OTP: npm rate limit hit. npm sent no Retry-After …
|
|
8
|
+
// 228000
|
|
9
|
+
//
|
|
10
|
+
// (measured on a real `gjsify onboard` sweep: a rate-limit notice from a
|
|
11
|
+
// concurrent worker spliced into the digits being entered, leaving the typed
|
|
12
|
+
// code visually split across two lines).
|
|
13
|
+
//
|
|
14
|
+
// The rule is arbitration, not silence: a message that arrives during a prompt
|
|
15
|
+
// is HELD and flushed the moment the prompt closes. Dropping it would trade a
|
|
16
|
+
// cosmetic problem for a lost one, and the messages that show up here are
|
|
17
|
+
// exactly the ones explaining why the sweep is slow.
|
|
18
|
+
let depth = 0;
|
|
19
|
+
const held = [];
|
|
20
|
+
/** How many prompts are currently on screen. Exposed for tests. */
|
|
21
|
+
export function promptDepth() {
|
|
22
|
+
return depth;
|
|
23
|
+
}
|
|
24
|
+
/** Mark a prompt as open; every `writeAroundPrompt` is held until it closes. */
|
|
25
|
+
export function beginPrompt() {
|
|
26
|
+
depth++;
|
|
27
|
+
}
|
|
28
|
+
/** Close a prompt and flush anything held while it was open. */
|
|
29
|
+
export function endPrompt(write = defaultWrite) {
|
|
30
|
+
depth = Math.max(0, depth - 1);
|
|
31
|
+
if (depth > 0)
|
|
32
|
+
return;
|
|
33
|
+
if (held.length === 0)
|
|
34
|
+
return;
|
|
35
|
+
const pendingLines = held.splice(0, held.length);
|
|
36
|
+
for (const line of pendingLines)
|
|
37
|
+
write(line);
|
|
38
|
+
}
|
|
39
|
+
/** Write now, or hold until the open prompt closes. */
|
|
40
|
+
export function writeAroundPrompt(text, write = defaultWrite) {
|
|
41
|
+
if (depth > 0) {
|
|
42
|
+
held.push(text);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
write(text);
|
|
46
|
+
}
|
|
47
|
+
function defaultWrite(text) {
|
|
48
|
+
process.stdout.write(text);
|
|
49
|
+
}
|
|
50
|
+
/** Test seam: forget any held output and reset the depth. */
|
|
51
|
+
export function resetPromptOutput() {
|
|
52
|
+
depth = 0;
|
|
53
|
+
held.length = 0;
|
|
54
|
+
}
|
package/lib/utils/prompt.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
//
|
|
16
16
|
// Non-TTY stdin (piped input, CI) keeps a plain line read so
|
|
17
17
|
// `printf 'user\npass\n' | gjsify login` still works.
|
|
18
|
+
import { beginPrompt, endPrompt } from './prompt-output.js';
|
|
18
19
|
const CTRL_C = '\x03'; // ETX (Ctrl-C)
|
|
19
20
|
const DEL = '\x7f'; // DEL (Backspace on most terminals)
|
|
20
21
|
const BACKSPACE = '\x08'; // BS
|
|
@@ -98,8 +99,18 @@ async function runRawSession(fn) {
|
|
|
98
99
|
stdin.setEncoding('utf-8');
|
|
99
100
|
stdin.on('data', onData);
|
|
100
101
|
const read = (question, mask) => new Promise((resolve) => {
|
|
102
|
+
// A prompt owns the last line: hold back every other writer until it
|
|
103
|
+
// closes, or their output lands inside the digits being typed.
|
|
104
|
+
beginPrompt();
|
|
101
105
|
out.write(question);
|
|
102
|
-
pending = {
|
|
106
|
+
pending = {
|
|
107
|
+
mask,
|
|
108
|
+
buf: '',
|
|
109
|
+
resolve: (value) => {
|
|
110
|
+
endPrompt();
|
|
111
|
+
resolve(value);
|
|
112
|
+
},
|
|
113
|
+
};
|
|
103
114
|
});
|
|
104
115
|
try {
|
|
105
116
|
return await fn(read);
|
|
@@ -138,8 +149,17 @@ function readLine() {
|
|
|
138
149
|
/** Print a question and read one visible line. */
|
|
139
150
|
export async function promptLine(question) {
|
|
140
151
|
if (!isTtyStdin()) {
|
|
152
|
+
// Non-TTY holds back other writers too: the interleaving is not visible
|
|
153
|
+
// here, but a transcript in which a notice sits between the question and
|
|
154
|
+
// the answer is just as hard to read afterwards.
|
|
155
|
+
beginPrompt();
|
|
141
156
|
process.stdout.write(question);
|
|
142
|
-
|
|
157
|
+
try {
|
|
158
|
+
return await readLine();
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
endPrompt();
|
|
162
|
+
}
|
|
143
163
|
}
|
|
144
164
|
return runRawSession((read) => read(question, false));
|
|
145
165
|
}
|
|
@@ -4,7 +4,7 @@ import type { Workspace } from '@gjsify/workspace';
|
|
|
4
4
|
* workspaces. Extend this the moment a new such group appears — the list is the
|
|
5
5
|
* only thing standing between one and a silently broken first release.
|
|
6
6
|
*/
|
|
7
|
-
export declare const NON_WORKSPACE_PUBLISHABLE_DIRS: readonly [readonly ["packages", "node-gi"], readonly ["packages", "napi"]];
|
|
7
|
+
export declare const NON_WORKSPACE_PUBLISHABLE_DIRS: readonly [readonly ["packages", "node-gi"], readonly ["packages", "napi"], readonly ["packages", "node-runtime"]];
|
|
8
8
|
/**
|
|
9
9
|
* Publishable packages in {@link NON_WORKSPACE_PUBLISHABLE_DIRS}, shaped as
|
|
10
10
|
* `Workspace` entries so callers can treat them like any other package.
|
|
@@ -19,6 +19,7 @@ import { dirname, join } from 'node:path';
|
|
|
19
19
|
export const NON_WORKSPACE_PUBLISHABLE_DIRS = [
|
|
20
20
|
['packages', 'node-gi'], // @gjsify/node-gi + the @gjsify/gtk-runtime-* bundles
|
|
21
21
|
['packages', 'napi'], // @gjsify/napi (the N-API host in GJS, own napi.yml)
|
|
22
|
+
['packages', 'node-runtime'], // the @gjsify/node-runtime-* bundled Node interpreters
|
|
22
23
|
];
|
|
23
24
|
/**
|
|
24
25
|
* Publishable packages in {@link NON_WORKSPACE_PUBLISHABLE_DIRS}, shaped as
|