@westbayberry/dg 2.3.2 → 2.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -208
- package/dist/agents/copilot-cli.js +7 -2
- package/dist/agents/persistence.js +83 -8
- package/dist/agents/registry.js +5 -2
- package/dist/agents/routing.js +31 -14
- package/dist/api/analyze.js +138 -41
- package/dist/audit/deep.js +2 -1
- package/dist/audit/detectors.js +23 -2
- package/dist/audit/rules.js +4 -1
- package/dist/audit-ui/export.js +5 -4
- package/dist/auth/device-login.js +33 -11
- package/dist/auth/login-app.js +17 -12
- package/dist/auth/store.js +65 -9
- package/dist/bin/dg.js +1 -1
- package/dist/commands/audit.js +12 -20
- package/dist/commands/{guard-commit.js → commit-guard.js} +12 -12
- package/dist/commands/cooldown.js +2 -1
- package/dist/commands/decisions.js +8 -7
- package/dist/commands/help.js +11 -3
- package/dist/commands/licenses.js +17 -8
- package/dist/commands/login.js +58 -21
- package/dist/commands/router.js +12 -2
- package/dist/commands/sbom.js +20 -5
- package/dist/commands/scan.js +5 -4
- package/dist/commands/service.js +1 -1
- package/dist/commands/setup.js +12 -12
- package/dist/commands/status.js +4 -4
- package/dist/commands/uninstall.js +1 -1
- package/dist/commands/update.js +20 -0
- package/dist/commands/verify.js +4 -4
- package/dist/config/settings.js +81 -12
- package/dist/export-ui/ExportDialog.js +1 -1
- package/dist/gate/cooldown-request.js +22 -0
- package/dist/gate/egress-guard.js +185 -0
- package/dist/gate/host-match.js +48 -0
- package/dist/gate/verdict-core.js +101 -0
- package/dist/gate/verdict-fetch.js +214 -0
- package/dist/launcher/agent-check.js +1096 -42
- package/dist/launcher/agent-hook-exec.js +8 -1
- package/dist/launcher/agent-hook-io.js +5 -3
- package/dist/launcher/classify.js +110 -26
- package/dist/launcher/env.js +84 -18
- package/dist/launcher/install-preflight.js +26 -3
- package/dist/launcher/live-install.js +4 -2
- package/dist/launcher/manifest-screen.js +88 -7
- package/dist/launcher/output-redaction.js +153 -15
- package/dist/launcher/preflight-prompt.js +1 -1
- package/dist/launcher/resolve-real-binary.js +8 -3
- package/dist/launcher/run.js +237 -40
- package/dist/origin/artifact-cache.js +55 -0
- package/dist/origin/cargo-config.js +12 -0
- package/dist/origin/gate-mode.js +7 -0
- package/dist/origin/platform-paths.js +14 -0
- package/dist/origin/precedence.js +345 -0
- package/dist/origin/server.js +611 -0
- package/dist/origin/spike-worker.js +35 -0
- package/dist/origin/worker.js +116 -0
- package/dist/presentation/package-page.js +21 -1
- package/dist/presentation/safe-version.js +11 -0
- package/dist/presentation/sarif.js +109 -0
- package/dist/project/dgfile.js +54 -17
- package/dist/project/override-trust.js +0 -0
- package/dist/proxy/enforcement.js +27 -7
- package/dist/proxy/metadata-map.js +33 -7
- package/dist/proxy/preverified.js +0 -3
- package/dist/proxy/server.js +104 -446
- package/dist/proxy/upstream-proxy.js +23 -4
- package/dist/proxy/worker.js +3 -2
- package/dist/publish-set/collect.js +6 -0
- package/dist/runtime/fatal.js +2 -1
- package/dist/runtime/first-run.js +5 -1
- package/dist/runtime/node-version.js +51 -0
- package/dist/runtime/nudges.js +27 -1
- package/dist/sbom/cyclonedx.js +120 -33
- package/dist/sbom/graph.js +236 -0
- package/dist/sbom/spdx.js +53 -0
- package/dist/sbom-ui/SbomApp.js +5 -3
- package/dist/sbom-ui/inventory.js +8 -10
- package/dist/scan/analyze-worker.js +3 -1
- package/dist/scan/collect.js +38 -8
- package/dist/scan/command.js +77 -27
- package/dist/scan/discovery.js +18 -7
- package/dist/scan/installed.js +202 -0
- package/dist/scan/manifest-coverage.js +128 -0
- package/dist/scan/render.js +125 -40
- package/dist/scan/scanner-report.js +92 -22
- package/dist/scan/staged.js +68 -15
- package/dist/scan/types.js +7 -1
- package/dist/scan-ui/LegacyApp.js +3 -3
- package/dist/scan-ui/components/InteractiveResultsView.js +17 -14
- package/dist/scan-ui/hooks/useScan.js +37 -11
- package/dist/scan-ui/launch.js +2 -2
- package/dist/scan-ui/logo.js +6 -2
- package/dist/scan-ui/shims.js +6 -6
- package/dist/scripts/gate.js +53 -35
- package/dist/security/csv.js +9 -0
- package/dist/security/sanitize.js +11 -2
- package/dist/service/state.js +120 -37
- package/dist/service/trust-refresh.js +12 -1
- package/dist/service/trust-store.js +36 -5
- package/dist/service/worker.js +5 -7
- package/dist/setup/git-hook.js +2 -2
- package/dist/setup/install-method.js +36 -0
- package/dist/setup/plan.js +119 -57
- package/dist/setup/uninstall-standalone.js +7 -2
- package/dist/setup-ui/gate.js +1 -8
- package/dist/setup-ui/offer.js +14 -7
- package/dist/setup-ui/selector.js +1 -1
- package/dist/setup-ui/wizard.js +73 -78
- package/dist/standalone/uninstall.mjs +1114 -386
- package/dist/state/cleanup-registry.js +14 -3
- package/dist/state/cooldown-held.js +36 -13
- package/dist/state/locks.js +239 -34
- package/dist/state/sessions.js +2 -2
- package/dist/util/sh-escape.js +6 -0
- package/dist/util/tty-prompt.js +13 -4
- package/dist/verify/local.js +60 -10
- package/dist/verify/package-check.js +35 -49
- package/dist/verify/preflight.js +59 -33
- package/dist/verify/render.js +11 -12
- package/npm-shrinkwrap.json +714 -213
- package/package.json +4 -2
|
@@ -0,0 +1,611 @@
|
|
|
1
|
+
import { createHash, createHmac, randomBytes, timingSafeEqual } from "node:crypto";
|
|
2
|
+
import { mkdirSync, renameSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { Agent as HttpAgent, createServer, request as httpRequest } from "node:http";
|
|
5
|
+
import { Agent as HttpsAgent, request as httpsRequest } from "node:https";
|
|
6
|
+
import { collectBounded } from "../proxy/buffer-budget.js";
|
|
7
|
+
import { isLinkLocalHost, isPrivateNetworkHost, lookupForTarget } from "../gate/egress-guard.js";
|
|
8
|
+
import { lookupVerdict } from "../gate/verdict-fetch.js";
|
|
9
|
+
import { enforceProtectedInstall } from "../proxy/enforcement.js";
|
|
10
|
+
import { artifactDisplayName, extractRegistryMetadataIdentities, isRegistryIndexRequest, resolveArtifactIdentity } from "../proxy/metadata-map.js";
|
|
11
|
+
import { redactSecrets } from "../launcher/output-redaction.js";
|
|
12
|
+
import { lookupCachedArtifact, storeCachedArtifact } from "./artifact-cache.js";
|
|
13
|
+
const DEFAULT_UPSTREAMS = {
|
|
14
|
+
npmRegistry: "https://registry.npmjs.org",
|
|
15
|
+
pypiSimple: "https://pypi.org",
|
|
16
|
+
pypiFiles: "https://files.pythonhosted.org",
|
|
17
|
+
cargoIndex: "https://index.crates.io",
|
|
18
|
+
cargoStatic: "https://static.crates.io"
|
|
19
|
+
};
|
|
20
|
+
const upstreamHttpAgent = new HttpAgent({ keepAlive: true, maxSockets: 64 });
|
|
21
|
+
const upstreamHttpsAgent = new HttpsAgent({ keepAlive: true, maxSockets: 64 });
|
|
22
|
+
export async function startOriginServer(options) {
|
|
23
|
+
const host = options.host ?? "127.0.0.1";
|
|
24
|
+
const requests = [];
|
|
25
|
+
const server = createServer();
|
|
26
|
+
await listen(server, host, options.port ?? 0);
|
|
27
|
+
const address = server.address();
|
|
28
|
+
if (typeof address !== "object" || address === null) {
|
|
29
|
+
throw new Error("origin server did not bind a TCP port");
|
|
30
|
+
}
|
|
31
|
+
const base = `http://${host}:${address.port}`;
|
|
32
|
+
const resolved = {
|
|
33
|
+
ecosystem: options.ecosystem,
|
|
34
|
+
strategy: options.npmTarballStrategy ?? "canonical",
|
|
35
|
+
upstreams: { ...DEFAULT_UPSTREAMS, ...options.upstreams },
|
|
36
|
+
base,
|
|
37
|
+
...(options.onRequest ? { onRequest: options.onRequest } : {}),
|
|
38
|
+
...(options.gate ? { gate: options.gate } : {}),
|
|
39
|
+
...(options.onDecision ? { onDecision: options.onDecision } : {}),
|
|
40
|
+
...(options.cacheDir ? { cacheDir: options.cacheDir } : {}),
|
|
41
|
+
...(options.stateFile ? { stateFile: options.stateFile } : {}),
|
|
42
|
+
downloadSecret: randomBytes(32),
|
|
43
|
+
requests,
|
|
44
|
+
identities: [],
|
|
45
|
+
decisions: [],
|
|
46
|
+
hashes: [],
|
|
47
|
+
events: [],
|
|
48
|
+
port: address.port
|
|
49
|
+
};
|
|
50
|
+
// Initial ready state so a launcher polling the state file (the proxy's
|
|
51
|
+
// waitForProxyReady) detects the origin is bound before routing an install.
|
|
52
|
+
writeOriginState(resolved);
|
|
53
|
+
server.on("request", (request, response) => {
|
|
54
|
+
const path = request.url ?? "/";
|
|
55
|
+
requests.push(`${request.method} ${path}`);
|
|
56
|
+
resolved.onRequest?.(request.method ?? "GET", path);
|
|
57
|
+
handle(resolved, request, response).catch((error) => {
|
|
58
|
+
sendError(response, 502, `origin error: ${error instanceof Error ? error.message : String(error)}`);
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
return {
|
|
62
|
+
port: address.port,
|
|
63
|
+
url: base,
|
|
64
|
+
requests,
|
|
65
|
+
close: () => new Promise((resolve) => server.close(() => resolve()))
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
function listen(server, host, port) {
|
|
69
|
+
return new Promise((resolve, reject) => {
|
|
70
|
+
server.once("error", reject);
|
|
71
|
+
server.listen(port, host, () => {
|
|
72
|
+
server.off("error", reject);
|
|
73
|
+
resolve();
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
async function handle(origin, request, response) {
|
|
78
|
+
const path = request.url ?? "/";
|
|
79
|
+
// A distinctive marker so a doctor probe can prove THIS port is dg's origin
|
|
80
|
+
// (routing target), not the real registry — liveness alone does not.
|
|
81
|
+
if (path === "/-/dg-origin-health") {
|
|
82
|
+
serveJson(response, 200, { origin: "dependency-guardian", ecosystem: origin.ecosystem, gated: Boolean(origin.gate) });
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const target = new URL(`${origin.base}${path}`);
|
|
86
|
+
const accept = headerValue(request.headers.accept);
|
|
87
|
+
if (origin.ecosystem === "npm") {
|
|
88
|
+
return handleNpm(origin, target, accept, response);
|
|
89
|
+
}
|
|
90
|
+
if (origin.ecosystem === "pypi") {
|
|
91
|
+
return handlePypi(origin, target, accept, response);
|
|
92
|
+
}
|
|
93
|
+
return handleCargo(origin, target, accept, response);
|
|
94
|
+
}
|
|
95
|
+
// Metadata (packument / simple index / config.json) is re-serialized in memory
|
|
96
|
+
// (JSON.parse, string split), so cap it well below the 1 GiB artifact budget — a
|
|
97
|
+
// near-gigabyte packument from a malicious registry would otherwise OOM the
|
|
98
|
+
// worker. Legit packuments are KB-to-MB; the proxy caps its decode at 32 MiB.
|
|
99
|
+
function maxMetadataBytes(env) {
|
|
100
|
+
return positiveInt(env.DG_ORIGIN_MAX_METADATA_BYTES, 64 * 1024 * 1024);
|
|
101
|
+
}
|
|
102
|
+
function metadataTooLarge(origin, upstream, response) {
|
|
103
|
+
const limit = maxMetadataBytes(origin.gate?.env ?? process.env);
|
|
104
|
+
if (upstream.body.length > limit) {
|
|
105
|
+
sendError(response, 502, `registry metadata exceeded the ${limit}-byte limit`);
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
async function handleNpm(origin, target, accept, response) {
|
|
111
|
+
const path = target.pathname;
|
|
112
|
+
const upstreamUrl = new URL(`${origin.upstreams.npmRegistry}${path}${target.search}`);
|
|
113
|
+
// npm reserves /-/ for registry API endpoints (/-/npm/v1/keys, audits, search):
|
|
114
|
+
// normally metadata. But the URL shape alone is not trustworthy — a hostile /
|
|
115
|
+
// MITM packument can point a (rewritten) tarball at a /-/-prefixed path, so
|
|
116
|
+
// classify by the RESPONSE: pass a JSON API body through, gate any non-JSON body
|
|
117
|
+
// as an artifact instead of serving it ungated.
|
|
118
|
+
if (path.startsWith("/-/")) {
|
|
119
|
+
const apiResponse = await guardedFetch(upstreamUrl, accept, origin.gate?.env);
|
|
120
|
+
if (metadataTooLarge(origin, apiResponse, response)) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
// Don't trust the upstream Content-Type — an artifact mislabeled
|
|
124
|
+
// application/json must still be gated. Require the body to actually parse as
|
|
125
|
+
// JSON to serve it as a metadata/API response; otherwise gate it as an artifact.
|
|
126
|
+
if (!bodyIsParseableJson(apiResponse)) {
|
|
127
|
+
return gateBytes(origin, target, apiResponse, response);
|
|
128
|
+
}
|
|
129
|
+
return serve(response, apiResponse);
|
|
130
|
+
}
|
|
131
|
+
if (serveCacheHit(origin, target, response)) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
const upstream = await guardedFetch(upstreamUrl, accept, origin.gate?.env);
|
|
135
|
+
// Classify by the RESPONSE, not the URL: only a JSON body is a packument
|
|
136
|
+
// (metadata, re-serialized). ANY non-JSON body is an artifact (tarball) and is
|
|
137
|
+
// gated — a malicious/MITM packument can point dist.tarball at an off-path URL
|
|
138
|
+
// lacking the /-/ marker, so the URL shape cannot be trusted to find tarballs.
|
|
139
|
+
if (!isJson(upstream)) {
|
|
140
|
+
return gateBytes(origin, target, upstream, response);
|
|
141
|
+
}
|
|
142
|
+
if (metadataTooLarge(origin, upstream, response)) {
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
rememberIdentities(origin, target, upstream);
|
|
146
|
+
const doc = JSON.parse(upstream.body.toString("utf8"));
|
|
147
|
+
if (origin.strategy === "loopback") {
|
|
148
|
+
rewriteNpmTarballs(doc, origin.base);
|
|
149
|
+
}
|
|
150
|
+
serveJson(response, upstream.status, doc);
|
|
151
|
+
}
|
|
152
|
+
async function handlePypi(origin, target, accept, response) {
|
|
153
|
+
const path = target.pathname;
|
|
154
|
+
// Download URLs the origin rewrote into the simple index route back here, so
|
|
155
|
+
// EVERY wheel/sdist on ANY host is gated — a compromised or MITM'd index can
|
|
156
|
+
// list files off files.pythonhosted.org, and the manager would fetch those
|
|
157
|
+
// directly, un-gated. The original URL is base64url-encoded in the path.
|
|
158
|
+
if (path.startsWith("/__dl/")) {
|
|
159
|
+
const rest = path.slice("/__dl/".length);
|
|
160
|
+
const separator = rest.indexOf("/");
|
|
161
|
+
const base = separator < 0 ? null : verifiedDownloadBase(origin, rest.slice(0, separator));
|
|
162
|
+
if (!base) {
|
|
163
|
+
sendError(response, 400, "invalid download reference");
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
let filename = rest.slice(separator + 1);
|
|
167
|
+
const isSidecar = filename.endsWith(".metadata");
|
|
168
|
+
if (isSidecar) {
|
|
169
|
+
filename = filename.slice(0, -".metadata".length);
|
|
170
|
+
}
|
|
171
|
+
let original;
|
|
172
|
+
try {
|
|
173
|
+
original = new URL(`${filename}${target.search}`, base);
|
|
174
|
+
}
|
|
175
|
+
catch {
|
|
176
|
+
sendError(response, 400, "invalid download reference");
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
// The filename echoes back what the origin itself wrote into the index; pin it
|
|
180
|
+
// to the encoded host so a crafted path can never redirect the fetch off-host.
|
|
181
|
+
if (original.origin !== base.origin || (original.protocol !== "http:" && original.protocol !== "https:")) {
|
|
182
|
+
sendError(response, 400, "invalid download reference");
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
// A public index that lists a download on a private/internal address is an
|
|
186
|
+
// SSRF attempt — the origin must not be turned into a request proxy into the
|
|
187
|
+
// internal network. A fully-private upstream (a corporate index, or a test
|
|
188
|
+
// server) legitimately serves private downloads, so only refuse when the
|
|
189
|
+
// index host itself is public (mirrors the redirect guard's public->private
|
|
190
|
+
// rule).
|
|
191
|
+
if (isPrivateNetworkHost(original) && !isPrivateNetworkHost(new URL(origin.upstreams.pypiSimple))) {
|
|
192
|
+
sendError(response, 400, "refusing to fetch a download from a private address listed by a public index");
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
if (isSidecar) {
|
|
196
|
+
// PEP 658 metadata sidecar — resolution metadata, pass through ungated. The
|
|
197
|
+
// sidecar lives at the artifact PATH + ".metadata"; append to the pathname,
|
|
198
|
+
// not the full URL string (which would drop ".metadata" into a query string
|
|
199
|
+
// when the download URL carries one).
|
|
200
|
+
const sidecarUrl = new URL(original.toString());
|
|
201
|
+
sidecarUrl.pathname += ".metadata";
|
|
202
|
+
return serve(response, await guardedFetch(sidecarUrl, accept, origin.gate?.env));
|
|
203
|
+
}
|
|
204
|
+
return gateOrServe(origin, original, original, accept, response);
|
|
205
|
+
}
|
|
206
|
+
if (path.startsWith("/simple/")) {
|
|
207
|
+
const indexUrl = new URL(`${origin.upstreams.pypiSimple}${path}${target.search}`);
|
|
208
|
+
const upstream = await guardedFetch(indexUrl, accept, origin.gate?.env);
|
|
209
|
+
if (metadataTooLarge(origin, upstream, response)) {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
rememberIdentities(origin, target, upstream);
|
|
213
|
+
const body = rewritePypiIndex(origin, upstream, indexUrl);
|
|
214
|
+
serve(response, { status: upstream.status, headers: { "content-type": headerValue(upstream.headers["content-type"]) || "text/html" }, body });
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
if (isRegistryIndexRequest(target)) {
|
|
218
|
+
return serve(response, await guardedFetch(new URL(`${origin.upstreams.pypiFiles}${path}${target.search}`), accept, origin.gate?.env));
|
|
219
|
+
}
|
|
220
|
+
return gateOrServe(origin, new URL(`${origin.upstreams.pypiFiles}${path}${target.search}`), target, accept, response);
|
|
221
|
+
}
|
|
222
|
+
// The /__dl indirection is a loopback forward-fetcher; without proof the origin
|
|
223
|
+
// itself emitted the reference, any local process could drive it to fetch an
|
|
224
|
+
// arbitrary host (an SSRF open relay). The base host is HMAC-bound to a
|
|
225
|
+
// per-process secret, so only hosts this origin actually rewrote into a served
|
|
226
|
+
// index are fetchable. The host-pin + private-network guards below still apply.
|
|
227
|
+
function verifiedDownloadBase(origin, segment) {
|
|
228
|
+
const dot = segment.lastIndexOf(".");
|
|
229
|
+
if (dot <= 0) {
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
|
+
const encoded = segment.slice(0, dot);
|
|
233
|
+
if (!signaturesMatch(segment.slice(dot + 1), signDownloadBase(origin.downloadSecret, encoded))) {
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
return decodeDownloadBase(encoded);
|
|
237
|
+
}
|
|
238
|
+
function signDownloadBase(secret, encoded) {
|
|
239
|
+
return createHmac("sha256", secret).update(encoded).digest("base64url");
|
|
240
|
+
}
|
|
241
|
+
function signaturesMatch(provided, expected) {
|
|
242
|
+
const a = Buffer.from(provided);
|
|
243
|
+
const b = Buffer.from(expected);
|
|
244
|
+
return a.length === b.length && timingSafeEqual(a, b);
|
|
245
|
+
}
|
|
246
|
+
function decodeDownloadBase(encoded) {
|
|
247
|
+
try {
|
|
248
|
+
const url = new URL(Buffer.from(decodeURIComponent(encoded), "base64url").toString("utf8"));
|
|
249
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
return url;
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
// Rewrite EVERY download href in a simple index (PEP 503 HTML or PEP 691 JSON) to
|
|
259
|
+
// the origin's /__dl/ indirection, preserving the #sha256 fragment / hashes pip
|
|
260
|
+
// binds to, so no wheel can be fetched off-origin regardless of its host.
|
|
261
|
+
function rewritePypiIndex(origin, upstream, indexUrl) {
|
|
262
|
+
if (/json/i.test(headerValue(upstream.headers["content-type"]))) {
|
|
263
|
+
try {
|
|
264
|
+
const doc = JSON.parse(upstream.body.toString("utf8"));
|
|
265
|
+
if (Array.isArray(doc.files)) {
|
|
266
|
+
for (const file of doc.files) {
|
|
267
|
+
if (file && typeof file === "object" && typeof file.url === "string") {
|
|
268
|
+
file.url = rewritePypiHref(file.url, indexUrl, origin);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return Buffer.from(JSON.stringify(doc));
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
return upstream.body;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
// Match href values in any quote style HTML permits — double, single, and
|
|
279
|
+
// unquoted — because pip's tolerant parser accepts all three; a malicious index
|
|
280
|
+
// that single-quotes its hrefs must not slip a wheel past the rewrite. The
|
|
281
|
+
// rewritten value is always emitted double-quoted.
|
|
282
|
+
const html = upstream.body
|
|
283
|
+
.toString("utf8")
|
|
284
|
+
.replace(/(href\s*=\s*)("([^"]*)"|'([^']*)'|([^\s"'>]+))/gi, (_m, pre, _value, dq, sq, uq) => {
|
|
285
|
+
const href = dq ?? sq ?? uq ?? "";
|
|
286
|
+
return `${pre}"${rewritePypiHref(href, indexUrl, origin)}"`;
|
|
287
|
+
});
|
|
288
|
+
return Buffer.from(html);
|
|
289
|
+
}
|
|
290
|
+
function rewritePypiHref(href, indexUrl, origin) {
|
|
291
|
+
const hashIndex = href.indexOf("#");
|
|
292
|
+
const urlPart = hashIndex >= 0 ? href.slice(0, hashIndex) : href;
|
|
293
|
+
const fragment = hashIndex >= 0 ? href.slice(hashIndex) : "";
|
|
294
|
+
let absolute;
|
|
295
|
+
try {
|
|
296
|
+
absolute = new URL(urlPart, indexUrl);
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
return href;
|
|
300
|
+
}
|
|
301
|
+
if (absolute.protocol !== "http:" && absolute.protocol !== "https:") {
|
|
302
|
+
return href;
|
|
303
|
+
}
|
|
304
|
+
// Keep the real filename as the LAST path segment — pip/uv parse the package
|
|
305
|
+
// name + version from it — and encode only the directory prefix (host + dirs).
|
|
306
|
+
const segments = absolute.pathname.split("/");
|
|
307
|
+
const filename = segments.pop() ?? "";
|
|
308
|
+
if (!filename) {
|
|
309
|
+
return href;
|
|
310
|
+
}
|
|
311
|
+
const base = `${absolute.origin}${segments.join("/")}/`;
|
|
312
|
+
const encoded = Buffer.from(base, "utf8").toString("base64url");
|
|
313
|
+
const signature = signDownloadBase(origin.downloadSecret, encoded);
|
|
314
|
+
return `${origin.base}/__dl/${encoded}.${signature}/${filename}${absolute.search}${fragment}`;
|
|
315
|
+
}
|
|
316
|
+
async function handleCargo(origin, target, accept, response) {
|
|
317
|
+
const path = target.pathname;
|
|
318
|
+
if (path === "/config.json") {
|
|
319
|
+
const upstream = await guardedFetch(new URL(`${origin.upstreams.cargoIndex}/config.json`), accept, origin.gate?.env);
|
|
320
|
+
if (metadataTooLarge(origin, upstream, response)) {
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
const config = JSON.parse(upstream.body.toString("utf8"));
|
|
324
|
+
config.dl = `${origin.base}/crates`;
|
|
325
|
+
serveJson(response, upstream.status, config);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
if (path.startsWith("/crates/")) {
|
|
329
|
+
return gateOrServe(origin, new URL(`${origin.upstreams.cargoStatic}${path}${target.search}`), target, accept, response);
|
|
330
|
+
}
|
|
331
|
+
const upstream = await guardedFetch(new URL(`${origin.upstreams.cargoIndex}${path}${target.search}`), accept, origin.gate?.env);
|
|
332
|
+
serve(response, upstream);
|
|
333
|
+
}
|
|
334
|
+
// Buffer the artifact fully (bounded), verdict-gate it, and release only on an
|
|
335
|
+
// allow. A block is a clean fetch error — never an altered or truncated body
|
|
336
|
+
// masquerading as the artifact. Without gate options the bytes pass through.
|
|
337
|
+
// Serve a previously-allowed artifact from the warm cache (no re-download, no
|
|
338
|
+
// re-scan); the content hash is the proof it passed. Records a pass decision so
|
|
339
|
+
// finalize accounts for it. Returns true when it served a hit.
|
|
340
|
+
function serveCacheHit(origin, clientTarget, response) {
|
|
341
|
+
if (!origin.gate || !origin.cacheDir) {
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
344
|
+
const cached = lookupCachedArtifact(origin.cacheDir, clientTarget.toString());
|
|
345
|
+
if (!cached) {
|
|
346
|
+
return false;
|
|
347
|
+
}
|
|
348
|
+
recordOriginVerdict(origin, clientTarget, cached.sha256, {
|
|
349
|
+
verdict: "pass",
|
|
350
|
+
cause: "pass",
|
|
351
|
+
packageName: clientTarget.pathname,
|
|
352
|
+
reason: "already verified (content-addressed cache hit)"
|
|
353
|
+
});
|
|
354
|
+
response.writeHead(200, { "content-type": "application/octet-stream", "content-length": cached.bytes.length });
|
|
355
|
+
response.end(cached.bytes);
|
|
356
|
+
return true;
|
|
357
|
+
}
|
|
358
|
+
async function gateOrServe(origin, upstreamUrl, clientTarget, accept, response) {
|
|
359
|
+
if (serveCacheHit(origin, clientTarget, response)) {
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const upstream = await guardedFetch(upstreamUrl, accept, origin.gate?.env);
|
|
363
|
+
return gateBytes(origin, clientTarget, upstream, response);
|
|
364
|
+
}
|
|
365
|
+
// Verdict-gate an ALREADY-FETCHED artifact body. A block is a clean 403, never
|
|
366
|
+
// altered or truncated bytes. Used both for known artifact paths (via gateOrServe)
|
|
367
|
+
// and for any non-JSON npm response (so an off-path dist.tarball cannot evade the
|
|
368
|
+
// gate). Without gate options the bytes pass through.
|
|
369
|
+
async function gateBytes(origin, clientTarget, upstream, response) {
|
|
370
|
+
if (origin.gate && upstream.status === 206) {
|
|
371
|
+
// A partial artifact cannot be verified — the proxy blocks 206 too.
|
|
372
|
+
recordOriginVerdict(origin, clientTarget, "", {
|
|
373
|
+
verdict: "block",
|
|
374
|
+
cause: "policy",
|
|
375
|
+
packageName: clientTarget.pathname,
|
|
376
|
+
reason: "registry returned partial content — a partial artifact cannot be verified"
|
|
377
|
+
});
|
|
378
|
+
sendError(response, 403, `Dependency Guardian blocked ${redactSecrets(clientTarget.pathname)}: registry returned partial content`);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
if (!origin.gate || upstream.status < 200 || upstream.status >= 300) {
|
|
382
|
+
serve(response, upstream);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
const gate = origin.gate;
|
|
386
|
+
const sha256 = createHash("sha256").update(upstream.body).digest("hex");
|
|
387
|
+
const resolution = resolveArtifactIdentity(clientTarget, origin.identities, gate.classification);
|
|
388
|
+
const identity = resolution.kind === "resolved" ? resolution.identity : undefined;
|
|
389
|
+
const verdict = identity
|
|
390
|
+
? await lookupVerdict(gate, clientTarget, sha256, { statusCode: upstream.status, headers: upstream.headers, body: upstream.body }, identity).catch((error) => ({
|
|
391
|
+
verdict: "block",
|
|
392
|
+
packageName: artifactDisplayName(identity),
|
|
393
|
+
cause: "api-timeout",
|
|
394
|
+
reason: error instanceof Error ? error.message : "verdict lookup failed"
|
|
395
|
+
}))
|
|
396
|
+
: { verdict: "block", packageName: resolution.kind === "ambiguous" ? resolution.packageName : clientTarget.pathname, cause: "policy", reason: resolution.kind === "ambiguous" ? resolution.reason : "could not resolve artifact identity" };
|
|
397
|
+
const decision = enforceProtectedInstall({ classification: gate.classification, env: gate.env, proxyVerdict: verdict });
|
|
398
|
+
origin.decisions.push(decision);
|
|
399
|
+
origin.hashes.push({ url: redactSecrets(clientTarget.toString()), sha256 });
|
|
400
|
+
origin.events.push(`${decision.action}:${decision.cause}:${redactSecrets(decision.packageName)}`);
|
|
401
|
+
writeOriginState(origin);
|
|
402
|
+
origin.onDecision?.(decision);
|
|
403
|
+
if (decision.action === "block") {
|
|
404
|
+
sendError(response, 403, `Dependency Guardian blocked ${redactSecrets(decision.packageName)}: ${redactSecrets(decision.reason)}`);
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
if (origin.cacheDir) {
|
|
408
|
+
storeCachedArtifact(origin.cacheDir, clientTarget.toString(), upstream.body, sha256);
|
|
409
|
+
}
|
|
410
|
+
serve(response, upstream);
|
|
411
|
+
}
|
|
412
|
+
function recordOriginVerdict(origin, target, sha256, verdict) {
|
|
413
|
+
if (!origin.gate) {
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
const decision = enforceProtectedInstall({ classification: origin.gate.classification, env: origin.gate.env, proxyVerdict: verdict });
|
|
417
|
+
origin.decisions.push(decision);
|
|
418
|
+
origin.hashes.push({ url: redactSecrets(target.toString()), sha256 });
|
|
419
|
+
origin.events.push(`${decision.action}:${decision.cause}:${redactSecrets(decision.packageName)}`);
|
|
420
|
+
writeOriginState(origin);
|
|
421
|
+
origin.onDecision?.(decision);
|
|
422
|
+
}
|
|
423
|
+
function rememberIdentities(origin, metadataUrl, upstream) {
|
|
424
|
+
if (!origin.gate) {
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
const found = extractRegistryMetadataIdentities(metadataUrl, { headers: upstream.headers, body: upstream.body });
|
|
428
|
+
for (const identity of found) {
|
|
429
|
+
if (!origin.identities.some((existing) => existing.tarballUrl === identity.tarballUrl && existing.version === identity.version)) {
|
|
430
|
+
origin.identities.push(identity);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
function rewriteNpmTarballs(doc, base) {
|
|
435
|
+
const versions = doc.versions;
|
|
436
|
+
if (!isRecord(versions)) {
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
for (const version of Object.values(versions)) {
|
|
440
|
+
if (!isRecord(version)) {
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
const dist = version.dist;
|
|
444
|
+
if (isRecord(dist) && typeof dist.tarball === "string") {
|
|
445
|
+
dist.tarball = `${base}${new URL(dist.tarball).pathname}`;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
const REDIRECT_LIMIT = 5;
|
|
450
|
+
function upstreamIdleTimeoutMs(env) {
|
|
451
|
+
return positiveInt(env.DG_UPSTREAM_IDLE_TIMEOUT_MS, 30_000);
|
|
452
|
+
}
|
|
453
|
+
function upstreamTotalTimeoutMs(env) {
|
|
454
|
+
return positiveInt(env.DG_UPSTREAM_TOTAL_TIMEOUT_MS, 300_000);
|
|
455
|
+
}
|
|
456
|
+
function positiveInt(raw, fallback) {
|
|
457
|
+
const parsed = Number.parseInt(raw ?? "", 10);
|
|
458
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
459
|
+
}
|
|
460
|
+
// Fetch upstream over a guarded DNS lookup (refuses a public host that resolves
|
|
461
|
+
// to an internal/metadata IP — DNS-rebinding SSRF), with an idle + total timeout
|
|
462
|
+
// so a hanging or slow-drip upstream cannot stall the install forever, following
|
|
463
|
+
// redirects ourselves (with the private-redirect guard the proxy uses) so a 3xx
|
|
464
|
+
// is never handed to the manager to follow un-gated. Identity encoding so the
|
|
465
|
+
// bytes are never mislabelled when re-served.
|
|
466
|
+
async function guardedFetch(target, accept, env = process.env) {
|
|
467
|
+
let current = target;
|
|
468
|
+
for (let hop = 0;; hop += 1) {
|
|
469
|
+
const response = await guardedFetchOnce(current, accept, env);
|
|
470
|
+
if (![301, 302, 303, 307, 308].includes(response.status)) {
|
|
471
|
+
return response;
|
|
472
|
+
}
|
|
473
|
+
if (hop >= REDIRECT_LIMIT) {
|
|
474
|
+
throw new Error(`registry redirect chain exceeded ${REDIRECT_LIMIT} hops for ${current.host}`);
|
|
475
|
+
}
|
|
476
|
+
const location = headerValue(response.headers.location);
|
|
477
|
+
if (!location) {
|
|
478
|
+
throw new Error(`registry returned a ${response.status} redirect with no Location for ${current.host}`);
|
|
479
|
+
}
|
|
480
|
+
const next = new URL(location, current);
|
|
481
|
+
if (next.protocol !== "http:" && next.protocol !== "https:") {
|
|
482
|
+
throw new Error(`registry redirected to unsupported protocol ${next.protocol}`);
|
|
483
|
+
}
|
|
484
|
+
if (isPrivateNetworkHost(next) && !isPrivateNetworkHost(target)) {
|
|
485
|
+
throw new Error(`registry redirected a public artifact request into a private address (${next.hostname})`);
|
|
486
|
+
}
|
|
487
|
+
current = next;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
function guardedFetchOnce(target, accept, env) {
|
|
491
|
+
if (isLinkLocalHost(target)) {
|
|
492
|
+
return Promise.reject(new Error(`refusing to fetch from link-local/metadata address ${target.hostname}`));
|
|
493
|
+
}
|
|
494
|
+
const isHttps = target.protocol === "https:";
|
|
495
|
+
const requester = isHttps ? httpsRequest : httpRequest;
|
|
496
|
+
return new Promise((resolve, reject) => {
|
|
497
|
+
let stream = null;
|
|
498
|
+
const request = requester({
|
|
499
|
+
hostname: target.hostname,
|
|
500
|
+
port: target.port ? Number(target.port) : isHttps ? 443 : 80,
|
|
501
|
+
path: `${target.pathname}${target.search}`,
|
|
502
|
+
method: "GET",
|
|
503
|
+
agent: isHttps ? upstreamHttpsAgent : upstreamHttpAgent,
|
|
504
|
+
lookup: lookupForTarget(target, env),
|
|
505
|
+
headers: { "accept-encoding": "identity", ...(accept ? { accept } : {}) }
|
|
506
|
+
}, (upstream) => {
|
|
507
|
+
stream = upstream;
|
|
508
|
+
collectBounded(upstream, { label: target.toString(), env })
|
|
509
|
+
.then((body) => resolve({ status: upstream.statusCode ?? 502, headers: responseHeaders(upstream), body }))
|
|
510
|
+
.catch(reject);
|
|
511
|
+
});
|
|
512
|
+
const expire = (detail) => {
|
|
513
|
+
const error = new Error(`upstream registry request for ${target.host} timed out: ${detail}`);
|
|
514
|
+
stream?.destroy(error);
|
|
515
|
+
request.destroy(error);
|
|
516
|
+
reject(error);
|
|
517
|
+
};
|
|
518
|
+
request.setTimeout(upstreamIdleTimeoutMs(env), () => expire(`no data for ${upstreamIdleTimeoutMs(env)}ms`));
|
|
519
|
+
const totalTimer = setTimeout(() => expire(`exceeded the ${upstreamTotalTimeoutMs(env)}ms total budget`), upstreamTotalTimeoutMs(env));
|
|
520
|
+
totalTimer.unref();
|
|
521
|
+
request.once("close", () => clearTimeout(totalTimer));
|
|
522
|
+
request.once("error", reject);
|
|
523
|
+
request.end();
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
function serve(response, upstream) {
|
|
527
|
+
const headers = {
|
|
528
|
+
"content-type": headerValue(upstream.headers["content-type"]) || "application/octet-stream",
|
|
529
|
+
"content-length": upstream.body.length
|
|
530
|
+
};
|
|
531
|
+
const etag = headerValue(upstream.headers["etag"]);
|
|
532
|
+
if (etag) {
|
|
533
|
+
headers.etag = etag;
|
|
534
|
+
}
|
|
535
|
+
response.writeHead(upstream.status, headers);
|
|
536
|
+
response.end(upstream.body);
|
|
537
|
+
}
|
|
538
|
+
function serveJson(response, status, doc) {
|
|
539
|
+
const body = Buffer.from(JSON.stringify(doc));
|
|
540
|
+
response.writeHead(status, { "content-type": "application/json", "content-length": body.length });
|
|
541
|
+
response.end(body);
|
|
542
|
+
}
|
|
543
|
+
// Emit the proxy's ProxySessionState shape (readProxySessionState reads it), so
|
|
544
|
+
// dg doctor and telemetry see one decision/hash/event stream whichever door ran.
|
|
545
|
+
function writeOriginState(origin) {
|
|
546
|
+
if (!origin.stateFile) {
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
const state = {
|
|
550
|
+
ready: true,
|
|
551
|
+
port: origin.port,
|
|
552
|
+
decisions: origin.decisions,
|
|
553
|
+
inflight: [],
|
|
554
|
+
hashes: origin.hashes,
|
|
555
|
+
identities: origin.identities,
|
|
556
|
+
events: origin.events
|
|
557
|
+
};
|
|
558
|
+
try {
|
|
559
|
+
mkdirSync(dirname(origin.stateFile), { recursive: true, mode: 0o700 });
|
|
560
|
+
const tmp = `${origin.stateFile}.${process.pid}.tmp`;
|
|
561
|
+
writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
562
|
+
renameSync(tmp, origin.stateFile);
|
|
563
|
+
}
|
|
564
|
+
catch {
|
|
565
|
+
// telemetry is best-effort and must never break an install.
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
function sendError(response, status, message) {
|
|
569
|
+
if (response.headersSent) {
|
|
570
|
+
response.destroy();
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
const body = `${message}\n`;
|
|
574
|
+
response.writeHead(status, { "content-type": "text/plain; charset=utf-8", "content-length": Buffer.byteLength(body) });
|
|
575
|
+
response.end(body);
|
|
576
|
+
}
|
|
577
|
+
function responseHeaders(response) {
|
|
578
|
+
const headers = {};
|
|
579
|
+
for (const [key, value] of Object.entries(response.headers)) {
|
|
580
|
+
if (value !== undefined) {
|
|
581
|
+
headers[key] = Array.isArray(value) ? [...value] : value;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
return headers;
|
|
585
|
+
}
|
|
586
|
+
function isJson(upstream) {
|
|
587
|
+
return /json/i.test(headerValue(upstream.headers["content-type"]));
|
|
588
|
+
}
|
|
589
|
+
// Content-Type is upstream-controlled and spoofable; the /-/ pass-through proves
|
|
590
|
+
// a body is metadata by actually parsing it (the size cap is applied first).
|
|
591
|
+
function bodyIsParseableJson(upstream) {
|
|
592
|
+
try {
|
|
593
|
+
JSON.parse(upstream.body.toString("utf8"));
|
|
594
|
+
return true;
|
|
595
|
+
}
|
|
596
|
+
catch {
|
|
597
|
+
return false;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
function headerValue(value) {
|
|
601
|
+
if (Array.isArray(value)) {
|
|
602
|
+
return value.join(",");
|
|
603
|
+
}
|
|
604
|
+
if (typeof value === "number") {
|
|
605
|
+
return String(value);
|
|
606
|
+
}
|
|
607
|
+
return value ?? "";
|
|
608
|
+
}
|
|
609
|
+
function isRecord(value) {
|
|
610
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
611
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { appendFileSync } from "node:fs";
|
|
2
|
+
import { startOriginServer } from "./server.js";
|
|
3
|
+
// Standalone entry so the P0 spike origin runs as its OWN process. The spawned
|
|
4
|
+
// package manager connects to this process's loopback port; a manager spawned by
|
|
5
|
+
// the test worker cannot reach a port the test worker itself holds, so the origin
|
|
6
|
+
// must not be in-process. Prints `READY <url>` once bound.
|
|
7
|
+
async function main() {
|
|
8
|
+
const ecosystem = (process.argv[2] ?? "npm");
|
|
9
|
+
const strategy = (process.argv[3] ?? "canonical");
|
|
10
|
+
// The test reads request routing from this file AFTER spawnSync returns. A
|
|
11
|
+
// file (not stderr) is required because spawnSync blocks the event loop, so
|
|
12
|
+
// the worker's async stderr is not processed until after the test's
|
|
13
|
+
// synchronous assertions run.
|
|
14
|
+
const requestLog = process.env.DG_ORIGIN_REQUEST_LOG;
|
|
15
|
+
const handle = await startOriginServer({
|
|
16
|
+
ecosystem,
|
|
17
|
+
npmTarballStrategy: strategy,
|
|
18
|
+
onRequest: (method, path) => {
|
|
19
|
+
process.stderr.write(`REQ ${method} ${path}\n`);
|
|
20
|
+
if (requestLog) {
|
|
21
|
+
appendFileSync(requestLog, `${method} ${path}\n`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
process.stdout.write(`READY ${handle.url}\n`);
|
|
26
|
+
const shutdown = () => {
|
|
27
|
+
handle.close().finally(() => process.exit(0));
|
|
28
|
+
};
|
|
29
|
+
process.on("SIGTERM", shutdown);
|
|
30
|
+
process.on("SIGINT", shutdown);
|
|
31
|
+
}
|
|
32
|
+
main().catch((error) => {
|
|
33
|
+
process.stderr.write(`origin worker failed: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
34
|
+
process.exit(1);
|
|
35
|
+
});
|