@skydiveai/git-cache 0.1.0-beta.1480
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.mjs +43 -0
- package/dist/index.d.mts +179 -0
- package/dist/index.mjs +2 -0
- package/dist/server-B5u5DETT.mjs +1829 -0
- package/package.json +56 -0
|
@@ -0,0 +1,1829 @@
|
|
|
1
|
+
import http from "node:http";
|
|
2
|
+
import https from "node:https";
|
|
3
|
+
import { createHmac, randomBytes } from "node:crypto";
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { mkdir, readFile, readdir, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
|
|
6
|
+
import { hostname } from "node:os";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { createGunzip, gunzip } from "node:zlib";
|
|
9
|
+
import { promisify } from "node:util";
|
|
10
|
+
//#region src/server.ts
|
|
11
|
+
/**
|
|
12
|
+
* Sandbox-local Git smart-HTTP cache.
|
|
13
|
+
*
|
|
14
|
+
* NOT a hosted git proxy or remote: this is a loopback-only daemon that runs
|
|
15
|
+
* on the same machine as the git clients it serves, and accelerates
|
|
16
|
+
* clone/fetch of any HTTPS git repo by keeping a local mirror. Different
|
|
17
|
+
* machine, different layer, different purpose than a deployed git remote.
|
|
18
|
+
*
|
|
19
|
+
* Data paths
|
|
20
|
+
* ----------
|
|
21
|
+
* System git config rewrites every `https://` fetch/clone URL to this daemon,
|
|
22
|
+
* preserving the true authority as the first path segment:
|
|
23
|
+
*
|
|
24
|
+
* https://github.com/owner/repo.git
|
|
25
|
+
* -> http://127.0.0.1:PORT/github.com/owner/repo.git
|
|
26
|
+
*
|
|
27
|
+
* Pushes are restored to direct HTTPS by a matching `pushInsteadOf`, so this
|
|
28
|
+
* daemon only ever serves upload-pack (never receive-pack).
|
|
29
|
+
*
|
|
30
|
+
* Cold miss : stream the origin's upload-pack response to the first client
|
|
31
|
+
* byte-for-byte while opportunistically ingesting the sideband
|
|
32
|
+
* pack into a quarantined bare repo. If the ingest cannot form a
|
|
33
|
+
* complete standalone mirror (filtered / shallow / have-based /
|
|
34
|
+
* too large / disconnected), it is abandoned — the client's clone
|
|
35
|
+
* is never affected. On success the mirror is atomically published
|
|
36
|
+
* for warm reuse.
|
|
37
|
+
* Warm hit : serve `git-upload-pack` from the local mirror.
|
|
38
|
+
* Refresh : stale mirrors are served immediately (stale-while-revalidate)
|
|
39
|
+
* and refreshed in the background.
|
|
40
|
+
*
|
|
41
|
+
* Security: binds loopback only; upstream host/path are strictly validated
|
|
42
|
+
* (SSRF surface); credentials come from the platform git credential helper and
|
|
43
|
+
* never touch the client hop, the mirror config, URLs, argv, or logs.
|
|
44
|
+
*
|
|
45
|
+
* Zero runtime dependencies (stdlib only) so it can run in the system layer
|
|
46
|
+
* with no npm install.
|
|
47
|
+
*/
|
|
48
|
+
const gunzip$1 = promisify(gunzip);
|
|
49
|
+
/**
|
|
50
|
+
* An unset/empty env resolves to trusted-client (the daemon's original
|
|
51
|
+
* contract); anything other than the two known modes is a hard startup
|
|
52
|
+
* error rather than a silent fallback — a typo'd GIT_CACHE_AUTH_MODE on a
|
|
53
|
+
* shared deployment must never quietly run without authorization.
|
|
54
|
+
*/
|
|
55
|
+
function parseAuthMode(raw) {
|
|
56
|
+
if (!raw) return "trusted-client";
|
|
57
|
+
if (raw === "trusted-client" || raw === "check-through") return raw;
|
|
58
|
+
throw new Error(`git-cache: unknown GIT_CACHE_AUTH_MODE '${raw}' (expected 'trusted-client' or 'check-through')`);
|
|
59
|
+
}
|
|
60
|
+
const config = {
|
|
61
|
+
port: Number(process.env.GIT_CACHE_PORT || process.env.PORT || 38996),
|
|
62
|
+
bindHost: process.env.GIT_CACHE_BIND || "127.0.0.1",
|
|
63
|
+
cacheDir: process.env.GIT_CACHE_DIR || "/home/user/.cache/git-cache",
|
|
64
|
+
ttlSeconds: Number(process.env.GIT_CACHE_TTL_SECONDS || 300),
|
|
65
|
+
credentialHelper: process.env.GIT_CACHE_CREDENTIAL_HELPER || "",
|
|
66
|
+
authMode: parseAuthMode(process.env.GIT_CACHE_AUTH_MODE),
|
|
67
|
+
maxPackBytes: Number(process.env.GIT_CACHE_MAX_PACK_BYTES || 8 * 1024 * 1024 * 1024),
|
|
68
|
+
maxImporterQueueBytes: Number(process.env.GIT_CACHE_MAX_IMPORTER_QUEUE_BYTES || 256 * 1024 * 1024),
|
|
69
|
+
maxCacheBytes: Number(process.env.GIT_CACHE_MAX_CACHE_BYTES || 0),
|
|
70
|
+
hostAllowlist: (process.env.GIT_CACHE_HOST_ALLOWLIST || "").split(",").map((h) => h.trim().toLowerCase()).filter(Boolean),
|
|
71
|
+
logLevel: process.env.GIT_CACHE_LOG_LEVEL || "info",
|
|
72
|
+
testOriginBase: (process.env.GIT_CACHE_TEST_ORIGIN_BASE || "").replace(/\/+$/, "")
|
|
73
|
+
};
|
|
74
|
+
const LOG_LEVELS = {
|
|
75
|
+
error: 0,
|
|
76
|
+
warn: 1,
|
|
77
|
+
info: 2,
|
|
78
|
+
debug: 3
|
|
79
|
+
};
|
|
80
|
+
const GIT_ENV = {
|
|
81
|
+
...process.env,
|
|
82
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
83
|
+
GIT_ASKPASS: "/bin/true",
|
|
84
|
+
SSH_ASKPASS: "/bin/true",
|
|
85
|
+
GCM_INTERACTIVE: "never",
|
|
86
|
+
GIT_CONFIG_NOSYSTEM: process.env.GIT_CONFIG_NOSYSTEM || "1"
|
|
87
|
+
};
|
|
88
|
+
const activeLevel = LOG_LEVELS[config.logLevel] ?? LOG_LEVELS.info;
|
|
89
|
+
function log(level, event, fields = {}) {
|
|
90
|
+
if ((LOG_LEVELS[level] ?? 2) > activeLevel) return;
|
|
91
|
+
const record = {
|
|
92
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
93
|
+
service: "git-cache",
|
|
94
|
+
level,
|
|
95
|
+
event,
|
|
96
|
+
...fields
|
|
97
|
+
};
|
|
98
|
+
const line = JSON.stringify(record);
|
|
99
|
+
if (level === "error" || level === "warn") process.stderr.write(line + "\n");
|
|
100
|
+
else process.stdout.write(line + "\n");
|
|
101
|
+
}
|
|
102
|
+
const metrics = {
|
|
103
|
+
cold_requests: 0,
|
|
104
|
+
cold_followers: 0,
|
|
105
|
+
warm_hits: 0,
|
|
106
|
+
refreshes: 0,
|
|
107
|
+
publications: 0,
|
|
108
|
+
abandoned_ineligible: 0,
|
|
109
|
+
abandoned_oversize: 0,
|
|
110
|
+
abandoned_queue_overflow: 0,
|
|
111
|
+
abandoned_import_error: 0,
|
|
112
|
+
abandoned_disconnect: 0,
|
|
113
|
+
passthrough_ineligible: 0,
|
|
114
|
+
passthrough_missing_want: 0,
|
|
115
|
+
invalidations: 0,
|
|
116
|
+
evictions: 0,
|
|
117
|
+
refresh_failures: 0,
|
|
118
|
+
refresh_deferred: 0,
|
|
119
|
+
publish_deferred: 0,
|
|
120
|
+
credential_url_redirects: 0,
|
|
121
|
+
lfs_redirects: 0,
|
|
122
|
+
upstream_errors: 0,
|
|
123
|
+
bad_requests: 0,
|
|
124
|
+
cold_proxy_bytes: 0,
|
|
125
|
+
captured_pack_bytes: 0,
|
|
126
|
+
cold_build_failures: 0,
|
|
127
|
+
backfills: 0,
|
|
128
|
+
backfill_failures: 0,
|
|
129
|
+
auth_checks: 0,
|
|
130
|
+
auth_memo_hits: 0,
|
|
131
|
+
auth_denials: 0
|
|
132
|
+
};
|
|
133
|
+
/** The counter payload `/metrics` serves. Copied so callers can't mutate it. */
|
|
134
|
+
function metricsSnapshot() {
|
|
135
|
+
return { ...metrics };
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Record the outcome of a finished cold build. A failure leaves the mirror
|
|
139
|
+
* unpublished, so every later clone silently pays full upstream cost — it must
|
|
140
|
+
* move a counter and not just write a log line, or a cache that never publishes
|
|
141
|
+
* reads as healthy on /metrics.
|
|
142
|
+
*/
|
|
143
|
+
function recordColdBuildOutcome(error) {
|
|
144
|
+
if (!error) return;
|
|
145
|
+
metrics.cold_build_failures++;
|
|
146
|
+
}
|
|
147
|
+
/** repo -> Promise resolving when an in-flight background refresh finishes. */
|
|
148
|
+
const inflightRefresh = /* @__PURE__ */ new Map();
|
|
149
|
+
/** Set of repos with a cold-stream leader currently populating the cache. */
|
|
150
|
+
const coldLeaders = /* @__PURE__ */ new Set();
|
|
151
|
+
/** repo -> parsed advertisement refs from the most recent cold info/refs. */
|
|
152
|
+
const advertisements = /* @__PURE__ */ new Map();
|
|
153
|
+
/** Wall-clock ms; any mirror last-fetched before this is treated as stale. */
|
|
154
|
+
let invalidateBeforeMs = 0;
|
|
155
|
+
function upstreamGitArgs(args) {
|
|
156
|
+
return config.credentialHelper ? [
|
|
157
|
+
"-c",
|
|
158
|
+
`credential.helper=${config.credentialHelper}`,
|
|
159
|
+
...args
|
|
160
|
+
] : args;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* @param {string[]} args
|
|
164
|
+
* @param {import('node:child_process').SpawnOptions} [options]
|
|
165
|
+
* @returns {Promise<{ stdout: string, stderr: string }>}
|
|
166
|
+
*/
|
|
167
|
+
function runGit(args, options = {}) {
|
|
168
|
+
return new Promise((resolve, reject) => {
|
|
169
|
+
const child = spawn("git", args, {
|
|
170
|
+
env: GIT_ENV,
|
|
171
|
+
...options,
|
|
172
|
+
stdio: [
|
|
173
|
+
"ignore",
|
|
174
|
+
"pipe",
|
|
175
|
+
"pipe"
|
|
176
|
+
]
|
|
177
|
+
});
|
|
178
|
+
let stdout = "";
|
|
179
|
+
let stderr = "";
|
|
180
|
+
let outBytes = 0;
|
|
181
|
+
let errBytes = 0;
|
|
182
|
+
const CAP = 256 * 1024;
|
|
183
|
+
child.stdout.on("data", (d) => {
|
|
184
|
+
if (outBytes < CAP) {
|
|
185
|
+
stdout += d;
|
|
186
|
+
outBytes += d.length;
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
child.stderr.on("data", (d) => {
|
|
190
|
+
if (errBytes < CAP) {
|
|
191
|
+
stderr += d;
|
|
192
|
+
errBytes += d.length;
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
child.on("error", reject);
|
|
196
|
+
child.on("close", (code) => {
|
|
197
|
+
if (code === 0) resolve({
|
|
198
|
+
stdout,
|
|
199
|
+
stderr
|
|
200
|
+
});
|
|
201
|
+
else reject(/* @__PURE__ */ new Error(`git ${args[0]} exited ${code}: ${stderr.trim()}`));
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
/** host -> Promise<string|null> Basic auth header ("Basic base64(user:pass)"). */
|
|
206
|
+
const credentialCache = /* @__PURE__ */ new Map();
|
|
207
|
+
async function fetchCredentialHeader(host) {
|
|
208
|
+
if (!config.credentialHelper) return null;
|
|
209
|
+
if (credentialCache.has(host)) return credentialCache.get(host);
|
|
210
|
+
const p = (async () => {
|
|
211
|
+
try {
|
|
212
|
+
/** @type {string} */
|
|
213
|
+
const stdout = await new Promise((resolve, reject) => {
|
|
214
|
+
const child = spawn("git", [
|
|
215
|
+
"-c",
|
|
216
|
+
`credential.helper=${config.credentialHelper}`,
|
|
217
|
+
"credential",
|
|
218
|
+
"fill"
|
|
219
|
+
], {
|
|
220
|
+
stdio: [
|
|
221
|
+
"pipe",
|
|
222
|
+
"pipe",
|
|
223
|
+
"pipe"
|
|
224
|
+
],
|
|
225
|
+
env: GIT_ENV
|
|
226
|
+
});
|
|
227
|
+
let out = "";
|
|
228
|
+
let err = "";
|
|
229
|
+
child.stdout.on("data", (d) => out += d);
|
|
230
|
+
child.stderr.on("data", (d) => {
|
|
231
|
+
if (err.length < 64 * 1024) err += d;
|
|
232
|
+
});
|
|
233
|
+
child.on("error", reject);
|
|
234
|
+
child.on("close", (code) => {
|
|
235
|
+
if (code === 0) resolve(out);
|
|
236
|
+
else reject(/* @__PURE__ */ new Error(`git credential fill exited ${code}: ${err.trim()}`));
|
|
237
|
+
});
|
|
238
|
+
child.stdin.end(`protocol=https\nhost=${host}\n\n`);
|
|
239
|
+
});
|
|
240
|
+
const fields = {};
|
|
241
|
+
for (const line of stdout.split("\n")) {
|
|
242
|
+
const eq = line.indexOf("=");
|
|
243
|
+
if (eq > 0) fields[line.slice(0, eq)] = line.slice(eq + 1);
|
|
244
|
+
}
|
|
245
|
+
if (!fields.username && !fields.password) return null;
|
|
246
|
+
return `Basic ${Buffer.from(`${fields.username || ""}:${fields.password || ""}`).toString("base64")}`;
|
|
247
|
+
} catch (_error) {
|
|
248
|
+
log("debug", "credential_fill_miss", { host });
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
})();
|
|
252
|
+
credentialCache.set(host, p);
|
|
253
|
+
return p;
|
|
254
|
+
}
|
|
255
|
+
function dropCredential(host) {
|
|
256
|
+
credentialCache.delete(host);
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Guard against non-numeric env values explicitly: NaN would only be
|
|
260
|
+
* fail-safe through the accident of `elapsed < NaN` being false, and any
|
|
261
|
+
* later refactor to a `>= ttl` comparison would silently flip that into
|
|
262
|
+
* memoize-forever.
|
|
263
|
+
*/
|
|
264
|
+
function parseTtlMs(raw, fallback) {
|
|
265
|
+
const n = Number(raw);
|
|
266
|
+
return raw !== void 0 && Number.isFinite(n) && n >= 0 ? n : fallback;
|
|
267
|
+
}
|
|
268
|
+
const AUTH_ALLOW_TTL_MS = parseTtlMs(process.env.GIT_CACHE_AUTH_ALLOW_TTL_MS, 6e4);
|
|
269
|
+
const AUTH_DENY_TTL_MS = parseTtlMs(process.env.GIT_CACHE_AUTH_DENY_TTL_MS, 5e3);
|
|
270
|
+
const AUTH_CACHE_MAX = 1e4;
|
|
271
|
+
let AUTH_HMAC_KEY = randomBytes(32);
|
|
272
|
+
/** memoKey -> { at, allowed, status } */
|
|
273
|
+
const authCache = /* @__PURE__ */ new Map();
|
|
274
|
+
/** memoKey -> in-flight validation. A clone is a GET+POST pair and a herd is
|
|
275
|
+
* many clients on one repo; coalescing means each (credential, repo) costs one
|
|
276
|
+
* upstream round-trip per TTL window, not one per request. */
|
|
277
|
+
const authInflight = /* @__PURE__ */ new Map();
|
|
278
|
+
function authMemoKey(credHeader, repo) {
|
|
279
|
+
return `${createHmac("sha256", AUTH_HMAC_KEY).update(credHeader).digest("hex")} ${repo}`;
|
|
280
|
+
}
|
|
281
|
+
function evictAuthCache() {
|
|
282
|
+
if (authCache.size <= AUTH_CACHE_MAX) return;
|
|
283
|
+
const now = Date.now();
|
|
284
|
+
for (const [key, entry] of authCache) {
|
|
285
|
+
const ttl = entry.allowed ? AUTH_ALLOW_TTL_MS : AUTH_DENY_TTL_MS;
|
|
286
|
+
if (now - entry.at > ttl) authCache.delete(key);
|
|
287
|
+
}
|
|
288
|
+
if (authCache.size > AUTH_CACHE_MAX) {
|
|
289
|
+
const iter = authCache.keys();
|
|
290
|
+
while (authCache.size > AUTH_CACHE_MAX) {
|
|
291
|
+
const next = iter.next();
|
|
292
|
+
if (next.done) break;
|
|
293
|
+
authCache.delete(next.value);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
function validateUpstream(credHeader, upstreamUrl) {
|
|
298
|
+
return new Promise((resolve) => {
|
|
299
|
+
const request = requestUpstream(new URL(`${upstreamUrl}/info/refs?service=git-upload-pack`), {
|
|
300
|
+
method: "GET",
|
|
301
|
+
headers: {
|
|
302
|
+
authorization: credHeader,
|
|
303
|
+
"user-agent": "git/git-cache-authz"
|
|
304
|
+
}
|
|
305
|
+
}, (res) => {
|
|
306
|
+
res.resume();
|
|
307
|
+
const status = res.statusCode ?? 502;
|
|
308
|
+
if (status === 200) {
|
|
309
|
+
resolve({
|
|
310
|
+
allowed: true,
|
|
311
|
+
status: 200
|
|
312
|
+
});
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
if (status >= 300 && status < 400) {
|
|
316
|
+
resolve({
|
|
317
|
+
allowed: false,
|
|
318
|
+
status: 404
|
|
319
|
+
});
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
resolve({
|
|
323
|
+
allowed: false,
|
|
324
|
+
status: status === 401 || status === 403 || status === 404 ? status : 502
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
request.setTimeout(1e4, () => {
|
|
328
|
+
request.destroy();
|
|
329
|
+
resolve({
|
|
330
|
+
allowed: false,
|
|
331
|
+
status: 502
|
|
332
|
+
});
|
|
333
|
+
});
|
|
334
|
+
request.on("error", () => resolve({
|
|
335
|
+
allowed: false,
|
|
336
|
+
status: 502
|
|
337
|
+
}));
|
|
338
|
+
request.end();
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
async function authorizeCheckThrough(credHeader, repo, upstreamUrl) {
|
|
342
|
+
const key = authMemoKey(credHeader, repo);
|
|
343
|
+
const cached = authCache.get(key);
|
|
344
|
+
if (cached) {
|
|
345
|
+
const ttl = cached.allowed ? AUTH_ALLOW_TTL_MS : AUTH_DENY_TTL_MS;
|
|
346
|
+
if (Date.now() - cached.at < ttl) {
|
|
347
|
+
metrics.auth_memo_hits++;
|
|
348
|
+
return cached;
|
|
349
|
+
}
|
|
350
|
+
authCache.delete(key);
|
|
351
|
+
}
|
|
352
|
+
const inflight = authInflight.get(key);
|
|
353
|
+
if (inflight) return inflight;
|
|
354
|
+
const p = (async () => {
|
|
355
|
+
const verdict = await validateUpstream(credHeader, upstreamUrl);
|
|
356
|
+
metrics.auth_checks++;
|
|
357
|
+
if (!verdict.allowed) metrics.auth_denials++;
|
|
358
|
+
if (verdict.status !== 502) {
|
|
359
|
+
authCache.set(key, {
|
|
360
|
+
at: Date.now(),
|
|
361
|
+
...verdict
|
|
362
|
+
});
|
|
363
|
+
evictAuthCache();
|
|
364
|
+
}
|
|
365
|
+
return verdict;
|
|
366
|
+
})().finally(() => authInflight.delete(key));
|
|
367
|
+
authInflight.set(key, p);
|
|
368
|
+
return p;
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Env fragment handing a client's credential to a spawned git subprocess via
|
|
372
|
+
* config-over-environment — never argv (visible in /proc) and never on-disk
|
|
373
|
+
* config. Composes with GIT_ENV: GIT_CONFIG_NOSYSTEM does not affect
|
|
374
|
+
* GIT_CONFIG_COUNT entries.
|
|
375
|
+
*/
|
|
376
|
+
function upstreamGitCredentialEnv(credHeader) {
|
|
377
|
+
if (!credHeader) return {};
|
|
378
|
+
return {
|
|
379
|
+
GIT_CONFIG_COUNT: "1",
|
|
380
|
+
GIT_CONFIG_KEY_0: "http.extraHeader",
|
|
381
|
+
GIT_CONFIG_VALUE_0: `Authorization: ${credHeader}`
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
const HOSTNAME_RE = /^(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$/i;
|
|
385
|
+
/**
|
|
386
|
+
* True for anything that looks like an IPv4 dotted-quad or an IPv6 literal
|
|
387
|
+
* (with or without brackets). We refuse IP-literal upstream hosts entirely.
|
|
388
|
+
*/
|
|
389
|
+
function isIpLiteral(host) {
|
|
390
|
+
const h = host.replace(/^\[|\]$/g, "");
|
|
391
|
+
if (/:/.test(h)) return true;
|
|
392
|
+
const m = h.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
393
|
+
if (m) return m.slice(1).every((o) => Number(o) <= 255);
|
|
394
|
+
const last = h.split(".").pop();
|
|
395
|
+
return /^\d+$/.test(last || "");
|
|
396
|
+
}
|
|
397
|
+
function parseUpstream(rawPathname, rawSearch = "") {
|
|
398
|
+
if (/[\u0000-\u001f\u007f]/.test(rawPathname)) return null;
|
|
399
|
+
let pathname;
|
|
400
|
+
try {
|
|
401
|
+
pathname = decodeURIComponent(rawPathname);
|
|
402
|
+
} catch (_error) {
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
try {
|
|
406
|
+
if (decodeURIComponent(pathname) !== pathname) return null;
|
|
407
|
+
} catch (_error) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
const match = pathname.match(/^\/([^/]+)\/(.+?)(\.git)?\/(info\/refs|git-upload-pack)$/);
|
|
411
|
+
if (!match) {
|
|
412
|
+
const lfs = pathname.match(/^\/([^/@]+)\/(.+?\/info\/lfs(?:\/.*)?)$/);
|
|
413
|
+
if (lfs && !isIpLiteral(lfs[1].split(":")[0])) {
|
|
414
|
+
const authority = lfs[1];
|
|
415
|
+
const host = authority.split(":")[0].toLowerCase();
|
|
416
|
+
if (HOSTNAME_RE.test(host) && host !== "localhost" && !host.endsWith(".localhost")) return {
|
|
417
|
+
redirect: `https://${authority}/${lfs[2]}${rawSearch || ""}`,
|
|
418
|
+
operation: "lfs"
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
const authority = match[1];
|
|
424
|
+
const at = authority.lastIndexOf("@");
|
|
425
|
+
if (at !== -1) {
|
|
426
|
+
const hostPort = authority.slice(at + 1);
|
|
427
|
+
const rest = pathname.slice(1 + authority.length);
|
|
428
|
+
if (hostPort && authority.indexOf("@") === at && /^[A-Za-z0-9.:-]+$/.test(hostPort)) return {
|
|
429
|
+
redirect: `https://${authority}${rest}${rawSearch || ""}`,
|
|
430
|
+
operation: match[4]
|
|
431
|
+
};
|
|
432
|
+
return null;
|
|
433
|
+
}
|
|
434
|
+
const repoPath = match[2].replace(/\.git$/, "");
|
|
435
|
+
const hadDotGit = Boolean(match[3]);
|
|
436
|
+
const operation = match[4];
|
|
437
|
+
let host = authority;
|
|
438
|
+
let port = 443;
|
|
439
|
+
const colon = authority.lastIndexOf(":");
|
|
440
|
+
if (colon !== -1) {
|
|
441
|
+
host = authority.slice(0, colon);
|
|
442
|
+
const p = authority.slice(colon + 1);
|
|
443
|
+
if (!/^\d{1,5}$/.test(p)) return null;
|
|
444
|
+
port = Number(p);
|
|
445
|
+
if (port !== 443) return null;
|
|
446
|
+
}
|
|
447
|
+
host = host.toLowerCase();
|
|
448
|
+
if (isIpLiteral(host)) return null;
|
|
449
|
+
if (!HOSTNAME_RE.test(host)) return null;
|
|
450
|
+
if (host === "localhost" || host.endsWith(".localhost")) return null;
|
|
451
|
+
if (!/\.[a-z]{2,}$/.test(host)) return null;
|
|
452
|
+
if (config.hostAllowlist.length && !config.hostAllowlist.includes(host)) return null;
|
|
453
|
+
if (repoPath.includes("..") || repoPath.startsWith("/") || repoPath.endsWith("/") || repoPath.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) return null;
|
|
454
|
+
if (!/^[A-Za-z0-9._~+@/-]+$/.test(repoPath)) return null;
|
|
455
|
+
const repo = `${host}/${repoPath}`;
|
|
456
|
+
const suffix = hadDotGit ? ".git" : "";
|
|
457
|
+
return {
|
|
458
|
+
repo,
|
|
459
|
+
upstreamUrl: config.testOriginBase ? `${config.testOriginBase}/${repo}${suffix}` : `https://${authority}/${repoPath}${suffix}`,
|
|
460
|
+
operation
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Best-effort human explanation for a request parseUpstream refused, used
|
|
465
|
+
* ONLY in the 404 body so an agent whose (legitimate but unproxyable) remote
|
|
466
|
+
* hits a security rejection gets an actionable error instead of an opaque
|
|
467
|
+
* "git cache endpoint". Diagnostic only — the actual gate is parseUpstream;
|
|
468
|
+
* this never influences serving decisions, so a miss here is harmless.
|
|
469
|
+
*/
|
|
470
|
+
function explainRejection(rawPathname) {
|
|
471
|
+
let pathname;
|
|
472
|
+
try {
|
|
473
|
+
pathname = decodeURIComponent(rawPathname);
|
|
474
|
+
} catch (_error) {
|
|
475
|
+
return null;
|
|
476
|
+
}
|
|
477
|
+
const m = pathname.match(/^\/([^/]+)\//);
|
|
478
|
+
if (!m) return null;
|
|
479
|
+
const authority = m[1];
|
|
480
|
+
if (authority === void 0 || authority.includes("@")) return null;
|
|
481
|
+
const colon = authority.lastIndexOf(":");
|
|
482
|
+
const host = (colon === -1 ? authority : authority.slice(0, colon)).replace(/^\[|\]$/g, "").toLowerCase();
|
|
483
|
+
const port = colon === -1 ? null : authority.slice(colon + 1);
|
|
484
|
+
if (port !== null && /^\d{1,5}$/.test(port) && Number(port) !== 443) return `refusing upstream on non-443 port :${port}`;
|
|
485
|
+
if (isIpLiteral(host)) return "refusing IP-literal upstream host (DNS names only)";
|
|
486
|
+
if (host === "localhost" || host.endsWith(".localhost")) return "refusing localhost upstream";
|
|
487
|
+
if (!host.includes(".")) return "refusing single-label upstream host";
|
|
488
|
+
return null;
|
|
489
|
+
}
|
|
490
|
+
function validateMethod(req, operation) {
|
|
491
|
+
if (operation === "info/refs") return req.method === "GET";
|
|
492
|
+
if (operation === "git-upload-pack") return req.method === "POST";
|
|
493
|
+
return false;
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* How long a held lock may go untouched before another writer may break it.
|
|
497
|
+
* A live holder refreshes its lock every LOCK_HEARTBEAT_MS, so only a crashed
|
|
498
|
+
* or wedged process falls this far behind.
|
|
499
|
+
*/
|
|
500
|
+
const LOCK_STALE_MS = 180 * 1e3;
|
|
501
|
+
const LOCK_HEARTBEAT_MS = 20 * 1e3;
|
|
502
|
+
/** Read a filesystem or stream error's `code` without asserting a type. */
|
|
503
|
+
function errnoCode(error) {
|
|
504
|
+
if (typeof error === "object" && error !== null && "code" in error) {
|
|
505
|
+
const { code } = error;
|
|
506
|
+
return typeof code === "string" ? code : void 0;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
function lockPath(repo, name) {
|
|
510
|
+
return path.join(config.cacheDir, "locks", `${repo.replace(/\//g, "__")}.${name}.lock`);
|
|
511
|
+
}
|
|
512
|
+
/**
|
|
513
|
+
* Run `fn` while holding a named lock for `repo`, or return `null` without
|
|
514
|
+
* running it if another writer holds that lock.
|
|
515
|
+
*
|
|
516
|
+
* The in-memory guards elsewhere in this file (`inflightRefresh`,
|
|
517
|
+
* `coldLeaders`, `backfilling`) only serialize writers inside ONE process,
|
|
518
|
+
* which is the whole story when each process owns its own disk. On a mirror
|
|
519
|
+
* store shared between tasks they are blind to each other, so two tasks will
|
|
520
|
+
* happily fetch into, publish over, or evict the same mirror at the same
|
|
521
|
+
* time. This lock is what makes those operations mutually exclusive across
|
|
522
|
+
* every writer of the store.
|
|
523
|
+
*
|
|
524
|
+
* `mkdir` is the primitive because its create-or-fail is atomic on a single
|
|
525
|
+
* disk and over NFS alike. A lock whose mtime has stopped advancing is
|
|
526
|
+
* assumed abandoned and broken, so a task that dies mid-operation cannot
|
|
527
|
+
* wedge a repo forever.
|
|
528
|
+
*/
|
|
529
|
+
async function withRepoLock(repo, name, fn) {
|
|
530
|
+
const dir = lockPath(repo, name);
|
|
531
|
+
await mkdir(path.dirname(dir), { recursive: true });
|
|
532
|
+
const acquire = async () => {
|
|
533
|
+
try {
|
|
534
|
+
await mkdir(dir);
|
|
535
|
+
return true;
|
|
536
|
+
} catch (error) {
|
|
537
|
+
if (errnoCode(error) !== "EEXIST") throw error;
|
|
538
|
+
return false;
|
|
539
|
+
}
|
|
540
|
+
};
|
|
541
|
+
let held = await acquire();
|
|
542
|
+
if (!held) try {
|
|
543
|
+
const info = await stat(dir);
|
|
544
|
+
if (Date.now() - info.mtimeMs > LOCK_STALE_MS) {
|
|
545
|
+
log("warn", "lock_broken", {
|
|
546
|
+
repo,
|
|
547
|
+
name
|
|
548
|
+
});
|
|
549
|
+
await rm(dir, {
|
|
550
|
+
recursive: true,
|
|
551
|
+
force: true
|
|
552
|
+
});
|
|
553
|
+
held = await acquire();
|
|
554
|
+
}
|
|
555
|
+
} catch (_error) {
|
|
556
|
+
held = await acquire();
|
|
557
|
+
}
|
|
558
|
+
if (!held) return null;
|
|
559
|
+
await writeFile(path.join(dir, "owner"), INSTANCE_ID, "utf8").catch((_error) => log("debug", "lock_owner_write_failed", {
|
|
560
|
+
repo,
|
|
561
|
+
name
|
|
562
|
+
}));
|
|
563
|
+
const heartbeat = setInterval(() => {
|
|
564
|
+
const now = /* @__PURE__ */ new Date();
|
|
565
|
+
utimes(dir, now, now).catch((error) => log("debug", "lock_heartbeat_failed", {
|
|
566
|
+
repo,
|
|
567
|
+
name,
|
|
568
|
+
error: String(error)
|
|
569
|
+
}));
|
|
570
|
+
}, LOCK_HEARTBEAT_MS);
|
|
571
|
+
heartbeat.unref?.();
|
|
572
|
+
try {
|
|
573
|
+
return await fn();
|
|
574
|
+
} finally {
|
|
575
|
+
clearInterval(heartbeat);
|
|
576
|
+
await rm(dir, {
|
|
577
|
+
recursive: true,
|
|
578
|
+
force: true
|
|
579
|
+
}).catch((_error) => log("debug", "lock_release_failed", {
|
|
580
|
+
repo,
|
|
581
|
+
name
|
|
582
|
+
}));
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Identifies this process among every writer sharing the mirror store.
|
|
587
|
+
*
|
|
588
|
+
* When the store is a shared filesystem, several tasks write to it at once and
|
|
589
|
+
* a pid is no longer unique — two tasks routinely run the same pid. Temp paths
|
|
590
|
+
* and lock ownership records mix in the host and a random suffix so they
|
|
591
|
+
* cannot collide.
|
|
592
|
+
*/
|
|
593
|
+
const INSTANCE_ID = `${hostname()}-${process.pid}-${randomBytes(4).toString("hex")}`;
|
|
594
|
+
function localPath(repo) {
|
|
595
|
+
return path.join(config.cacheDir, "mirrors", `${repo}.git`);
|
|
596
|
+
}
|
|
597
|
+
const metaPath = (local) => `${local}.meta.json`;
|
|
598
|
+
const servedMarkerPath = (local) => `${local}.served`;
|
|
599
|
+
/**
|
|
600
|
+
* How recently a mirror must have been served to be exempt from eviction.
|
|
601
|
+
*
|
|
602
|
+
* Eviction orders by last upstream fetch, which says nothing about whether
|
|
603
|
+
* clients are reading a mirror right now. A busy repo whose refreshes are all
|
|
604
|
+
* cheap no-ops can therefore sort to the front of the LRU list while it is
|
|
605
|
+
* actively streaming.
|
|
606
|
+
*/
|
|
607
|
+
const EVICTION_QUIET_MS = 900 * 1e3;
|
|
608
|
+
/**
|
|
609
|
+
* Record that this mirror just served a client.
|
|
610
|
+
*
|
|
611
|
+
* A marker file rather than an in-memory timestamp because the reader and the
|
|
612
|
+
* evictor can be different tasks. Best-effort on purpose: failing to record a
|
|
613
|
+
* serve must never fail the serve itself, and the cost of missing one is a
|
|
614
|
+
* mirror that becomes eligible for eviction slightly early.
|
|
615
|
+
*/
|
|
616
|
+
function markServed(local) {
|
|
617
|
+
const now = /* @__PURE__ */ new Date();
|
|
618
|
+
writeFile(servedMarkerPath(local), "", "utf8").then(() => utimes(servedMarkerPath(local), now, now)).catch((error) => log("debug", "served_mark_failed", {
|
|
619
|
+
local,
|
|
620
|
+
error: String(error)
|
|
621
|
+
}));
|
|
622
|
+
}
|
|
623
|
+
async function readLastServedMs(local) {
|
|
624
|
+
try {
|
|
625
|
+
return (await stat(servedMarkerPath(local))).mtimeMs;
|
|
626
|
+
} catch (_error) {
|
|
627
|
+
return 0;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
async function pathExists(p) {
|
|
631
|
+
try {
|
|
632
|
+
await stat(p);
|
|
633
|
+
return true;
|
|
634
|
+
} catch (_error) {
|
|
635
|
+
return false;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
function copyRequestHeaders(headers) {
|
|
639
|
+
const result = {};
|
|
640
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
641
|
+
const lower = name.toLowerCase();
|
|
642
|
+
if ([
|
|
643
|
+
"host",
|
|
644
|
+
"connection",
|
|
645
|
+
"authorization",
|
|
646
|
+
"proxy-authorization",
|
|
647
|
+
"transfer-encoding",
|
|
648
|
+
"content-length",
|
|
649
|
+
"content-encoding"
|
|
650
|
+
].includes(lower)) continue;
|
|
651
|
+
result[name] = value;
|
|
652
|
+
}
|
|
653
|
+
return result;
|
|
654
|
+
}
|
|
655
|
+
/** Merge copied client headers with the cache's own upstream credential. */
|
|
656
|
+
function withCredential(headers, credHeader) {
|
|
657
|
+
const merged = { ...headers };
|
|
658
|
+
if (credHeader) merged["authorization"] = credHeader;
|
|
659
|
+
return merged;
|
|
660
|
+
}
|
|
661
|
+
/**
|
|
662
|
+
* Scheme-aware upstream request: https in production, http only for the
|
|
663
|
+
* test-origin override (config.testOriginBase). Everything else is identical.
|
|
664
|
+
*/
|
|
665
|
+
function requestUpstream(target, options, callback) {
|
|
666
|
+
return (target.protocol === "http:" ? http : https).request(target, options, callback);
|
|
667
|
+
}
|
|
668
|
+
function openUpstream(req, upstreamUrl, operation, callback, credHeader) {
|
|
669
|
+
const up = requestUpstream(new URL(upstreamUrl + (operation === "info/refs" ? "/info/refs?service=git-upload-pack" : "/git-upload-pack")), {
|
|
670
|
+
method: req.method,
|
|
671
|
+
headers: withCredential(copyRequestHeaders(req.headers), credHeader)
|
|
672
|
+
}, (res) => callback(res, null));
|
|
673
|
+
up.on("error", (error) => callback(null, error));
|
|
674
|
+
if (req.method === "POST") req.pipe(up);
|
|
675
|
+
else up.end();
|
|
676
|
+
return up;
|
|
677
|
+
}
|
|
678
|
+
function parseAdvertisement(buffer) {
|
|
679
|
+
const refs = [];
|
|
680
|
+
let offset = 0;
|
|
681
|
+
let sha256 = false;
|
|
682
|
+
while (offset + 4 <= buffer.length) {
|
|
683
|
+
const header = buffer.subarray(offset, offset + 4).toString("ascii");
|
|
684
|
+
if (!/^[0-9a-f]{4}$/i.test(header)) break;
|
|
685
|
+
const size = Number.parseInt(header, 16);
|
|
686
|
+
offset += 4;
|
|
687
|
+
if (size === 0 || size === 1 || size === 2) continue;
|
|
688
|
+
if (size < 4 || offset + size - 4 > buffer.length) break;
|
|
689
|
+
let line = buffer.subarray(offset, offset + size - 4);
|
|
690
|
+
offset += size - 4;
|
|
691
|
+
const nul = line.indexOf(0);
|
|
692
|
+
if (nul >= 0 && line.includes(Buffer.from("object-format=sha256"))) sha256 = true;
|
|
693
|
+
if (nul >= 0) line = line.subarray(0, nul);
|
|
694
|
+
const [, oid, ref] = line.toString("utf8").trim().match(/^([0-9a-f]{40}) (refs\/[A-Za-z0-9._/-]+|HEAD)$/) ?? [];
|
|
695
|
+
if (oid && ref && !ref.endsWith("^{}")) refs.push({
|
|
696
|
+
oid,
|
|
697
|
+
ref
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
return {
|
|
701
|
+
refs,
|
|
702
|
+
sha256
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* Inspect a client's git-upload-pack request body to decide cache eligibility.
|
|
707
|
+
* We only ingest packs that can form a COMPLETE standalone mirror: no filter,
|
|
708
|
+
* no shallow/deepen, and no client-provided `have` lines (which would make the
|
|
709
|
+
* server omit objects the client already had). Everything else is passed
|
|
710
|
+
* through with ingestion disabled — correctness over coverage.
|
|
711
|
+
*
|
|
712
|
+
* Returns { eligible, reason }.
|
|
713
|
+
*/
|
|
714
|
+
/**
|
|
715
|
+
* True when a `git-upload-pack` POST body speaks protocol v2.
|
|
716
|
+
*
|
|
717
|
+
* v2 replaces v0's bare want/have lines with a leading `command=<verb>` line,
|
|
718
|
+
* so the first pkt-line identifies the dialect unambiguously.
|
|
719
|
+
*/
|
|
720
|
+
function isProtocolV2Request(buffer) {
|
|
721
|
+
if (buffer.length < 4) return false;
|
|
722
|
+
const header = buffer.subarray(0, 4).toString("ascii");
|
|
723
|
+
if (!/^[0-9a-f]{4}$/i.test(header)) return false;
|
|
724
|
+
const size = Number.parseInt(header, 16);
|
|
725
|
+
if (size < 4 || size > buffer.length) return false;
|
|
726
|
+
return buffer.subarray(4, size).toString("utf8").startsWith("command=");
|
|
727
|
+
}
|
|
728
|
+
function classifyUploadRequest(buffer) {
|
|
729
|
+
const protocolV2 = isProtocolV2Request(buffer);
|
|
730
|
+
let offset = 0;
|
|
731
|
+
let sawWant = false;
|
|
732
|
+
const reasons = [];
|
|
733
|
+
const wants = [];
|
|
734
|
+
while (offset + 4 <= buffer.length) {
|
|
735
|
+
const header = buffer.subarray(offset, offset + 4).toString("ascii");
|
|
736
|
+
if (!/^[0-9a-f]{4}$/i.test(header)) break;
|
|
737
|
+
const size = Number.parseInt(header, 16);
|
|
738
|
+
offset += 4;
|
|
739
|
+
if (size === 0 || size === 1 || size === 2) continue;
|
|
740
|
+
if (size < 4 || offset + size - 4 > buffer.length) break;
|
|
741
|
+
const line = buffer.subarray(offset, offset + size - 4).toString("utf8");
|
|
742
|
+
offset += size - 4;
|
|
743
|
+
const token = line.trim();
|
|
744
|
+
if (token.startsWith("want ")) {
|
|
745
|
+
sawWant = true;
|
|
746
|
+
const oid = token.slice(5).split(" ")[0].split("\0")[0];
|
|
747
|
+
if (/^[0-9a-f]{40}$/i.test(oid)) wants.push(oid.toLowerCase());
|
|
748
|
+
const caps = line.includes("\0") ? line.split("\0")[1] || "" : "";
|
|
749
|
+
if (/(^|\s)filter(\s|=|$)/.test(caps)) reasons.push("filter-cap");
|
|
750
|
+
if (/(^|\s)deepen/.test(caps)) reasons.push("deepen-cap");
|
|
751
|
+
} else if (token.startsWith("have ")) reasons.push("have");
|
|
752
|
+
else if (token.startsWith("shallow ")) reasons.push("shallow");
|
|
753
|
+
else if (token.startsWith("deepen")) reasons.push("deepen");
|
|
754
|
+
else if (token.startsWith("filter ")) reasons.push("filter");
|
|
755
|
+
}
|
|
756
|
+
if (protocolV2) return {
|
|
757
|
+
eligible: false,
|
|
758
|
+
reason: "protocol-v2",
|
|
759
|
+
wants
|
|
760
|
+
};
|
|
761
|
+
if (!sawWant) return {
|
|
762
|
+
eligible: false,
|
|
763
|
+
reason: "no-want",
|
|
764
|
+
wants
|
|
765
|
+
};
|
|
766
|
+
if (reasons.length) return {
|
|
767
|
+
eligible: false,
|
|
768
|
+
reason: [...new Set(reasons)].join(","),
|
|
769
|
+
wants
|
|
770
|
+
};
|
|
771
|
+
return {
|
|
772
|
+
eligible: true,
|
|
773
|
+
reason: "full-clone",
|
|
774
|
+
wants
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
var SidebandPackExtractor = class {
|
|
778
|
+
writable;
|
|
779
|
+
buffer;
|
|
780
|
+
sawPack;
|
|
781
|
+
inPackSection;
|
|
782
|
+
needsDrain;
|
|
783
|
+
errored;
|
|
784
|
+
constructor(writable) {
|
|
785
|
+
this.writable = writable;
|
|
786
|
+
this.buffer = Buffer.alloc(0);
|
|
787
|
+
this.sawPack = false;
|
|
788
|
+
this.inPackSection = false;
|
|
789
|
+
this.needsDrain = false;
|
|
790
|
+
this.errored = false;
|
|
791
|
+
}
|
|
792
|
+
push(chunk) {
|
|
793
|
+
this.buffer = this.buffer.length ? Buffer.concat([this.buffer, chunk]) : chunk;
|
|
794
|
+
while (this.buffer.length >= 4) {
|
|
795
|
+
const header = this.buffer.subarray(0, 4).toString("ascii");
|
|
796
|
+
if (!/^[0-9a-f]{4}$/i.test(header)) throw new Error("invalid upload-pack pkt-line header");
|
|
797
|
+
const size = Number.parseInt(header, 16);
|
|
798
|
+
if (size === 0 || size === 1 || size === 2) {
|
|
799
|
+
this.buffer = this.buffer.subarray(4);
|
|
800
|
+
continue;
|
|
801
|
+
}
|
|
802
|
+
if (size < 5 || this.buffer.length < size) return;
|
|
803
|
+
const packet = this.buffer.subarray(4, size);
|
|
804
|
+
this.buffer = this.buffer.subarray(size);
|
|
805
|
+
if (packet.toString("utf8") === "packfile\n") {
|
|
806
|
+
this.inPackSection = true;
|
|
807
|
+
continue;
|
|
808
|
+
}
|
|
809
|
+
if (packet[0] === 1 && (this.inPackSection || packet.subarray(1, 5).toString("ascii") === "PACK")) {
|
|
810
|
+
this.inPackSection = true;
|
|
811
|
+
this.sawPack = true;
|
|
812
|
+
if (!this.writable.write(packet.subarray(1))) this.needsDrain = true;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
return !this.needsDrain;
|
|
816
|
+
}
|
|
817
|
+
drained() {
|
|
818
|
+
this.needsDrain = false;
|
|
819
|
+
}
|
|
820
|
+
end() {
|
|
821
|
+
if (!this.errored) this.writable.end();
|
|
822
|
+
}
|
|
823
|
+
};
|
|
824
|
+
async function writeMeta(local, extra = {}) {
|
|
825
|
+
const tmp = `${metaPath(local)}.tmp.${INSTANCE_ID}`;
|
|
826
|
+
await writeFile(tmp, JSON.stringify({
|
|
827
|
+
lastFetchMs: Date.now(),
|
|
828
|
+
...extra
|
|
829
|
+
}), "utf8");
|
|
830
|
+
await rename(tmp, metaPath(local));
|
|
831
|
+
}
|
|
832
|
+
async function readMeta(local) {
|
|
833
|
+
try {
|
|
834
|
+
return JSON.parse(await readFile(metaPath(local), "utf8"));
|
|
835
|
+
} catch (_error) {
|
|
836
|
+
return null;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Stop git from repacking or pruning this mirror on its own.
|
|
841
|
+
*
|
|
842
|
+
* `git fetch` runs auto-maintenance afterwards, which repacks and then
|
|
843
|
+
* DELETES the packfiles it superseded. On one machine that is invisible:
|
|
844
|
+
* a reader already holding the file keeps reading it after the unlink. When
|
|
845
|
+
* the store is shared between tasks that guarantee is gone — the deleting
|
|
846
|
+
* task and the reading task are different NFS clients, and the reader's next
|
|
847
|
+
* read of a file deleted underneath it fails with a stale handle mid-stream.
|
|
848
|
+
*
|
|
849
|
+
* Packfiles are only immutable while they exist; maintenance is what removes
|
|
850
|
+
* them. Turning it off here keeps mirror contents append-only, which is what
|
|
851
|
+
* makes concurrent readers safe. Repacking still has to happen eventually,
|
|
852
|
+
* but as a deliberate operation under the mirror lock rather than as a side
|
|
853
|
+
* effect of whichever task happened to fetch.
|
|
854
|
+
*/
|
|
855
|
+
async function disableAutoMaintenance(dir) {
|
|
856
|
+
await runGit([
|
|
857
|
+
"--git-dir",
|
|
858
|
+
dir,
|
|
859
|
+
"config",
|
|
860
|
+
"gc.auto",
|
|
861
|
+
"0"
|
|
862
|
+
]);
|
|
863
|
+
await runGit([
|
|
864
|
+
"--git-dir",
|
|
865
|
+
dir,
|
|
866
|
+
"config",
|
|
867
|
+
"maintenance.auto",
|
|
868
|
+
"false"
|
|
869
|
+
]);
|
|
870
|
+
}
|
|
871
|
+
async function configureMirror(dir, upstreamUrl) {
|
|
872
|
+
await runGit([
|
|
873
|
+
"--git-dir",
|
|
874
|
+
dir,
|
|
875
|
+
"config",
|
|
876
|
+
"uploadpack.allowFilter",
|
|
877
|
+
"true"
|
|
878
|
+
]);
|
|
879
|
+
await runGit([
|
|
880
|
+
"--git-dir",
|
|
881
|
+
dir,
|
|
882
|
+
"config",
|
|
883
|
+
"uploadpack.allowAnySHA1InWant",
|
|
884
|
+
"true"
|
|
885
|
+
]);
|
|
886
|
+
await disableAutoMaintenance(dir);
|
|
887
|
+
await runGit([
|
|
888
|
+
"--git-dir",
|
|
889
|
+
dir,
|
|
890
|
+
"config",
|
|
891
|
+
"remote.origin.url",
|
|
892
|
+
upstreamUrl
|
|
893
|
+
]);
|
|
894
|
+
await runGit([
|
|
895
|
+
"--git-dir",
|
|
896
|
+
dir,
|
|
897
|
+
"config",
|
|
898
|
+
"--replace-all",
|
|
899
|
+
"remote.origin.fetch",
|
|
900
|
+
"+refs/heads/*:refs/heads/*"
|
|
901
|
+
]);
|
|
902
|
+
await runGit([
|
|
903
|
+
"--git-dir",
|
|
904
|
+
dir,
|
|
905
|
+
"config",
|
|
906
|
+
"--add",
|
|
907
|
+
"remote.origin.fetch",
|
|
908
|
+
"+refs/tags/*:refs/tags/*"
|
|
909
|
+
]);
|
|
910
|
+
await runGit([
|
|
911
|
+
"--git-dir",
|
|
912
|
+
dir,
|
|
913
|
+
"config",
|
|
914
|
+
"remote.origin.tagOpt",
|
|
915
|
+
"--tags"
|
|
916
|
+
]);
|
|
917
|
+
}
|
|
918
|
+
async function completeCapturedMirror(dir, upstreamUrl, refs, credHeader = null) {
|
|
919
|
+
await configureMirror(dir, upstreamUrl);
|
|
920
|
+
await runGit(upstreamGitArgs([
|
|
921
|
+
"--git-dir",
|
|
922
|
+
dir,
|
|
923
|
+
"fetch",
|
|
924
|
+
"--quiet",
|
|
925
|
+
"--prune",
|
|
926
|
+
"--prune-tags",
|
|
927
|
+
"origin"
|
|
928
|
+
]), { env: {
|
|
929
|
+
...GIT_ENV,
|
|
930
|
+
...upstreamGitCredentialEnv(credHeader)
|
|
931
|
+
} });
|
|
932
|
+
const advertisedHead = (refs || []).find((r) => r.ref === "HEAD")?.oid;
|
|
933
|
+
const candidates = advertisedHead ? (refs || []).filter((r) => r.ref.startsWith("refs/heads/") && r.oid === advertisedHead) : [];
|
|
934
|
+
if (candidates.length === 1) await runGit([
|
|
935
|
+
"--git-dir",
|
|
936
|
+
dir,
|
|
937
|
+
"symbolic-ref",
|
|
938
|
+
"HEAD",
|
|
939
|
+
candidates[0].ref
|
|
940
|
+
]);
|
|
941
|
+
else try {
|
|
942
|
+
const { stdout } = await runGit(upstreamGitArgs([
|
|
943
|
+
"--git-dir",
|
|
944
|
+
dir,
|
|
945
|
+
"ls-remote",
|
|
946
|
+
"--symref",
|
|
947
|
+
"origin",
|
|
948
|
+
"HEAD"
|
|
949
|
+
]), { env: {
|
|
950
|
+
...GIT_ENV,
|
|
951
|
+
...upstreamGitCredentialEnv(credHeader)
|
|
952
|
+
} });
|
|
953
|
+
const match = stdout.match(/^ref:\s+(refs\/heads\/\S+)\s+HEAD$/m);
|
|
954
|
+
if (match) await runGit([
|
|
955
|
+
"--git-dir",
|
|
956
|
+
dir,
|
|
957
|
+
"symbolic-ref",
|
|
958
|
+
"HEAD",
|
|
959
|
+
match[1]
|
|
960
|
+
]);
|
|
961
|
+
} catch (_error) {}
|
|
962
|
+
await runGit([
|
|
963
|
+
"--git-dir",
|
|
964
|
+
dir,
|
|
965
|
+
"fsck",
|
|
966
|
+
"--connectivity-only",
|
|
967
|
+
"--no-dangling"
|
|
968
|
+
]);
|
|
969
|
+
}
|
|
970
|
+
/** Repos with a background backfill in flight, so two never race. */
|
|
971
|
+
const backfilling = /* @__PURE__ */ new Set();
|
|
972
|
+
/**
|
|
973
|
+
* Whether a request that could NOT be cached is worth building a mirror for
|
|
974
|
+
* out-of-band. Only uncacheable requests need it — a cacheable one already
|
|
975
|
+
* populated the cache — and a sha256 repo is excluded because the
|
|
976
|
+
* advertisement parser cannot read its 64-hex OIDs, so no correct mirror can
|
|
977
|
+
* be completed for it at all.
|
|
978
|
+
*/
|
|
979
|
+
function shouldBackfillAfterPassthrough({ cacheable, sha256 }) {
|
|
980
|
+
return !cacheable && !sha256;
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Build a mirror for a repo whose client request could not seed one.
|
|
984
|
+
*
|
|
985
|
+
* A filtered / shallow / incremental fetch receives a deliberately incomplete
|
|
986
|
+
* pack, so it passes through and populates nothing. Left there it is a dead
|
|
987
|
+
* end, and the worst kind: the agent prompt steers LARGE repos toward
|
|
988
|
+
* `--filter=blob:none`, so the repos the cache would help most are exactly the
|
|
989
|
+
* ones that would never get a mirror. Fetch our own complete copy instead,
|
|
990
|
+
* independently of what the client asked for.
|
|
991
|
+
*
|
|
992
|
+
* Runs AFTER the client's passthrough has finished, so this fetch never
|
|
993
|
+
* competes for bandwidth with the clone an agent is waiting on. Once published,
|
|
994
|
+
* later clones are served locally — filtered ones included, since a complete
|
|
995
|
+
* mirror can satisfy any request shape (`uploadpack.allowFilter` is set on it).
|
|
996
|
+
*/
|
|
997
|
+
async function backfillMirror(repo, upstreamUrl, credHeader = null) {
|
|
998
|
+
if (backfilling.has(repo) || coldLeaders.has(repo)) return;
|
|
999
|
+
const local = localPath(repo);
|
|
1000
|
+
if (await pathExists(local)) return;
|
|
1001
|
+
backfilling.add(repo);
|
|
1002
|
+
const temp = `${config.cacheDir}/quarantine/${repo.replace(/\//g, "__")}.backfill-${INSTANCE_ID}-${Date.now()}`;
|
|
1003
|
+
const startedMs = Date.now();
|
|
1004
|
+
try {
|
|
1005
|
+
log("info", "backfill_start", { repo });
|
|
1006
|
+
await mkdir(path.dirname(local), { recursive: true });
|
|
1007
|
+
await mkdir(path.dirname(temp), { recursive: true });
|
|
1008
|
+
await rm(temp, {
|
|
1009
|
+
recursive: true,
|
|
1010
|
+
force: true
|
|
1011
|
+
});
|
|
1012
|
+
await runGit([
|
|
1013
|
+
"init",
|
|
1014
|
+
"--bare",
|
|
1015
|
+
"--quiet",
|
|
1016
|
+
temp
|
|
1017
|
+
]);
|
|
1018
|
+
await completeCapturedMirror(temp, upstreamUrl, null, credHeader);
|
|
1019
|
+
const published = await withRepoLock(repo, "write", async () => {
|
|
1020
|
+
if (await pathExists(local)) return false;
|
|
1021
|
+
await rename(temp, local);
|
|
1022
|
+
await writeMeta(local, { publishedMs: Date.now() });
|
|
1023
|
+
return true;
|
|
1024
|
+
});
|
|
1025
|
+
if (published !== true) {
|
|
1026
|
+
if (published === null) {
|
|
1027
|
+
metrics.publish_deferred++;
|
|
1028
|
+
log("info", "publish_deferred", { repo });
|
|
1029
|
+
}
|
|
1030
|
+
await rm(temp, {
|
|
1031
|
+
recursive: true,
|
|
1032
|
+
force: true
|
|
1033
|
+
});
|
|
1034
|
+
return;
|
|
1035
|
+
}
|
|
1036
|
+
metrics.publications++;
|
|
1037
|
+
metrics.backfills++;
|
|
1038
|
+
log("info", "backfill_ready", {
|
|
1039
|
+
repo,
|
|
1040
|
+
ms: Date.now() - startedMs
|
|
1041
|
+
});
|
|
1042
|
+
await maybeEvict();
|
|
1043
|
+
} catch (error) {
|
|
1044
|
+
metrics.backfill_failures++;
|
|
1045
|
+
log("error", "backfill_failed", {
|
|
1046
|
+
repo,
|
|
1047
|
+
error: String(error)
|
|
1048
|
+
});
|
|
1049
|
+
await rm(temp, {
|
|
1050
|
+
recursive: true,
|
|
1051
|
+
force: true
|
|
1052
|
+
}).catch((_error) => log("debug", "cleanup_failed", { path: temp }));
|
|
1053
|
+
} finally {
|
|
1054
|
+
backfilling.delete(repo);
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
function startColdStream(req, res, upstreamUrl, repo, requestBody, credHeader) {
|
|
1058
|
+
const local = localPath(repo);
|
|
1059
|
+
const temp = `${config.cacheDir}/quarantine/${repo.replace(/\//g, "__")}.stream-${INSTANCE_ID}-${Date.now()}`;
|
|
1060
|
+
metrics.cold_requests++;
|
|
1061
|
+
coldLeaders.add(repo);
|
|
1062
|
+
log("info", "cold_start", { repo });
|
|
1063
|
+
const eligibility = classifyUploadRequest(requestBody);
|
|
1064
|
+
const isSha256 = advertisements.get(repo)?.sha256 === true;
|
|
1065
|
+
const wantCache = eligibility.eligible && !isSha256;
|
|
1066
|
+
if (!wantCache) {
|
|
1067
|
+
metrics.passthrough_ineligible++;
|
|
1068
|
+
log("info", "cold_ineligible", {
|
|
1069
|
+
repo,
|
|
1070
|
+
reason: isSha256 ? "sha256-object-format" : eligibility.reason
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
let settle;
|
|
1074
|
+
const done = new Promise((resolve, reject) => {
|
|
1075
|
+
settle = {
|
|
1076
|
+
resolve,
|
|
1077
|
+
reject
|
|
1078
|
+
};
|
|
1079
|
+
});
|
|
1080
|
+
(async () => {
|
|
1081
|
+
await mkdir(path.dirname(local), { recursive: true });
|
|
1082
|
+
await mkdir(path.dirname(temp), { recursive: true });
|
|
1083
|
+
if (wantCache) {
|
|
1084
|
+
await rm(temp, {
|
|
1085
|
+
recursive: true,
|
|
1086
|
+
force: true
|
|
1087
|
+
});
|
|
1088
|
+
await runGit([
|
|
1089
|
+
"init",
|
|
1090
|
+
"--bare",
|
|
1091
|
+
"--quiet",
|
|
1092
|
+
temp
|
|
1093
|
+
]);
|
|
1094
|
+
}
|
|
1095
|
+
const up = requestUpstream(new URL(upstreamUrl + "/git-upload-pack"), {
|
|
1096
|
+
method: "POST",
|
|
1097
|
+
headers: withCredential(copyRequestHeaders(req.headers), credHeader)
|
|
1098
|
+
}, (upRes) => {
|
|
1099
|
+
res.writeHead(upRes.statusCode, upRes.headers);
|
|
1100
|
+
if (upRes.statusCode !== 200) {
|
|
1101
|
+
if (upRes.statusCode === 401 || upRes.statusCode === 403) dropCredential(repo.split("/")[0]);
|
|
1102
|
+
metrics.upstream_errors++;
|
|
1103
|
+
upRes.pipe(res);
|
|
1104
|
+
upRes.on("end", () => settle.reject(/* @__PURE__ */ new Error(`upstream HTTP ${upRes.statusCode}`)));
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
if (!wantCache) {
|
|
1108
|
+
upRes.on("data", (c) => {
|
|
1109
|
+
metrics.cold_proxy_bytes += c.length;
|
|
1110
|
+
});
|
|
1111
|
+
upRes.pipe(res);
|
|
1112
|
+
upRes.on("end", () => settle.resolve(null));
|
|
1113
|
+
upRes.on("error", (e) => settle.reject(e));
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
const index = spawn("git", [
|
|
1117
|
+
"--git-dir",
|
|
1118
|
+
temp,
|
|
1119
|
+
"index-pack",
|
|
1120
|
+
"--stdin",
|
|
1121
|
+
"--fix-thin",
|
|
1122
|
+
"--keep=git-cache-cold"
|
|
1123
|
+
], {
|
|
1124
|
+
stdio: [
|
|
1125
|
+
"pipe",
|
|
1126
|
+
"pipe",
|
|
1127
|
+
"pipe"
|
|
1128
|
+
],
|
|
1129
|
+
env: GIT_ENV
|
|
1130
|
+
});
|
|
1131
|
+
let indexOut = "";
|
|
1132
|
+
let indexErr = "";
|
|
1133
|
+
index.stdout.on("data", (d) => indexOut += d);
|
|
1134
|
+
index.stderr.on("data", (d) => {
|
|
1135
|
+
if (indexErr.length < 256 * 1024) indexErr += d;
|
|
1136
|
+
});
|
|
1137
|
+
const extractor = new SidebandPackExtractor(index.stdin);
|
|
1138
|
+
let streamError = null;
|
|
1139
|
+
let responseNeedsDrain = false;
|
|
1140
|
+
let capturedBytes = 0;
|
|
1141
|
+
let queuedBytes = 0;
|
|
1142
|
+
let abandoned = false;
|
|
1143
|
+
let clientGone = false;
|
|
1144
|
+
const abandonCache = (reason, metric) => {
|
|
1145
|
+
if (abandoned) return;
|
|
1146
|
+
abandoned = true;
|
|
1147
|
+
extractor.errored = true;
|
|
1148
|
+
metrics[metric]++;
|
|
1149
|
+
log("warn", "cold_abandon", {
|
|
1150
|
+
repo,
|
|
1151
|
+
reason
|
|
1152
|
+
});
|
|
1153
|
+
try {
|
|
1154
|
+
index.stdin.destroy();
|
|
1155
|
+
} catch (_error) {}
|
|
1156
|
+
try {
|
|
1157
|
+
index.kill("SIGTERM");
|
|
1158
|
+
} catch (_error) {}
|
|
1159
|
+
};
|
|
1160
|
+
const resumeUpstream = () => {
|
|
1161
|
+
if (!responseNeedsDrain && !extractor.needsDrain) upRes.resume();
|
|
1162
|
+
};
|
|
1163
|
+
res.on("drain", () => {
|
|
1164
|
+
responseNeedsDrain = false;
|
|
1165
|
+
resumeUpstream();
|
|
1166
|
+
});
|
|
1167
|
+
index.stdin.on("drain", () => {
|
|
1168
|
+
queuedBytes = 0;
|
|
1169
|
+
extractor.drained();
|
|
1170
|
+
resumeUpstream();
|
|
1171
|
+
});
|
|
1172
|
+
index.stdin.on("error", () => {});
|
|
1173
|
+
upRes.on("data", (chunk) => {
|
|
1174
|
+
metrics.cold_proxy_bytes += chunk.length;
|
|
1175
|
+
if (!clientGone && !res.write(chunk)) responseNeedsDrain = true;
|
|
1176
|
+
if (!abandoned) {
|
|
1177
|
+
capturedBytes += chunk.length;
|
|
1178
|
+
if (config.maxPackBytes && capturedBytes > config.maxPackBytes) abandonCache("oversize", "abandoned_oversize");
|
|
1179
|
+
else try {
|
|
1180
|
+
if (!extractor.push(chunk)) {
|
|
1181
|
+
queuedBytes += chunk.length;
|
|
1182
|
+
if (queuedBytes > config.maxImporterQueueBytes) abandonCache("queue-overflow", "abandoned_queue_overflow");
|
|
1183
|
+
}
|
|
1184
|
+
} catch (error) {
|
|
1185
|
+
streamError = error instanceof Error ? error : new Error(String(error));
|
|
1186
|
+
abandonCache("parse-error", "abandoned_import_error");
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
if (responseNeedsDrain) upRes.pause();
|
|
1190
|
+
});
|
|
1191
|
+
upRes.on("end", () => {
|
|
1192
|
+
if (!clientGone) res.end();
|
|
1193
|
+
if (!abandoned) extractor.end();
|
|
1194
|
+
});
|
|
1195
|
+
upRes.on("error", (error) => {
|
|
1196
|
+
streamError = error;
|
|
1197
|
+
res.destroy(error);
|
|
1198
|
+
abandonCache("upstream-error", "abandoned_import_error");
|
|
1199
|
+
});
|
|
1200
|
+
res.on("close", () => {
|
|
1201
|
+
if (!res.writableEnded) {
|
|
1202
|
+
clientGone = true;
|
|
1203
|
+
responseNeedsDrain = false;
|
|
1204
|
+
resumeUpstream();
|
|
1205
|
+
log("debug", "client_disconnect_continue_warm", { repo });
|
|
1206
|
+
}
|
|
1207
|
+
});
|
|
1208
|
+
index.on("close", async (code) => {
|
|
1209
|
+
try {
|
|
1210
|
+
if (abandoned) {
|
|
1211
|
+
await rm(temp, {
|
|
1212
|
+
recursive: true,
|
|
1213
|
+
force: true
|
|
1214
|
+
});
|
|
1215
|
+
settle.resolve(null);
|
|
1216
|
+
return;
|
|
1217
|
+
}
|
|
1218
|
+
if (streamError) throw streamError instanceof Error ? streamError : new Error(String(streamError));
|
|
1219
|
+
if (!extractor.sawPack) {
|
|
1220
|
+
await rm(temp, {
|
|
1221
|
+
recursive: true,
|
|
1222
|
+
force: true
|
|
1223
|
+
});
|
|
1224
|
+
settle.resolve(null);
|
|
1225
|
+
return;
|
|
1226
|
+
}
|
|
1227
|
+
if (code !== 0) throw new Error(`index-pack exited ${code}: ${indexErr.trim()}`);
|
|
1228
|
+
metrics.captured_pack_bytes += capturedBytes;
|
|
1229
|
+
await completeCapturedMirror(temp, upstreamUrl, advertisements.get(repo)?.refs, credHeader);
|
|
1230
|
+
const published = await withRepoLock(repo, "write", async () => {
|
|
1231
|
+
if (await pathExists(local)) return false;
|
|
1232
|
+
await rename(temp, local);
|
|
1233
|
+
await writeMeta(local, { publishedMs: Date.now() });
|
|
1234
|
+
return true;
|
|
1235
|
+
});
|
|
1236
|
+
if (published === true) {
|
|
1237
|
+
metrics.publications++;
|
|
1238
|
+
log("info", "cold_ready", {
|
|
1239
|
+
repo,
|
|
1240
|
+
pack: indexOut.trim()
|
|
1241
|
+
});
|
|
1242
|
+
await maybeEvict();
|
|
1243
|
+
} else {
|
|
1244
|
+
if (published === null) {
|
|
1245
|
+
metrics.publish_deferred++;
|
|
1246
|
+
log("info", "publish_deferred", { repo });
|
|
1247
|
+
}
|
|
1248
|
+
await rm(temp, {
|
|
1249
|
+
recursive: true,
|
|
1250
|
+
force: true
|
|
1251
|
+
});
|
|
1252
|
+
}
|
|
1253
|
+
settle.resolve(local);
|
|
1254
|
+
} catch (error) {
|
|
1255
|
+
settle.reject(error);
|
|
1256
|
+
}
|
|
1257
|
+
});
|
|
1258
|
+
});
|
|
1259
|
+
up.on("error", (error) => settle.reject(error));
|
|
1260
|
+
up.end(requestBody);
|
|
1261
|
+
})().catch(settle.reject);
|
|
1262
|
+
done.catch(async (error) => {
|
|
1263
|
+
recordColdBuildOutcome(error);
|
|
1264
|
+
if (error) log("error", "cold_build_failed", {
|
|
1265
|
+
repo,
|
|
1266
|
+
error: String(error)
|
|
1267
|
+
});
|
|
1268
|
+
await rm(temp, {
|
|
1269
|
+
recursive: true,
|
|
1270
|
+
force: true
|
|
1271
|
+
}).catch((_error) => log("debug", "cleanup_failed", { path: temp }));
|
|
1272
|
+
}).finally(() => {
|
|
1273
|
+
coldLeaders.delete(repo);
|
|
1274
|
+
advertisements.delete(repo);
|
|
1275
|
+
if (shouldBackfillAfterPassthrough({
|
|
1276
|
+
cacheable: wantCache,
|
|
1277
|
+
sha256: isSha256
|
|
1278
|
+
})) backfillMirror(repo, upstreamUrl, credHeader);
|
|
1279
|
+
});
|
|
1280
|
+
}
|
|
1281
|
+
async function refreshMirror(repo, local, credHeader = null) {
|
|
1282
|
+
await runGit([
|
|
1283
|
+
"--git-dir",
|
|
1284
|
+
local,
|
|
1285
|
+
"config",
|
|
1286
|
+
"uploadpack.allowFilter",
|
|
1287
|
+
"true"
|
|
1288
|
+
]);
|
|
1289
|
+
await runGit([
|
|
1290
|
+
"--git-dir",
|
|
1291
|
+
local,
|
|
1292
|
+
"config",
|
|
1293
|
+
"uploadpack.allowAnySHA1InWant",
|
|
1294
|
+
"true"
|
|
1295
|
+
]);
|
|
1296
|
+
await disableAutoMaintenance(local);
|
|
1297
|
+
await runGit(upstreamGitArgs([
|
|
1298
|
+
"--git-dir",
|
|
1299
|
+
local,
|
|
1300
|
+
"fetch",
|
|
1301
|
+
"--quiet",
|
|
1302
|
+
"--prune",
|
|
1303
|
+
"--prune-tags",
|
|
1304
|
+
"origin"
|
|
1305
|
+
]), { env: {
|
|
1306
|
+
...GIT_ENV,
|
|
1307
|
+
...upstreamGitCredentialEnv(credHeader)
|
|
1308
|
+
} });
|
|
1309
|
+
await writeMeta(local);
|
|
1310
|
+
metrics.refreshes++;
|
|
1311
|
+
log("info", "refresh", { repo });
|
|
1312
|
+
return local;
|
|
1313
|
+
}
|
|
1314
|
+
async function ensureWarm(repo, credHeader = null) {
|
|
1315
|
+
const local = localPath(repo);
|
|
1316
|
+
const lastFetchMs = (await readMeta(local))?.lastFetchMs ?? 0;
|
|
1317
|
+
if (!((Date.now() - lastFetchMs) / 1e3 >= config.ttlSeconds || lastFetchMs < invalidateBeforeMs)) {
|
|
1318
|
+
metrics.warm_hits++;
|
|
1319
|
+
return {
|
|
1320
|
+
local,
|
|
1321
|
+
cacheState: "HIT"
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1324
|
+
const mustBlock = lastFetchMs < invalidateBeforeMs;
|
|
1325
|
+
if (!inflightRefresh.has(repo)) inflightRefresh.set(repo, withRepoLock(repo, "write", () => refreshMirror(repo, local, credHeader)).then((result) => {
|
|
1326
|
+
if (result === null) {
|
|
1327
|
+
metrics.refresh_deferred++;
|
|
1328
|
+
log("info", "refresh_deferred", { repo });
|
|
1329
|
+
}
|
|
1330
|
+
}).catch((error) => {
|
|
1331
|
+
metrics.refresh_failures++;
|
|
1332
|
+
log("warn", "refresh_failed", {
|
|
1333
|
+
repo,
|
|
1334
|
+
error: String(error)
|
|
1335
|
+
});
|
|
1336
|
+
}).finally(() => inflightRefresh.delete(repo)));
|
|
1337
|
+
if (mustBlock) {
|
|
1338
|
+
await inflightRefresh.get(repo);
|
|
1339
|
+
if (((await readMeta(local))?.lastFetchMs ?? 0) < invalidateBeforeMs) throw new Error("mirror refresh failed after push invalidation");
|
|
1340
|
+
return {
|
|
1341
|
+
local,
|
|
1342
|
+
cacheState: "REFRESH"
|
|
1343
|
+
};
|
|
1344
|
+
}
|
|
1345
|
+
return {
|
|
1346
|
+
local,
|
|
1347
|
+
cacheState: "STALE"
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
/**
|
|
1351
|
+
* Translate the client's `Git-Protocol` header into the environment variable
|
|
1352
|
+
* `git-upload-pack` reads it from. Absent header means v0, which is git's own
|
|
1353
|
+
* default, so nothing is set.
|
|
1354
|
+
*/
|
|
1355
|
+
function gitProtocolEnv(headers) {
|
|
1356
|
+
const raw = headers["git-protocol"];
|
|
1357
|
+
const value = Array.isArray(raw) ? raw[0] : raw;
|
|
1358
|
+
return value ? { GIT_PROTOCOL: value } : {};
|
|
1359
|
+
}
|
|
1360
|
+
/**
|
|
1361
|
+
* True when every OID is already present in the mirror.
|
|
1362
|
+
*
|
|
1363
|
+
* `git-upload-pack` refuses the whole request with `not our ref <oid>` if a
|
|
1364
|
+
* single wanted object is missing, so this is the difference between serving
|
|
1365
|
+
* a fetch and failing it. A mirror can legitimately lack a wanted object: the
|
|
1366
|
+
* client negotiated against a fresher advertisement — from a sibling task
|
|
1367
|
+
* holding its own mirror set, or from the sandbox-local cache tier — and asks
|
|
1368
|
+
* for a commit this mirror has not fetched yet.
|
|
1369
|
+
*/
|
|
1370
|
+
async function mirrorHasObjects(local, oids) {
|
|
1371
|
+
if (oids.length === 0) return true;
|
|
1372
|
+
const unique = [...new Set(oids)];
|
|
1373
|
+
try {
|
|
1374
|
+
const { stdout } = await runGitWithInput([
|
|
1375
|
+
"--git-dir",
|
|
1376
|
+
local,
|
|
1377
|
+
"cat-file",
|
|
1378
|
+
"--batch-check=%(objectname) %(objecttype)"
|
|
1379
|
+
], unique.map((oid) => `${oid}\n`).join(""));
|
|
1380
|
+
return !/\bmissing\b/.test(stdout);
|
|
1381
|
+
} catch (_error) {
|
|
1382
|
+
return false;
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
/** runGit, but with a body written to the child's stdin. */
|
|
1386
|
+
function runGitWithInput(args, input) {
|
|
1387
|
+
return new Promise((resolve, reject) => {
|
|
1388
|
+
const child = spawn("git", args, {
|
|
1389
|
+
env: GIT_ENV,
|
|
1390
|
+
stdio: [
|
|
1391
|
+
"pipe",
|
|
1392
|
+
"pipe",
|
|
1393
|
+
"pipe"
|
|
1394
|
+
]
|
|
1395
|
+
});
|
|
1396
|
+
let stdout = "";
|
|
1397
|
+
let stderr = "";
|
|
1398
|
+
child.stdout.on("data", (d) => {
|
|
1399
|
+
if (stdout.length < 256 * 1024) stdout += d;
|
|
1400
|
+
});
|
|
1401
|
+
child.stderr.on("data", (d) => {
|
|
1402
|
+
if (stderr.length < 256 * 1024) stderr += d;
|
|
1403
|
+
});
|
|
1404
|
+
child.on("error", reject);
|
|
1405
|
+
child.on("close", (code) => {
|
|
1406
|
+
if (code === 0) resolve({
|
|
1407
|
+
stdout,
|
|
1408
|
+
stderr
|
|
1409
|
+
});
|
|
1410
|
+
else reject(/* @__PURE__ */ new Error(`git exited ${code}: ${stderr.trim()}`));
|
|
1411
|
+
});
|
|
1412
|
+
child.stdin.on("error", (error) => {
|
|
1413
|
+
if (errnoCode(error) !== "EPIPE") reject(error);
|
|
1414
|
+
});
|
|
1415
|
+
child.stdin.end(input);
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
/**
|
|
1419
|
+
* Forward an already-buffered `git-upload-pack` POST to origin and stream the
|
|
1420
|
+
* response straight back, for a fetch this cache cannot serve itself.
|
|
1421
|
+
*/
|
|
1422
|
+
function proxyUploadPackWithBody(res, upstreamUrl, reqHeaders, body, credHeader) {
|
|
1423
|
+
const up = requestUpstream(new URL(upstreamUrl + "/git-upload-pack"), {
|
|
1424
|
+
method: "POST",
|
|
1425
|
+
headers: withCredential(copyRequestHeaders(reqHeaders), credHeader)
|
|
1426
|
+
}, (upRes) => {
|
|
1427
|
+
res.writeHead(upRes.statusCode, upRes.headers);
|
|
1428
|
+
upRes.pipe(res);
|
|
1429
|
+
});
|
|
1430
|
+
up.on("error", () => {
|
|
1431
|
+
metrics.upstream_errors++;
|
|
1432
|
+
if (!res.headersSent) res.writeHead(502);
|
|
1433
|
+
res.end();
|
|
1434
|
+
});
|
|
1435
|
+
up.end(body);
|
|
1436
|
+
}
|
|
1437
|
+
function serveUploadPack(req, res, local, advertise, cacheState, body = null) {
|
|
1438
|
+
const args = ["--stateless-rpc"];
|
|
1439
|
+
if (advertise) args.push("--advertise-refs");
|
|
1440
|
+
args.push(local);
|
|
1441
|
+
const child = spawn("git-upload-pack", args, {
|
|
1442
|
+
stdio: [
|
|
1443
|
+
"pipe",
|
|
1444
|
+
"pipe",
|
|
1445
|
+
"pipe"
|
|
1446
|
+
],
|
|
1447
|
+
env: {
|
|
1448
|
+
...GIT_ENV,
|
|
1449
|
+
...gitProtocolEnv(req.headers)
|
|
1450
|
+
}
|
|
1451
|
+
});
|
|
1452
|
+
res.statusCode = 200;
|
|
1453
|
+
res.setHeader("Cache-Control", "no-store");
|
|
1454
|
+
res.setHeader("X-Git-Cache", cacheState);
|
|
1455
|
+
res.setHeader("Content-Type", advertise ? "application/x-git-upload-pack-advertisement" : "application/x-git-upload-pack-result");
|
|
1456
|
+
if (advertise) {
|
|
1457
|
+
res.write("001e# service=git-upload-pack\n0000");
|
|
1458
|
+
child.stdin.end();
|
|
1459
|
+
} else if (body !== null) child.stdin.end(body);
|
|
1460
|
+
else if ((req.headers["content-encoding"] || "").toLowerCase() === "gzip") {
|
|
1461
|
+
const gunzip = createGunzip();
|
|
1462
|
+
req.pipe(gunzip).pipe(child.stdin);
|
|
1463
|
+
} else req.pipe(child.stdin);
|
|
1464
|
+
child.stdout.pipe(res);
|
|
1465
|
+
let stderr = "";
|
|
1466
|
+
let finished = false;
|
|
1467
|
+
child.stderr.on("data", (d) => {
|
|
1468
|
+
if (stderr.length < 64 * 1024) stderr += d;
|
|
1469
|
+
});
|
|
1470
|
+
child.on("close", (code) => {
|
|
1471
|
+
finished = true;
|
|
1472
|
+
if (code !== 0) log("error", "upload_pack_failed", {
|
|
1473
|
+
local,
|
|
1474
|
+
code,
|
|
1475
|
+
stderr: stderr.trim()
|
|
1476
|
+
});
|
|
1477
|
+
if (!res.writableEnded) res.end();
|
|
1478
|
+
});
|
|
1479
|
+
res.on("close", () => {
|
|
1480
|
+
if (!finished && !child.killed) child.kill("SIGTERM");
|
|
1481
|
+
});
|
|
1482
|
+
}
|
|
1483
|
+
async function dirSize(dir) {
|
|
1484
|
+
let total = 0;
|
|
1485
|
+
const stack = [dir];
|
|
1486
|
+
while (stack.length) {
|
|
1487
|
+
const cur = stack.pop();
|
|
1488
|
+
if (cur === void 0) continue;
|
|
1489
|
+
let entries;
|
|
1490
|
+
try {
|
|
1491
|
+
entries = await readdir(cur, { withFileTypes: true });
|
|
1492
|
+
} catch (_error) {
|
|
1493
|
+
continue;
|
|
1494
|
+
}
|
|
1495
|
+
for (const e of entries) {
|
|
1496
|
+
const p = path.join(cur, e.name);
|
|
1497
|
+
if (e.isDirectory()) stack.push(p);
|
|
1498
|
+
else try {
|
|
1499
|
+
total += (await stat(p)).size;
|
|
1500
|
+
} catch (_error) {}
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
return total;
|
|
1504
|
+
}
|
|
1505
|
+
/**
|
|
1506
|
+
* Invert localPath(): recover the cache's repo key ("host/owner/repo") from a
|
|
1507
|
+
* mirror's filesystem path. coldLeaders/inflightRefresh are keyed by repo, so
|
|
1508
|
+
* eviction must compare in repo-space — comparing the raw path against those
|
|
1509
|
+
* sets silently never matches, and eviction could rm a mirror that is mid-cold
|
|
1510
|
+
* publication or mid-refresh.
|
|
1511
|
+
*/
|
|
1512
|
+
function repoFromMirrorPath(mirrorPath) {
|
|
1513
|
+
const root = path.join(config.cacheDir, "mirrors");
|
|
1514
|
+
return path.relative(root, mirrorPath).replace(/\.git$/, "");
|
|
1515
|
+
}
|
|
1516
|
+
async function listMirrors() {
|
|
1517
|
+
const root = path.join(config.cacheDir, "mirrors");
|
|
1518
|
+
const out = [];
|
|
1519
|
+
const stack = [root];
|
|
1520
|
+
while (stack.length) {
|
|
1521
|
+
const cur = stack.pop();
|
|
1522
|
+
if (cur === void 0) continue;
|
|
1523
|
+
let entries;
|
|
1524
|
+
try {
|
|
1525
|
+
entries = await readdir(cur, { withFileTypes: true });
|
|
1526
|
+
} catch (_error) {
|
|
1527
|
+
continue;
|
|
1528
|
+
}
|
|
1529
|
+
for (const e of entries) {
|
|
1530
|
+
const p = path.join(cur, e.name);
|
|
1531
|
+
if (e.isDirectory() && e.name.endsWith(".git")) out.push(p);
|
|
1532
|
+
else if (e.isDirectory()) stack.push(p);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
return out;
|
|
1536
|
+
}
|
|
1537
|
+
let evicting = false;
|
|
1538
|
+
async function maybeEvict() {
|
|
1539
|
+
if (!config.maxCacheBytes || evicting) return;
|
|
1540
|
+
evicting = true;
|
|
1541
|
+
try {
|
|
1542
|
+
const mirrors = await listMirrors();
|
|
1543
|
+
let total = 0;
|
|
1544
|
+
const sized = [];
|
|
1545
|
+
for (const m of mirrors) {
|
|
1546
|
+
const size = await dirSize(m);
|
|
1547
|
+
const meta = await readMeta(m);
|
|
1548
|
+
total += size;
|
|
1549
|
+
sized.push({
|
|
1550
|
+
m,
|
|
1551
|
+
size,
|
|
1552
|
+
lastFetchMs: meta?.lastFetchMs ?? 0,
|
|
1553
|
+
lastServedMs: await readLastServedMs(m)
|
|
1554
|
+
});
|
|
1555
|
+
}
|
|
1556
|
+
if (total <= config.maxCacheBytes) return;
|
|
1557
|
+
sized.sort((a, b) => a.lastFetchMs - b.lastFetchMs);
|
|
1558
|
+
for (const entry of sized) {
|
|
1559
|
+
if (total <= config.maxCacheBytes) break;
|
|
1560
|
+
const repo = repoFromMirrorPath(entry.m);
|
|
1561
|
+
if (coldLeaders.has(repo) || inflightRefresh.has(repo)) continue;
|
|
1562
|
+
if (Date.now() - entry.lastServedMs < EVICTION_QUIET_MS) continue;
|
|
1563
|
+
if (await withRepoLock(repo, "write", async () => {
|
|
1564
|
+
await rm(entry.m, {
|
|
1565
|
+
recursive: true,
|
|
1566
|
+
force: true
|
|
1567
|
+
});
|
|
1568
|
+
await rm(metaPath(entry.m), { force: true }).catch((_error) => log("debug", "meta_cleanup_failed", {}));
|
|
1569
|
+
await rm(servedMarkerPath(entry.m), { force: true }).catch((_error) => log("debug", "served_cleanup_failed", {}));
|
|
1570
|
+
return true;
|
|
1571
|
+
}) === null) continue;
|
|
1572
|
+
total -= entry.size;
|
|
1573
|
+
metrics.evictions++;
|
|
1574
|
+
log("info", "evict", {
|
|
1575
|
+
mirror: entry.m,
|
|
1576
|
+
freed: entry.size
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
} finally {
|
|
1580
|
+
evicting = false;
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
async function cleanupQuarantine() {
|
|
1584
|
+
const q = path.join(config.cacheDir, "quarantine");
|
|
1585
|
+
await rm(q, {
|
|
1586
|
+
recursive: true,
|
|
1587
|
+
force: true
|
|
1588
|
+
}).catch((_error) => log("debug", "quarantine_clear_failed", {}));
|
|
1589
|
+
await mkdir(q, { recursive: true }).catch((_error) => log("debug", "quarantine_mkdir_failed", {}));
|
|
1590
|
+
}
|
|
1591
|
+
function readBody(req, limit) {
|
|
1592
|
+
return new Promise((resolve, reject) => {
|
|
1593
|
+
const chunks = [];
|
|
1594
|
+
let size = 0;
|
|
1595
|
+
req.on("data", (c) => {
|
|
1596
|
+
size += c.length;
|
|
1597
|
+
if (size > limit) {
|
|
1598
|
+
reject(/* @__PURE__ */ new Error("request body too large"));
|
|
1599
|
+
req.destroy();
|
|
1600
|
+
return;
|
|
1601
|
+
}
|
|
1602
|
+
chunks.push(c);
|
|
1603
|
+
});
|
|
1604
|
+
req.on("end", () => resolve(Buffer.concat(chunks)));
|
|
1605
|
+
req.on("error", reject);
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1608
|
+
/** True when the TCP peer of this request is the local machine itself. */
|
|
1609
|
+
function isLoopbackPeer(req) {
|
|
1610
|
+
const addr = req.socket.remoteAddress;
|
|
1611
|
+
return addr === "127.0.0.1" || addr === "::1" || addr === "::ffff:127.0.0.1";
|
|
1612
|
+
}
|
|
1613
|
+
async function handle(req, res) {
|
|
1614
|
+
const parsed = new URL(req.url, "http://git-cache.local");
|
|
1615
|
+
if (parsed.pathname === "/healthz") {
|
|
1616
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1617
|
+
return res.end(JSON.stringify({
|
|
1618
|
+
ok: true,
|
|
1619
|
+
bind: config.bindHost,
|
|
1620
|
+
cacheDir: config.cacheDir,
|
|
1621
|
+
ttlSeconds: config.ttlSeconds,
|
|
1622
|
+
privateRepos: Boolean(config.credentialHelper),
|
|
1623
|
+
authMode: config.authMode,
|
|
1624
|
+
coldMode: "stream-through",
|
|
1625
|
+
invalidateBeforeMs
|
|
1626
|
+
}));
|
|
1627
|
+
}
|
|
1628
|
+
if (parsed.pathname === "/metrics") {
|
|
1629
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1630
|
+
return res.end(JSON.stringify(metricsSnapshot()));
|
|
1631
|
+
}
|
|
1632
|
+
if (parsed.pathname === "/invalidate-all" && req.method === "POST") {
|
|
1633
|
+
if (config.authMode === "check-through" && !isLoopbackPeer(req)) {
|
|
1634
|
+
metrics.auth_denials++;
|
|
1635
|
+
log("info", "invalidate_all_denied", { peer: req.socket.remoteAddress ?? null });
|
|
1636
|
+
res.writeHead(403, { "Content-Type": "text/plain" });
|
|
1637
|
+
return res.end("invalidate-all is operator-only on a shared cache\n");
|
|
1638
|
+
}
|
|
1639
|
+
invalidateBeforeMs = Date.now();
|
|
1640
|
+
metrics.invalidations++;
|
|
1641
|
+
log("info", "invalidate_all", {});
|
|
1642
|
+
res.writeHead(204);
|
|
1643
|
+
return res.end();
|
|
1644
|
+
}
|
|
1645
|
+
const up = parseUpstream(parsed.pathname, parsed.search);
|
|
1646
|
+
if (up && up.redirect !== void 0) {
|
|
1647
|
+
if (up.operation === "lfs") metrics.lfs_redirects++;
|
|
1648
|
+
else metrics.credential_url_redirects++;
|
|
1649
|
+
log("info", "upstream_redirect", { op: up.operation });
|
|
1650
|
+
res.writeHead(307, { Location: up.redirect });
|
|
1651
|
+
return res.end();
|
|
1652
|
+
}
|
|
1653
|
+
if (!up || !("repo" in up) || !validateMethod(req, up.operation)) {
|
|
1654
|
+
metrics.bad_requests++;
|
|
1655
|
+
log("debug", "bad_request", {
|
|
1656
|
+
pathname: parsed.pathname,
|
|
1657
|
+
method: req.method,
|
|
1658
|
+
parsed: up ? up.operation : null
|
|
1659
|
+
});
|
|
1660
|
+
const hint = up ? null : explainRejection(parsed.pathname);
|
|
1661
|
+
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
1662
|
+
return res.end(hint ? `git-cache: ${hint}; this remote cannot be proxied — run \`git-cache disable\` to have git talk to it directly\n` : "git cache endpoint\n");
|
|
1663
|
+
}
|
|
1664
|
+
try {
|
|
1665
|
+
let clientCred = null;
|
|
1666
|
+
if (config.authMode === "check-through") {
|
|
1667
|
+
const header = req.headers["authorization"];
|
|
1668
|
+
const raw = Array.isArray(header) ? header[0] : header;
|
|
1669
|
+
if (!raw) {
|
|
1670
|
+
metrics.auth_denials++;
|
|
1671
|
+
res.writeHead(401, {
|
|
1672
|
+
"Content-Type": "text/plain",
|
|
1673
|
+
"WWW-Authenticate": "Basic realm=\"git-cache\""
|
|
1674
|
+
});
|
|
1675
|
+
return res.end("authentication required\n");
|
|
1676
|
+
}
|
|
1677
|
+
const verdict = await authorizeCheckThrough(raw, up.repo, up.upstreamUrl);
|
|
1678
|
+
if (!verdict.allowed) {
|
|
1679
|
+
log("info", "auth_denied", {
|
|
1680
|
+
repo: up.repo,
|
|
1681
|
+
status: verdict.status
|
|
1682
|
+
});
|
|
1683
|
+
res.writeHead(verdict.status, { "Content-Type": "text/plain" });
|
|
1684
|
+
return res.end("not authorized for this repository\n");
|
|
1685
|
+
}
|
|
1686
|
+
clientCred = raw;
|
|
1687
|
+
}
|
|
1688
|
+
const host = up.repo.split("/")[0];
|
|
1689
|
+
let uploadBody = null;
|
|
1690
|
+
let uploadWants = [];
|
|
1691
|
+
if (up.operation === "git-upload-pack") {
|
|
1692
|
+
try {
|
|
1693
|
+
uploadBody = await readBody(req, 16 * 1024 * 1024);
|
|
1694
|
+
} catch (_error) {
|
|
1695
|
+
metrics.bad_requests++;
|
|
1696
|
+
if (!res.headersSent) res.writeHead(413);
|
|
1697
|
+
return res.end("request too large\n");
|
|
1698
|
+
}
|
|
1699
|
+
if ((req.headers["content-encoding"] || "").toLowerCase() === "gzip") try {
|
|
1700
|
+
uploadBody = Buffer.from(await gunzip$1(uploadBody));
|
|
1701
|
+
} catch (_error) {
|
|
1702
|
+
metrics.bad_requests++;
|
|
1703
|
+
if (!res.headersSent) res.writeHead(400);
|
|
1704
|
+
return res.end("bad gzip body\n");
|
|
1705
|
+
}
|
|
1706
|
+
uploadWants = classifyUploadRequest(uploadBody).wants;
|
|
1707
|
+
}
|
|
1708
|
+
if (await pathExists(localPath(up.repo))) {
|
|
1709
|
+
const warm = await ensureWarm(up.repo, clientCred);
|
|
1710
|
+
markServed(warm.local);
|
|
1711
|
+
if (up.operation === "git-upload-pack" && !await mirrorHasObjects(warm.local, uploadWants)) {
|
|
1712
|
+
await refreshMirror(up.repo, warm.local, clientCred).catch((error) => {
|
|
1713
|
+
metrics.refresh_failures++;
|
|
1714
|
+
log("warn", "refresh_failed", {
|
|
1715
|
+
repo: up.repo,
|
|
1716
|
+
error: String(error)
|
|
1717
|
+
});
|
|
1718
|
+
});
|
|
1719
|
+
if (!await mirrorHasObjects(warm.local, uploadWants)) {
|
|
1720
|
+
metrics.passthrough_missing_want++;
|
|
1721
|
+
log("info", "passthrough_missing_want", { repo: up.repo });
|
|
1722
|
+
const credHeader = clientCred ?? await fetchCredentialHeader(host);
|
|
1723
|
+
return proxyUploadPackWithBody(res, up.upstreamUrl, req.headers, uploadBody, credHeader);
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
log("info", "serve", {
|
|
1727
|
+
repo: up.repo,
|
|
1728
|
+
op: up.operation,
|
|
1729
|
+
state: warm.cacheState
|
|
1730
|
+
});
|
|
1731
|
+
return serveUploadPack(req, res, warm.local, up.operation === "info/refs", warm.cacheState, uploadBody);
|
|
1732
|
+
}
|
|
1733
|
+
if (up.operation === "info/refs") {
|
|
1734
|
+
const credHeader = clientCred ?? await fetchCredentialHeader(host);
|
|
1735
|
+
return await new Promise((resolve, reject) => {
|
|
1736
|
+
openUpstream(req, up.upstreamUrl, up.operation, (upRes, error) => {
|
|
1737
|
+
if (error) {
|
|
1738
|
+
reject(error);
|
|
1739
|
+
return;
|
|
1740
|
+
}
|
|
1741
|
+
if (upRes.statusCode === 401 || upRes.statusCode === 403) dropCredential(host);
|
|
1742
|
+
res.writeHead(upRes.statusCode, upRes.headers);
|
|
1743
|
+
const chunks = [];
|
|
1744
|
+
let bytes = 0;
|
|
1745
|
+
const CAP = 32 * 1024 * 1024;
|
|
1746
|
+
upRes.on("data", (chunk) => {
|
|
1747
|
+
res.write(chunk);
|
|
1748
|
+
if (bytes < CAP) {
|
|
1749
|
+
chunks.push(chunk);
|
|
1750
|
+
bytes += chunk.length;
|
|
1751
|
+
}
|
|
1752
|
+
});
|
|
1753
|
+
upRes.on("end", () => {
|
|
1754
|
+
res.end();
|
|
1755
|
+
if (upRes.statusCode === 200 && bytes < CAP && gitProtocolEnv(req.headers).GIT_PROTOCOL === void 0) advertisements.set(up.repo, parseAdvertisement(Buffer.concat(chunks)));
|
|
1756
|
+
resolve(void 0);
|
|
1757
|
+
});
|
|
1758
|
+
upRes.on("error", reject);
|
|
1759
|
+
}, credHeader);
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
const body = uploadBody ?? Buffer.alloc(0);
|
|
1763
|
+
const credHeader = clientCred ?? await fetchCredentialHeader(host);
|
|
1764
|
+
if (coldLeaders.has(up.repo)) return proxyColdFollowerWithBody(res, up.upstreamUrl, req.headers, body, up.repo, credHeader);
|
|
1765
|
+
return startColdStream(req, res, up.upstreamUrl, up.repo, body, credHeader);
|
|
1766
|
+
} catch (error) {
|
|
1767
|
+
metrics.upstream_errors++;
|
|
1768
|
+
log("error", "request_failed", { error: String(error) });
|
|
1769
|
+
if (!res.headersSent) res.writeHead(502, { "Content-Type": "text/plain" });
|
|
1770
|
+
res.end("unable to serve repository\n");
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
/** A follower whose body we already buffered (leader path buffers too). */
|
|
1774
|
+
function proxyColdFollowerWithBody(res, upstreamUrl, reqHeaders, body, repo, credHeader) {
|
|
1775
|
+
metrics.cold_followers++;
|
|
1776
|
+
log("info", "cold_follower", { repo });
|
|
1777
|
+
const up = requestUpstream(new URL(upstreamUrl + "/git-upload-pack"), {
|
|
1778
|
+
method: "POST",
|
|
1779
|
+
headers: withCredential(copyRequestHeaders(reqHeaders), credHeader)
|
|
1780
|
+
}, (upRes) => {
|
|
1781
|
+
res.writeHead(upRes.statusCode, upRes.headers);
|
|
1782
|
+
upRes.pipe(res);
|
|
1783
|
+
});
|
|
1784
|
+
up.on("error", () => {
|
|
1785
|
+
metrics.upstream_errors++;
|
|
1786
|
+
if (!res.headersSent) res.writeHead(502);
|
|
1787
|
+
res.end();
|
|
1788
|
+
});
|
|
1789
|
+
up.end(body);
|
|
1790
|
+
}
|
|
1791
|
+
async function startServer(options = {}) {
|
|
1792
|
+
if (options.authMode !== void 0) config.authMode = options.authMode;
|
|
1793
|
+
if (options.bindHost !== void 0) config.bindHost = options.bindHost;
|
|
1794
|
+
if (options.port !== void 0) config.port = options.port;
|
|
1795
|
+
if (options.cacheDir !== void 0) config.cacheDir = options.cacheDir;
|
|
1796
|
+
if (options.maxCacheBytes !== void 0) config.maxCacheBytes = options.maxCacheBytes;
|
|
1797
|
+
if (options.credentialHelper !== void 0) config.credentialHelper = options.credentialHelper;
|
|
1798
|
+
if (!(config.bindHost === "127.0.0.1" || config.bindHost === "::1" || config.bindHost === "localhost") && config.authMode !== "check-through") throw new Error(`git-cache: refusing to bind ${config.bindHost} without check-through authorization — a non-loopback trusted-client cache would serve every mirrored repo to anyone who can reach it. Set authMode: 'check-through' (or GIT_CACHE_AUTH_MODE=check-through), or bind loopback.`);
|
|
1799
|
+
if (config.authMode === "check-through" && config.credentialHelper) throw new Error("git-cache: credentialHelper is incompatible with check-through authorization — upstream requests authenticate with each client's own credential.");
|
|
1800
|
+
await mkdir(path.join(config.cacheDir, "mirrors"), { recursive: true });
|
|
1801
|
+
await cleanupQuarantine();
|
|
1802
|
+
const server = http.createServer((req, res) => {
|
|
1803
|
+
handle(req, res).catch((error) => {
|
|
1804
|
+
log("error", "unhandled", { error: String(error) });
|
|
1805
|
+
if (!res.headersSent) res.writeHead(500);
|
|
1806
|
+
res.end();
|
|
1807
|
+
});
|
|
1808
|
+
});
|
|
1809
|
+
server.requestTimeout = 360 * 60 * 1e3;
|
|
1810
|
+
server.headersTimeout = 60 * 1e3;
|
|
1811
|
+
server.keepAliveTimeout = 65 * 1e3;
|
|
1812
|
+
await new Promise((resolve) => server.listen(config.port, config.bindHost, () => resolve(void 0)));
|
|
1813
|
+
log("info", "listening", {
|
|
1814
|
+
bind: `${config.bindHost}:${config.port}`,
|
|
1815
|
+
cacheDir: config.cacheDir,
|
|
1816
|
+
ttlSeconds: config.ttlSeconds,
|
|
1817
|
+
private: Boolean(config.credentialHelper),
|
|
1818
|
+
authMode: config.authMode
|
|
1819
|
+
});
|
|
1820
|
+
const shutdown = () => {
|
|
1821
|
+
log("info", "shutdown", {});
|
|
1822
|
+
server.close(() => process.exit(0));
|
|
1823
|
+
setTimeout(() => process.exit(0), 5e3).unref();
|
|
1824
|
+
};
|
|
1825
|
+
process.on("SIGTERM", shutdown);
|
|
1826
|
+
process.on("SIGINT", shutdown);
|
|
1827
|
+
}
|
|
1828
|
+
//#endregion
|
|
1829
|
+
export { isProtocolV2Request as a, parseAdvertisement as c, shouldBackfillAfterPassthrough as d, startServer as f, explainRejection as i, parseUpstream as l, classifyUploadRequest as n, metricsSnapshot as o, copyRequestHeaders as r, mirrorHasObjects as s, SidebandPackExtractor as t, recordColdBuildOutcome as u };
|