@skydiveai/git-cache 0.1.0 → 0.1.251-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +1 -1
- package/dist/index.d.mts +29 -0
- package/dist/index.mjs +1 -1
- package/dist/{server-B5u5DETT.mjs → server-DjJs3KcN.mjs} +197 -64
- package/package.json +2 -12
package/dist/cli.mjs
CHANGED
package/dist/index.d.mts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import http from "node:http";
|
|
1
2
|
import * as _$node_stream0 from "node:stream";
|
|
2
3
|
|
|
3
4
|
//#region src/server.d.ts
|
|
@@ -19,6 +20,8 @@ declare function metricsSnapshot(): {
|
|
|
19
20
|
evictions: number;
|
|
20
21
|
refresh_failures: number;
|
|
21
22
|
refresh_deferred: number;
|
|
23
|
+
revalidations: number;
|
|
24
|
+
revalidate_failures: number;
|
|
22
25
|
publish_deferred: number;
|
|
23
26
|
credential_url_redirects: number;
|
|
24
27
|
lfs_redirects: number;
|
|
@@ -148,6 +151,32 @@ declare function shouldBackfillAfterPassthrough({
|
|
|
148
151
|
cacheable: any;
|
|
149
152
|
sha256: any;
|
|
150
153
|
}): boolean;
|
|
154
|
+
/**
|
|
155
|
+
* Decide how to serve a warm mirror for one request.
|
|
156
|
+
*
|
|
157
|
+
* `revalidateRefs` is set for the `info/refs` half of a fetch — the ref
|
|
158
|
+
* advertisement, where the client learns the remote's HEAD and branch tips.
|
|
159
|
+
* That answer must never be stale: serving a pre-merge advertisement is how a
|
|
160
|
+
* `git fetch` right after an upstream squash-merge silently returns the old
|
|
161
|
+
* `origin/<branch>` (the object-reuse win of the cache is not worth lying about
|
|
162
|
+
* where HEAD is). So for `info/refs` a stale mirror BLOCKS on a synchronous
|
|
163
|
+
* refresh before it answers, exactly as an explicit push invalidation does.
|
|
164
|
+
* The `git-upload-pack` half keeps stale-while-revalidate: its wants come from
|
|
165
|
+
* the just-served fresh advertisement, so the client only asks for objects the
|
|
166
|
+
* fresh refs named, and the caller's `mirrorHasObjects` gate already falls back
|
|
167
|
+
* to origin for any the mirror is missing (the new objects a merge introduced).
|
|
168
|
+
*
|
|
169
|
+
* A blocked refresh that FAILS (network blip, expired token, upstream 5xx)
|
|
170
|
+
* returns `cacheState: 'STALE_REVALIDATE_FAILED'` rather than throwing, so the
|
|
171
|
+
* caller can proxy `info/refs` straight to origin — fresh refs from upstream,
|
|
172
|
+
* never a 502 and never the stale advertisement.
|
|
173
|
+
*/
|
|
174
|
+
/**
|
|
175
|
+
* Whether a mirror stamped `lastFetchMs` is fresh: past the invalidation fence
|
|
176
|
+
* AND within the TTL window. `lastFetchMs` is a fetch-START time (see
|
|
177
|
+
* writeMeta), so a fetch that began before the fence is stale no matter when
|
|
178
|
+
* it completed — the data it holds predates the invalidation by construction.
|
|
179
|
+
*/
|
|
151
180
|
/**
|
|
152
181
|
* True when every OID is already present in the mirror.
|
|
153
182
|
*
|
package/dist/index.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { a as isProtocolV2Request, c as parseAdvertisement, d as shouldBackfillAfterPassthrough, f as startServer, i as explainRejection, l as parseUpstream, n as classifyUploadRequest, o as metricsSnapshot, r as copyRequestHeaders, s as mirrorHasObjects, t as SidebandPackExtractor, u as recordColdBuildOutcome } from "./server-
|
|
1
|
+
import { a as isProtocolV2Request, c as parseAdvertisement, d as shouldBackfillAfterPassthrough, f as startServer, i as explainRejection, l as parseUpstream, n as classifyUploadRequest, o as metricsSnapshot, r as copyRequestHeaders, s as mirrorHasObjects, t as SidebandPackExtractor, u as recordColdBuildOutcome } from "./server-DjJs3KcN.mjs";
|
|
2
2
|
export { SidebandPackExtractor, classifyUploadRequest, copyRequestHeaders, explainRejection, isProtocolV2Request, metricsSnapshot, mirrorHasObjects, parseAdvertisement, parseUpstream, recordColdBuildOutcome, shouldBackfillAfterPassthrough, startServer };
|
|
@@ -5,6 +5,7 @@ import { spawn } from "node:child_process";
|
|
|
5
5
|
import { mkdir, readFile, readdir, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
|
|
6
6
|
import { hostname } from "node:os";
|
|
7
7
|
import path from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
8
9
|
import { createGunzip, gunzip } from "node:zlib";
|
|
9
10
|
import { promisify } from "node:util";
|
|
10
11
|
//#region src/server.ts
|
|
@@ -46,6 +47,16 @@ import { promisify } from "node:util";
|
|
|
46
47
|
* with no npm install.
|
|
47
48
|
*/
|
|
48
49
|
const gunzip$1 = promisify(gunzip);
|
|
50
|
+
let runningVersion = null;
|
|
51
|
+
async function readOwnPackageVersion() {
|
|
52
|
+
try {
|
|
53
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
54
|
+
const manifest = JSON.parse(await readFile(path.join(here, "..", "package.json"), "utf8"));
|
|
55
|
+
return typeof manifest.version === "string" ? manifest.version : null;
|
|
56
|
+
} catch (_error) {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
49
60
|
/**
|
|
50
61
|
* An unset/empty env resolves to trusted-client (the daemon's original
|
|
51
62
|
* contract); anything other than the two known modes is a hard startup
|
|
@@ -67,6 +78,7 @@ const config = {
|
|
|
67
78
|
maxPackBytes: Number(process.env.GIT_CACHE_MAX_PACK_BYTES || 8 * 1024 * 1024 * 1024),
|
|
68
79
|
maxImporterQueueBytes: Number(process.env.GIT_CACHE_MAX_IMPORTER_QUEUE_BYTES || 256 * 1024 * 1024),
|
|
69
80
|
maxCacheBytes: Number(process.env.GIT_CACHE_MAX_CACHE_BYTES || 0),
|
|
81
|
+
upstreamTimeoutMs: Number(process.env.GIT_CACHE_UPSTREAM_TIMEOUT_MS || 3e4),
|
|
70
82
|
hostAllowlist: (process.env.GIT_CACHE_HOST_ALLOWLIST || "").split(",").map((h) => h.trim().toLowerCase()).filter(Boolean),
|
|
71
83
|
logLevel: process.env.GIT_CACHE_LOG_LEVEL || "info",
|
|
72
84
|
testOriginBase: (process.env.GIT_CACHE_TEST_ORIGIN_BASE || "").replace(/\/+$/, "")
|
|
@@ -83,7 +95,9 @@ const GIT_ENV = {
|
|
|
83
95
|
GIT_ASKPASS: "/bin/true",
|
|
84
96
|
SSH_ASKPASS: "/bin/true",
|
|
85
97
|
GCM_INTERACTIVE: "never",
|
|
86
|
-
GIT_CONFIG_NOSYSTEM: process.env.GIT_CONFIG_NOSYSTEM || "1"
|
|
98
|
+
GIT_CONFIG_NOSYSTEM: process.env.GIT_CONFIG_NOSYSTEM || "1",
|
|
99
|
+
GIT_HTTP_LOW_SPEED_LIMIT: "1024",
|
|
100
|
+
GIT_HTTP_LOW_SPEED_TIME: "60"
|
|
87
101
|
};
|
|
88
102
|
const activeLevel = LOG_LEVELS[config.logLevel] ?? LOG_LEVELS.info;
|
|
89
103
|
function log(level, event, fields = {}) {
|
|
@@ -116,6 +130,8 @@ const metrics = {
|
|
|
116
130
|
evictions: 0,
|
|
117
131
|
refresh_failures: 0,
|
|
118
132
|
refresh_deferred: 0,
|
|
133
|
+
revalidations: 0,
|
|
134
|
+
revalidate_failures: 0,
|
|
119
135
|
publish_deferred: 0,
|
|
120
136
|
credential_url_redirects: 0,
|
|
121
137
|
lfs_redirects: 0,
|
|
@@ -296,12 +312,12 @@ function evictAuthCache() {
|
|
|
296
312
|
}
|
|
297
313
|
function validateUpstream(credHeader, upstreamUrl) {
|
|
298
314
|
return new Promise((resolve) => {
|
|
299
|
-
const
|
|
315
|
+
const target = new URL(`${upstreamUrl}/info/refs?service=git-upload-pack`);
|
|
316
|
+
const headers = { "user-agent": "git/git-cache-authz" };
|
|
317
|
+
if (credHeader !== null) headers["authorization"] = credHeader;
|
|
318
|
+
const request = requestUpstream(target, {
|
|
300
319
|
method: "GET",
|
|
301
|
-
headers
|
|
302
|
-
authorization: credHeader,
|
|
303
|
-
"user-agent": "git/git-cache-authz"
|
|
304
|
-
}
|
|
320
|
+
headers
|
|
305
321
|
}, (res) => {
|
|
306
322
|
res.resume();
|
|
307
323
|
const status = res.statusCode ?? 502;
|
|
@@ -339,7 +355,7 @@ function validateUpstream(credHeader, upstreamUrl) {
|
|
|
339
355
|
});
|
|
340
356
|
}
|
|
341
357
|
async function authorizeCheckThrough(credHeader, repo, upstreamUrl) {
|
|
342
|
-
const key = authMemoKey(credHeader, repo);
|
|
358
|
+
const key = authMemoKey(credHeader ?? "", repo);
|
|
343
359
|
const cached = authCache.get(key);
|
|
344
360
|
if (cached) {
|
|
345
361
|
const ttl = cached.allowed ? AUTH_ALLOW_TTL_MS : AUTH_DENY_TTL_MS;
|
|
@@ -435,14 +451,12 @@ function parseUpstream(rawPathname, rawSearch = "") {
|
|
|
435
451
|
const hadDotGit = Boolean(match[3]);
|
|
436
452
|
const operation = match[4];
|
|
437
453
|
let host = authority;
|
|
438
|
-
let port = 443;
|
|
439
454
|
const colon = authority.lastIndexOf(":");
|
|
440
455
|
if (colon !== -1) {
|
|
441
456
|
host = authority.slice(0, colon);
|
|
442
457
|
const p = authority.slice(colon + 1);
|
|
443
458
|
if (!/^\d{1,5}$/.test(p)) return null;
|
|
444
|
-
|
|
445
|
-
if (port !== 443) return null;
|
|
459
|
+
if (Number(p) !== 443) return null;
|
|
446
460
|
}
|
|
447
461
|
host = host.toLowerCase();
|
|
448
462
|
if (isIpLiteral(host)) return null;
|
|
@@ -665,12 +679,35 @@ function withCredential(headers, credHeader) {
|
|
|
665
679
|
function requestUpstream(target, options, callback) {
|
|
666
680
|
return (target.protocol === "http:" ? http : https).request(target, options, callback);
|
|
667
681
|
}
|
|
682
|
+
/**
|
|
683
|
+
* Arm the idle-socket ceiling on an upstream request. Fires only when the
|
|
684
|
+
* socket is SILENT for config.upstreamTimeoutMs (any activity resets it), so a
|
|
685
|
+
* healthy transfer — even a slow one — is never interrupted. Must be armed on
|
|
686
|
+
* EVERY upstream request: a hung upstream surfaces as a fast error to the
|
|
687
|
+
* caller instead of a client `git fetch` that never returns.
|
|
688
|
+
*/
|
|
689
|
+
function armUpstreamIdleTimeout(up, onTimeout) {
|
|
690
|
+
up.setTimeout(config.upstreamTimeoutMs, () => {
|
|
691
|
+
log("error", "upstream_idle_timeout", { ms: config.upstreamTimeoutMs });
|
|
692
|
+
onTimeout();
|
|
693
|
+
});
|
|
694
|
+
}
|
|
668
695
|
function openUpstream(req, upstreamUrl, operation, callback, credHeader) {
|
|
669
|
-
const
|
|
696
|
+
const target = new URL(upstreamUrl + (operation === "info/refs" ? "/info/refs?service=git-upload-pack" : "/git-upload-pack"));
|
|
697
|
+
let settled = false;
|
|
698
|
+
const settle = (resOrNull, error) => {
|
|
699
|
+
if (settled) return;
|
|
700
|
+
settled = true;
|
|
701
|
+
callback(resOrNull, error);
|
|
702
|
+
};
|
|
703
|
+
const up = requestUpstream(target, {
|
|
670
704
|
method: req.method,
|
|
671
705
|
headers: withCredential(copyRequestHeaders(req.headers), credHeader)
|
|
672
|
-
}, (res) =>
|
|
673
|
-
up.on("error", (error) =>
|
|
706
|
+
}, (res) => settle(res, null));
|
|
707
|
+
up.on("error", (error) => settle(null, error));
|
|
708
|
+
armUpstreamIdleTimeout(up, () => {
|
|
709
|
+
up.destroy(/* @__PURE__ */ new Error(`upstream idle for ${config.upstreamTimeoutMs}ms`));
|
|
710
|
+
});
|
|
674
711
|
if (req.method === "POST") req.pipe(up);
|
|
675
712
|
else up.end();
|
|
676
713
|
return up;
|
|
@@ -821,10 +858,10 @@ var SidebandPackExtractor = class {
|
|
|
821
858
|
if (!this.errored) this.writable.end();
|
|
822
859
|
}
|
|
823
860
|
};
|
|
824
|
-
async function writeMeta(local, extra = {}) {
|
|
861
|
+
async function writeMeta(local, fetchStartMs, extra = {}) {
|
|
825
862
|
const tmp = `${metaPath(local)}.tmp.${INSTANCE_ID}`;
|
|
826
863
|
await writeFile(tmp, JSON.stringify({
|
|
827
|
-
lastFetchMs:
|
|
864
|
+
lastFetchMs: fetchStartMs,
|
|
828
865
|
...extra
|
|
829
866
|
}), "utf8");
|
|
830
867
|
await rename(tmp, metaPath(local));
|
|
@@ -1019,7 +1056,7 @@ async function backfillMirror(repo, upstreamUrl, credHeader = null) {
|
|
|
1019
1056
|
const published = await withRepoLock(repo, "write", async () => {
|
|
1020
1057
|
if (await pathExists(local)) return false;
|
|
1021
1058
|
await rename(temp, local);
|
|
1022
|
-
await writeMeta(local, { publishedMs: Date.now() });
|
|
1059
|
+
await writeMeta(local, startedMs, { publishedMs: Date.now() });
|
|
1023
1060
|
return true;
|
|
1024
1061
|
});
|
|
1025
1062
|
if (published !== true) {
|
|
@@ -1056,6 +1093,7 @@ async function backfillMirror(repo, upstreamUrl, credHeader = null) {
|
|
|
1056
1093
|
}
|
|
1057
1094
|
function startColdStream(req, res, upstreamUrl, repo, requestBody, credHeader) {
|
|
1058
1095
|
const local = localPath(repo);
|
|
1096
|
+
const captureStartedMs = Date.now();
|
|
1059
1097
|
const temp = `${config.cacheDir}/quarantine/${repo.replace(/\//g, "__")}.stream-${INSTANCE_ID}-${Date.now()}`;
|
|
1060
1098
|
metrics.cold_requests++;
|
|
1061
1099
|
coldLeaders.add(repo);
|
|
@@ -1230,7 +1268,7 @@ function startColdStream(req, res, upstreamUrl, repo, requestBody, credHeader) {
|
|
|
1230
1268
|
const published = await withRepoLock(repo, "write", async () => {
|
|
1231
1269
|
if (await pathExists(local)) return false;
|
|
1232
1270
|
await rename(temp, local);
|
|
1233
|
-
await writeMeta(local, { publishedMs: Date.now() });
|
|
1271
|
+
await writeMeta(local, captureStartedMs, { publishedMs: Date.now() });
|
|
1234
1272
|
return true;
|
|
1235
1273
|
});
|
|
1236
1274
|
if (published === true) {
|
|
@@ -1257,6 +1295,13 @@ function startColdStream(req, res, upstreamUrl, repo, requestBody, credHeader) {
|
|
|
1257
1295
|
});
|
|
1258
1296
|
});
|
|
1259
1297
|
up.on("error", (error) => settle.reject(error));
|
|
1298
|
+
armUpstreamIdleTimeout(up, () => {
|
|
1299
|
+
if (!res.headersSent) {
|
|
1300
|
+
res.writeHead(502, { "Content-Type": "text/plain" });
|
|
1301
|
+
res.end("unable to serve repository\n");
|
|
1302
|
+
}
|
|
1303
|
+
up.destroy(/* @__PURE__ */ new Error(`upstream idle for ${config.upstreamTimeoutMs}ms`));
|
|
1304
|
+
});
|
|
1260
1305
|
up.end(requestBody);
|
|
1261
1306
|
})().catch(settle.reject);
|
|
1262
1307
|
done.catch(async (error) => {
|
|
@@ -1294,6 +1339,7 @@ async function refreshMirror(repo, local, credHeader = null) {
|
|
|
1294
1339
|
"true"
|
|
1295
1340
|
]);
|
|
1296
1341
|
await disableAutoMaintenance(local);
|
|
1342
|
+
const fetchStartMs = Date.now();
|
|
1297
1343
|
await runGit(upstreamGitArgs([
|
|
1298
1344
|
"--git-dir",
|
|
1299
1345
|
local,
|
|
@@ -1306,22 +1352,54 @@ async function refreshMirror(repo, local, credHeader = null) {
|
|
|
1306
1352
|
...GIT_ENV,
|
|
1307
1353
|
...upstreamGitCredentialEnv(credHeader)
|
|
1308
1354
|
} });
|
|
1309
|
-
await writeMeta(local);
|
|
1355
|
+
await writeMeta(local, fetchStartMs);
|
|
1310
1356
|
metrics.refreshes++;
|
|
1311
1357
|
log("info", "refresh", { repo });
|
|
1312
1358
|
return local;
|
|
1313
1359
|
}
|
|
1314
|
-
|
|
1360
|
+
/**
|
|
1361
|
+
* Decide how to serve a warm mirror for one request.
|
|
1362
|
+
*
|
|
1363
|
+
* `revalidateRefs` is set for the `info/refs` half of a fetch — the ref
|
|
1364
|
+
* advertisement, where the client learns the remote's HEAD and branch tips.
|
|
1365
|
+
* That answer must never be stale: serving a pre-merge advertisement is how a
|
|
1366
|
+
* `git fetch` right after an upstream squash-merge silently returns the old
|
|
1367
|
+
* `origin/<branch>` (the object-reuse win of the cache is not worth lying about
|
|
1368
|
+
* where HEAD is). So for `info/refs` a stale mirror BLOCKS on a synchronous
|
|
1369
|
+
* refresh before it answers, exactly as an explicit push invalidation does.
|
|
1370
|
+
* The `git-upload-pack` half keeps stale-while-revalidate: its wants come from
|
|
1371
|
+
* the just-served fresh advertisement, so the client only asks for objects the
|
|
1372
|
+
* fresh refs named, and the caller's `mirrorHasObjects` gate already falls back
|
|
1373
|
+
* to origin for any the mirror is missing (the new objects a merge introduced).
|
|
1374
|
+
*
|
|
1375
|
+
* A blocked refresh that FAILS (network blip, expired token, upstream 5xx)
|
|
1376
|
+
* returns `cacheState: 'STALE_REVALIDATE_FAILED'` rather than throwing, so the
|
|
1377
|
+
* caller can proxy `info/refs` straight to origin — fresh refs from upstream,
|
|
1378
|
+
* never a 502 and never the stale advertisement.
|
|
1379
|
+
*/
|
|
1380
|
+
/**
|
|
1381
|
+
* Whether a mirror stamped `lastFetchMs` is fresh: past the invalidation fence
|
|
1382
|
+
* AND within the TTL window. `lastFetchMs` is a fetch-START time (see
|
|
1383
|
+
* writeMeta), so a fetch that began before the fence is stale no matter when
|
|
1384
|
+
* it completed — the data it holds predates the invalidation by construction.
|
|
1385
|
+
*/
|
|
1386
|
+
function isMetaFresh(lastFetchMs, fenceMs, ttlSeconds, nowMs) {
|
|
1387
|
+
if (lastFetchMs <= 0) return false;
|
|
1388
|
+
if (lastFetchMs > nowMs) return false;
|
|
1389
|
+
return lastFetchMs >= fenceMs && (nowMs - lastFetchMs) / 1e3 < ttlSeconds;
|
|
1390
|
+
}
|
|
1391
|
+
async function ensureWarm(repo, credHeader = null, revalidateRefs = false) {
|
|
1315
1392
|
const local = localPath(repo);
|
|
1316
1393
|
const lastFetchMs = (await readMeta(local))?.lastFetchMs ?? 0;
|
|
1317
|
-
if (
|
|
1394
|
+
if (!!isMetaFresh(lastFetchMs, invalidateBeforeMs, config.ttlSeconds, Date.now())) {
|
|
1318
1395
|
metrics.warm_hits++;
|
|
1319
1396
|
return {
|
|
1320
1397
|
local,
|
|
1321
1398
|
cacheState: "HIT"
|
|
1322
1399
|
};
|
|
1323
1400
|
}
|
|
1324
|
-
const
|
|
1401
|
+
const fencedByPush = lastFetchMs < invalidateBeforeMs;
|
|
1402
|
+
const mustBlock = fencedByPush || revalidateRefs;
|
|
1325
1403
|
if (!inflightRefresh.has(repo)) inflightRefresh.set(repo, withRepoLock(repo, "write", () => refreshMirror(repo, local, credHeader)).then((result) => {
|
|
1326
1404
|
if (result === null) {
|
|
1327
1405
|
metrics.refresh_deferred++;
|
|
@@ -1336,7 +1414,18 @@ async function ensureWarm(repo, credHeader = null) {
|
|
|
1336
1414
|
}).finally(() => inflightRefresh.delete(repo)));
|
|
1337
1415
|
if (mustBlock) {
|
|
1338
1416
|
await inflightRefresh.get(repo);
|
|
1339
|
-
if (((await readMeta(local))?.lastFetchMs ?? 0
|
|
1417
|
+
if (!isMetaFresh((await readMeta(local))?.lastFetchMs ?? 0, invalidateBeforeMs, config.ttlSeconds, Date.now())) {
|
|
1418
|
+
if (revalidateRefs && !fencedByPush) {
|
|
1419
|
+
metrics.revalidate_failures++;
|
|
1420
|
+
log("warn", "revalidate_failed", { repo });
|
|
1421
|
+
return {
|
|
1422
|
+
local,
|
|
1423
|
+
cacheState: "STALE_REVALIDATE_FAILED"
|
|
1424
|
+
};
|
|
1425
|
+
}
|
|
1426
|
+
throw new Error("mirror refresh failed after invalidation");
|
|
1427
|
+
}
|
|
1428
|
+
if (revalidateRefs) metrics.revalidations++;
|
|
1340
1429
|
return {
|
|
1341
1430
|
local,
|
|
1342
1431
|
cacheState: "REFRESH"
|
|
@@ -1432,6 +1521,9 @@ function proxyUploadPackWithBody(res, upstreamUrl, reqHeaders, body, credHeader)
|
|
|
1432
1521
|
if (!res.headersSent) res.writeHead(502);
|
|
1433
1522
|
res.end();
|
|
1434
1523
|
});
|
|
1524
|
+
armUpstreamIdleTimeout(up, () => {
|
|
1525
|
+
up.destroy(/* @__PURE__ */ new Error(`upstream idle for ${config.upstreamTimeoutMs}ms`));
|
|
1526
|
+
});
|
|
1435
1527
|
up.end(body);
|
|
1436
1528
|
}
|
|
1437
1529
|
function serveUploadPack(req, res, local, advertise, cacheState, body = null) {
|
|
@@ -1610,12 +1702,59 @@ function isLoopbackPeer(req) {
|
|
|
1610
1702
|
const addr = req.socket.remoteAddress;
|
|
1611
1703
|
return addr === "127.0.0.1" || addr === "::1" || addr === "::ffff:127.0.0.1";
|
|
1612
1704
|
}
|
|
1705
|
+
/**
|
|
1706
|
+
* Proxy an `info/refs` advertisement straight to origin and stream it back to
|
|
1707
|
+
* the client, retaining a bounded copy of a v0 advertisement for later ref
|
|
1708
|
+
* completion. Used for a cold miss AND when a warm mirror's blocking
|
|
1709
|
+
* revalidation failed — either way the client gets a FRESH advertisement from
|
|
1710
|
+
* upstream, never the mirror's stale refs.
|
|
1711
|
+
*/
|
|
1712
|
+
async function proxyInfoRefs(req, res, repo, upstreamUrl, host, credHeader) {
|
|
1713
|
+
const cred = credHeader ?? await fetchCredentialHeader(host);
|
|
1714
|
+
await new Promise((resolve, reject) => {
|
|
1715
|
+
openUpstream(req, upstreamUrl, "info/refs", (upRes, error) => {
|
|
1716
|
+
if (error) {
|
|
1717
|
+
reject(error);
|
|
1718
|
+
return;
|
|
1719
|
+
}
|
|
1720
|
+
if (upRes.statusCode === 401 || upRes.statusCode === 403) dropCredential(host);
|
|
1721
|
+
res.writeHead(upRes.statusCode, upRes.headers);
|
|
1722
|
+
const chunks = [];
|
|
1723
|
+
let bytes = 0;
|
|
1724
|
+
const CAP = 32 * 1024 * 1024;
|
|
1725
|
+
upRes.on("data", (chunk) => {
|
|
1726
|
+
res.write(chunk);
|
|
1727
|
+
if (bytes < CAP) {
|
|
1728
|
+
chunks.push(chunk);
|
|
1729
|
+
bytes += chunk.length;
|
|
1730
|
+
}
|
|
1731
|
+
});
|
|
1732
|
+
upRes.on("end", () => {
|
|
1733
|
+
res.end();
|
|
1734
|
+
if (upRes.statusCode === 200 && bytes < CAP && gitProtocolEnv(req.headers).GIT_PROTOCOL === void 0) advertisements.set(repo, parseAdvertisement(Buffer.concat(chunks)));
|
|
1735
|
+
resolve();
|
|
1736
|
+
});
|
|
1737
|
+
upRes.on("error", reject);
|
|
1738
|
+
}, cred);
|
|
1739
|
+
});
|
|
1740
|
+
}
|
|
1741
|
+
function requestErrorContext(req) {
|
|
1742
|
+
const rawTarget = req.url ?? "";
|
|
1743
|
+
const target = (rawTarget.split("?", 1)[0] ?? "").replace(/^\/[^/?#@]+@/, "/[redacted]@").replace(/^(https?:\/\/)[^/?#@]+@/, "$1[redacted]@");
|
|
1744
|
+
return {
|
|
1745
|
+
method: req.method ?? null,
|
|
1746
|
+
target,
|
|
1747
|
+
targetLength: rawTarget.length,
|
|
1748
|
+
peer: req.socket?.remoteAddress ?? null
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1613
1751
|
async function handle(req, res) {
|
|
1614
1752
|
const parsed = new URL(req.url, "http://git-cache.local");
|
|
1615
1753
|
if (parsed.pathname === "/healthz") {
|
|
1616
1754
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
1617
1755
|
return res.end(JSON.stringify({
|
|
1618
1756
|
ok: true,
|
|
1757
|
+
version: runningVersion,
|
|
1619
1758
|
bind: config.bindHost,
|
|
1620
1759
|
cacheDir: config.cacheDir,
|
|
1621
1760
|
ttlSeconds: config.ttlSeconds,
|
|
@@ -1666,16 +1805,15 @@ async function handle(req, res) {
|
|
|
1666
1805
|
if (config.authMode === "check-through") {
|
|
1667
1806
|
const header = req.headers["authorization"];
|
|
1668
1807
|
const raw = Array.isArray(header) ? header[0] : header;
|
|
1669
|
-
|
|
1670
|
-
metrics.auth_denials++;
|
|
1671
|
-
res.writeHead(401, {
|
|
1672
|
-
"Content-Type": "text/plain",
|
|
1673
|
-
"WWW-Authenticate": "Basic realm=\"git-cache\""
|
|
1674
|
-
});
|
|
1675
|
-
return res.end("authentication required\n");
|
|
1676
|
-
}
|
|
1677
|
-
const verdict = await authorizeCheckThrough(raw, up.repo, up.upstreamUrl);
|
|
1808
|
+
const verdict = await authorizeCheckThrough(raw ?? null, up.repo, up.upstreamUrl);
|
|
1678
1809
|
if (!verdict.allowed) {
|
|
1810
|
+
if (!raw && (verdict.status === 401 || verdict.status === 403)) {
|
|
1811
|
+
res.writeHead(401, {
|
|
1812
|
+
"Content-Type": "text/plain",
|
|
1813
|
+
"WWW-Authenticate": "Basic realm=\"git-cache\""
|
|
1814
|
+
});
|
|
1815
|
+
return res.end("authentication required\n");
|
|
1816
|
+
}
|
|
1679
1817
|
log("info", "auth_denied", {
|
|
1680
1818
|
repo: up.repo,
|
|
1681
1819
|
status: verdict.status
|
|
@@ -1683,9 +1821,9 @@ async function handle(req, res) {
|
|
|
1683
1821
|
res.writeHead(verdict.status, { "Content-Type": "text/plain" });
|
|
1684
1822
|
return res.end("not authorized for this repository\n");
|
|
1685
1823
|
}
|
|
1686
|
-
clientCred = raw;
|
|
1824
|
+
clientCred = raw ?? null;
|
|
1687
1825
|
}
|
|
1688
|
-
const host = up.repo.split("/")[0];
|
|
1826
|
+
const host = up.repo.split("/")[0] ?? "";
|
|
1689
1827
|
let uploadBody = null;
|
|
1690
1828
|
let uploadWants = [];
|
|
1691
1829
|
if (up.operation === "git-upload-pack") {
|
|
@@ -1706,8 +1844,16 @@ async function handle(req, res) {
|
|
|
1706
1844
|
uploadWants = classifyUploadRequest(uploadBody).wants;
|
|
1707
1845
|
}
|
|
1708
1846
|
if (await pathExists(localPath(up.repo))) {
|
|
1709
|
-
const warm = await ensureWarm(up.repo, clientCred);
|
|
1847
|
+
const warm = await ensureWarm(up.repo, clientCred, up.operation === "info/refs");
|
|
1710
1848
|
markServed(warm.local);
|
|
1849
|
+
if (warm.cacheState === "STALE_REVALIDATE_FAILED") {
|
|
1850
|
+
log("info", "serve", {
|
|
1851
|
+
repo: up.repo,
|
|
1852
|
+
op: up.operation,
|
|
1853
|
+
state: warm.cacheState
|
|
1854
|
+
});
|
|
1855
|
+
return await proxyInfoRefs(req, res, up.repo, up.upstreamUrl, host, clientCred);
|
|
1856
|
+
}
|
|
1711
1857
|
if (up.operation === "git-upload-pack" && !await mirrorHasObjects(warm.local, uploadWants)) {
|
|
1712
1858
|
await refreshMirror(up.repo, warm.local, clientCred).catch((error) => {
|
|
1713
1859
|
metrics.refresh_failures++;
|
|
@@ -1730,35 +1876,7 @@ async function handle(req, res) {
|
|
|
1730
1876
|
});
|
|
1731
1877
|
return serveUploadPack(req, res, warm.local, up.operation === "info/refs", warm.cacheState, uploadBody);
|
|
1732
1878
|
}
|
|
1733
|
-
if (up.operation === "info/refs")
|
|
1734
|
-
const credHeader = clientCred ?? await fetchCredentialHeader(host);
|
|
1735
|
-
return await new Promise((resolve, reject) => {
|
|
1736
|
-
openUpstream(req, up.upstreamUrl, up.operation, (upRes, error) => {
|
|
1737
|
-
if (error) {
|
|
1738
|
-
reject(error);
|
|
1739
|
-
return;
|
|
1740
|
-
}
|
|
1741
|
-
if (upRes.statusCode === 401 || upRes.statusCode === 403) dropCredential(host);
|
|
1742
|
-
res.writeHead(upRes.statusCode, upRes.headers);
|
|
1743
|
-
const chunks = [];
|
|
1744
|
-
let bytes = 0;
|
|
1745
|
-
const CAP = 32 * 1024 * 1024;
|
|
1746
|
-
upRes.on("data", (chunk) => {
|
|
1747
|
-
res.write(chunk);
|
|
1748
|
-
if (bytes < CAP) {
|
|
1749
|
-
chunks.push(chunk);
|
|
1750
|
-
bytes += chunk.length;
|
|
1751
|
-
}
|
|
1752
|
-
});
|
|
1753
|
-
upRes.on("end", () => {
|
|
1754
|
-
res.end();
|
|
1755
|
-
if (upRes.statusCode === 200 && bytes < CAP && gitProtocolEnv(req.headers).GIT_PROTOCOL === void 0) advertisements.set(up.repo, parseAdvertisement(Buffer.concat(chunks)));
|
|
1756
|
-
resolve(void 0);
|
|
1757
|
-
});
|
|
1758
|
-
upRes.on("error", reject);
|
|
1759
|
-
}, credHeader);
|
|
1760
|
-
});
|
|
1761
|
-
}
|
|
1879
|
+
if (up.operation === "info/refs") return await proxyInfoRefs(req, res, up.repo, up.upstreamUrl, host, clientCred);
|
|
1762
1880
|
const body = uploadBody ?? Buffer.alloc(0);
|
|
1763
1881
|
const credHeader = clientCred ?? await fetchCredentialHeader(host);
|
|
1764
1882
|
if (coldLeaders.has(up.repo)) return proxyColdFollowerWithBody(res, up.upstreamUrl, req.headers, body, up.repo, credHeader);
|
|
@@ -1786,6 +1904,9 @@ function proxyColdFollowerWithBody(res, upstreamUrl, reqHeaders, body, repo, cre
|
|
|
1786
1904
|
if (!res.headersSent) res.writeHead(502);
|
|
1787
1905
|
res.end();
|
|
1788
1906
|
});
|
|
1907
|
+
armUpstreamIdleTimeout(up, () => {
|
|
1908
|
+
up.destroy(/* @__PURE__ */ new Error(`upstream idle for ${config.upstreamTimeoutMs}ms`));
|
|
1909
|
+
});
|
|
1789
1910
|
up.end(body);
|
|
1790
1911
|
}
|
|
1791
1912
|
async function startServer(options = {}) {
|
|
@@ -1799,9 +1920,21 @@ async function startServer(options = {}) {
|
|
|
1799
1920
|
if (config.authMode === "check-through" && config.credentialHelper) throw new Error("git-cache: credentialHelper is incompatible with check-through authorization — upstream requests authenticate with each client's own credential.");
|
|
1800
1921
|
await mkdir(path.join(config.cacheDir, "mirrors"), { recursive: true });
|
|
1801
1922
|
await cleanupQuarantine();
|
|
1923
|
+
runningVersion = await readOwnPackageVersion();
|
|
1802
1924
|
const server = http.createServer((req, res) => {
|
|
1925
|
+
const context = requestErrorContext(req);
|
|
1926
|
+
res.on("finish", () => {
|
|
1927
|
+
if (res.statusCode >= 400 && res.statusCode < 500) log("info", "client_error", {
|
|
1928
|
+
...context,
|
|
1929
|
+
status: res.statusCode
|
|
1930
|
+
});
|
|
1931
|
+
});
|
|
1803
1932
|
handle(req, res).catch((error) => {
|
|
1804
|
-
log("error", "unhandled", {
|
|
1933
|
+
log("error", "unhandled", {
|
|
1934
|
+
error: String(error),
|
|
1935
|
+
stack: error instanceof Error ? error.stack : null,
|
|
1936
|
+
...context
|
|
1937
|
+
});
|
|
1805
1938
|
if (!res.headersSent) res.writeHead(500);
|
|
1806
1939
|
res.end();
|
|
1807
1940
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skydiveai/git-cache",
|
|
3
|
-
"version": "0.1.0",
|
|
3
|
+
"version": "0.1.251-beta.0",
|
|
4
4
|
"description": "Sandbox-local Git smart-HTTP clone/fetch cache: a loopback daemon that transparently mirrors HTTPS git reads so repeated clone/fetch/ls-remote skip the origin round-trip. Runs inside an agent sandbox; pushes stay direct.",
|
|
5
5
|
"homepage": "https://skydive.com",
|
|
6
6
|
"license": "MIT",
|
|
@@ -20,15 +20,6 @@
|
|
|
20
20
|
}
|
|
21
21
|
},
|
|
22
22
|
"publishConfig": {
|
|
23
|
-
"bin": {
|
|
24
|
-
"git-cache-daemon": "./dist/cli.mjs"
|
|
25
|
-
},
|
|
26
|
-
"exports": {
|
|
27
|
-
".": {
|
|
28
|
-
"types": "./dist/index.d.mts",
|
|
29
|
-
"default": "./dist/index.mjs"
|
|
30
|
-
}
|
|
31
|
-
},
|
|
32
23
|
"access": "public",
|
|
33
24
|
"registry": "https://registry.npmjs.org"
|
|
34
25
|
},
|
|
@@ -39,8 +30,7 @@
|
|
|
39
30
|
"test:git-cache-e2e": "yarn build && bash test/e2e.sh",
|
|
40
31
|
"bench": "yarn build && bash test/bench.sh",
|
|
41
32
|
"test:unit": "yarn build && vitest run --passWithNoTests && yarn lint:shell && bash test/local-e2e.sh && bash test/perf-check.sh",
|
|
42
|
-
"test:ci": "yarn build && vitest run --coverage --coverage.reporter=lcovonly --reporter=default --reporter=github-actions --minWorkers=1 --maxWorkers=2 --passWithNoTests && yarn lint:shell && bash test/local-e2e.sh && bash test/perf-check.sh"
|
|
43
|
-
"publish:system-artifacts": "yarn build && doppler run --preserve-env -- node ../../scripts/anyone/publish-system-artifact.mjs"
|
|
33
|
+
"test:ci": "yarn build && vitest run --coverage --coverage.reporter=lcovonly --reporter=default --reporter=github-actions --minWorkers=1 --maxWorkers=2 --passWithNoTests && yarn lint:shell && bash test/local-e2e.sh && bash test/perf-check.sh"
|
|
44
34
|
},
|
|
45
35
|
"devDependencies": {
|
|
46
36
|
"@types/node": "^24.0.0",
|