maven-proxy 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -10
- package/bin/maven-proxy.js +6 -4
- package/package.json +3 -3
- package/src/cache/cache-path.js +162 -1
- package/src/cache/{maven-affinity-index.js → maven-negative-index.js} +27 -143
- package/src/config/config.js +33 -8
- package/src/index.js +11 -9
- package/src/proxy/proxy-http-handler.js +58 -25
- package/src/repo/repo-server.js +142 -9
package/README.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
[English](README.md) | [中文](README-zh.md)
|
|
4
4
|
|
|
5
|
+
[English ChangeLog](ChangeLog.md) | [中文更新日志](ChangeLog-zh.md)
|
|
6
|
+
|
|
5
7
|
[](https://www.npmjs.com/package/maven-proxy)
|
|
6
8
|
|
|
7
9
|
## 1. Project Positioning
|
|
@@ -275,6 +277,8 @@ Environment variables:
|
|
|
275
277
|
- REPO_FALLBACK_REPOS: repository fallback list.
|
|
276
278
|
- NPM_REGISTRY_DOMAINS: npm domains for ecosystem routing (wildcards supported).
|
|
277
279
|
- MAVEN_REPO_DOMAINS: maven domains for ecosystem routing (wildcards supported).
|
|
280
|
+
- MAVEN_CACHE_USE_DOMAIN_DIR: whether Maven cache uses hostname as the first-level directory. Default false.
|
|
281
|
+
- MAVEN_CACHE_IGNORE_PATH_PREFIXES: Maven cache path-prefix ignore rules (comma-separated, supports host/path and host:port/path). Default: repo1.maven.org/maven2,repo.maven.apache.org/maven2,jitpack.io/,plugins.gradle.org/m2,dl.google.com/dl/android/maven2,dl.google.com/dl/google/maven.
|
|
278
282
|
- HTTPS_MITM_DOMAINS: MITM domain list (includes registry.npmjs.org by default, wildcards supported).
|
|
279
283
|
- DOWNLOAD_LOG_DIR: log directory.
|
|
280
284
|
- LOG_RETENTION: log retention duration (supports s/m/h/d), for example 7d.
|
|
@@ -284,11 +288,11 @@ Environment variables:
|
|
|
284
288
|
- OUTBOUND_KEEP_ALIVE_INTERVAL: keep-alive interval (supports s/m/h/d), for example 1s.
|
|
285
289
|
- OUTBOUND_MAX_SOCKETS: max outbound sockets per origin.
|
|
286
290
|
- OUTBOUND_MAX_FREE_SOCKETS: max idle outbound sockets per origin.
|
|
287
|
-
-
|
|
288
|
-
-
|
|
289
|
-
- MAVEN_NEGATIVE_CACHE_TTL: negative cache TTL (supports s/m/h/d), for example 24h.
|
|
290
|
-
-
|
|
291
|
-
-
|
|
291
|
+
- MAVEN_NEGATIVE_ENABLED: enable Maven negative index. Default true.
|
|
292
|
+
- MAVEN_NEGATIVE_INDEX_DIR: Maven negative index directory. Default data/index.
|
|
293
|
+
- MAVEN_NEGATIVE_CACHE_TTL: negative cache TTL (supports s/m/h/d), for example 24h.
|
|
294
|
+
- MAVEN_NEGATIVE_FLUSH_INTERVAL: flush interval for negative event log (supports s/m/h/d), for example 5s.
|
|
295
|
+
- MAVEN_NEGATIVE_EVENT_MAX_MB: max size threshold for negative event log compaction in MB.
|
|
292
296
|
- MAVEN_PROXY_CONFIG_MODE: development or user.
|
|
293
297
|
- MAVEN_PROXY_CONFIG_FILE: explicit config file path.
|
|
294
298
|
- EXISTING_TRUST_STORE_PATH: optional existing truststore path. If present, truststore init prefers it as source.
|
|
@@ -325,6 +329,8 @@ Priority:
|
|
|
325
329
|
- `HTTPS_PASSTHROUGH_FOR_UNMATCHED`: Whether unmatched HTTPS domains are tunneled directly. Default `false`.
|
|
326
330
|
- `NPM_REGISTRY_DOMAINS`: Domains treated as npm ecosystem for cache routing (wildcards supported).
|
|
327
331
|
- `MAVEN_REPO_DOMAINS`: Domains treated as Maven ecosystem for cache routing (wildcards supported).
|
|
332
|
+
- `MAVEN_CACHE_USE_DOMAIN_DIR`: Whether Maven cache uses hostname as the first-level directory. Default `false`.
|
|
333
|
+
- `MAVEN_CACHE_IGNORE_PATH_PREFIXES`: Maven cache path-prefix ignore rules (comma-separated, supports `host/path` and `host:port/path`). Default `repo1.maven.org/maven2,repo.maven.apache.org/maven2,jitpack.io/,plugins.gradle.org/m2,dl.google.com/dl/android/maven2,dl.google.com/dl/google/maven`.
|
|
328
334
|
- `MULTI_THREAD_DOMAINS`: Domains allowed to use multi-thread download (wildcards supported).
|
|
329
335
|
- `MULTI_THREAD_COUNT`: Number of download threads for ranged downloads.
|
|
330
336
|
- `MULTI_THREAD_MIN_SIZE_MB`: Minimum size threshold to trigger multi-thread download (MB).
|
|
@@ -337,11 +343,11 @@ Priority:
|
|
|
337
343
|
- `OUTBOUND_KEEP_ALIVE_INTERVAL`: Keep-alive interval (supports `s/m/h/d`). Default `1s`.
|
|
338
344
|
- `OUTBOUND_MAX_SOCKETS`: Max outbound sockets per origin. Default `64`.
|
|
339
345
|
- `OUTBOUND_MAX_FREE_SOCKETS`: Max idle outbound sockets per origin. Default `16`.
|
|
340
|
-
- `
|
|
341
|
-
- `
|
|
342
|
-
- `MAVEN_NEGATIVE_CACHE_TTL`: Negative cache TTL (supports `s/m/h/d`). Default `24h`.
|
|
343
|
-
- `
|
|
344
|
-
- `
|
|
346
|
+
- `MAVEN_NEGATIVE_ENABLED`: Enable Maven negative cache index. Default `true`.
|
|
347
|
+
- `MAVEN_NEGATIVE_INDEX_DIR`: Maven negative index directory. Default `data/index`.
|
|
348
|
+
- `MAVEN_NEGATIVE_CACHE_TTL`: Negative cache TTL (supports `s/m/h/d`). Default `24h`.
|
|
349
|
+
- `MAVEN_NEGATIVE_FLUSH_INTERVAL`: Flush interval for negative event log (supports `s/m/h/d`). Default `5s`.
|
|
350
|
+
- `MAVEN_NEGATIVE_EVENT_MAX_MB`: Max size threshold for negative event log compaction in MB. Default `8`.
|
|
345
351
|
- `UPSTREAM_PROXY_URL`: Generic upstream proxy URL (fallback for HTTP/HTTPS).
|
|
346
352
|
- `UPSTREAM_HTTP_PROXY_URL`: Upstream proxy URL for HTTP requests.
|
|
347
353
|
- `UPSTREAM_HTTPS_PROXY_URL`: Upstream proxy URL for HTTPS requests.
|
package/bin/maven-proxy.js
CHANGED
|
@@ -179,6 +179,8 @@ function getDefaultConfigTemplate() {
|
|
|
179
179
|
appendEntry("HTTPS_PASSTHROUGH_FOR_UNMATCHED", "false", "未命中 MITM 域名时是否允许隧道透传,默认 false。", "Allow tunnel passthrough for unmatched MITM domains. Default: false.");
|
|
180
180
|
appendEntry("NPM_REGISTRY_DOMAINS", "registry.npmjs.org,registry.npmmirror.com,npm.pkg.github.com", "识别为 npm 生态并分流缓存的域名列表(支持通配符)。", "Domains recognized as npm ecosystem for cache routing (wildcard supported).");
|
|
181
181
|
appendEntry("MAVEN_REPO_DOMAINS", "repo1.maven.org,repo.maven.apache.org,jitpack.io,plugins.gradle.org,dl.google.com", "识别为 Maven 生态并分流缓存的域名列表(支持通配符)。", "Domains recognized as Maven ecosystem for cache routing (wildcard supported).");
|
|
182
|
+
appendEntry("MAVEN_CACHE_USE_DOMAIN_DIR", "false", "Maven 缓存是否按域名作为一级目录,默认 false。", "Whether Maven cache uses hostname as the first-level directory. Default: false.");
|
|
183
|
+
appendEntry("MAVEN_CACHE_IGNORE_PATH_PREFIXES", "repo1.maven.org/maven2,repo.maven.apache.org/maven2,jitpack.io/,plugins.gradle.org/m2,dl.google.com/dl/android/maven2,dl.google.com/dl/google/maven", "Maven 缓存应忽略的路径前缀规则(逗号分隔,支持 host/path 与 host:port/path)。", "Maven cache path-prefix ignore rules (comma-separated, supports host/path and host:port/path).");
|
|
182
184
|
appendEntry("MULTI_THREAD_DOMAINS", "repo1.maven.org", "启用多线程下载的域名列表(支持通配符)。", "Domains that enable multi-thread download (wildcard supported).");
|
|
183
185
|
appendEntry("MULTI_THREAD_COUNT", "8", "多线程下载线程数,默认 8。", "Thread count for multi-thread download. Default: 8.");
|
|
184
186
|
appendEntry("MULTI_THREAD_MIN_SIZE_MB", "1", "触发多线程下载的最小文件大小阈值(MB),默认 1。", "Minimum file size threshold in MB to trigger multi-thread download. Default: 1.");
|
|
@@ -191,11 +193,11 @@ function getDefaultConfigTemplate() {
|
|
|
191
193
|
appendEntry("OUTBOUND_KEEP_ALIVE_INTERVAL", "1s", "keep-alive 间隔(支持 s/m/h/d),默认 1s。", "Keep-alive interval (supports s/m/h/d). Default: 1s.");
|
|
192
194
|
appendEntry("OUTBOUND_MAX_SOCKETS", "64", "每个源站最大出站连接数,默认 64。", "Maximum outbound sockets per upstream host. Default: 64.");
|
|
193
195
|
appendEntry("OUTBOUND_MAX_FREE_SOCKETS", "16", "每个源站可保留空闲连接上限,默认 16。", "Maximum free outbound sockets kept per upstream host. Default: 16.");
|
|
194
|
-
appendEntry("
|
|
195
|
-
appendEntry("
|
|
196
|
+
appendEntry("MAVEN_NEGATIVE_ENABLED", "true", "是否启用 Maven negative 索引,默认 true。", "Enable Maven negative index. Default: true.");
|
|
197
|
+
appendEntry("MAVEN_NEGATIVE_INDEX_DIR", "data/index", "Maven negative 索引目录,默认 data/index。", "Maven negative index directory. Default: data/index.");
|
|
196
198
|
appendEntry("MAVEN_NEGATIVE_CACHE_TTL", "24h", "负缓存 TTL(支持 s/m/h/d),默认 24h。", "Negative cache TTL (supports s/m/h/d). Default: 24h.");
|
|
197
|
-
appendEntry("
|
|
198
|
-
appendEntry("
|
|
199
|
+
appendEntry("MAVEN_NEGATIVE_FLUSH_INTERVAL", "5s", "negative 事件日志 flush 周期(支持 s/m/h/d),默认 5s。", "Negative event log flush interval (supports s/m/h/d). Default: 5s.");
|
|
200
|
+
appendEntry("MAVEN_NEGATIVE_EVENT_MAX_MB", "8", "negative 事件日志压缩阈值(MB),默认 8。", "Negative event log compaction threshold in MB. Default: 8.");
|
|
199
201
|
appendEntry("UPSTREAM_PROXY_URL", "", "通用上级代理地址(HTTP/HTTPS 兜底)。", "Generic upstream proxy URL fallback for HTTP/HTTPS.");
|
|
200
202
|
appendEntry("UPSTREAM_HTTP_PROXY_URL", "", "HTTP 请求使用的上级代理地址。", "Upstream proxy URL for HTTP requests.");
|
|
201
203
|
appendEntry("UPSTREAM_HTTPS_PROXY_URL", "", "HTTPS 请求使用的上级代理地址。", "Upstream proxy URL for HTTPS requests.");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "maven-proxy",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.3",
|
|
4
4
|
"description": "Maven proxy with cache, HTTPS MITM for selected domains, and local repo publishing",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
"npm:version:patch": "npm version patch",
|
|
36
36
|
"npm:view:version": "npm view maven-proxy version",
|
|
37
37
|
"npm:view:dist-tags": "npm view maven-proxy dist-tags",
|
|
38
|
-
"replay:
|
|
39
|
-
"test:replay": "node --test test/replay-
|
|
38
|
+
"replay:negative": "node --test test/replay-negative.test.js",
|
|
39
|
+
"test:replay": "node --test test/replay-negative.test.js",
|
|
40
40
|
"test": "node --test test/*.test.js"
|
|
41
41
|
},
|
|
42
42
|
"keywords": [],
|
package/src/cache/cache-path.js
CHANGED
|
@@ -16,6 +16,162 @@ function looksLikeMavenVersionSegment(segment) {
|
|
|
16
16
|
return /^\d[0-9A-Za-z._-]*$/.test(String(segment || ""));
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
function normalizeSlashPath(rawPath) {
|
|
20
|
+
const decoded = safeDecode(rawPath || "/").replace(/\\/g, "/").replace(/\/+/g, "/");
|
|
21
|
+
if (!decoded) {
|
|
22
|
+
return "/";
|
|
23
|
+
}
|
|
24
|
+
return decoded.startsWith("/") ? decoded : `/${decoded}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function trimTrailingSlash(normalizedPath) {
|
|
28
|
+
if (normalizedPath.length > 1 && normalizedPath.endsWith("/")) {
|
|
29
|
+
return normalizedPath.slice(0, -1);
|
|
30
|
+
}
|
|
31
|
+
return normalizedPath;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function normalizeRulePrefixPath(rawPrefix) {
|
|
35
|
+
return trimTrailingSlash(normalizeSlashPath(rawPrefix || "/"));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function getEffectivePort(urlObj) {
|
|
39
|
+
if (urlObj?.port) {
|
|
40
|
+
return String(urlObj.port);
|
|
41
|
+
}
|
|
42
|
+
return urlObj?.protocol === "https:" ? "443" : "80";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function parseMavenCacheIgnorePathRule(rawRule) {
|
|
46
|
+
const text = String(rawRule || "").trim();
|
|
47
|
+
if (!text) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const firstSlash = text.indexOf("/");
|
|
52
|
+
if (firstSlash <= 0) {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const hostPortText = text.slice(0, firstSlash).trim().toLowerCase();
|
|
57
|
+
const prefixText = text.slice(firstSlash);
|
|
58
|
+
const hostMatch = hostPortText.match(/^([^:\/]+)(?::(\d+))?$/);
|
|
59
|
+
if (!hostMatch) {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const hostname = hostMatch[1];
|
|
64
|
+
const port = hostMatch[2] || "";
|
|
65
|
+
const pathPrefix = normalizeRulePrefixPath(prefixText);
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
hostname,
|
|
69
|
+
port,
|
|
70
|
+
pathPrefix,
|
|
71
|
+
raw: text,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function ruleMatchesUrl(rule, urlObj) {
|
|
76
|
+
if (!rule || !urlObj) {
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const hostname = String(urlObj.hostname || "").toLowerCase();
|
|
81
|
+
if (!hostname || hostname !== rule.hostname) {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (!rule.port) {
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return getEffectivePort(urlObj) === rule.port;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function parseMavenCacheIgnorePathPrefixes(rawRules) {
|
|
93
|
+
const input = Array.isArray(rawRules) ? rawRules.join(",") : String(rawRules || "");
|
|
94
|
+
const tokens = input
|
|
95
|
+
.split(",")
|
|
96
|
+
.map((item) => item.trim())
|
|
97
|
+
.filter(Boolean);
|
|
98
|
+
|
|
99
|
+
const parsed = [];
|
|
100
|
+
const dedupe = new Set();
|
|
101
|
+
|
|
102
|
+
for (const token of tokens) {
|
|
103
|
+
const rule = parseMavenCacheIgnorePathRule(token);
|
|
104
|
+
if (!rule) {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const key = `${rule.hostname}:${rule.port}|${rule.pathPrefix}`;
|
|
109
|
+
if (dedupe.has(key)) {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
dedupe.add(key);
|
|
114
|
+
parsed.push(rule);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return parsed;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function stripMavenIgnoredPathPrefix(pathname, urlObj, rules = []) {
|
|
121
|
+
const normalizedPath = normalizeSlashPath(pathname || "/");
|
|
122
|
+
const validRules = Array.isArray(rules) ? rules : [];
|
|
123
|
+
if (validRules.length === 0 || !urlObj) {
|
|
124
|
+
return normalizedPath;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const matchedRules = validRules
|
|
128
|
+
.filter((rule) => ruleMatchesUrl(rule, urlObj))
|
|
129
|
+
.sort((a, b) => b.pathPrefix.length - a.pathPrefix.length);
|
|
130
|
+
|
|
131
|
+
for (const rule of matchedRules) {
|
|
132
|
+
const prefix = rule.pathPrefix;
|
|
133
|
+
if (prefix === "/") {
|
|
134
|
+
return normalizedPath;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (normalizedPath === prefix) {
|
|
138
|
+
return "/";
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (normalizedPath.startsWith(`${prefix}/`)) {
|
|
142
|
+
const stripped = normalizedPath.slice(prefix.length);
|
|
143
|
+
return stripped || "/";
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return normalizedPath;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function buildMavenHostlessPathCandidates(pathname, rules = []) {
|
|
151
|
+
const normalizedPath = normalizeSlashPath(pathname || "/");
|
|
152
|
+
const candidates = new Set([normalizedPath]);
|
|
153
|
+
const validRules = Array.isArray(rules) ? rules : [];
|
|
154
|
+
|
|
155
|
+
for (const rule of validRules) {
|
|
156
|
+
const prefix = rule.pathPrefix;
|
|
157
|
+
if (!prefix || prefix === "/") {
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (normalizedPath === prefix) {
|
|
162
|
+
candidates.add("/");
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (normalizedPath.startsWith(`${prefix}/`)) {
|
|
167
|
+
const stripped = normalizedPath.slice(prefix.length);
|
|
168
|
+
candidates.add(stripped || "/");
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return [...candidates];
|
|
173
|
+
}
|
|
174
|
+
|
|
19
175
|
function isLikelyMavenFilePath(parts, normalizedPath) {
|
|
20
176
|
if (normalizedPath.endsWith("/") || parts.length === 0) {
|
|
21
177
|
return false;
|
|
@@ -63,9 +219,14 @@ function isLikelyMavenFilePath(parts, normalizedPath) {
|
|
|
63
219
|
export function getCacheFilePath(cacheDir, urlObj, options = {}) {
|
|
64
220
|
const ecosystem = sanitizeSegment(String(options.ecosystem || "generic").toLowerCase());
|
|
65
221
|
const includeHost = options.includeHost ?? ecosystem !== "maven";
|
|
222
|
+
const mavenIgnoreRules = options.mavenCacheIgnorePathPrefixRules || [];
|
|
66
223
|
|
|
67
224
|
const rawPathname = safeDecode(urlObj.pathname || "/");
|
|
68
|
-
|
|
225
|
+
let normalized = rawPathname.replace(/\\/g, "/");
|
|
226
|
+
if (ecosystem === "maven") {
|
|
227
|
+
normalized = stripMavenIgnoredPathPrefix(normalized, urlObj, mavenIgnoreRules);
|
|
228
|
+
}
|
|
229
|
+
|
|
69
230
|
const lowerNormalized = normalized.toLowerCase();
|
|
70
231
|
const parts = normalized.split("/").filter(Boolean);
|
|
71
232
|
|
|
@@ -31,16 +31,6 @@ function readJsonFile(filePath, fallback) {
|
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
function serializeSnapshot(positiveMap, negativeMap, conflictMap) {
|
|
35
|
-
return {
|
|
36
|
-
version: 1,
|
|
37
|
-
generatedAt: new Date().toISOString(),
|
|
38
|
-
positive: [...positiveMap.entries()],
|
|
39
|
-
negative: [...negativeMap.entries()],
|
|
40
|
-
conflicts: [...conflictMap.entries()],
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
|
|
44
34
|
function normalizeRequestPath(pathname) {
|
|
45
35
|
return String(pathname || "")
|
|
46
36
|
.replace(/\\/g, "/")
|
|
@@ -64,22 +54,26 @@ function buildNegativeKey(scope, canonicalKey) {
|
|
|
64
54
|
return `${String(scope || "").toLowerCase()}|${canonicalKey}`;
|
|
65
55
|
}
|
|
66
56
|
|
|
67
|
-
|
|
57
|
+
function serializeSnapshot(negativeMap) {
|
|
58
|
+
return {
|
|
59
|
+
version: 1,
|
|
60
|
+
generatedAt: new Date().toISOString(),
|
|
61
|
+
negative: [...negativeMap.entries()],
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export class MavenNegativeIndex {
|
|
68
66
|
constructor(config) {
|
|
69
|
-
this.enabled = toBool(config.
|
|
70
|
-
this.indexDir = config.
|
|
67
|
+
this.enabled = toBool(config.mavenNegativeEnabled, true);
|
|
68
|
+
this.indexDir = config.mavenNegativeIndexDir;
|
|
71
69
|
this.negativeTtlMs = toPositiveInt(config.mavenNegativeCacheTtlMs, 24 * 60 * 60 * 1000);
|
|
72
|
-
this.flushIntervalMs = toPositiveInt(config.
|
|
73
|
-
this.maxEventBytes = toPositiveInt(config.
|
|
70
|
+
this.flushIntervalMs = toPositiveInt(config.mavenNegativeFlushIntervalMs, 5000);
|
|
71
|
+
this.maxEventBytes = toPositiveInt(config.mavenNegativeEventMaxBytes, 8 * 1024 * 1024);
|
|
74
72
|
|
|
75
|
-
this.snapshotPath = path.join(this.indexDir, "maven-
|
|
76
|
-
this.eventLogPath = path.join(this.indexDir, "maven-
|
|
73
|
+
this.snapshotPath = path.join(this.indexDir, "maven-negative.snapshot.json");
|
|
74
|
+
this.eventLogPath = path.join(this.indexDir, "maven-negative.events.log");
|
|
77
75
|
|
|
78
|
-
// Positive entries are persistent and have no TTL. They are removed only
|
|
79
|
-
// when the cache file disappears or a conflict is detected.
|
|
80
|
-
this.positive = new Map();
|
|
81
76
|
this.negative = new Map();
|
|
82
|
-
this.conflicts = new Map();
|
|
83
77
|
|
|
84
78
|
this.pendingEvents = [];
|
|
85
79
|
this.flushTimer = null;
|
|
@@ -98,7 +92,7 @@ export class MavenAffinityIndex {
|
|
|
98
92
|
|
|
99
93
|
this.flushTimer = setInterval(() => {
|
|
100
94
|
this.flush().catch((error) => {
|
|
101
|
-
console.error(`[
|
|
95
|
+
console.error(`[maven-negative] flush failed: ${error.message}`);
|
|
102
96
|
});
|
|
103
97
|
}, this.flushIntervalMs);
|
|
104
98
|
|
|
@@ -113,20 +107,12 @@ export class MavenAffinityIndex {
|
|
|
113
107
|
return;
|
|
114
108
|
}
|
|
115
109
|
|
|
116
|
-
for (const [key, value] of snapshot.positive || []) {
|
|
117
|
-
this.positive.set(key, value);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
110
|
const currentTime = nowMs();
|
|
121
111
|
for (const [key, value] of snapshot.negative || []) {
|
|
122
112
|
if (value?.expireAt && value.expireAt > currentTime) {
|
|
123
113
|
this.negative.set(key, value);
|
|
124
114
|
}
|
|
125
115
|
}
|
|
126
|
-
|
|
127
|
-
for (const [key, value] of snapshot.conflicts || []) {
|
|
128
|
-
this.conflicts.set(key, value);
|
|
129
|
-
}
|
|
130
116
|
}
|
|
131
117
|
|
|
132
118
|
#replayEventLog() {
|
|
@@ -167,22 +153,6 @@ export class MavenAffinityIndex {
|
|
|
167
153
|
return;
|
|
168
154
|
}
|
|
169
155
|
|
|
170
|
-
if (type === "positive_upsert") {
|
|
171
|
-
this.positive.set(payload.key, payload.value);
|
|
172
|
-
if (append) {
|
|
173
|
-
this.#enqueueEvent(type, payload);
|
|
174
|
-
}
|
|
175
|
-
return;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
if (type === "positive_remove") {
|
|
179
|
-
this.positive.delete(payload.key);
|
|
180
|
-
if (append) {
|
|
181
|
-
this.#enqueueEvent(type, payload);
|
|
182
|
-
}
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
156
|
if (type === "negative_upsert") {
|
|
187
157
|
if (payload.value?.expireAt > nowMs()) {
|
|
188
158
|
this.negative.set(payload.key, payload.value);
|
|
@@ -202,50 +172,6 @@ export class MavenAffinityIndex {
|
|
|
202
172
|
}
|
|
203
173
|
return;
|
|
204
174
|
}
|
|
205
|
-
|
|
206
|
-
if (type === "conflict_set") {
|
|
207
|
-
this.conflicts.set(payload.key, payload.value);
|
|
208
|
-
if (append) {
|
|
209
|
-
this.#enqueueEvent(type, payload);
|
|
210
|
-
}
|
|
211
|
-
return;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
if (type === "conflict_clear") {
|
|
215
|
-
this.conflicts.delete(payload.key);
|
|
216
|
-
if (append) {
|
|
217
|
-
this.#enqueueEvent(type, payload);
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
async resolvePreferredCachePath(canonicalKey) {
|
|
223
|
-
if (!this.enabled || !canonicalKey) {
|
|
224
|
-
return "";
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
if (this.conflicts.has(canonicalKey)) {
|
|
228
|
-
return "";
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
const existing = this.positive.get(canonicalKey);
|
|
232
|
-
if (!existing?.cachePath) {
|
|
233
|
-
return "";
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
try {
|
|
237
|
-
const stats = await fs.promises.stat(existing.cachePath);
|
|
238
|
-
if (!stats.isFile()) {
|
|
239
|
-
throw new Error("not-file");
|
|
240
|
-
}
|
|
241
|
-
return existing.cachePath;
|
|
242
|
-
} catch {
|
|
243
|
-
this.#applyEvent({
|
|
244
|
-
type: "positive_remove",
|
|
245
|
-
payload: { key: canonicalKey },
|
|
246
|
-
});
|
|
247
|
-
return "";
|
|
248
|
-
}
|
|
249
175
|
}
|
|
250
176
|
|
|
251
177
|
shouldSkipRequest(canonicalKey, urlObj) {
|
|
@@ -271,58 +197,6 @@ export class MavenAffinityIndex {
|
|
|
271
197
|
return true;
|
|
272
198
|
}
|
|
273
199
|
|
|
274
|
-
recordSuccess({ canonicalKey, host, cachePath, fileName, urlObj = null }) {
|
|
275
|
-
if (!this.enabled || !canonicalKey || !cachePath || !fileName) {
|
|
276
|
-
return;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
const previous = this.positive.get(canonicalKey);
|
|
280
|
-
if (previous && previous.fileName !== fileName) {
|
|
281
|
-
this.#applyEvent({
|
|
282
|
-
type: "conflict_set",
|
|
283
|
-
payload: {
|
|
284
|
-
key: canonicalKey,
|
|
285
|
-
value: {
|
|
286
|
-
reason: "file-name-mismatch",
|
|
287
|
-
updatedAt: nowMs(),
|
|
288
|
-
previousFileName: previous.fileName,
|
|
289
|
-
currentFileName: fileName,
|
|
290
|
-
},
|
|
291
|
-
},
|
|
292
|
-
});
|
|
293
|
-
|
|
294
|
-
this.#applyEvent({
|
|
295
|
-
type: "positive_remove",
|
|
296
|
-
payload: { key: canonicalKey },
|
|
297
|
-
});
|
|
298
|
-
return;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
this.#applyEvent({
|
|
302
|
-
type: "positive_upsert",
|
|
303
|
-
payload: {
|
|
304
|
-
key: canonicalKey,
|
|
305
|
-
value: {
|
|
306
|
-
cachePath,
|
|
307
|
-
fileName,
|
|
308
|
-
host: String(host || "").toLowerCase(),
|
|
309
|
-
updatedAt: nowMs(),
|
|
310
|
-
},
|
|
311
|
-
},
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
const successScope = buildNegativeScope(urlObj);
|
|
315
|
-
if (successScope) {
|
|
316
|
-
const negativeKey = buildNegativeKey(successScope, canonicalKey);
|
|
317
|
-
if (this.negative.has(negativeKey)) {
|
|
318
|
-
this.#applyEvent({
|
|
319
|
-
type: "negative_remove",
|
|
320
|
-
payload: { key: negativeKey },
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
|
|
326
200
|
recordNegative({ canonicalKey, urlObj, statusCode = 404, ttlMs = this.negativeTtlMs }) {
|
|
327
201
|
const scope = buildNegativeScope(urlObj);
|
|
328
202
|
if (!this.enabled || !canonicalKey || !scope) {
|
|
@@ -345,6 +219,16 @@ export class MavenAffinityIndex {
|
|
|
345
219
|
});
|
|
346
220
|
}
|
|
347
221
|
|
|
222
|
+
clearNegative({ canonicalKey, urlObj }) {
|
|
223
|
+
const scope = buildNegativeScope(urlObj);
|
|
224
|
+
if (!this.enabled || !canonicalKey || !scope) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const key = buildNegativeKey(scope, canonicalKey);
|
|
229
|
+
this.#applyEvent({ type: "negative_remove", payload: { key } });
|
|
230
|
+
}
|
|
231
|
+
|
|
348
232
|
async flush() {
|
|
349
233
|
if (!this.enabled || this.flushing) {
|
|
350
234
|
return;
|
|
@@ -370,7 +254,7 @@ export class MavenAffinityIndex {
|
|
|
370
254
|
}
|
|
371
255
|
|
|
372
256
|
async #writeSnapshotAndResetEventLog() {
|
|
373
|
-
const snapshot = serializeSnapshot(this.
|
|
257
|
+
const snapshot = serializeSnapshot(this.negative);
|
|
374
258
|
const tempPath = `${this.snapshotPath}.tmp`;
|
|
375
259
|
await fs.promises.writeFile(tempPath, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
|
|
376
260
|
await fs.promises.rename(tempPath, this.snapshotPath);
|
package/src/config/config.js
CHANGED
|
@@ -3,6 +3,7 @@ import os from "node:os";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import dotenv from "dotenv";
|
|
5
5
|
import { detectJavaHome } from "../common/java-home.js";
|
|
6
|
+
import { parseMavenCacheIgnorePathPrefixes } from "../cache/cache-path.js";
|
|
6
7
|
|
|
7
8
|
const cwd = process.cwd();
|
|
8
9
|
const userConfigDir = path.resolve(os.homedir(), "maven-proxy");
|
|
@@ -196,11 +197,24 @@ const defaultRepoFallbackRepos = [
|
|
|
196
197
|
"https://dl.google.com",
|
|
197
198
|
];
|
|
198
199
|
|
|
200
|
+
const defaultMavenCacheIgnorePathPrefixes = [
|
|
201
|
+
"repo1.maven.org/maven2",
|
|
202
|
+
"repo.maven.apache.org/maven2",
|
|
203
|
+
"jitpack.io/",
|
|
204
|
+
"plugins.gradle.org/m2",
|
|
205
|
+
"dl.google.com/dl/android/maven2",
|
|
206
|
+
"dl.google.com/dl/google/maven",
|
|
207
|
+
];
|
|
208
|
+
|
|
199
209
|
const repoFallbackRepos = normalizeRepoList(
|
|
200
210
|
process.env.REPO_FALLBACK_REPOS,
|
|
201
211
|
defaultRepoFallbackRepos,
|
|
202
212
|
);
|
|
203
213
|
|
|
214
|
+
const mavenCacheIgnorePathPrefixes = String(process.env.MAVEN_CACHE_IGNORE_PATH_PREFIXES || "").trim()
|
|
215
|
+
|| defaultMavenCacheIgnorePathPrefixes.join(",");
|
|
216
|
+
const mavenCacheIgnorePathPrefixRules = parseMavenCacheIgnorePathPrefixes(mavenCacheIgnorePathPrefixes);
|
|
217
|
+
|
|
204
218
|
const defaultMavenRepoDomains = [
|
|
205
219
|
"repo1.maven.org",
|
|
206
220
|
"repo.maven.apache.org",
|
|
@@ -216,15 +230,16 @@ const multiThreadMinSizeBytes = Math.max(0, toInt(process.env.MULTI_THREAD_MIN_S
|
|
|
216
230
|
const downloadTimeout = process.env.DOWNLOAD_TIMEOUT || "60s";
|
|
217
231
|
const outboundKeepAliveInterval = process.env.OUTBOUND_KEEP_ALIVE_INTERVAL || "1s";
|
|
218
232
|
const mavenNegativeCacheTtl = process.env.MAVEN_NEGATIVE_CACHE_TTL || "24h";
|
|
219
|
-
|
|
233
|
+
// Prefer new MAVEN_NEGATIVE_* names but fall back to legacy MAVEN_AFFINITY_* for compatibility
|
|
234
|
+
const mavenNegativeFlushInterval = process.env.MAVEN_NEGATIVE_FLUSH_INTERVAL || process.env.MAVEN_AFFINITY_FLUSH_INTERVAL || "5s";
|
|
220
235
|
const logRetention = process.env.LOG_RETENTION || "7d";
|
|
221
236
|
|
|
222
237
|
const downloadTimeoutMs = Math.max(1, parseDurationToMs(downloadTimeout, 60 * 1000));
|
|
223
238
|
const outboundKeepAliveMsecs = Math.max(1, parseDurationToMs(outboundKeepAliveInterval, 1000));
|
|
224
239
|
const mavenNegativeCacheTtlMs = Math.max(1, parseDurationToMs(mavenNegativeCacheTtl, 24 * 60 * 60 * 1000));
|
|
225
|
-
const
|
|
240
|
+
const mavenNegativeFlushIntervalMs = Math.max(1, parseDurationToMs(mavenNegativeFlushInterval, 5 * 1000));
|
|
226
241
|
const logRetentionDays = Math.max(1, Math.ceil(parseDurationToMs(logRetention, 7 * 24 * 60 * 60 * 1000) / (24 * 60 * 60 * 1000)));
|
|
227
|
-
const
|
|
242
|
+
const mavenNegativeEventMaxBytes = Math.max(1, toInt(process.env.MAVEN_NEGATIVE_EVENT_MAX_MB ?? process.env.MAVEN_AFFINITY_EVENT_MAX_MB, 8)) * 1024 * 1024;
|
|
228
243
|
|
|
229
244
|
export const config = {
|
|
230
245
|
configMode,
|
|
@@ -241,6 +256,9 @@ export const config = {
|
|
|
241
256
|
httpsMitmDomains: toList(process.env.HTTPS_MITM_DOMAINS, ["repo1.maven.org", "repo.maven.apache.org", "registry.npmjs.org"]),
|
|
242
257
|
npmRegistryDomains: toList(process.env.NPM_REGISTRY_DOMAINS, ["registry.npmjs.org", "registry.npmmirror.com", "npm.pkg.github.com"]),
|
|
243
258
|
mavenRepoDomains: toList(process.env.MAVEN_REPO_DOMAINS, [...new Set(defaultMavenRepoDomains)]),
|
|
259
|
+
mavenCacheUseDomainDir: toBool(process.env.MAVEN_CACHE_USE_DOMAIN_DIR, false),
|
|
260
|
+
mavenCacheIgnorePathPrefixes,
|
|
261
|
+
mavenCacheIgnorePathPrefixRules,
|
|
244
262
|
multiThreadDomains: toList(process.env.MULTI_THREAD_DOMAINS, ["repo1.maven.org"]),
|
|
245
263
|
multiThreadCount: Math.max(1, toInt(process.env.MULTI_THREAD_COUNT, 4)),
|
|
246
264
|
multiThreadMinSizeBytes,
|
|
@@ -251,13 +269,20 @@ export const config = {
|
|
|
251
269
|
outboundKeepAliveMsecs,
|
|
252
270
|
outboundMaxSockets: Math.max(1, toInt(process.env.OUTBOUND_MAX_SOCKETS, 64)),
|
|
253
271
|
outboundMaxFreeSockets: Math.max(1, toInt(process.env.OUTBOUND_MAX_FREE_SOCKETS, 16)),
|
|
254
|
-
|
|
255
|
-
|
|
272
|
+
// Negative-only index configuration (new names)
|
|
273
|
+
mavenNegativeEnabled: toBool(process.env.MAVEN_NEGATIVE_ENABLED ?? process.env.MAVEN_AFFINITY_ENABLED, true),
|
|
274
|
+
mavenNegativeIndexDir: path.resolve(configBaseDir, process.env.MAVEN_NEGATIVE_INDEX_DIR || process.env.MAVEN_AFFINITY_INDEX_DIR || "data/index"),
|
|
256
275
|
mavenNegativeCacheTtl,
|
|
257
276
|
mavenNegativeCacheTtlMs,
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
277
|
+
mavenNegativeFlushInterval,
|
|
278
|
+
mavenNegativeFlushIntervalMs,
|
|
279
|
+
mavenNegativeEventMaxBytes,
|
|
280
|
+
// Backwards-compatible aliases for older code that still references affinity names
|
|
281
|
+
mavenAffinityEnabled: toBool(process.env.MAVEN_NEGATIVE_ENABLED ?? process.env.MAVEN_AFFINITY_ENABLED, true),
|
|
282
|
+
mavenAffinityIndexDir: path.resolve(configBaseDir, process.env.MAVEN_NEGATIVE_INDEX_DIR || process.env.MAVEN_AFFINITY_INDEX_DIR || "data/index"),
|
|
283
|
+
mavenAffinityFlushInterval: mavenNegativeFlushInterval,
|
|
284
|
+
mavenAffinityFlushIntervalMs: mavenNegativeFlushIntervalMs,
|
|
285
|
+
mavenAffinityEventMaxBytes: mavenNegativeEventMaxBytes,
|
|
261
286
|
cacheCleanupEnabled: toBool(process.env.CACHE_CLEANUP_ENABLED, true),
|
|
262
287
|
cacheCleanupDailyAt: process.env.CACHE_CLEANUP_DAILY_AT || "03:00",
|
|
263
288
|
cacheCleanupCheckMinInterval: process.env.CACHE_CLEANUP_CHECK_MIN_INTERVAL || "10m",
|
package/src/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { startProxyServer } from "./proxy/proxy-server.js";
|
|
|
7
7
|
import { startRepoServer } from "./repo/repo-server.js";
|
|
8
8
|
import { getTrustStoreCommands } from "./cert/truststore-utils.js";
|
|
9
9
|
import { UpstreamProxyManager } from "./proxy/upstream-proxy.js";
|
|
10
|
-
import {
|
|
10
|
+
import { MavenNegativeIndex } from "./cache/maven-negative-index.js";
|
|
11
11
|
import { CacheCleanupManager } from "./cache/cache-cleanup-manager.js";
|
|
12
12
|
import { installConsoleLogFileMirror, installGlobalErrorLogging } from "./common/console-log-file.js";
|
|
13
13
|
|
|
@@ -84,8 +84,8 @@ async function main() {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
const upstreamProxyManager = new UpstreamProxyManager(config, matchesDomain);
|
|
87
|
-
const
|
|
88
|
-
await
|
|
87
|
+
const mavenNegativeIndex = new MavenNegativeIndex(config);
|
|
88
|
+
await mavenNegativeIndex.init();
|
|
89
89
|
const cacheCleanupManager = new CacheCleanupManager(config);
|
|
90
90
|
await cacheCleanupManager.init();
|
|
91
91
|
|
|
@@ -97,7 +97,7 @@ async function main() {
|
|
|
97
97
|
downloader,
|
|
98
98
|
matchesDomain,
|
|
99
99
|
upstreamProxyManager,
|
|
100
|
-
|
|
100
|
+
mavenNegativeIndex,
|
|
101
101
|
cacheCleanupManager,
|
|
102
102
|
);
|
|
103
103
|
const repoServer = startRepoServer(config, downloader, cacheCleanupManager);
|
|
@@ -120,6 +120,8 @@ async function main() {
|
|
|
120
120
|
startupInfo(`[maven-proxy] repo port: ${config.repoPort}`);
|
|
121
121
|
startupInfo(`[maven-proxy] cache dir : ${config.cacheDir}`);
|
|
122
122
|
startupInfo(`[maven-proxy] cache maven: ${config.mavenCacheDir}`);
|
|
123
|
+
startupInfo(`[maven-proxy] maven cache use domain dir: ${config.mavenCacheUseDomainDir}`);
|
|
124
|
+
startupInfo(`[maven-proxy] maven cache ignore path prefixes: ${config.mavenCacheIgnorePathPrefixes || "(none)"}`);
|
|
123
125
|
startupInfo(`[maven-proxy] cache npm : ${config.npmCacheDir}`);
|
|
124
126
|
startupInfo(`[maven-proxy] cache other: ${config.genericCacheDir}`);
|
|
125
127
|
startupInfo(`[maven-proxy] log dir: ${config.downloadLogDir}`);
|
|
@@ -130,12 +132,12 @@ async function main() {
|
|
|
130
132
|
startupInfo(`[maven-proxy] outbound keepAlive interval: ${config.outboundKeepAliveInterval}`);
|
|
131
133
|
startupInfo(`[maven-proxy] outbound maxSockets: ${config.outboundMaxSockets}`);
|
|
132
134
|
startupInfo(`[maven-proxy] outbound maxFreeSockets: ${config.outboundMaxFreeSockets}`);
|
|
133
|
-
startupInfo(`[maven-proxy] maven
|
|
134
|
-
startupInfo(`[maven-proxy] maven
|
|
135
|
+
startupInfo(`[maven-proxy] maven negative index enabled: ${config.mavenNegativeEnabled}`);
|
|
136
|
+
startupInfo(`[maven-proxy] maven negative index dir: ${config.mavenNegativeIndexDir}`);
|
|
135
137
|
startupInfo(`[maven-proxy] maven negative cache ttl: ${config.mavenNegativeCacheTtl}`);
|
|
136
|
-
startupInfo(`[maven-proxy] maven
|
|
138
|
+
startupInfo(`[maven-proxy] maven negative flush interval: ${config.mavenNegativeFlushInterval}`);
|
|
137
139
|
startupInfo(`[maven-proxy] download timeout: ${config.downloadTimeout}`);
|
|
138
|
-
startupInfo(`[maven-proxy] maven
|
|
140
|
+
startupInfo(`[maven-proxy] maven negative event max(MB): ${config.mavenNegativeEventMaxBytes / (1024 * 1024)}`);
|
|
139
141
|
startupInfo(`[maven-proxy] cache cleanup enabled: ${config.cacheCleanupEnabled}`);
|
|
140
142
|
startupInfo(`[maven-proxy] cache cleanup daily at: ${config.cacheCleanupDailyAt}`);
|
|
141
143
|
startupInfo(`[maven-proxy] cache touch on hit: ${config.cacheTouchOnHit}`);
|
|
@@ -167,7 +169,7 @@ async function main() {
|
|
|
167
169
|
repoServer.close();
|
|
168
170
|
upstreamProxyManager.destroy();
|
|
169
171
|
void cacheCleanupManager.destroy();
|
|
170
|
-
void
|
|
172
|
+
void mavenNegativeIndex.destroy();
|
|
171
173
|
};
|
|
172
174
|
|
|
173
175
|
process.on("SIGINT", shutdown);
|
|
@@ -36,6 +36,44 @@ function pickClient(protocol) {
|
|
|
36
36
|
return protocol === "https:" ? https : http;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
function hasFileExtension(urlObj) {
|
|
40
|
+
try {
|
|
41
|
+
const pathname = String(urlObj?.pathname || "");
|
|
42
|
+
const base = path.basename(pathname || "").toLowerCase();
|
|
43
|
+
if (!base) return false;
|
|
44
|
+
|
|
45
|
+
const knownSuffixes = [
|
|
46
|
+
".pom",
|
|
47
|
+
".jar",
|
|
48
|
+
".aar",
|
|
49
|
+
".war",
|
|
50
|
+
".zip",
|
|
51
|
+
".module",
|
|
52
|
+
".xml",
|
|
53
|
+
".sha1",
|
|
54
|
+
".md5",
|
|
55
|
+
".sha256",
|
|
56
|
+
".sha512",
|
|
57
|
+
".asc",
|
|
58
|
+
".json",
|
|
59
|
+
".toml",
|
|
60
|
+
".klib",
|
|
61
|
+
".tgz",
|
|
62
|
+
".tar.gz",
|
|
63
|
+
];
|
|
64
|
+
|
|
65
|
+
if (knownSuffixes.some((s) => base.endsWith(s))) return true;
|
|
66
|
+
|
|
67
|
+
const ext = path.extname(base);
|
|
68
|
+
if (!ext) return false;
|
|
69
|
+
|
|
70
|
+
// Treat numeric-only extensions (like version segments) as NOT an extension.
|
|
71
|
+
return /[a-zA-Z]/.test(ext);
|
|
72
|
+
} catch {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
39
77
|
function sanitizeHeaders(headers = {}) {
|
|
40
78
|
const result = { ...headers };
|
|
41
79
|
const blocked = [
|
|
@@ -80,11 +118,7 @@ function sendErrorText(res, statusCode, message, context = "proxy") {
|
|
|
80
118
|
sendText(res, statusCode, message);
|
|
81
119
|
}
|
|
82
120
|
|
|
83
|
-
|
|
84
|
-
const lower = String(fileName || "").toLowerCase();
|
|
85
|
-
const base = lower.replace(/\.(sha1|sha256|sha512|md5|asc)$/i, "");
|
|
86
|
-
return /\.(jar|aar|war)$/i.test(base);
|
|
87
|
-
}
|
|
121
|
+
// Positive affinity removed: only negative index (404/410) is retained.
|
|
88
122
|
|
|
89
123
|
function buildUrl(req, forcedProtocol = null) {
|
|
90
124
|
const raw = req.url || "/";
|
|
@@ -200,10 +234,11 @@ export function createHttpRequestHandler({
|
|
|
200
234
|
ecosystem = detectPackageEcosystem(urlObj, config, matchesDomain);
|
|
201
235
|
cachePath = getCacheFilePath(config.cacheDir, urlObj, {
|
|
202
236
|
ecosystem,
|
|
203
|
-
includeHost: ecosystem !== "maven",
|
|
237
|
+
includeHost: ecosystem !== "maven" || config.mavenCacheUseDomainDir,
|
|
238
|
+
mavenCacheIgnorePathPrefixRules: config.mavenCacheIgnorePathPrefixRules,
|
|
204
239
|
});
|
|
205
240
|
|
|
206
|
-
if (ecosystem === "maven" && mavenAffinityIndex
|
|
241
|
+
if (ecosystem === "maven" && mavenAffinityIndex) {
|
|
207
242
|
canonical = parseMavenReleaseCanonical(urlObj);
|
|
208
243
|
}
|
|
209
244
|
} catch (error) {
|
|
@@ -220,22 +255,21 @@ export function createHttpRequestHandler({
|
|
|
220
255
|
}
|
|
221
256
|
|
|
222
257
|
if (canonical && mavenAffinityIndex) {
|
|
223
|
-
if (isPositiveAffinityEligible(canonical.fileName)) {
|
|
224
|
-
const preferredPath = await mavenAffinityIndex.resolvePreferredCachePath(canonical.canonicalKey);
|
|
225
|
-
if (preferredPath) {
|
|
226
|
-
console.log(`[proxy] affinity hit canonical=${canonical.canonicalKey} host=${urlObj.hostname}`);
|
|
227
|
-
await serveFile(res, req, preferredPath, cacheCleanupManager);
|
|
228
|
-
return;
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
|
|
232
258
|
if (mavenAffinityIndex.shouldSkipRequest(canonical.canonicalKey, urlObj)) {
|
|
233
|
-
console.log(`[proxy]
|
|
259
|
+
console.log(`[proxy] negative skip canonical=${canonical.canonicalKey} host=${urlObj.hostname}`);
|
|
234
260
|
sendText(res, 404, "Not Found");
|
|
235
261
|
return;
|
|
236
262
|
}
|
|
237
263
|
}
|
|
238
264
|
|
|
265
|
+
// If the requested resource has no file extension, do not cache it.
|
|
266
|
+
// If a file without extension already exists in cache, it was handled above.
|
|
267
|
+
if (!hasFileExtension(urlObj)) {
|
|
268
|
+
console.log(`[proxy] skip caching for extensionless path host=${urlObj.hostname} path=${urlObj.pathname}`);
|
|
269
|
+
forwardDirectRequest(req, res, urlObj, config.downloadTimeoutMs, upstreamProxyManager);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
|
|
239
273
|
try {
|
|
240
274
|
console.log(`[proxy] local cache miss host=${urlObj.hostname} path=${urlObj.pathname}`);
|
|
241
275
|
if (cacheCleanupManager) {
|
|
@@ -244,14 +278,13 @@ export function createHttpRequestHandler({
|
|
|
244
278
|
await fs.promises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
245
279
|
await downloader.ensureCached(urlObj, cachePath, req.headers);
|
|
246
280
|
|
|
247
|
-
if (canonical && mavenAffinityIndex
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
});
|
|
281
|
+
if (canonical && mavenAffinityIndex) {
|
|
282
|
+
try {
|
|
283
|
+
// Clear any negative entry for this request scope on successful fetch.
|
|
284
|
+
mavenAffinityIndex.clearNegative({ canonicalKey: canonical.canonicalKey, urlObj });
|
|
285
|
+
} catch (err) {
|
|
286
|
+
console.error(`[proxy] clearing negative index failed: ${err?.message || err}`);
|
|
287
|
+
}
|
|
255
288
|
}
|
|
256
289
|
|
|
257
290
|
res.setHeader("x-cache", "MISS");
|
package/src/repo/repo-server.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import http from "node:http";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import {
|
|
5
|
+
buildMavenHostlessPathCandidates,
|
|
6
|
+
stripMavenIgnoredPathPrefix,
|
|
7
|
+
} from "../cache/cache-path.js";
|
|
4
8
|
|
|
5
9
|
function safeJoin(baseDir, requestPath) {
|
|
6
10
|
const pathname = decodeURIComponent(requestPath || "/");
|
|
@@ -13,6 +17,58 @@ function safeJoin(baseDir, requestPath) {
|
|
|
13
17
|
return path.join(baseDir, normalized);
|
|
14
18
|
}
|
|
15
19
|
|
|
20
|
+
function sanitizeHostSegment(hostname) {
|
|
21
|
+
return String(hostname || "unknown").toLowerCase().replace(/[<>:\\"|?*]/g, "_");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function collectRepoHosts(repoBases = []) {
|
|
25
|
+
const hosts = new Set();
|
|
26
|
+
|
|
27
|
+
for (const repoBase of repoBases) {
|
|
28
|
+
try {
|
|
29
|
+
const parsed = new URL(repoBase);
|
|
30
|
+
hosts.add(sanitizeHostSegment(parsed.hostname));
|
|
31
|
+
} catch {
|
|
32
|
+
// ignore invalid URL
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return [...hosts];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function buildDomainScopedPath(mavenCacheDir, hostname, relativePath) {
|
|
40
|
+
const hostDir = sanitizeHostSegment(hostname);
|
|
41
|
+
return safeJoin(path.join(mavenCacheDir, hostDir), relativePath);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function normalizeRelativePathForHost(relativePath, hostname, rules = []) {
|
|
45
|
+
const inputPath = `/${String(relativePath || "").replace(/^\/+/, "")}`;
|
|
46
|
+
const normalized = stripMavenIgnoredPathPrefix(
|
|
47
|
+
inputPath,
|
|
48
|
+
{ protocol: "https:", hostname, port: "" },
|
|
49
|
+
rules,
|
|
50
|
+
);
|
|
51
|
+
return normalized.replace(/^\/+/, "");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function normalizeRelativePathCandidates(relativePath, rules = []) {
|
|
55
|
+
const inputPath = `/${String(relativePath || "").replace(/^\/+/, "")}`;
|
|
56
|
+
const candidates = buildMavenHostlessPathCandidates(inputPath, rules);
|
|
57
|
+
return candidates.map((item) => item.replace(/^\/+/, ""));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function buildDefaultRepoFilePath(config, relativePath) {
|
|
61
|
+
if (!config.mavenCacheUseDomainDir) {
|
|
62
|
+
const candidates = normalizeRelativePathCandidates(relativePath, config.mavenCacheIgnorePathPrefixRules);
|
|
63
|
+
return safeJoin(config.mavenCacheDir, candidates[0] || relativePath);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const hosts = collectRepoHosts(config.repoFallbackRepos || []);
|
|
67
|
+
const host = hosts[0] || "unknown";
|
|
68
|
+
const normalized = normalizeRelativePathForHost(relativePath, host, config.mavenCacheIgnorePathPrefixRules);
|
|
69
|
+
return buildDomainScopedPath(config.mavenCacheDir, host, normalized || relativePath);
|
|
70
|
+
}
|
|
71
|
+
|
|
16
72
|
async function statIfExists(filePath) {
|
|
17
73
|
try {
|
|
18
74
|
return await fs.promises.stat(filePath);
|
|
@@ -24,6 +80,68 @@ async function statIfExists(filePath) {
|
|
|
24
80
|
}
|
|
25
81
|
}
|
|
26
82
|
|
|
83
|
+
async function findCachedMavenFile(config, relativePath) {
|
|
84
|
+
const ignoreRules = config.mavenCacheIgnorePathPrefixRules || [];
|
|
85
|
+
|
|
86
|
+
if (!config.mavenCacheUseDomainDir) {
|
|
87
|
+
const candidates = normalizeRelativePathCandidates(relativePath, ignoreRules);
|
|
88
|
+
for (const candidate of candidates) {
|
|
89
|
+
const filePath = safeJoin(config.mavenCacheDir, candidate);
|
|
90
|
+
const stats = await statIfExists(filePath);
|
|
91
|
+
if (stats && stats.isFile()) {
|
|
92
|
+
return { filePath, stats };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const fallbackPath = safeJoin(config.mavenCacheDir, candidates[0] || relativePath);
|
|
97
|
+
return { filePath: fallbackPath, stats: null };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const checkedHosts = new Set();
|
|
101
|
+
const preferredHosts = collectRepoHosts(config.repoFallbackRepos || []);
|
|
102
|
+
|
|
103
|
+
for (const host of preferredHosts) {
|
|
104
|
+
checkedHosts.add(host);
|
|
105
|
+
const normalized = normalizeRelativePathForHost(relativePath, host, ignoreRules);
|
|
106
|
+
const filePath = buildDomainScopedPath(config.mavenCacheDir, host, normalized || relativePath);
|
|
107
|
+
const stats = await statIfExists(filePath);
|
|
108
|
+
if (stats && stats.isFile()) {
|
|
109
|
+
return { filePath, stats };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
let entries = [];
|
|
114
|
+
try {
|
|
115
|
+
entries = await fs.promises.readdir(config.mavenCacheDir, { withFileTypes: true });
|
|
116
|
+
} catch (error) {
|
|
117
|
+
if (error.code !== "ENOENT") {
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
for (const entry of entries) {
|
|
123
|
+
if (!entry.isDirectory()) {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if (checkedHosts.has(entry.name)) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const normalized = normalizeRelativePathForHost(relativePath, entry.name, ignoreRules);
|
|
132
|
+
const filePath = safeJoin(path.join(config.mavenCacheDir, entry.name), normalized || relativePath);
|
|
133
|
+
const stats = await statIfExists(filePath);
|
|
134
|
+
if (stats && stats.isFile()) {
|
|
135
|
+
return { filePath, stats };
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return {
|
|
140
|
+
filePath: buildDefaultRepoFilePath(config, relativePath),
|
|
141
|
+
stats: null,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
27
145
|
function buildRemoteUrl(repoBase, relativePath) {
|
|
28
146
|
const base = repoBase.endsWith("/") ? repoBase : `${repoBase}/`;
|
|
29
147
|
const relative = relativePath.replace(/^\/+/, "");
|
|
@@ -41,31 +159,45 @@ function buildCandidateRelativePaths(relativePath) {
|
|
|
41
159
|
return [...new Set(candidates.filter(Boolean))];
|
|
42
160
|
}
|
|
43
161
|
|
|
44
|
-
async function ensureFromRemoteRepos(config, downloader,
|
|
162
|
+
async function ensureFromRemoteRepos(config, downloader, relativePath, cacheCleanupManager = null) {
|
|
45
163
|
if (!downloader) {
|
|
46
|
-
return null;
|
|
164
|
+
return { filePath: buildDefaultRepoFilePath(config, relativePath), stats: null };
|
|
47
165
|
}
|
|
48
166
|
|
|
49
167
|
const repos = config.repoFallbackRepos || [];
|
|
50
168
|
if (repos.length === 0) {
|
|
51
|
-
return null;
|
|
169
|
+
return { filePath: buildDefaultRepoFilePath(config, relativePath), stats: null };
|
|
52
170
|
}
|
|
53
171
|
|
|
54
172
|
let hasNon404Error = false;
|
|
55
173
|
let lastError = null;
|
|
56
174
|
const candidatePaths = buildCandidateRelativePaths(relativePath);
|
|
175
|
+
const ignoreRules = config.mavenCacheIgnorePathPrefixRules || [];
|
|
57
176
|
|
|
58
177
|
for (const repoBase of repos) {
|
|
59
178
|
for (const candidatePath of candidatePaths) {
|
|
60
179
|
const remoteUrl = buildRemoteUrl(repoBase, candidatePath);
|
|
61
180
|
|
|
62
181
|
try {
|
|
182
|
+
const normalizedRelativePath = normalizeRelativePathForHost(
|
|
183
|
+
relativePath,
|
|
184
|
+
remoteUrl.hostname,
|
|
185
|
+
ignoreRules,
|
|
186
|
+
) || relativePath;
|
|
187
|
+
|
|
188
|
+
const targetPath = config.mavenCacheUseDomainDir
|
|
189
|
+
? buildDomainScopedPath(config.mavenCacheDir, remoteUrl.hostname, normalizedRelativePath)
|
|
190
|
+
: safeJoin(config.mavenCacheDir, normalizedRelativePath);
|
|
191
|
+
|
|
63
192
|
if (cacheCleanupManager) {
|
|
64
193
|
await cacheCleanupManager.checkAndCleanupIfNeeded("repo-cache-miss");
|
|
65
194
|
}
|
|
66
195
|
console.log(`[repo] cache miss, try remote ${remoteUrl.href}`);
|
|
67
|
-
await downloader.ensureCached(remoteUrl,
|
|
68
|
-
return
|
|
196
|
+
await downloader.ensureCached(remoteUrl, targetPath, {});
|
|
197
|
+
return {
|
|
198
|
+
filePath: targetPath,
|
|
199
|
+
stats: await statIfExists(targetPath),
|
|
200
|
+
};
|
|
69
201
|
} catch (error) {
|
|
70
202
|
lastError = error;
|
|
71
203
|
if (error.statusCode !== 404) {
|
|
@@ -79,7 +211,7 @@ async function ensureFromRemoteRepos(config, downloader, filePath, relativePath,
|
|
|
79
211
|
throw lastError;
|
|
80
212
|
}
|
|
81
213
|
|
|
82
|
-
return null;
|
|
214
|
+
return { filePath: buildDefaultRepoFilePath(config, relativePath), stats: null };
|
|
83
215
|
}
|
|
84
216
|
|
|
85
217
|
export function startRepoServer(config, downloader = null, cacheCleanupManager = null) {
|
|
@@ -87,11 +219,12 @@ export function startRepoServer(config, downloader = null, cacheCleanupManager =
|
|
|
87
219
|
try {
|
|
88
220
|
const urlObj = new URL(req.url || "/", `http://${req.headers.host || "localhost"}`);
|
|
89
221
|
const relativePath = path.posix.normalize(urlObj.pathname || "/").replace(/^\/+/, "");
|
|
90
|
-
|
|
91
|
-
let stats = await statIfExists(filePath);
|
|
222
|
+
let { filePath, stats } = await findCachedMavenFile(config, relativePath);
|
|
92
223
|
|
|
93
224
|
if (!stats || !stats.isFile()) {
|
|
94
|
-
|
|
225
|
+
const fetched = await ensureFromRemoteRepos(config, downloader, relativePath, cacheCleanupManager);
|
|
226
|
+
filePath = fetched.filePath;
|
|
227
|
+
stats = fetched.stats;
|
|
95
228
|
}
|
|
96
229
|
|
|
97
230
|
if (!stats || !stats.isFile()) {
|