maven-proxy 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -5
- package/bin/maven-proxy.js +70 -43
- package/package.json +2 -1
- package/src/cache/cache-cleanup-manager.js +416 -0
- package/src/config/config.js +63 -8
- package/src/index.js +21 -5
- package/src/proxy/proxy-http-handler.js +12 -4
- package/src/proxy/proxy-server.js +2 -0
- package/src/repo/repo-server.js +10 -3
package/README.md
CHANGED
|
@@ -256,6 +256,17 @@ curl.exe -k -sS -D - -o NUL -x http://127.0.0.1:8080 https://registry.npmjs.org/
|
|
|
256
256
|
### 8.4 Upstream Proxy Settings
|
|
257
257
|
|
|
258
258
|
Environment variables:
|
|
259
|
+
- CACHE_CLEANUP_ENABLED: enable automatic cache cleanup.
|
|
260
|
+
- CACHE_CLEANUP_DAILY_AT: daily cleanup check time in local timezone (HH:mm), for example 03:00.
|
|
261
|
+
- CACHE_CLEANUP_CHECK_MIN_INTERVAL: minimum interval between pressure checks (supports s/m/h/d), for example 10m.
|
|
262
|
+
- CACHE_TOUCH_ON_HIT: update file mtime when cache hit returns successfully.
|
|
263
|
+
- CACHE_TOUCH_MIN_INTERVAL: minimum interval between two touches for the same file (supports s/m/h/d), default 1d.
|
|
264
|
+
- CACHE_RETENTION_START: initial retention window for cleanup rounds (supports s/m/h/d), default 10d.
|
|
265
|
+
- CACHE_RETENTION_MIN: minimum retention window for cleanup rounds (supports s/m/h/d), default 1d.
|
|
266
|
+
- CACHE_DISK_FREE_TRIGGER: trigger cleanup when disk free bytes are below this value (supports K/M/G/T), for example 20G.
|
|
267
|
+
- CACHE_DISK_FREE_TARGET: stop cleanup when disk free bytes recover to this value (supports K/M/G/T), for example 25G.
|
|
268
|
+
- CACHE_MAX_SIZE: optional cache-size trigger threshold (supports K/M/G/T).
|
|
269
|
+
- CACHE_TARGET_SIZE: optional cache-size target to stop cleanup (supports K/M/G/T).
|
|
259
270
|
- UPSTREAM_PROXY_URL: generic upstream proxy URL.
|
|
260
271
|
- UPSTREAM_HTTP_PROXY_URL: upstream proxy for HTTP.
|
|
261
272
|
- UPSTREAM_HTTPS_PROXY_URL: upstream proxy for HTTPS.
|
|
@@ -266,13 +277,18 @@ Environment variables:
|
|
|
266
277
|
- MAVEN_REPO_DOMAINS: maven domains for ecosystem routing (wildcards supported).
|
|
267
278
|
- HTTPS_MITM_DOMAINS: MITM domain list (includes registry.npmjs.org by default, wildcards supported).
|
|
268
279
|
- DOWNLOAD_LOG_DIR: log directory.
|
|
269
|
-
-
|
|
280
|
+
- LOG_RETENTION: log retention duration (supports s/m/h/d), for example 7d.
|
|
270
281
|
- LOG_TO_STDOUT: whether to also print runtime logs to stdout/stderr; startup logs are always printed.
|
|
271
282
|
- LOG_CONNECT_EVENTS: whether to print verbose CONNECT/MITM handshake logs. Default false.
|
|
272
283
|
- OUTBOUND_KEEP_ALIVE: enable outbound keep-alive connection pooling.
|
|
273
|
-
-
|
|
284
|
+
- OUTBOUND_KEEP_ALIVE_INTERVAL: keep-alive interval (supports s/m/h/d), for example 1s.
|
|
274
285
|
- OUTBOUND_MAX_SOCKETS: max outbound sockets per origin.
|
|
275
286
|
- OUTBOUND_MAX_FREE_SOCKETS: max idle outbound sockets per origin.
|
|
287
|
+
- MAVEN_AFFINITY_ENABLED: enable Maven affinity index.
|
|
288
|
+
- MAVEN_AFFINITY_INDEX_DIR: Maven affinity index directory. Default data/index.
|
|
289
|
+
- MAVEN_NEGATIVE_CACHE_TTL: negative cache TTL (supports s/m/h/d), for example 24h.
|
|
290
|
+
- MAVEN_AFFINITY_FLUSH_INTERVAL: flush interval for affinity event log (supports s/m/h/d), for example 5s.
|
|
291
|
+
- MAVEN_AFFINITY_EVENT_MAX_MB: max size threshold for affinity event log compaction in MB.
|
|
276
292
|
- MAVEN_PROXY_CONFIG_MODE: development or user.
|
|
277
293
|
- MAVEN_PROXY_CONFIG_FILE: explicit config file path.
|
|
278
294
|
- EXISTING_TRUST_STORE_PATH: optional existing truststore path. If present, truststore init prefers it as source.
|
|
@@ -292,6 +308,17 @@ Priority:
|
|
|
292
308
|
- `PROXY_PORT`: Proxy server port. Default `8080`.
|
|
293
309
|
- `REPO_PORT`: Local repository server port. Default `8081`.
|
|
294
310
|
- `CACHE_DIR`: Base cache directory. Default `data/cache`.
|
|
311
|
+
- `CACHE_CLEANUP_ENABLED`: Enable automatic cache cleanup. Default `true`.
|
|
312
|
+
- `CACHE_CLEANUP_DAILY_AT`: Daily cleanup check time in local timezone (`HH:mm`). Default `03:00`.
|
|
313
|
+
- `CACHE_CLEANUP_CHECK_MIN_INTERVAL`: Minimum interval between pressure checks (supports `s/m/h/d`). Default `10m`.
|
|
314
|
+
- `CACHE_TOUCH_ON_HIT`: Update file mtime when cache hit returns successfully. Default `true`.
|
|
315
|
+
- `CACHE_TOUCH_MIN_INTERVAL`: Minimum interval between two touches for the same file (supports `s/m/h/d`). Default `1d`.
|
|
316
|
+
- `CACHE_RETENTION_START`: Initial retention window for cleanup rounds (supports `s/m/h/d`). Default `10d`.
|
|
317
|
+
- `CACHE_RETENTION_MIN`: Minimum retention window for cleanup rounds (supports `s/m/h/d`). Default `1d`.
|
|
318
|
+
- `CACHE_DISK_FREE_TRIGGER`: Trigger cleanup when disk free bytes are below this value (supports `K/M/G/T`). Default `20G`.
|
|
319
|
+
- `CACHE_DISK_FREE_TARGET`: Stop cleanup when disk free bytes recover to this value (supports `K/M/G/T`). Default `25G`.
|
|
320
|
+
- `CACHE_MAX_SIZE`: Optional cache-size trigger threshold (supports `K/M/G/T`). Default empty (disabled).
|
|
321
|
+
- `CACHE_TARGET_SIZE`: Optional cache-size target to stop cleanup (supports `K/M/G/T`). Default empty (disabled).
|
|
295
322
|
- `REPO_FALLBACK_REPOS`: Comma-separated fallback repository URLs for cache misses.
|
|
296
323
|
- `ENABLE_HTTPS_PROXY`: Enable HTTPS proxy handling (`true/false`).
|
|
297
324
|
- `HTTPS_MITM_DOMAINS`: Comma-separated domains to apply MITM certificate issuance (wildcards supported).
|
|
@@ -301,15 +328,20 @@ Priority:
|
|
|
301
328
|
- `MULTI_THREAD_DOMAINS`: Domains allowed to use multi-thread download (wildcards supported).
|
|
302
329
|
- `MULTI_THREAD_COUNT`: Number of download threads for ranged downloads.
|
|
303
330
|
- `MULTI_THREAD_MIN_SIZE_MB`: Minimum size threshold to trigger multi-thread download (MB).
|
|
304
|
-
- `
|
|
331
|
+
- `DOWNLOAD_TIMEOUT`: Upstream request timeout (supports `s/m/h/d`). Default `60s`.
|
|
305
332
|
- `DOWNLOAD_LOG_DIR`: Directory for unified app/error logs.
|
|
306
|
-
- `
|
|
333
|
+
- `LOG_RETENTION`: Log retention duration (supports `s/m/h/d`). Default `7d`.
|
|
307
334
|
- `LOG_TO_STDOUT`: Whether to also print runtime logs to stdout/stderr. Startup logs are always printed. Default `true`.
|
|
308
335
|
- `LOG_CONNECT_EVENTS`: Whether to print verbose CONNECT/MITM handshake logs. Default `false`.
|
|
309
336
|
- `OUTBOUND_KEEP_ALIVE`: Enable outbound keep-alive connection pooling. Default `true`.
|
|
310
|
-
- `
|
|
337
|
+
- `OUTBOUND_KEEP_ALIVE_INTERVAL`: Keep-alive interval (supports `s/m/h/d`). Default `1s`.
|
|
311
338
|
- `OUTBOUND_MAX_SOCKETS`: Max outbound sockets per origin. Default `64`.
|
|
312
339
|
- `OUTBOUND_MAX_FREE_SOCKETS`: Max idle outbound sockets per origin. Default `16`.
|
|
340
|
+
- `MAVEN_AFFINITY_ENABLED`: Enable Maven affinity cache index. Default `true`.
|
|
341
|
+
- `MAVEN_AFFINITY_INDEX_DIR`: Maven affinity index directory. Default `data/index`.
|
|
342
|
+
- `MAVEN_NEGATIVE_CACHE_TTL`: Negative cache TTL (supports `s/m/h/d`). Default `24h`.
|
|
343
|
+
- `MAVEN_AFFINITY_FLUSH_INTERVAL`: Flush interval for affinity event log (supports `s/m/h/d`). Default `5s`.
|
|
344
|
+
- `MAVEN_AFFINITY_EVENT_MAX_MB`: Max size threshold for affinity event log compaction in MB. Default `8`.
|
|
313
345
|
- `UPSTREAM_PROXY_URL`: Generic upstream proxy URL (fallback for HTTP/HTTPS).
|
|
314
346
|
- `UPSTREAM_HTTP_PROXY_URL`: Upstream proxy URL for HTTP requests.
|
|
315
347
|
- `UPSTREAM_HTTPS_PROXY_URL`: Upstream proxy URL for HTTPS requests.
|
package/bin/maven-proxy.js
CHANGED
|
@@ -140,52 +140,79 @@ function resolvePath(inputPath) {
|
|
|
140
140
|
}
|
|
141
141
|
|
|
142
142
|
function getDefaultConfigTemplate() {
|
|
143
|
-
|
|
143
|
+
const lines = [
|
|
144
144
|
"# Maven Proxy default user config",
|
|
145
|
+
"# Maven Proxy 用户模式默认配置",
|
|
145
146
|
"# This file is loaded by default in user mode.",
|
|
146
147
|
"",
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
"
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
"
|
|
172
|
-
"
|
|
173
|
-
"
|
|
174
|
-
"
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
148
|
+
];
|
|
149
|
+
|
|
150
|
+
const appendEntry = (key, value, zhComment, enComment) => {
|
|
151
|
+
lines.push(`# 中文: ${zhComment}`);
|
|
152
|
+
lines.push(`# English: ${enComment}`);
|
|
153
|
+
lines.push(`${key}=${value}`);
|
|
154
|
+
lines.push("");
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
appendEntry("PROXY_PORT", "8080", "代理服务端口,默认 8080。", "Proxy service port. Default: 8080.");
|
|
158
|
+
appendEntry("REPO_PORT", "8081", "本地仓库服务端口,默认 8081。", "Local repository service port. Default: 8081.");
|
|
159
|
+
appendEntry("CACHE_DIR", "data/cache", "缓存根目录,默认 data/cache。", "Cache root directory. Default: data/cache.");
|
|
160
|
+
appendEntry("CACHE_CLEANUP_ENABLED", "true", "是否启用自动缓存清理,默认 true。", "Enable automatic cache cleanup. Default: true.");
|
|
161
|
+
appendEntry("CACHE_CLEANUP_DAILY_AT", "03:00", "每日固定检查时间(本地时区,HH:mm),默认 03:00。", "Fixed daily check time in local timezone (HH:mm). Default: 03:00.");
|
|
162
|
+
appendEntry("CACHE_CLEANUP_CHECK_MIN_INTERVAL", "10m", "压力检测最小间隔(支持 s/m/h/d),默认 10m。", "Minimum interval for pressure checks (supports s/m/h/d). Default: 10m.");
|
|
163
|
+
appendEntry("CACHE_TOUCH_ON_HIT", "true", "缓存命中返回时是否更新文件 mtime,默认 true。", "Update file mtime when serving cache hit. Default: true.");
|
|
164
|
+
appendEntry("CACHE_TOUCH_MIN_INTERVAL", "1d", "同一文件两次 touch 最小间隔(支持 s/m/h/d),默认 1d。", "Minimum interval between touches for the same file (supports s/m/h/d). Default: 1d.");
|
|
165
|
+
appendEntry("CACHE_RETENTION_START", "10d", "清理轮次初始保留窗口(支持 s/m/h/d),默认 10d。", "Initial retention window for cleanup cycle (supports s/m/h/d). Default: 10d.");
|
|
166
|
+
appendEntry("CACHE_RETENTION_MIN", "1d", "清理轮次最小保留窗口(支持 s/m/h/d),默认 1d。", "Minimum retention window for cleanup cycle (supports s/m/h/d). Default: 1d.");
|
|
167
|
+
appendEntry("CACHE_DISK_FREE_TRIGGER", "20G", "磁盘剩余空间低于该值触发清理(支持 K/M/G/T),默认 20G。", "Trigger cleanup when free disk space is below this value (supports K/M/G/T). Default: 20G.");
|
|
168
|
+
appendEntry("CACHE_DISK_FREE_TARGET", "25G", "磁盘剩余空间恢复到该值可停止清理(支持 K/M/G/T),默认 25G。", "Stop cleanup when free disk space recovers to this value (supports K/M/G/T). Default: 25G.");
|
|
169
|
+
appendEntry("CACHE_MAX_SIZE", "", "可选:缓存总大小触发阈值(支持 K/M/G/T),默认空(禁用)。", "Optional: cache total size trigger threshold (supports K/M/G/T). Default: empty (disabled).");
|
|
170
|
+
appendEntry("CACHE_TARGET_SIZE", "", "可选:缓存总大小回落目标(支持 K/M/G/T),默认空(禁用)。", "Optional: cache size recovery target (supports K/M/G/T). Default: empty (disabled).");
|
|
171
|
+
appendEntry(
|
|
172
|
+
"REPO_FALLBACK_REPOS",
|
|
173
|
+
"https://repo1.maven.org/maven2,https://jitpack.io,https://plugins.gradle.org/m2,https://dl.google.com",
|
|
174
|
+
"缓存未命中时回源仓库列表(逗号分隔)。",
|
|
175
|
+
"Fallback repository list when cache misses (comma-separated).",
|
|
176
|
+
);
|
|
177
|
+
appendEntry("ENABLE_HTTPS_PROXY", "true", "是否启用 HTTPS 代理处理(true/false),默认 true。", "Enable HTTPS proxy handling (true/false). Default: true.");
|
|
178
|
+
appendEntry("HTTPS_MITM_DOMAINS", "repo1.maven.org,repo.maven.apache.org,registry.npmjs.org", "执行 MITM 证书签发的域名列表(逗号分隔,支持通配符)。", "Domains for MITM certificate issuance (comma-separated, wildcard supported).");
|
|
179
|
+
appendEntry("HTTPS_PASSTHROUGH_FOR_UNMATCHED", "false", "未命中 MITM 域名时是否允许隧道透传,默认 false。", "Allow tunnel passthrough for unmatched MITM domains. Default: false.");
|
|
180
|
+
appendEntry("NPM_REGISTRY_DOMAINS", "registry.npmjs.org,registry.npmmirror.com,npm.pkg.github.com", "识别为 npm 生态并分流缓存的域名列表(支持通配符)。", "Domains recognized as npm ecosystem for cache routing (wildcard supported).");
|
|
181
|
+
appendEntry("MAVEN_REPO_DOMAINS", "repo1.maven.org,repo.maven.apache.org,jitpack.io,plugins.gradle.org,dl.google.com", "识别为 Maven 生态并分流缓存的域名列表(支持通配符)。", "Domains recognized as Maven ecosystem for cache routing (wildcard supported).");
|
|
182
|
+
appendEntry("MULTI_THREAD_DOMAINS", "repo1.maven.org", "启用多线程下载的域名列表(支持通配符)。", "Domains that enable multi-thread download (wildcard supported).");
|
|
183
|
+
appendEntry("MULTI_THREAD_COUNT", "8", "多线程下载线程数,默认 8。", "Thread count for multi-thread download. Default: 8.");
|
|
184
|
+
appendEntry("MULTI_THREAD_MIN_SIZE_MB", "1", "触发多线程下载的最小文件大小阈值(MB),默认 1。", "Minimum file size threshold in MB to trigger multi-thread download. Default: 1.");
|
|
185
|
+
appendEntry("DOWNLOAD_TIMEOUT", "60s", "上游请求超时时间(支持 s/m/h/d),默认 60s。", "Upstream request timeout (supports s/m/h/d). Default: 60s.");
|
|
186
|
+
appendEntry("DOWNLOAD_LOG_DIR", "data/logs/downloads", "统一主日志与错误日志目录。", "Unified main and error log directory.");
|
|
187
|
+
appendEntry("LOG_RETENTION", "7d", "日志保留时长(支持 s/m/h/d),默认 7d。", "Log retention duration (supports s/m/h/d). Default: 7d.");
|
|
188
|
+
appendEntry("LOG_TO_STDOUT", "false", "是否输出运行期日志到命令行,默认 false。", "Output runtime logs to stdout. Default: false.");
|
|
189
|
+
appendEntry("LOG_CONNECT_EVENTS", "false", "是否输出详细 CONNECT/MITM 握手日志,默认 false。", "Output detailed CONNECT/MITM handshake logs. Default: false.");
|
|
190
|
+
appendEntry("OUTBOUND_KEEP_ALIVE", "true", "是否启用出站 keep-alive 连接复用池,默认 true。", "Enable outbound keep-alive connection pool. Default: true.");
|
|
191
|
+
appendEntry("OUTBOUND_KEEP_ALIVE_INTERVAL", "1s", "keep-alive 间隔(支持 s/m/h/d),默认 1s。", "Keep-alive interval (supports s/m/h/d). Default: 1s.");
|
|
192
|
+
appendEntry("OUTBOUND_MAX_SOCKETS", "64", "每个源站最大出站连接数,默认 64。", "Maximum outbound sockets per upstream host. Default: 64.");
|
|
193
|
+
appendEntry("OUTBOUND_MAX_FREE_SOCKETS", "16", "每个源站可保留空闲连接上限,默认 16。", "Maximum free outbound sockets kept per upstream host. Default: 16.");
|
|
194
|
+
appendEntry("MAVEN_AFFINITY_ENABLED", "true", "是否启用 Maven affinity 缓存索引,默认 true。", "Enable Maven affinity index. Default: true.");
|
|
195
|
+
appendEntry("MAVEN_AFFINITY_INDEX_DIR", "data/index", "Maven affinity 索引目录,默认 data/index。", "Maven affinity index directory. Default: data/index.");
|
|
196
|
+
appendEntry("MAVEN_NEGATIVE_CACHE_TTL", "24h", "负缓存 TTL(支持 s/m/h/d),默认 24h。", "Negative cache TTL (supports s/m/h/d). Default: 24h.");
|
|
197
|
+
appendEntry("MAVEN_AFFINITY_FLUSH_INTERVAL", "5s", "affinity 事件日志 flush 周期(支持 s/m/h/d),默认 5s。", "Affinity event log flush interval (supports s/m/h/d). Default: 5s.");
|
|
198
|
+
appendEntry("MAVEN_AFFINITY_EVENT_MAX_MB", "8", "affinity 事件日志压缩阈值(MB),默认 8。", "Affinity event log compaction threshold in MB. Default: 8.");
|
|
199
|
+
appendEntry("UPSTREAM_PROXY_URL", "", "通用上级代理地址(HTTP/HTTPS 兜底)。", "Generic upstream proxy URL fallback for HTTP/HTTPS.");
|
|
200
|
+
appendEntry("UPSTREAM_HTTP_PROXY_URL", "", "HTTP 请求使用的上级代理地址。", "Upstream proxy URL for HTTP requests.");
|
|
201
|
+
appendEntry("UPSTREAM_HTTPS_PROXY_URL", "", "HTTPS 请求使用的上级代理地址。", "Upstream proxy URL for HTTPS requests.");
|
|
202
|
+
appendEntry("UPSTREAM_NO_PROXY", "127.0.0.1,localhost", "不走上级代理的域名列表(逗号分隔,支持通配符,* 表示全部直连)。", "Domains that bypass upstream proxy (comma-separated, wildcard supported, * means direct for all).");
|
|
203
|
+
appendEntry("UPSTREAM_IGNORE_DOMAINS", "", "额外忽略上级代理的域名列表(支持通配符)。", "Additional domains ignored by upstream proxy (wildcard supported).");
|
|
204
|
+
appendEntry("CERT_DIR", "data/certs", "证书根目录,默认 data/certs。", "Certificate root directory. Default: data/certs.");
|
|
205
|
+
appendEntry("ROOT_CERT_PATH", "data/certs/root-ca.crt", "Root CA 证书路径。", "Root CA certificate path.");
|
|
206
|
+
appendEntry("ROOT_KEY_PATH", "data/certs/root-ca.key.pem", "Root CA 私钥路径。", "Root CA private key path.");
|
|
207
|
+
appendEntry("LEAF_CERT_DIR", "data/certs/leaf", "站点叶子证书目录。", "Leaf certificate directory.");
|
|
208
|
+
appendEntry("TRUST_STORE_PATH", "data/certs/proxy-truststore.jks", "Java trust store 文件路径。", "Java trust store file path.");
|
|
209
|
+
appendEntry("TRUST_STORE_ALIAS", "maven-proxy-root-ca", "导入 Root CA 到 trust store 时使用的别名。", "Alias used to import Root CA into trust store.");
|
|
210
|
+
appendEntry("TRUST_STORE_PASSWORD", "changeit", "trust store 密码。", "Trust store password.");
|
|
211
|
+
appendEntry("EXISTING_TRUST_STORE_PATH", "", "已有 truststore 路径(可选),初始化时可作为源。", "Existing truststore path (optional), used as source during init.");
|
|
212
|
+
appendEntry("EXISTING_TRUST_STORE_PASSWORD", "", "已有 truststore 密码(可选),用于读取 EXISTING_TRUST_STORE_PATH。", "Existing truststore password (optional), used to read EXISTING_TRUST_STORE_PATH.");
|
|
213
|
+
appendEntry("JAVA_HOME", "", "Java 安装路径;为空或无效时会自动探测。", "Java installation path; auto-detected when empty or invalid.");
|
|
214
|
+
|
|
215
|
+
return `${lines.join("\n").trimEnd()}\n`;
|
|
189
216
|
}
|
|
190
217
|
|
|
191
218
|
async function initConfigFile(configFile, force = false) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "maven-proxy",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "Maven proxy with cache, HTTPS MITM for selected domains, and local repo publishing",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
"cli:start": "node bin/maven-proxy.js start --mode development",
|
|
20
20
|
"cli:stop": "node bin/maven-proxy.js stop",
|
|
21
21
|
"cli:doctor": "node bin/maven-proxy.js doctor --mode development",
|
|
22
|
+
"cli:doctor:user": "node bin/maven-proxy.js doctor --mode user",
|
|
22
23
|
"cli:truststore:print": "node bin/maven-proxy.js truststore print --mode development",
|
|
23
24
|
"cli:truststore:init": "node bin/maven-proxy.js truststore init --mode development",
|
|
24
25
|
"cli:truststore:merge": "node bin/maven-proxy.js truststore merge --mode development",
|
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
const SECOND_MS = 1000;
|
|
5
|
+
const MINUTE_MS = 60 * SECOND_MS;
|
|
6
|
+
const HOUR_MS = 60 * MINUTE_MS;
|
|
7
|
+
const DAY_MS = 24 * HOUR_MS;
|
|
8
|
+
|
|
9
|
+
function toBool(value, fallback) {
|
|
10
|
+
if (value == null || value === "") {
|
|
11
|
+
return fallback;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
return ["1", "true", "yes", "on"].includes(String(value).toLowerCase());
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function parseDurationToMs(raw, fallbackMs) {
|
|
18
|
+
const text = String(raw || "").trim();
|
|
19
|
+
if (!text) {
|
|
20
|
+
return fallbackMs;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const match = text.match(/^(\d+)([smhd])$/i);
|
|
24
|
+
if (!match) {
|
|
25
|
+
return fallbackMs;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const value = Number.parseInt(match[1], 10);
|
|
29
|
+
const unit = match[2].toLowerCase();
|
|
30
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
31
|
+
return fallbackMs;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (unit === "s") {
|
|
35
|
+
return value * SECOND_MS;
|
|
36
|
+
}
|
|
37
|
+
if (unit === "m") {
|
|
38
|
+
return value * MINUTE_MS;
|
|
39
|
+
}
|
|
40
|
+
if (unit === "h") {
|
|
41
|
+
return value * HOUR_MS;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return value * DAY_MS;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function parseSizeToBytes(raw, fallbackBytes = 0) {
|
|
48
|
+
const text = String(raw || "").trim();
|
|
49
|
+
if (!text) {
|
|
50
|
+
return fallbackBytes;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const match = text.match(/^(\d+)([KMGT]?)$/i);
|
|
54
|
+
if (!match) {
|
|
55
|
+
return fallbackBytes;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const value = Number.parseInt(match[1], 10);
|
|
59
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
60
|
+
return fallbackBytes;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const unit = String(match[2] || "").toUpperCase();
|
|
64
|
+
const unitPow = {
|
|
65
|
+
"": 0,
|
|
66
|
+
K: 1,
|
|
67
|
+
M: 2,
|
|
68
|
+
G: 3,
|
|
69
|
+
T: 4,
|
|
70
|
+
}[unit];
|
|
71
|
+
|
|
72
|
+
if (unitPow == null) {
|
|
73
|
+
return fallbackBytes;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return value * (1024 ** unitPow);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function parseDailyAt(text, fallback = { hour: 3, minute: 0 }) {
|
|
80
|
+
const raw = String(text || "").trim();
|
|
81
|
+
const match = raw.match(/^(\d{1,2}):(\d{2})$/);
|
|
82
|
+
if (!match) {
|
|
83
|
+
return fallback;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const hour = Number.parseInt(match[1], 10);
|
|
87
|
+
const minute = Number.parseInt(match[2], 10);
|
|
88
|
+
|
|
89
|
+
if (hour < 0 || hour > 23 || minute < 0 || minute > 59) {
|
|
90
|
+
return fallback;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return { hour, minute };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function statIfFile(filePath) {
|
|
97
|
+
try {
|
|
98
|
+
const stats = await fs.promises.stat(filePath);
|
|
99
|
+
return stats.isFile() ? stats : null;
|
|
100
|
+
} catch (error) {
|
|
101
|
+
if (error.code === "ENOENT") {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function getDiskFreeBytes(targetDir) {
|
|
109
|
+
if (typeof fs.promises.statfs !== "function") {
|
|
110
|
+
return Number.POSITIVE_INFINITY;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const info = await fs.promises.statfs(targetDir);
|
|
114
|
+
return Number(info.bavail || 0) * Number(info.bsize || 0);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function walkFiles(dirPath, onFile) {
|
|
118
|
+
const entries = await fs.promises.readdir(dirPath, { withFileTypes: true });
|
|
119
|
+
|
|
120
|
+
for (const entry of entries) {
|
|
121
|
+
const fullPath = path.join(dirPath, entry.name);
|
|
122
|
+
|
|
123
|
+
if (entry.isDirectory()) {
|
|
124
|
+
await walkFiles(fullPath, onFile);
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (!entry.isFile()) {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
await onFile(fullPath);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function getDirSizeBytes(dirPath) {
|
|
137
|
+
let total = 0;
|
|
138
|
+
|
|
139
|
+
try {
|
|
140
|
+
await walkFiles(dirPath, async (filePath) => {
|
|
141
|
+
const stats = await statIfFile(filePath);
|
|
142
|
+
if (stats) {
|
|
143
|
+
total += stats.size;
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
} catch (error) {
|
|
147
|
+
if (error.code !== "ENOENT") {
|
|
148
|
+
throw error;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return total;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function formatBytes(bytes) {
|
|
156
|
+
const value = Number(bytes || 0);
|
|
157
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
158
|
+
return "0B";
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (value >= 1024 ** 4) {
|
|
162
|
+
return `${(value / (1024 ** 4)).toFixed(2)}TB`;
|
|
163
|
+
}
|
|
164
|
+
if (value >= 1024 ** 3) {
|
|
165
|
+
return `${(value / (1024 ** 3)).toFixed(2)}GB`;
|
|
166
|
+
}
|
|
167
|
+
if (value >= 1024 ** 2) {
|
|
168
|
+
return `${(value / (1024 ** 2)).toFixed(2)}MB`;
|
|
169
|
+
}
|
|
170
|
+
if (value >= 1024) {
|
|
171
|
+
return `${(value / 1024).toFixed(2)}KB`;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return `${Math.floor(value)}B`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export class CacheCleanupManager {
|
|
178
|
+
constructor(config) {
|
|
179
|
+
this.config = config;
|
|
180
|
+
this.enabled = toBool(config.cacheCleanupEnabled, true);
|
|
181
|
+
this.touchOnHit = toBool(config.cacheTouchOnHit, true);
|
|
182
|
+
this.touchMinIntervalMs = Math.max(0, parseDurationToMs(config.cacheTouchMinInterval, DAY_MS));
|
|
183
|
+
this.retentionStartDays = Math.max(1, Math.floor(parseDurationToMs(config.cacheRetentionStart, 10 * DAY_MS) / DAY_MS));
|
|
184
|
+
this.retentionMinDays = Math.max(1, Math.floor(parseDurationToMs(config.cacheRetentionMin, DAY_MS) / DAY_MS));
|
|
185
|
+
this.diskFreeTriggerBytes = Math.max(0, parseSizeToBytes(config.cacheDiskFreeTrigger, 20 * 1024 ** 3));
|
|
186
|
+
this.diskFreeTargetBytes = Math.max(
|
|
187
|
+
this.diskFreeTriggerBytes,
|
|
188
|
+
parseSizeToBytes(config.cacheDiskFreeTarget, 25 * 1024 ** 3),
|
|
189
|
+
);
|
|
190
|
+
this.cacheMaxSizeBytes = Math.max(0, parseSizeToBytes(config.cacheMaxSize, 0));
|
|
191
|
+
this.cacheTargetSizeBytes = Math.max(
|
|
192
|
+
0,
|
|
193
|
+
parseSizeToBytes(config.cacheTargetSize, this.cacheMaxSizeBytes || 0),
|
|
194
|
+
);
|
|
195
|
+
this.checkMinIntervalMs = Math.max(MINUTE_MS, parseDurationToMs(config.cacheCleanupCheckMinInterval, 10 * MINUTE_MS));
|
|
196
|
+
this.dailyAt = parseDailyAt(config.cacheCleanupDailyAt, { hour: 3, minute: 0 });
|
|
197
|
+
|
|
198
|
+
this.lastTouchAt = new Map();
|
|
199
|
+
this.lastPressureCheckAt = 0;
|
|
200
|
+
this.cleanupRunning = false;
|
|
201
|
+
this.dailyTimer = null;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async init() {
|
|
205
|
+
if (!this.enabled) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
this.#scheduleDailyCheck();
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
async destroy() {
|
|
213
|
+
if (this.dailyTimer) {
|
|
214
|
+
clearTimeout(this.dailyTimer);
|
|
215
|
+
this.dailyTimer = null;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
touchFileOnHit(filePath) {
|
|
220
|
+
if (!this.enabled || !this.touchOnHit || !filePath) {
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const now = Date.now();
|
|
225
|
+
const last = this.lastTouchAt.get(filePath) || 0;
|
|
226
|
+
if (now - last < this.touchMinIntervalMs) {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
this.lastTouchAt.set(filePath, now);
|
|
231
|
+
|
|
232
|
+
fs.promises.utimes(filePath, new Date(now), new Date(now)).catch((error) => {
|
|
233
|
+
if (error.code !== "ENOENT") {
|
|
234
|
+
console.warn(`[cache-cleanup] touch failed path=${filePath} message=${error.message}`);
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async checkAndCleanupIfNeeded(reason = "manual", force = false) {
|
|
240
|
+
if (!this.enabled) {
|
|
241
|
+
return { triggered: false, reason: "disabled" };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const now = Date.now();
|
|
245
|
+
if (!force && now - this.lastPressureCheckAt < this.checkMinIntervalMs) {
|
|
246
|
+
return { triggered: false, reason: "throttled" };
|
|
247
|
+
}
|
|
248
|
+
this.lastPressureCheckAt = now;
|
|
249
|
+
|
|
250
|
+
const metrics = await this.#collectMetrics();
|
|
251
|
+
const overLimit = this.#isOverLimit(metrics);
|
|
252
|
+
|
|
253
|
+
if (!overLimit) {
|
|
254
|
+
return {
|
|
255
|
+
triggered: false,
|
|
256
|
+
reason: "below-limit",
|
|
257
|
+
metrics,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
return this.#runCleanup(reason, metrics);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async #runCleanup(reason, beforeMetrics) {
|
|
265
|
+
if (this.cleanupRunning) {
|
|
266
|
+
return { triggered: false, reason: "already-running" };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
this.cleanupRunning = true;
|
|
270
|
+
|
|
271
|
+
try {
|
|
272
|
+
console.warn(
|
|
273
|
+
`[cache-cleanup] start reason=${reason} free=${formatBytes(beforeMetrics.diskFreeBytes)} cache=${formatBytes(beforeMetrics.cacheSizeBytes)}`,
|
|
274
|
+
);
|
|
275
|
+
|
|
276
|
+
const rounds = [];
|
|
277
|
+
const startDay = Math.max(this.retentionStartDays, this.retentionMinDays);
|
|
278
|
+
const endDay = this.retentionMinDays;
|
|
279
|
+
|
|
280
|
+
for (let day = startDay; day >= endDay; day -= 1) {
|
|
281
|
+
const cutoffMs = Date.now() - (day * DAY_MS);
|
|
282
|
+
const result = await this.#deleteOlderThan(cutoffMs);
|
|
283
|
+
const metrics = await this.#collectMetrics();
|
|
284
|
+
|
|
285
|
+
rounds.push({
|
|
286
|
+
day,
|
|
287
|
+
deletedFiles: result.deletedFiles,
|
|
288
|
+
releasedBytes: result.releasedBytes,
|
|
289
|
+
diskFreeBytes: metrics.diskFreeBytes,
|
|
290
|
+
cacheSizeBytes: metrics.cacheSizeBytes,
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
console.warn(
|
|
294
|
+
`[cache-cleanup] round day=${day} deleted=${result.deletedFiles} released=${formatBytes(result.releasedBytes)} free=${formatBytes(metrics.diskFreeBytes)} cache=${formatBytes(metrics.cacheSizeBytes)}`,
|
|
295
|
+
);
|
|
296
|
+
|
|
297
|
+
if (this.#meetsTarget(metrics)) {
|
|
298
|
+
console.warn(`[cache-cleanup] success reason=${reason} stop-day=${day}`);
|
|
299
|
+
return {
|
|
300
|
+
triggered: true,
|
|
301
|
+
success: true,
|
|
302
|
+
stopDay: day,
|
|
303
|
+
before: beforeMetrics,
|
|
304
|
+
after: metrics,
|
|
305
|
+
rounds,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const afterMetrics = await this.#collectMetrics();
|
|
311
|
+
console.error(
|
|
312
|
+
`[cache-cleanup] warn cannot meet target at min-day=${this.retentionMinDays} free=${formatBytes(afterMetrics.diskFreeBytes)} cache=${formatBytes(afterMetrics.cacheSizeBytes)}`,
|
|
313
|
+
);
|
|
314
|
+
|
|
315
|
+
return {
|
|
316
|
+
triggered: true,
|
|
317
|
+
success: false,
|
|
318
|
+
stopDay: this.retentionMinDays,
|
|
319
|
+
before: beforeMetrics,
|
|
320
|
+
after: afterMetrics,
|
|
321
|
+
rounds,
|
|
322
|
+
};
|
|
323
|
+
} finally {
|
|
324
|
+
this.cleanupRunning = false;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
async #deleteOlderThan(cutoffMs) {
|
|
329
|
+
let deletedFiles = 0;
|
|
330
|
+
let releasedBytes = 0;
|
|
331
|
+
|
|
332
|
+
try {
|
|
333
|
+
await walkFiles(this.config.cacheDir, async (filePath) => {
|
|
334
|
+
const stats = await statIfFile(filePath);
|
|
335
|
+
if (!stats) {
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
if (stats.mtimeMs >= cutoffMs) {
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
try {
|
|
344
|
+
await fs.promises.unlink(filePath);
|
|
345
|
+
deletedFiles += 1;
|
|
346
|
+
releasedBytes += stats.size;
|
|
347
|
+
this.lastTouchAt.delete(filePath);
|
|
348
|
+
} catch (error) {
|
|
349
|
+
if (error.code !== "ENOENT") {
|
|
350
|
+
console.warn(`[cache-cleanup] delete failed path=${filePath} message=${error.message}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
});
|
|
354
|
+
} catch (error) {
|
|
355
|
+
if (error.code !== "ENOENT") {
|
|
356
|
+
throw error;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return { deletedFiles, releasedBytes };
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
async #collectMetrics() {
|
|
364
|
+
const [diskFreeBytes, cacheSizeBytes] = await Promise.all([
|
|
365
|
+
getDiskFreeBytes(this.config.cacheDir),
|
|
366
|
+
getDirSizeBytes(this.config.cacheDir),
|
|
367
|
+
]);
|
|
368
|
+
|
|
369
|
+
return { diskFreeBytes, cacheSizeBytes };
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
#isOverLimit(metrics) {
|
|
373
|
+
const diskLow = this.diskFreeTriggerBytes > 0 && metrics.diskFreeBytes <= this.diskFreeTriggerBytes;
|
|
374
|
+
const cacheOver = this.cacheMaxSizeBytes > 0 && metrics.cacheSizeBytes >= this.cacheMaxSizeBytes;
|
|
375
|
+
return diskLow || cacheOver;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
#meetsTarget(metrics) {
|
|
379
|
+
const diskOk =
|
|
380
|
+
this.diskFreeTriggerBytes <= 0 ||
|
|
381
|
+
metrics.diskFreeBytes >= this.diskFreeTargetBytes;
|
|
382
|
+
|
|
383
|
+
const cacheTarget = this.cacheTargetSizeBytes > 0 ? this.cacheTargetSizeBytes : this.cacheMaxSizeBytes;
|
|
384
|
+
const cacheOk =
|
|
385
|
+
this.cacheMaxSizeBytes <= 0 ||
|
|
386
|
+
(cacheTarget > 0 && metrics.cacheSizeBytes <= cacheTarget);
|
|
387
|
+
|
|
388
|
+
return diskOk && cacheOk;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
#scheduleDailyCheck() {
|
|
392
|
+
const now = new Date();
|
|
393
|
+
const next = new Date(now);
|
|
394
|
+
next.setHours(this.dailyAt.hour, this.dailyAt.minute, 0, 0);
|
|
395
|
+
|
|
396
|
+
if (next.getTime() <= now.getTime()) {
|
|
397
|
+
next.setDate(next.getDate() + 1);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const delay = Math.max(1000, next.getTime() - now.getTime());
|
|
401
|
+
|
|
402
|
+
this.dailyTimer = setTimeout(() => {
|
|
403
|
+
this.checkAndCleanupIfNeeded("daily-check", true)
|
|
404
|
+
.catch((error) => {
|
|
405
|
+
console.error(`[cache-cleanup] daily check failed: ${error.message}`);
|
|
406
|
+
})
|
|
407
|
+
.finally(() => {
|
|
408
|
+
this.#scheduleDailyCheck();
|
|
409
|
+
});
|
|
410
|
+
}, delay);
|
|
411
|
+
|
|
412
|
+
if (typeof this.dailyTimer.unref === "function") {
|
|
413
|
+
this.dailyTimer.unref();
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
package/src/config/config.js
CHANGED
|
@@ -99,6 +99,38 @@ function toInt(value, defaultValue) {
|
|
|
99
99
|
return Number.isFinite(parsed) ? parsed : defaultValue;
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
function parseDurationToMs(value, fallbackMs) {
|
|
103
|
+
const text = String(value || "").trim();
|
|
104
|
+
if (!text) {
|
|
105
|
+
return fallbackMs;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const match = text.match(/^(\d+)([smhd])$/i);
|
|
109
|
+
if (!match) {
|
|
110
|
+
return fallbackMs;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const amount = Number.parseInt(match[1], 10);
|
|
114
|
+
const unit = String(match[2] || "").toLowerCase();
|
|
115
|
+
if (!Number.isFinite(amount) || amount < 0) {
|
|
116
|
+
return fallbackMs;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (unit === "s") {
|
|
120
|
+
return amount * 1000;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (unit === "m") {
|
|
124
|
+
return amount * 60 * 1000;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if (unit === "h") {
|
|
128
|
+
return amount * 60 * 60 * 1000;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return amount * 24 * 60 * 60 * 1000;
|
|
132
|
+
}
|
|
133
|
+
|
|
102
134
|
function toList(value, defaultValue = []) {
|
|
103
135
|
if (!value) {
|
|
104
136
|
return defaultValue;
|
|
@@ -161,7 +193,7 @@ const defaultRepoFallbackRepos = [
|
|
|
161
193
|
"https://repo1.maven.org/maven2",
|
|
162
194
|
"https://jitpack.io",
|
|
163
195
|
"https://plugins.gradle.org/m2",
|
|
164
|
-
"https://
|
|
196
|
+
"https://dl.google.com",
|
|
165
197
|
];
|
|
166
198
|
|
|
167
199
|
const repoFallbackRepos = normalizeRepoList(
|
|
@@ -174,17 +206,24 @@ const defaultMavenRepoDomains = [
|
|
|
174
206
|
"repo.maven.apache.org",
|
|
175
207
|
"jitpack.io",
|
|
176
208
|
"plugins.gradle.org",
|
|
177
|
-
"
|
|
209
|
+
"dl.google.com",
|
|
178
210
|
...extractHostsFromUrls(repoFallbackRepos),
|
|
179
211
|
];
|
|
180
212
|
|
|
181
213
|
const cacheDir = path.resolve(configBaseDir, process.env.CACHE_DIR || "data/cache");
|
|
182
214
|
|
|
183
215
|
const multiThreadMinSizeBytes = Math.max(0, toInt(process.env.MULTI_THREAD_MIN_SIZE_MB, 1)) * 1024 * 1024;
|
|
184
|
-
const
|
|
185
|
-
const
|
|
186
|
-
const
|
|
187
|
-
const
|
|
216
|
+
const downloadTimeout = process.env.DOWNLOAD_TIMEOUT || "60s";
|
|
217
|
+
const outboundKeepAliveInterval = process.env.OUTBOUND_KEEP_ALIVE_INTERVAL || "1s";
|
|
218
|
+
const mavenNegativeCacheTtl = process.env.MAVEN_NEGATIVE_CACHE_TTL || "24h";
|
|
219
|
+
const mavenAffinityFlushInterval = process.env.MAVEN_AFFINITY_FLUSH_INTERVAL || "5s";
|
|
220
|
+
const logRetention = process.env.LOG_RETENTION || "7d";
|
|
221
|
+
|
|
222
|
+
const downloadTimeoutMs = Math.max(1, parseDurationToMs(downloadTimeout, 60 * 1000));
|
|
223
|
+
const outboundKeepAliveMsecs = Math.max(1, parseDurationToMs(outboundKeepAliveInterval, 1000));
|
|
224
|
+
const mavenNegativeCacheTtlMs = Math.max(1, parseDurationToMs(mavenNegativeCacheTtl, 24 * 60 * 60 * 1000));
|
|
225
|
+
const mavenAffinityFlushIntervalMs = Math.max(1, parseDurationToMs(mavenAffinityFlushInterval, 5 * 1000));
|
|
226
|
+
const logRetentionDays = Math.max(1, Math.ceil(parseDurationToMs(logRetention, 7 * 24 * 60 * 60 * 1000) / (24 * 60 * 60 * 1000)));
|
|
188
227
|
const mavenAffinityEventMaxBytes = Math.max(1, toInt(process.env.MAVEN_AFFINITY_EVENT_MAX_MB, 8)) * 1024 * 1024;
|
|
189
228
|
|
|
190
229
|
export const config = {
|
|
@@ -205,18 +244,34 @@ export const config = {
|
|
|
205
244
|
multiThreadDomains: toList(process.env.MULTI_THREAD_DOMAINS, ["repo1.maven.org"]),
|
|
206
245
|
multiThreadCount: Math.max(1, toInt(process.env.MULTI_THREAD_COUNT, 4)),
|
|
207
246
|
multiThreadMinSizeBytes,
|
|
247
|
+
downloadTimeout,
|
|
208
248
|
downloadTimeoutMs,
|
|
209
249
|
outboundKeepAlive: toBool(process.env.OUTBOUND_KEEP_ALIVE, true),
|
|
250
|
+
outboundKeepAliveInterval,
|
|
210
251
|
outboundKeepAliveMsecs,
|
|
211
252
|
outboundMaxSockets: Math.max(1, toInt(process.env.OUTBOUND_MAX_SOCKETS, 64)),
|
|
212
253
|
outboundMaxFreeSockets: Math.max(1, toInt(process.env.OUTBOUND_MAX_FREE_SOCKETS, 16)),
|
|
213
254
|
mavenAffinityEnabled: toBool(process.env.MAVEN_AFFINITY_ENABLED, true),
|
|
214
|
-
mavenAffinityIndexDir: path.resolve(
|
|
255
|
+
mavenAffinityIndexDir: path.resolve(configBaseDir, process.env.MAVEN_AFFINITY_INDEX_DIR || "data/index"),
|
|
256
|
+
mavenNegativeCacheTtl,
|
|
215
257
|
mavenNegativeCacheTtlMs,
|
|
258
|
+
mavenAffinityFlushInterval,
|
|
216
259
|
mavenAffinityFlushIntervalMs,
|
|
217
260
|
mavenAffinityEventMaxBytes,
|
|
261
|
+
cacheCleanupEnabled: toBool(process.env.CACHE_CLEANUP_ENABLED, true),
|
|
262
|
+
cacheCleanupDailyAt: process.env.CACHE_CLEANUP_DAILY_AT || "03:00",
|
|
263
|
+
cacheCleanupCheckMinInterval: process.env.CACHE_CLEANUP_CHECK_MIN_INTERVAL || "10m",
|
|
264
|
+
cacheTouchOnHit: toBool(process.env.CACHE_TOUCH_ON_HIT, true),
|
|
265
|
+
cacheTouchMinInterval: process.env.CACHE_TOUCH_MIN_INTERVAL || "1d",
|
|
266
|
+
cacheRetentionStart: process.env.CACHE_RETENTION_START || "10d",
|
|
267
|
+
cacheRetentionMin: process.env.CACHE_RETENTION_MIN || "1d",
|
|
268
|
+
cacheDiskFreeTrigger: process.env.CACHE_DISK_FREE_TRIGGER || "20G",
|
|
269
|
+
cacheDiskFreeTarget: process.env.CACHE_DISK_FREE_TARGET || "25G",
|
|
270
|
+
cacheMaxSize: process.env.CACHE_MAX_SIZE || "",
|
|
271
|
+
cacheTargetSize: process.env.CACHE_TARGET_SIZE || "",
|
|
218
272
|
downloadLogDir: path.resolve(configBaseDir, process.env.DOWNLOAD_LOG_DIR || "data/logs/downloads"),
|
|
219
|
-
|
|
273
|
+
logRetention,
|
|
274
|
+
logRetentionDays,
|
|
220
275
|
logToStdout: toBool(process.env.LOG_TO_STDOUT, true),
|
|
221
276
|
logConnectEvents: toBool(process.env.LOG_CONNECT_EVENTS, false),
|
|
222
277
|
certDir: path.resolve(configBaseDir, process.env.CERT_DIR || "data/certs"),
|
package/src/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import { startRepoServer } from "./repo/repo-server.js";
|
|
|
8
8
|
import { getTrustStoreCommands } from "./cert/truststore-utils.js";
|
|
9
9
|
import { UpstreamProxyManager } from "./proxy/upstream-proxy.js";
|
|
10
10
|
import { MavenAffinityIndex } from "./cache/maven-affinity-index.js";
|
|
11
|
+
import { CacheCleanupManager } from "./cache/cache-cleanup-manager.js";
|
|
11
12
|
import { installConsoleLogFileMirror, installGlobalErrorLogging } from "./common/console-log-file.js";
|
|
12
13
|
|
|
13
14
|
installConsoleLogFileMirror({
|
|
@@ -85,6 +86,8 @@ async function main() {
|
|
|
85
86
|
const upstreamProxyManager = new UpstreamProxyManager(config, matchesDomain);
|
|
86
87
|
const mavenAffinityIndex = new MavenAffinityIndex(config);
|
|
87
88
|
await mavenAffinityIndex.init();
|
|
89
|
+
const cacheCleanupManager = new CacheCleanupManager(config);
|
|
90
|
+
await cacheCleanupManager.init();
|
|
88
91
|
|
|
89
92
|
const downloader = new Downloader(config, matchesDomain, upstreamProxyManager);
|
|
90
93
|
|
|
@@ -95,8 +98,9 @@ async function main() {
|
|
|
95
98
|
matchesDomain,
|
|
96
99
|
upstreamProxyManager,
|
|
97
100
|
mavenAffinityIndex,
|
|
101
|
+
cacheCleanupManager,
|
|
98
102
|
);
|
|
99
|
-
const repoServer = startRepoServer(config, downloader);
|
|
103
|
+
const repoServer = startRepoServer(config, downloader, cacheCleanupManager);
|
|
100
104
|
|
|
101
105
|
await Promise.all([
|
|
102
106
|
waitForServerListening(proxyServer, "proxy server"),
|
|
@@ -119,18 +123,29 @@ async function main() {
|
|
|
119
123
|
startupInfo(`[maven-proxy] cache npm : ${config.npmCacheDir}`);
|
|
120
124
|
startupInfo(`[maven-proxy] cache other: ${config.genericCacheDir}`);
|
|
121
125
|
startupInfo(`[maven-proxy] log dir: ${config.downloadLogDir}`);
|
|
122
|
-
startupInfo(`[maven-proxy] log retention
|
|
126
|
+
startupInfo(`[maven-proxy] log retention: ${config.logRetention} (${config.logRetentionDays}d)`);
|
|
123
127
|
startupInfo(`[maven-proxy] log to stdout: ${config.logToStdout}`);
|
|
124
128
|
startupInfo(`[maven-proxy] log connect events: ${config.logConnectEvents}`);
|
|
125
129
|
startupInfo(`[maven-proxy] outbound keep-alive: ${config.outboundKeepAlive}`);
|
|
126
|
-
startupInfo(`[maven-proxy] outbound keepAlive
|
|
130
|
+
startupInfo(`[maven-proxy] outbound keepAlive interval: ${config.outboundKeepAliveInterval}`);
|
|
127
131
|
startupInfo(`[maven-proxy] outbound maxSockets: ${config.outboundMaxSockets}`);
|
|
128
132
|
startupInfo(`[maven-proxy] outbound maxFreeSockets: ${config.outboundMaxFreeSockets}`);
|
|
129
133
|
startupInfo(`[maven-proxy] maven affinity enabled: ${config.mavenAffinityEnabled}`);
|
|
130
134
|
startupInfo(`[maven-proxy] maven affinity index dir: ${config.mavenAffinityIndexDir}`);
|
|
131
|
-
startupInfo(`[maven-proxy] maven negative cache ttl
|
|
132
|
-
startupInfo(`[maven-proxy] maven affinity flush interval
|
|
135
|
+
startupInfo(`[maven-proxy] maven negative cache ttl: ${config.mavenNegativeCacheTtl}`);
|
|
136
|
+
startupInfo(`[maven-proxy] maven affinity flush interval: ${config.mavenAffinityFlushInterval}`);
|
|
137
|
+
startupInfo(`[maven-proxy] download timeout: ${config.downloadTimeout}`);
|
|
133
138
|
startupInfo(`[maven-proxy] maven affinity event max(MB): ${config.mavenAffinityEventMaxBytes / (1024 * 1024)}`);
|
|
139
|
+
startupInfo(`[maven-proxy] cache cleanup enabled: ${config.cacheCleanupEnabled}`);
|
|
140
|
+
startupInfo(`[maven-proxy] cache cleanup daily at: ${config.cacheCleanupDailyAt}`);
|
|
141
|
+
startupInfo(`[maven-proxy] cache touch on hit: ${config.cacheTouchOnHit}`);
|
|
142
|
+
startupInfo(`[maven-proxy] cache touch min interval: ${config.cacheTouchMinInterval}`);
|
|
143
|
+
startupInfo(`[maven-proxy] cache retention start: ${config.cacheRetentionStart}`);
|
|
144
|
+
startupInfo(`[maven-proxy] cache retention min: ${config.cacheRetentionMin}`);
|
|
145
|
+
startupInfo(`[maven-proxy] cache disk free trigger: ${config.cacheDiskFreeTrigger}`);
|
|
146
|
+
startupInfo(`[maven-proxy] cache disk free target: ${config.cacheDiskFreeTarget}`);
|
|
147
|
+
startupInfo(`[maven-proxy] cache max size: ${config.cacheMaxSize || "(disabled)"}`);
|
|
148
|
+
startupInfo(`[maven-proxy] cache target size: ${config.cacheTargetSize || "(disabled)"}`);
|
|
134
149
|
startupInfo(`[maven-proxy] root cert : ${config.rootCertPath}`);
|
|
135
150
|
startupInfo(`[maven-proxy] repo fallback repos: ${(config.repoFallbackRepos || []).join(",") || "(none)"}`);
|
|
136
151
|
if (config.upstreamProxyUrl || config.upstreamHttpProxyUrl || config.upstreamHttpsProxyUrl) {
|
|
@@ -151,6 +166,7 @@ async function main() {
|
|
|
151
166
|
mitmHttpServer.close();
|
|
152
167
|
repoServer.close();
|
|
153
168
|
upstreamProxyManager.destroy();
|
|
169
|
+
void cacheCleanupManager.destroy();
|
|
154
170
|
void mavenAffinityIndex.destroy();
|
|
155
171
|
};
|
|
156
172
|
|
|
@@ -101,13 +101,17 @@ function buildUrl(req, forcedProtocol = null) {
|
|
|
101
101
|
return new URL(`${protocol}//${host}${raw}`);
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
async function serveFile(res, req, filePath) {
|
|
104
|
+
async function serveFile(res, req, filePath, cacheCleanupManager = null) {
|
|
105
105
|
const stats = await statIfFile(filePath);
|
|
106
106
|
if (!stats) {
|
|
107
107
|
sendText(res, 404, "Not Found");
|
|
108
108
|
return;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
if (cacheCleanupManager) {
|
|
112
|
+
cacheCleanupManager.touchFileOnHit(filePath);
|
|
113
|
+
}
|
|
114
|
+
|
|
111
115
|
res.setHeader("content-length", String(stats.size));
|
|
112
116
|
if (!res.hasHeader("x-cache")) {
|
|
113
117
|
res.setHeader("x-cache", "HIT");
|
|
@@ -171,6 +175,7 @@ export function createHttpRequestHandler({
|
|
|
171
175
|
upstreamProxyManager = null,
|
|
172
176
|
matchesDomain,
|
|
173
177
|
mavenAffinityIndex = null,
|
|
178
|
+
cacheCleanupManager = null,
|
|
174
179
|
}) {
|
|
175
180
|
return async function handleHttpRequestPath(req, res, forcedProtocol = null) {
|
|
176
181
|
let urlObj;
|
|
@@ -210,7 +215,7 @@ export function createHttpRequestHandler({
|
|
|
210
215
|
const existing = await statIfFile(cachePath);
|
|
211
216
|
if (existing) {
|
|
212
217
|
console.log(`[proxy] local cache hit host=${urlObj.hostname} path=${urlObj.pathname}`);
|
|
213
|
-
await serveFile(res, req, cachePath);
|
|
218
|
+
await serveFile(res, req, cachePath, cacheCleanupManager);
|
|
214
219
|
return;
|
|
215
220
|
}
|
|
216
221
|
|
|
@@ -219,7 +224,7 @@ export function createHttpRequestHandler({
|
|
|
219
224
|
const preferredPath = await mavenAffinityIndex.resolvePreferredCachePath(canonical.canonicalKey);
|
|
220
225
|
if (preferredPath) {
|
|
221
226
|
console.log(`[proxy] affinity hit canonical=${canonical.canonicalKey} host=${urlObj.hostname}`);
|
|
222
|
-
await serveFile(res, req, preferredPath);
|
|
227
|
+
await serveFile(res, req, preferredPath, cacheCleanupManager);
|
|
223
228
|
return;
|
|
224
229
|
}
|
|
225
230
|
}
|
|
@@ -233,6 +238,9 @@ export function createHttpRequestHandler({
|
|
|
233
238
|
|
|
234
239
|
try {
|
|
235
240
|
console.log(`[proxy] local cache miss host=${urlObj.hostname} path=${urlObj.pathname}`);
|
|
241
|
+
if (cacheCleanupManager) {
|
|
242
|
+
await cacheCleanupManager.checkAndCleanupIfNeeded("cache-miss");
|
|
243
|
+
}
|
|
236
244
|
await fs.promises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
237
245
|
await downloader.ensureCached(urlObj, cachePath, req.headers);
|
|
238
246
|
|
|
@@ -247,7 +255,7 @@ export function createHttpRequestHandler({
|
|
|
247
255
|
}
|
|
248
256
|
|
|
249
257
|
res.setHeader("x-cache", "MISS");
|
|
250
|
-
await serveFile(res, req, cachePath);
|
|
258
|
+
await serveFile(res, req, cachePath, cacheCleanupManager);
|
|
251
259
|
} catch (error) {
|
|
252
260
|
if (
|
|
253
261
|
canonical &&
|
|
@@ -19,6 +19,7 @@ export function startProxyServer(
|
|
|
19
19
|
matchesDomain,
|
|
20
20
|
upstreamProxyManager = null,
|
|
21
21
|
mavenAffinityIndex = null,
|
|
22
|
+
cacheCleanupManager = null,
|
|
22
23
|
) {
|
|
23
24
|
const handleHttpRequestPath = createHttpRequestHandler({
|
|
24
25
|
config,
|
|
@@ -26,6 +27,7 @@ export function startProxyServer(
|
|
|
26
27
|
upstreamProxyManager,
|
|
27
28
|
matchesDomain,
|
|
28
29
|
mavenAffinityIndex,
|
|
30
|
+
cacheCleanupManager,
|
|
29
31
|
});
|
|
30
32
|
const mitmHttpServer = createMitmHttpServer(handleHttpRequestPath);
|
|
31
33
|
|
package/src/repo/repo-server.js
CHANGED
|
@@ -41,7 +41,7 @@ function buildCandidateRelativePaths(relativePath) {
|
|
|
41
41
|
return [...new Set(candidates.filter(Boolean))];
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
async function ensureFromRemoteRepos(config, downloader, filePath, relativePath) {
|
|
44
|
+
async function ensureFromRemoteRepos(config, downloader, filePath, relativePath, cacheCleanupManager = null) {
|
|
45
45
|
if (!downloader) {
|
|
46
46
|
return null;
|
|
47
47
|
}
|
|
@@ -60,6 +60,9 @@ async function ensureFromRemoteRepos(config, downloader, filePath, relativePath)
|
|
|
60
60
|
const remoteUrl = buildRemoteUrl(repoBase, candidatePath);
|
|
61
61
|
|
|
62
62
|
try {
|
|
63
|
+
if (cacheCleanupManager) {
|
|
64
|
+
await cacheCleanupManager.checkAndCleanupIfNeeded("repo-cache-miss");
|
|
65
|
+
}
|
|
63
66
|
console.log(`[repo] cache miss, try remote ${remoteUrl.href}`);
|
|
64
67
|
await downloader.ensureCached(remoteUrl, filePath, {});
|
|
65
68
|
return await statIfExists(filePath);
|
|
@@ -79,7 +82,7 @@ async function ensureFromRemoteRepos(config, downloader, filePath, relativePath)
|
|
|
79
82
|
return null;
|
|
80
83
|
}
|
|
81
84
|
|
|
82
|
-
export function startRepoServer(config, downloader = null) {
|
|
85
|
+
export function startRepoServer(config, downloader = null, cacheCleanupManager = null) {
|
|
83
86
|
const server = http.createServer(async (req, res) => {
|
|
84
87
|
try {
|
|
85
88
|
const urlObj = new URL(req.url || "/", `http://${req.headers.host || "localhost"}`);
|
|
@@ -88,7 +91,7 @@ export function startRepoServer(config, downloader = null) {
|
|
|
88
91
|
let stats = await statIfExists(filePath);
|
|
89
92
|
|
|
90
93
|
if (!stats || !stats.isFile()) {
|
|
91
|
-
stats = await ensureFromRemoteRepos(config, downloader, filePath, relativePath);
|
|
94
|
+
stats = await ensureFromRemoteRepos(config, downloader, filePath, relativePath, cacheCleanupManager);
|
|
92
95
|
}
|
|
93
96
|
|
|
94
97
|
if (!stats || !stats.isFile()) {
|
|
@@ -100,6 +103,10 @@ export function startRepoServer(config, downloader = null) {
|
|
|
100
103
|
res.setHeader("content-length", String(stats.size));
|
|
101
104
|
res.setHeader("cache-control", "public, max-age=3600");
|
|
102
105
|
|
|
106
|
+
if (cacheCleanupManager) {
|
|
107
|
+
cacheCleanupManager.touchFileOnHit(filePath);
|
|
108
|
+
}
|
|
109
|
+
|
|
103
110
|
if (req.method === "HEAD") {
|
|
104
111
|
res.writeHead(200);
|
|
105
112
|
res.end();
|