maven-proxy 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -6
- package/bin/maven-proxy.js +16 -6
- package/package.json +4 -2
- package/src/cache/downloader.js +10 -3
- package/src/cache/maven-affinity-index.js +393 -0
- package/src/common/console-log-file.js +39 -9
- package/src/common/maven-canonical.js +151 -0
- package/src/config/config.js +21 -5
- package/src/index.js +18 -1
- package/src/proxy/proxy-http-handler.js +52 -2
- package/src/proxy/proxy-server.js +9 -1
- package/src/proxy/upstream-proxy.js +41 -3
- package/src/common/download-log-writer.js +0 -27
package/README.md
CHANGED
|
@@ -185,8 +185,8 @@ npm install
|
|
|
185
185
|
2. Configure environment variables as needed.
|
|
186
186
|
|
|
187
187
|
Notes:
|
|
188
|
-
- Development mode (default): npm start loads .
|
|
189
|
-
- User mode (CLI default): npx maven-proxy or global command uses ~/maven-proxy/config.
|
|
188
|
+
- Development mode (default): npm start loads config.properties in the project root.
|
|
189
|
+
- User mode (CLI default): npx maven-proxy or global command uses ~/maven-proxy/config.properties.
|
|
190
190
|
- Override mode with MAVEN_PROXY_CONFIG_MODE as development or user.
|
|
191
191
|
- Override config file path with MAVEN_PROXY_CONFIG_FILE.
|
|
192
192
|
- JAVA_HOME supports auto-detection:
|
|
@@ -267,6 +267,11 @@ Environment variables:
|
|
|
267
267
|
- HTTPS_MITM_DOMAINS: MITM domain list (includes registry.npmjs.org by default, wildcards supported).
|
|
268
268
|
- DOWNLOAD_LOG_DIR: log directory.
|
|
269
269
|
- LOG_RETENTION_DAYS: number of days to retain logs.
|
|
270
|
+
- LOG_TO_STDOUT: whether to also print logs to stdout/stderr.
|
|
271
|
+
- OUTBOUND_KEEP_ALIVE: enable outbound keep-alive connection pooling.
|
|
272
|
+
- OUTBOUND_KEEP_ALIVE_SECONDS: keep-alive interval in seconds.
|
|
273
|
+
- OUTBOUND_MAX_SOCKETS: max outbound sockets per origin.
|
|
274
|
+
- OUTBOUND_MAX_FREE_SOCKETS: max idle outbound sockets per origin.
|
|
270
275
|
- MAVEN_PROXY_CONFIG_MODE: development or user.
|
|
271
276
|
- MAVEN_PROXY_CONFIG_FILE: explicit config file path.
|
|
272
277
|
- EXISTING_TRUST_STORE_PATH: optional existing truststore path. If present, truststore init prefers it as source.
|
|
@@ -294,10 +299,15 @@ Priority:
|
|
|
294
299
|
- `MAVEN_REPO_DOMAINS`: Domains treated as Maven ecosystem for cache routing (wildcards supported).
|
|
295
300
|
- `MULTI_THREAD_DOMAINS`: Domains allowed to use multi-thread download (wildcards supported).
|
|
296
301
|
- `MULTI_THREAD_COUNT`: Number of download threads for ranged downloads.
|
|
297
|
-
- `
|
|
298
|
-
- `
|
|
299
|
-
- `DOWNLOAD_LOG_DIR`: Directory for
|
|
302
|
+
- `MULTI_THREAD_MIN_SIZE_MB`: Minimum size threshold to trigger multi-thread download (MB).
|
|
303
|
+
- `DOWNLOAD_TIMEOUT_SECONDS`: Upstream request timeout in seconds.
|
|
304
|
+
- `DOWNLOAD_LOG_DIR`: Directory for unified app/error logs.
|
|
300
305
|
- `LOG_RETENTION_DAYS`: Number of days to keep log files.
|
|
306
|
+
- `LOG_TO_STDOUT`: Whether to also print logs to stdout/stderr. Default `true`.
|
|
307
|
+
- `OUTBOUND_KEEP_ALIVE`: Enable outbound keep-alive connection pooling. Default `true`.
|
|
308
|
+
- `OUTBOUND_KEEP_ALIVE_SECONDS`: Keep-alive interval in seconds. Default `1`.
|
|
309
|
+
- `OUTBOUND_MAX_SOCKETS`: Max outbound sockets per origin. Default `64`.
|
|
310
|
+
- `OUTBOUND_MAX_FREE_SOCKETS`: Max idle outbound sockets per origin. Default `16`.
|
|
301
311
|
- `UPSTREAM_PROXY_URL`: Generic upstream proxy URL (fallback for HTTP/HTTPS).
|
|
302
312
|
- `UPSTREAM_HTTP_PROXY_URL`: Upstream proxy URL for HTTP requests.
|
|
303
313
|
- `UPSTREAM_HTTPS_PROXY_URL`: Upstream proxy URL for HTTPS requests.
|
|
@@ -388,7 +398,7 @@ maven-proxy
|
|
|
388
398
|
```
|
|
389
399
|
|
|
390
400
|
Default CLI config path:
|
|
391
|
-
- ~/maven-proxy/config
|
|
401
|
+
- ~/maven-proxy/config.properties
|
|
392
402
|
|
|
393
403
|
Common commands:
|
|
394
404
|
- maven-proxy --config /path/to/config
|
package/bin/maven-proxy.js
CHANGED
|
@@ -6,7 +6,7 @@ import net from "node:net";
|
|
|
6
6
|
import { spawnSync } from "node:child_process";
|
|
7
7
|
|
|
8
8
|
const defaultConfigDir = path.resolve(os.homedir(), "maven-proxy");
|
|
9
|
-
const defaultConfigFile = path.join(defaultConfigDir, "config");
|
|
9
|
+
const defaultConfigFile = path.join(defaultConfigDir, "config.properties");
|
|
10
10
|
|
|
11
11
|
function normalizeMode(value) {
|
|
12
12
|
const normalized = String(value || "").trim().toLowerCase();
|
|
@@ -44,7 +44,7 @@ function resolveEffectiveMode(options) {
|
|
|
44
44
|
return forced;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
// CLI defaults to user mode to load ~/maven-proxy/config unless explicitly overridden.
|
|
47
|
+
// CLI defaults to user mode to load ~/maven-proxy/config.properties unless explicitly overridden.
|
|
48
48
|
return "user";
|
|
49
49
|
}
|
|
50
50
|
|
|
@@ -62,7 +62,7 @@ function printHelp() {
|
|
|
62
62
|
console.log(" npx maven-proxy");
|
|
63
63
|
console.log(" maven-proxy init-config");
|
|
64
64
|
console.log(" maven-proxy start --mode development");
|
|
65
|
-
console.log(" maven-proxy --config ~/maven-proxy/config");
|
|
65
|
+
console.log(" maven-proxy --config ~/maven-proxy/config.properties");
|
|
66
66
|
console.log(" maven-proxy truststore print");
|
|
67
67
|
console.log(" maven-proxy truststore merge --source ./a.jks --target ./b.jks");
|
|
68
68
|
console.log(" maven-proxy doctor");
|
|
@@ -149,10 +149,20 @@ function getDefaultConfigTemplate() {
|
|
|
149
149
|
"MAVEN_REPO_DOMAINS=repo1.maven.org,repo.maven.apache.org,jitpack.io,plugins.gradle.org,maven.google.com",
|
|
150
150
|
"MULTI_THREAD_DOMAINS=repo1.maven.org",
|
|
151
151
|
"MULTI_THREAD_COUNT=8",
|
|
152
|
-
"
|
|
153
|
-
"
|
|
152
|
+
"MULTI_THREAD_MIN_SIZE_MB=1",
|
|
153
|
+
"DOWNLOAD_TIMEOUT_SECONDS=60",
|
|
154
|
+
"OUTBOUND_KEEP_ALIVE=true",
|
|
155
|
+
"OUTBOUND_KEEP_ALIVE_SECONDS=1",
|
|
156
|
+
"OUTBOUND_MAX_SOCKETS=64",
|
|
157
|
+
"OUTBOUND_MAX_FREE_SOCKETS=16",
|
|
158
|
+
"MAVEN_AFFINITY_ENABLED=true",
|
|
159
|
+
"MAVEN_AFFINITY_INDEX_DIR=.index",
|
|
160
|
+
"MAVEN_NEGATIVE_CACHE_TTL_HOURS=24",
|
|
161
|
+
"MAVEN_AFFINITY_FLUSH_INTERVAL_SECONDS=5",
|
|
162
|
+
"MAVEN_AFFINITY_EVENT_MAX_MB=8",
|
|
154
163
|
"DOWNLOAD_LOG_DIR=data/logs/downloads",
|
|
155
164
|
"LOG_RETENTION_DAYS=7",
|
|
165
|
+
"LOG_TO_STDOUT=false",
|
|
156
166
|
"UPSTREAM_PROXY_URL=",
|
|
157
167
|
"UPSTREAM_HTTP_PROXY_URL=",
|
|
158
168
|
"UPSTREAM_HTTPS_PROXY_URL=",
|
|
@@ -304,7 +314,7 @@ async function runDoctor(options) {
|
|
|
304
314
|
printDoctorLine("WARN", "config file", `not found, expected ${config.defaultUserConfigPath}`);
|
|
305
315
|
} else {
|
|
306
316
|
warnings.push("development config file missing");
|
|
307
|
-
printDoctorLine("WARN", "config file", "no .
|
|
317
|
+
printDoctorLine("WARN", "config file", "no config.properties loaded, using defaults");
|
|
308
318
|
}
|
|
309
319
|
|
|
310
320
|
const keytool = checkKeytool();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "maven-proxy",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Maven proxy with cache, HTTPS MITM for selected domains, and local repo publishing",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -33,7 +33,9 @@
|
|
|
33
33
|
"npm:version:patch": "npm version patch",
|
|
34
34
|
"npm:view:version": "npm view maven-proxy version",
|
|
35
35
|
"npm:view:dist-tags": "npm view maven-proxy dist-tags",
|
|
36
|
-
"
|
|
36
|
+
"replay:affinity": "node --test test/replay-affinity.test.js",
|
|
37
|
+
"test:replay": "node --test test/replay-affinity.test.js",
|
|
38
|
+
"test": "node --test test/*.test.js"
|
|
37
39
|
},
|
|
38
40
|
"keywords": [],
|
|
39
41
|
"author": "",
|
package/src/cache/downloader.js
CHANGED
|
@@ -3,7 +3,6 @@ import http from "node:http";
|
|
|
3
3
|
import https from "node:https";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { pipeline } from "node:stream/promises";
|
|
6
|
-
import { DownloadLogWriter } from "../common/download-log-writer.js";
|
|
7
6
|
|
|
8
7
|
const REDIRECT_STATUS = new Set([301, 302, 303, 307, 308]);
|
|
9
8
|
const MAX_REDIRECTS = 5;
|
|
@@ -288,7 +287,6 @@ export class Downloader {
|
|
|
288
287
|
this.domainMatcher = domainMatcher;
|
|
289
288
|
this.upstreamProxyManager = upstreamProxyManager;
|
|
290
289
|
this.inflight = new Map();
|
|
291
|
-
this.downloadLogWriter = new DownloadLogWriter(config.downloadLogDir, config.logRetentionDays);
|
|
292
290
|
}
|
|
293
291
|
|
|
294
292
|
logDownload(event, urlObj, details = {}) {
|
|
@@ -299,7 +297,6 @@ export class Downloader {
|
|
|
299
297
|
.join(" ");
|
|
300
298
|
|
|
301
299
|
console.log(`[downloader] ${event} url=${url}${detailText ? ` ${detailText}` : ""}`);
|
|
302
|
-
this.downloadLogWriter.write(event, url, details);
|
|
303
300
|
}
|
|
304
301
|
|
|
305
302
|
async ensureCached(urlObj, finalPath, requestHeaders = {}) {
|
|
@@ -326,6 +323,8 @@ export class Downloader {
|
|
|
326
323
|
}
|
|
327
324
|
|
|
328
325
|
async #downloadAtomic(urlObj, finalPath, requestHeaders) {
|
|
326
|
+
const startedAt = Date.now();
|
|
327
|
+
|
|
329
328
|
await fs.promises.mkdir(path.dirname(finalPath), { recursive: true });
|
|
330
329
|
const tempPath = `${finalPath}.temp`;
|
|
331
330
|
await removeIfExists(tempPath);
|
|
@@ -388,6 +387,14 @@ export class Downloader {
|
|
|
388
387
|
|
|
389
388
|
await verifyFileSize(tempPath, metadata.contentLength);
|
|
390
389
|
await fs.promises.rename(tempPath, finalPath);
|
|
390
|
+
|
|
391
|
+
const finalStats = await fs.promises.stat(finalPath);
|
|
392
|
+
this.logDownload("download succeeded", downloadUrl, {
|
|
393
|
+
host: hostname,
|
|
394
|
+
targetPath: finalPath,
|
|
395
|
+
size: finalStats.size,
|
|
396
|
+
elapsedMs: Date.now() - startedAt,
|
|
397
|
+
});
|
|
391
398
|
} catch (error) {
|
|
392
399
|
if (isLocalFsWriteError(error)) {
|
|
393
400
|
if (!error.statusCode) {
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
function toPositiveInt(value, fallback) {
|
|
5
|
+
const parsed = Number.parseInt(value, 10);
|
|
6
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function toBool(value, fallback) {
|
|
10
|
+
if (value == null || value === "") {
|
|
11
|
+
return fallback;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
return ["1", "true", "yes", "on"].includes(String(value).toLowerCase());
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function nowMs() {
|
|
18
|
+
return Date.now();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function readJsonFile(filePath, fallback) {
|
|
22
|
+
if (!fs.existsSync(filePath)) {
|
|
23
|
+
return fallback;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
const text = fs.readFileSync(filePath, "utf8");
|
|
28
|
+
return JSON.parse(text);
|
|
29
|
+
} catch {
|
|
30
|
+
return fallback;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function serializeSnapshot(positiveMap, negativeMap, conflictMap) {
|
|
35
|
+
return {
|
|
36
|
+
version: 1,
|
|
37
|
+
generatedAt: new Date().toISOString(),
|
|
38
|
+
positive: [...positiveMap.entries()],
|
|
39
|
+
negative: [...negativeMap.entries()],
|
|
40
|
+
conflicts: [...conflictMap.entries()],
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function normalizeRequestPath(pathname) {
|
|
45
|
+
return String(pathname || "")
|
|
46
|
+
.replace(/\\/g, "/")
|
|
47
|
+
.replace(/\/+/g, "/")
|
|
48
|
+
.replace(/^\/+/, "")
|
|
49
|
+
.trim();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function buildNegativeScope(urlObj) {
|
|
53
|
+
if (!urlObj || typeof urlObj !== "object") {
|
|
54
|
+
return "";
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const protocol = urlObj.protocol === "https:" ? "https:" : "http:";
|
|
58
|
+
const host = String(urlObj.host || "").toLowerCase();
|
|
59
|
+
const pathname = normalizeRequestPath(urlObj.pathname || "");
|
|
60
|
+
return `${protocol}//${host}/${pathname}`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function buildNegativeKey(scope, canonicalKey) {
|
|
64
|
+
return `${String(scope || "").toLowerCase()}|${canonicalKey}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export class MavenAffinityIndex {
|
|
68
|
+
constructor(config) {
|
|
69
|
+
this.enabled = toBool(config.mavenAffinityEnabled, true);
|
|
70
|
+
this.indexDir = config.mavenAffinityIndexDir;
|
|
71
|
+
this.negativeTtlMs = toPositiveInt(config.mavenNegativeCacheTtlMs, 24 * 60 * 60 * 1000);
|
|
72
|
+
this.flushIntervalMs = toPositiveInt(config.mavenAffinityFlushIntervalMs, 5000);
|
|
73
|
+
this.maxEventBytes = toPositiveInt(config.mavenAffinityEventMaxBytes, 8 * 1024 * 1024);
|
|
74
|
+
|
|
75
|
+
this.snapshotPath = path.join(this.indexDir, "maven-affinity.snapshot.json");
|
|
76
|
+
this.eventLogPath = path.join(this.indexDir, "maven-affinity.events.log");
|
|
77
|
+
|
|
78
|
+
this.positive = new Map();
|
|
79
|
+
this.negative = new Map();
|
|
80
|
+
this.conflicts = new Map();
|
|
81
|
+
|
|
82
|
+
this.pendingEvents = [];
|
|
83
|
+
this.flushTimer = null;
|
|
84
|
+
this.flushing = false;
|
|
85
|
+
this.dirtySinceSnapshot = false;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async init() {
|
|
89
|
+
if (!this.enabled) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
await fs.promises.mkdir(this.indexDir, { recursive: true });
|
|
94
|
+
this.#loadSnapshot();
|
|
95
|
+
this.#replayEventLog();
|
|
96
|
+
|
|
97
|
+
this.flushTimer = setInterval(() => {
|
|
98
|
+
this.flush().catch((error) => {
|
|
99
|
+
console.error(`[affinity] flush failed: ${error.message}`);
|
|
100
|
+
});
|
|
101
|
+
}, this.flushIntervalMs);
|
|
102
|
+
|
|
103
|
+
if (typeof this.flushTimer.unref === "function") {
|
|
104
|
+
this.flushTimer.unref();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
#loadSnapshot() {
|
|
109
|
+
const snapshot = readJsonFile(this.snapshotPath, null);
|
|
110
|
+
if (!snapshot || typeof snapshot !== "object") {
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
for (const [key, value] of snapshot.positive || []) {
|
|
115
|
+
this.positive.set(key, value);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const currentTime = nowMs();
|
|
119
|
+
for (const [key, value] of snapshot.negative || []) {
|
|
120
|
+
if (value?.expireAt && value.expireAt > currentTime) {
|
|
121
|
+
this.negative.set(key, value);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
for (const [key, value] of snapshot.conflicts || []) {
|
|
126
|
+
this.conflicts.set(key, value);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
#replayEventLog() {
|
|
131
|
+
if (!fs.existsSync(this.eventLogPath)) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
let raw = "";
|
|
136
|
+
try {
|
|
137
|
+
raw = fs.readFileSync(this.eventLogPath, "utf8");
|
|
138
|
+
} catch {
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const lines = raw.split(/\r?\n/).filter(Boolean);
|
|
143
|
+
for (const line of lines) {
|
|
144
|
+
try {
|
|
145
|
+
const event = JSON.parse(line);
|
|
146
|
+
this.#applyEvent(event, false);
|
|
147
|
+
} catch {
|
|
148
|
+
// ignore invalid line
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
#enqueueEvent(type, payload) {
|
|
154
|
+
this.pendingEvents.push(JSON.stringify({ t: nowMs(), type, payload }));
|
|
155
|
+
this.dirtySinceSnapshot = true;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
#applyEvent(event, append = true) {
|
|
159
|
+
if (!event || typeof event !== "object") {
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const { type, payload } = event;
|
|
164
|
+
if (!type || !payload || typeof payload !== "object") {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if (type === "positive_upsert") {
|
|
169
|
+
this.positive.set(payload.key, payload.value);
|
|
170
|
+
if (append) {
|
|
171
|
+
this.#enqueueEvent(type, payload);
|
|
172
|
+
}
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
if (type === "positive_remove") {
|
|
177
|
+
this.positive.delete(payload.key);
|
|
178
|
+
if (append) {
|
|
179
|
+
this.#enqueueEvent(type, payload);
|
|
180
|
+
}
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (type === "negative_upsert") {
|
|
185
|
+
if (payload.value?.expireAt > nowMs()) {
|
|
186
|
+
this.negative.set(payload.key, payload.value);
|
|
187
|
+
} else {
|
|
188
|
+
this.negative.delete(payload.key);
|
|
189
|
+
}
|
|
190
|
+
if (append) {
|
|
191
|
+
this.#enqueueEvent(type, payload);
|
|
192
|
+
}
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (type === "negative_remove") {
|
|
197
|
+
this.negative.delete(payload.key);
|
|
198
|
+
if (append) {
|
|
199
|
+
this.#enqueueEvent(type, payload);
|
|
200
|
+
}
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (type === "conflict_set") {
|
|
205
|
+
this.conflicts.set(payload.key, payload.value);
|
|
206
|
+
if (append) {
|
|
207
|
+
this.#enqueueEvent(type, payload);
|
|
208
|
+
}
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (type === "conflict_clear") {
|
|
213
|
+
this.conflicts.delete(payload.key);
|
|
214
|
+
if (append) {
|
|
215
|
+
this.#enqueueEvent(type, payload);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async resolvePreferredCachePath(canonicalKey) {
|
|
221
|
+
if (!this.enabled || !canonicalKey) {
|
|
222
|
+
return "";
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
if (this.conflicts.has(canonicalKey)) {
|
|
226
|
+
return "";
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const existing = this.positive.get(canonicalKey);
|
|
230
|
+
if (!existing?.cachePath) {
|
|
231
|
+
return "";
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
const stats = await fs.promises.stat(existing.cachePath);
|
|
236
|
+
if (!stats.isFile()) {
|
|
237
|
+
throw new Error("not-file");
|
|
238
|
+
}
|
|
239
|
+
return existing.cachePath;
|
|
240
|
+
} catch {
|
|
241
|
+
this.#applyEvent({
|
|
242
|
+
type: "positive_remove",
|
|
243
|
+
payload: { key: canonicalKey },
|
|
244
|
+
});
|
|
245
|
+
return "";
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
shouldSkipRequest(canonicalKey, urlObj) {
|
|
250
|
+
const scope = buildNegativeScope(urlObj);
|
|
251
|
+
if (!this.enabled || !canonicalKey || !scope) {
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const key = buildNegativeKey(scope, canonicalKey);
|
|
256
|
+
const entry = this.negative.get(key);
|
|
257
|
+
if (!entry) {
|
|
258
|
+
return false;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
if (entry.expireAt <= nowMs()) {
|
|
262
|
+
this.#applyEvent({
|
|
263
|
+
type: "negative_remove",
|
|
264
|
+
payload: { key },
|
|
265
|
+
});
|
|
266
|
+
return false;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
recordSuccess({ canonicalKey, host, cachePath, fileName }) {
|
|
273
|
+
if (!this.enabled || !canonicalKey || !cachePath || !fileName) {
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const previous = this.positive.get(canonicalKey);
|
|
278
|
+
if (previous && previous.fileName !== fileName) {
|
|
279
|
+
this.#applyEvent({
|
|
280
|
+
type: "conflict_set",
|
|
281
|
+
payload: {
|
|
282
|
+
key: canonicalKey,
|
|
283
|
+
value: {
|
|
284
|
+
reason: "file-name-mismatch",
|
|
285
|
+
updatedAt: nowMs(),
|
|
286
|
+
previousFileName: previous.fileName,
|
|
287
|
+
currentFileName: fileName,
|
|
288
|
+
},
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
this.#applyEvent({
|
|
293
|
+
type: "positive_remove",
|
|
294
|
+
payload: { key: canonicalKey },
|
|
295
|
+
});
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
this.#applyEvent({
|
|
300
|
+
type: "positive_upsert",
|
|
301
|
+
payload: {
|
|
302
|
+
key: canonicalKey,
|
|
303
|
+
value: {
|
|
304
|
+
cachePath,
|
|
305
|
+
fileName,
|
|
306
|
+
host: String(host || "").toLowerCase(),
|
|
307
|
+
updatedAt: nowMs(),
|
|
308
|
+
},
|
|
309
|
+
},
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
if (host) {
|
|
313
|
+
const negativeKey = buildNegativeKey(host, canonicalKey);
|
|
314
|
+
if (this.negative.has(negativeKey)) {
|
|
315
|
+
this.#applyEvent({
|
|
316
|
+
type: "negative_remove",
|
|
317
|
+
payload: { key: negativeKey },
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
recordNegative({ canonicalKey, urlObj, statusCode = 404, ttlMs = this.negativeTtlMs }) {
|
|
324
|
+
const scope = buildNegativeScope(urlObj);
|
|
325
|
+
if (!this.enabled || !canonicalKey || !scope) {
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const expireAt = nowMs() + toPositiveInt(ttlMs, this.negativeTtlMs);
|
|
330
|
+
const key = buildNegativeKey(scope, canonicalKey);
|
|
331
|
+
this.#applyEvent({
|
|
332
|
+
type: "negative_upsert",
|
|
333
|
+
payload: {
|
|
334
|
+
key,
|
|
335
|
+
value: {
|
|
336
|
+
scope,
|
|
337
|
+
statusCode,
|
|
338
|
+
expireAt,
|
|
339
|
+
updatedAt: nowMs(),
|
|
340
|
+
},
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async flush() {
|
|
346
|
+
if (!this.enabled || this.flushing) {
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
this.flushing = true;
|
|
351
|
+
try {
|
|
352
|
+
if (this.pendingEvents.length > 0) {
|
|
353
|
+
const text = `${this.pendingEvents.join("\n")}\n`;
|
|
354
|
+
this.pendingEvents = [];
|
|
355
|
+
await fs.promises.appendFile(this.eventLogPath, text, "utf8");
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const stats = await fs.promises.stat(this.eventLogPath).catch(() => null);
|
|
359
|
+
const needsSnapshot = this.dirtySinceSnapshot && (!stats || stats.size >= this.maxEventBytes);
|
|
360
|
+
|
|
361
|
+
if (needsSnapshot) {
|
|
362
|
+
await this.#writeSnapshotAndResetEventLog();
|
|
363
|
+
}
|
|
364
|
+
} finally {
|
|
365
|
+
this.flushing = false;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async #writeSnapshotAndResetEventLog() {
|
|
370
|
+
const snapshot = serializeSnapshot(this.positive, this.negative, this.conflicts);
|
|
371
|
+
const tempPath = `${this.snapshotPath}.tmp`;
|
|
372
|
+
await fs.promises.writeFile(tempPath, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
|
|
373
|
+
await fs.promises.rename(tempPath, this.snapshotPath);
|
|
374
|
+
await fs.promises.writeFile(this.eventLogPath, "", "utf8");
|
|
375
|
+
this.dirtySinceSnapshot = false;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async destroy() {
|
|
379
|
+
if (!this.enabled) {
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
if (this.flushTimer) {
|
|
384
|
+
clearInterval(this.flushTimer);
|
|
385
|
+
this.flushTimer = null;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
await this.flush();
|
|
389
|
+
if (this.dirtySinceSnapshot) {
|
|
390
|
+
await this.#writeSnapshotAndResetEventLog();
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
}
|
|
@@ -4,45 +4,75 @@ import { DailyLogFile } from "./daily-log-file.js";
|
|
|
4
4
|
const MIRROR_INSTALLED = Symbol.for("maven-proxy.console-log-file.installed");
|
|
5
5
|
const GLOBAL_ERROR_HOOK_INSTALLED = Symbol.for("maven-proxy.global-error-hook.installed");
|
|
6
6
|
|
|
7
|
-
function mirrorConsoleMethod({
|
|
7
|
+
function mirrorConsoleMethod({
|
|
8
|
+
level,
|
|
9
|
+
originalMethod,
|
|
10
|
+
appLogFile,
|
|
11
|
+
errorLogFile,
|
|
12
|
+
outputToConsole,
|
|
13
|
+
}) {
|
|
8
14
|
return (...args) => {
|
|
9
|
-
|
|
15
|
+
if (outputToConsole) {
|
|
16
|
+
originalMethod(...args);
|
|
17
|
+
}
|
|
10
18
|
|
|
11
19
|
const line = `[${new Date().toISOString()}] [${level}] ${util.format(...args)}`;
|
|
12
|
-
|
|
20
|
+
appLogFile.appendLine(line).catch((error) => {
|
|
13
21
|
process.stderr.write(`[maven-proxy] write console log failed: ${error.message}\n`);
|
|
14
22
|
});
|
|
23
|
+
|
|
24
|
+
if (level === "ERROR") {
|
|
25
|
+
errorLogFile.appendLine(line).catch((error) => {
|
|
26
|
+
process.stderr.write(`[maven-proxy] write error log failed: ${error.message}\n`);
|
|
27
|
+
});
|
|
28
|
+
}
|
|
15
29
|
};
|
|
16
30
|
}
|
|
17
31
|
|
|
18
|
-
export function installConsoleLogFileMirror({
|
|
32
|
+
export function installConsoleLogFileMirror({
|
|
33
|
+
logDir,
|
|
34
|
+
retentionDays = 7,
|
|
35
|
+
outputToConsole = true,
|
|
36
|
+
}) {
|
|
19
37
|
if (globalThis[MIRROR_INSTALLED]) {
|
|
20
38
|
return;
|
|
21
39
|
}
|
|
22
40
|
globalThis[MIRROR_INSTALLED] = true;
|
|
23
41
|
|
|
24
|
-
const
|
|
42
|
+
const appLogFile = new DailyLogFile({
|
|
43
|
+
logDir,
|
|
44
|
+
filePrefix: "app",
|
|
45
|
+
retentionDays,
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
const errorLogFile = new DailyLogFile({
|
|
25
49
|
logDir,
|
|
26
|
-
filePrefix: "
|
|
50
|
+
filePrefix: "error",
|
|
27
51
|
retentionDays,
|
|
28
52
|
});
|
|
29
53
|
|
|
30
54
|
console.log = mirrorConsoleMethod({
|
|
31
55
|
level: "INFO",
|
|
32
56
|
originalMethod: console.log.bind(console),
|
|
33
|
-
|
|
57
|
+
appLogFile,
|
|
58
|
+
errorLogFile,
|
|
59
|
+
outputToConsole,
|
|
34
60
|
});
|
|
35
61
|
|
|
36
62
|
console.warn = mirrorConsoleMethod({
|
|
37
63
|
level: "WARN",
|
|
38
64
|
originalMethod: console.warn.bind(console),
|
|
39
|
-
|
|
65
|
+
appLogFile,
|
|
66
|
+
errorLogFile,
|
|
67
|
+
outputToConsole,
|
|
40
68
|
});
|
|
41
69
|
|
|
42
70
|
console.error = mirrorConsoleMethod({
|
|
43
71
|
level: "ERROR",
|
|
44
72
|
originalMethod: console.error.bind(console),
|
|
45
|
-
|
|
73
|
+
appLogFile,
|
|
74
|
+
errorLogFile,
|
|
75
|
+
outputToConsole,
|
|
46
76
|
});
|
|
47
77
|
}
|
|
48
78
|
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
function safeDecode(pathname) {
|
|
2
|
+
try {
|
|
3
|
+
return decodeURIComponent(pathname || "/");
|
|
4
|
+
} catch {
|
|
5
|
+
return pathname || "/";
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function escapeRegex(value) {
|
|
10
|
+
return String(value || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function normalizePathname(pathname) {
|
|
14
|
+
return String(pathname || "")
|
|
15
|
+
.replace(/\\/g, "/")
|
|
16
|
+
.replace(/^\/+/, "")
|
|
17
|
+
.replace(/\/+/g, "/")
|
|
18
|
+
.trim();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function stripKnownPrefixes(relativePath) {
|
|
22
|
+
const raw = normalizePathname(relativePath);
|
|
23
|
+
if (!raw) {
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const candidates = new Set([raw]);
|
|
28
|
+
|
|
29
|
+
if (raw.toLowerCase().startsWith("maven2/")) {
|
|
30
|
+
candidates.add(raw.slice("maven2/".length));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (raw.toLowerCase().startsWith("m2/")) {
|
|
34
|
+
candidates.add(raw.slice("m2/".length));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const maven2Marker = raw.toLowerCase().indexOf("/maven2/");
|
|
38
|
+
if (maven2Marker >= 0) {
|
|
39
|
+
candidates.add(raw.slice(maven2Marker + "/maven2/".length));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const m2Marker = raw.toLowerCase().indexOf("/m2/");
|
|
43
|
+
if (m2Marker >= 0) {
|
|
44
|
+
candidates.add(raw.slice(m2Marker + "/m2/".length));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const patterns = [
|
|
48
|
+
/^repository\/[^/]+\/(.+)$/i,
|
|
49
|
+
/^artifactory\/[^/]+\/(.+)$/i,
|
|
50
|
+
/^nexus\/content\/repositories\/[^/]+\/(.+)$/i,
|
|
51
|
+
/^repositories\/[^/]+\/(.+)$/i,
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
for (const pattern of patterns) {
|
|
55
|
+
const match = raw.match(pattern);
|
|
56
|
+
if (match?.[1]) {
|
|
57
|
+
candidates.add(match[1]);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const normalizedCandidates = [...candidates].map((item) => normalizePathname(item)).filter(Boolean);
|
|
62
|
+
normalizedCandidates.sort((left, right) => left.split("/").length - right.split("/").length);
|
|
63
|
+
return normalizedCandidates;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function isSafePathSegment(segment) {
|
|
67
|
+
return /^[A-Za-z0-9_.+-]+$/.test(String(segment || ""));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function isReleaseVersion(version) {
|
|
71
|
+
return !String(version || "").toUpperCase().endsWith("-SNAPSHOT");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function matchReleaseFileName(artifact, version, fileName) {
|
|
75
|
+
if (/-SNAPSHOT(?=\.|-)/i.test(fileName)) {
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const escapedArtifact = escapeRegex(artifact);
|
|
80
|
+
const escapedVersion = escapeRegex(version);
|
|
81
|
+
const pattern = new RegExp(
|
|
82
|
+
`^${escapedArtifact}-${escapedVersion}(?:-[A-Za-z0-9_.+:-]+)?\\.(pom|jar|module|aar|war)(?:\\.(sha1|sha256|sha512|md5|asc))?$`,
|
|
83
|
+
"i",
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
return pattern.test(fileName);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function tryParseCandidate(relativePath) {
|
|
90
|
+
const normalized = normalizePathname(relativePath);
|
|
91
|
+
const parts = normalized.split("/").filter(Boolean);
|
|
92
|
+
if (parts.length < 4) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const fileName = parts[parts.length - 1];
|
|
97
|
+
const version = parts[parts.length - 2];
|
|
98
|
+
const artifact = parts[parts.length - 3];
|
|
99
|
+
const groupParts = parts.slice(0, -3);
|
|
100
|
+
|
|
101
|
+
if (!artifact || !version || !fileName || groupParts.length === 0) {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (!isSafePathSegment(artifact) || !isSafePathSegment(version) || !isSafePathSegment(fileName)) {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (!groupParts.every((segment) => isSafePathSegment(segment))) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (!isReleaseVersion(version)) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (!matchReleaseFileName(artifact, version, fileName)) {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const groupPath = groupParts.join("/");
|
|
122
|
+
const canonicalPath = `${groupPath}/${artifact}/${version}/${fileName}`;
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
canonicalPath,
|
|
126
|
+
canonicalKey: canonicalPath,
|
|
127
|
+
groupPath,
|
|
128
|
+
artifact,
|
|
129
|
+
version,
|
|
130
|
+
fileName,
|
|
131
|
+
isRelease: true,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function parseMavenReleaseCanonical(urlObj) {
|
|
136
|
+
if (!urlObj || typeof urlObj !== "object") {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const decodedPath = safeDecode(urlObj.pathname || "/");
|
|
141
|
+
const candidates = stripKnownPrefixes(decodedPath);
|
|
142
|
+
|
|
143
|
+
for (const candidate of candidates) {
|
|
144
|
+
const parsed = tryParseCandidate(candidate);
|
|
145
|
+
if (parsed) {
|
|
146
|
+
return parsed;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return null;
|
|
151
|
+
}
|
package/src/config/config.js
CHANGED
|
@@ -6,7 +6,7 @@ import { detectJavaHome } from "../common/java-home.js";
|
|
|
6
6
|
|
|
7
7
|
const cwd = process.cwd();
|
|
8
8
|
const userConfigDir = path.resolve(os.homedir(), "maven-proxy");
|
|
9
|
-
const defaultUserConfigPath = path.join(userConfigDir, "config");
|
|
9
|
+
const defaultUserConfigPath = path.join(userConfigDir, "config.properties");
|
|
10
10
|
|
|
11
11
|
function normalizeConfigMode(value) {
|
|
12
12
|
const normalized = String(value || "").trim().toLowerCase();
|
|
@@ -55,8 +55,7 @@ function resolveConfigFilePath(configMode) {
|
|
|
55
55
|
|
|
56
56
|
if (configMode === "development") {
|
|
57
57
|
const devCandidates = [
|
|
58
|
-
path.resolve(cwd, ".
|
|
59
|
-
path.resolve(cwd, ".evn"),
|
|
58
|
+
path.resolve(cwd, "config.properties"),
|
|
60
59
|
];
|
|
61
60
|
|
|
62
61
|
for (const candidate of devCandidates) {
|
|
@@ -181,6 +180,13 @@ const defaultMavenRepoDomains = [
|
|
|
181
180
|
|
|
182
181
|
const cacheDir = path.resolve(configBaseDir, process.env.CACHE_DIR || "data/cache");
|
|
183
182
|
|
|
183
|
+
const multiThreadMinSizeBytes = Math.max(0, toInt(process.env.MULTI_THREAD_MIN_SIZE_MB, 1)) * 1024 * 1024;
|
|
184
|
+
const downloadTimeoutMs = Math.max(1, toInt(process.env.DOWNLOAD_TIMEOUT_SECONDS, 60)) * 1000;
|
|
185
|
+
const outboundKeepAliveMsecs = Math.max(1, toInt(process.env.OUTBOUND_KEEP_ALIVE_SECONDS, 1)) * 1000;
|
|
186
|
+
const mavenNegativeCacheTtlMs = Math.max(1, toInt(process.env.MAVEN_NEGATIVE_CACHE_TTL_HOURS, 24)) * 60 * 60 * 1000;
|
|
187
|
+
const mavenAffinityFlushIntervalMs = Math.max(1, toInt(process.env.MAVEN_AFFINITY_FLUSH_INTERVAL_SECONDS, 5)) * 1000;
|
|
188
|
+
const mavenAffinityEventMaxBytes = Math.max(1, toInt(process.env.MAVEN_AFFINITY_EVENT_MAX_MB, 8)) * 1024 * 1024;
|
|
189
|
+
|
|
184
190
|
export const config = {
|
|
185
191
|
configMode,
|
|
186
192
|
configBaseDir,
|
|
@@ -198,10 +204,20 @@ export const config = {
|
|
|
198
204
|
mavenRepoDomains: toList(process.env.MAVEN_REPO_DOMAINS, [...new Set(defaultMavenRepoDomains)]),
|
|
199
205
|
multiThreadDomains: toList(process.env.MULTI_THREAD_DOMAINS, ["repo1.maven.org"]),
|
|
200
206
|
multiThreadCount: Math.max(1, toInt(process.env.MULTI_THREAD_COUNT, 4)),
|
|
201
|
-
multiThreadMinSizeBytes
|
|
202
|
-
downloadTimeoutMs
|
|
207
|
+
multiThreadMinSizeBytes,
|
|
208
|
+
downloadTimeoutMs,
|
|
209
|
+
outboundKeepAlive: toBool(process.env.OUTBOUND_KEEP_ALIVE, true),
|
|
210
|
+
outboundKeepAliveMsecs,
|
|
211
|
+
outboundMaxSockets: Math.max(1, toInt(process.env.OUTBOUND_MAX_SOCKETS, 64)),
|
|
212
|
+
outboundMaxFreeSockets: Math.max(1, toInt(process.env.OUTBOUND_MAX_FREE_SOCKETS, 16)),
|
|
213
|
+
mavenAffinityEnabled: toBool(process.env.MAVEN_AFFINITY_ENABLED, true),
|
|
214
|
+
mavenAffinityIndexDir: path.resolve(cacheDir, process.env.MAVEN_AFFINITY_INDEX_DIR || ".index"),
|
|
215
|
+
mavenNegativeCacheTtlMs,
|
|
216
|
+
mavenAffinityFlushIntervalMs,
|
|
217
|
+
mavenAffinityEventMaxBytes,
|
|
203
218
|
downloadLogDir: path.resolve(configBaseDir, process.env.DOWNLOAD_LOG_DIR || "data/logs/downloads"),
|
|
204
219
|
logRetentionDays: Math.max(1, toInt(process.env.LOG_RETENTION_DAYS, 7)),
|
|
220
|
+
logToStdout: toBool(process.env.LOG_TO_STDOUT, true),
|
|
205
221
|
certDir: path.resolve(configBaseDir, process.env.CERT_DIR || "data/certs"),
|
|
206
222
|
rootCertPath: path.resolve(configBaseDir, process.env.ROOT_CERT_PATH || "data/certs/root-ca.crt"),
|
|
207
223
|
rootKeyPath: path.resolve(configBaseDir, process.env.ROOT_KEY_PATH || "data/certs/root-ca.key.pem"),
|
package/src/index.js
CHANGED
|
@@ -7,11 +7,13 @@ import { startProxyServer } from "./proxy/proxy-server.js";
|
|
|
7
7
|
import { startRepoServer } from "./repo/repo-server.js";
|
|
8
8
|
import { getTrustStoreCommands } from "./cert/truststore-utils.js";
|
|
9
9
|
import { UpstreamProxyManager } from "./proxy/upstream-proxy.js";
|
|
10
|
+
import { MavenAffinityIndex } from "./cache/maven-affinity-index.js";
|
|
10
11
|
import { installConsoleLogFileMirror, installGlobalErrorLogging } from "./common/console-log-file.js";
|
|
11
12
|
|
|
12
13
|
installConsoleLogFileMirror({
|
|
13
14
|
logDir: config.downloadLogDir,
|
|
14
15
|
retentionDays: config.logRetentionDays,
|
|
16
|
+
outputToConsole: config.logToStdout,
|
|
15
17
|
});
|
|
16
18
|
installGlobalErrorLogging();
|
|
17
19
|
|
|
@@ -34,6 +36,8 @@ async function main() {
|
|
|
34
36
|
}
|
|
35
37
|
|
|
36
38
|
const upstreamProxyManager = new UpstreamProxyManager(config, matchesDomain);
|
|
39
|
+
const mavenAffinityIndex = new MavenAffinityIndex(config);
|
|
40
|
+
await mavenAffinityIndex.init();
|
|
37
41
|
|
|
38
42
|
const downloader = new Downloader(config, matchesDomain, upstreamProxyManager);
|
|
39
43
|
|
|
@@ -43,6 +47,7 @@ async function main() {
|
|
|
43
47
|
downloader,
|
|
44
48
|
matchesDomain,
|
|
45
49
|
upstreamProxyManager,
|
|
50
|
+
mavenAffinityIndex,
|
|
46
51
|
);
|
|
47
52
|
const repoServer = startRepoServer(config, downloader);
|
|
48
53
|
|
|
@@ -61,8 +66,18 @@ async function main() {
|
|
|
61
66
|
console.log(`[maven-proxy] cache maven: ${config.mavenCacheDir}`);
|
|
62
67
|
console.log(`[maven-proxy] cache npm : ${config.npmCacheDir}`);
|
|
63
68
|
console.log(`[maven-proxy] cache other: ${config.genericCacheDir}`);
|
|
64
|
-
console.log(`[maven-proxy]
|
|
69
|
+
console.log(`[maven-proxy] log dir: ${config.downloadLogDir}`);
|
|
65
70
|
console.log(`[maven-proxy] log retention days: ${config.logRetentionDays}`);
|
|
71
|
+
console.log(`[maven-proxy] log to stdout: ${config.logToStdout}`);
|
|
72
|
+
console.log(`[maven-proxy] outbound keep-alive: ${config.outboundKeepAlive}`);
|
|
73
|
+
console.log(`[maven-proxy] outbound keepAlive(seconds): ${config.outboundKeepAliveMsecs / 1000}`);
|
|
74
|
+
console.log(`[maven-proxy] outbound maxSockets: ${config.outboundMaxSockets}`);
|
|
75
|
+
console.log(`[maven-proxy] outbound maxFreeSockets: ${config.outboundMaxFreeSockets}`);
|
|
76
|
+
console.log(`[maven-proxy] maven affinity enabled: ${config.mavenAffinityEnabled}`);
|
|
77
|
+
console.log(`[maven-proxy] maven affinity index dir: ${config.mavenAffinityIndexDir}`);
|
|
78
|
+
console.log(`[maven-proxy] maven negative cache ttl(hours): ${config.mavenNegativeCacheTtlMs / (60 * 60 * 1000)}`);
|
|
79
|
+
console.log(`[maven-proxy] maven affinity flush interval(seconds): ${config.mavenAffinityFlushIntervalMs / 1000}`);
|
|
80
|
+
console.log(`[maven-proxy] maven affinity event max(MB): ${config.mavenAffinityEventMaxBytes / (1024 * 1024)}`);
|
|
66
81
|
console.log(`[maven-proxy] root cert : ${config.rootCertPath}`);
|
|
67
82
|
console.log(`[maven-proxy] repo fallback repos: ${(config.repoFallbackRepos || []).join(",") || "(none)"}`);
|
|
68
83
|
if (config.upstreamProxyUrl || config.upstreamHttpProxyUrl || config.upstreamHttpsProxyUrl) {
|
|
@@ -81,6 +96,8 @@ async function main() {
|
|
|
81
96
|
proxyServer.close();
|
|
82
97
|
mitmHttpServer.close();
|
|
83
98
|
repoServer.close();
|
|
99
|
+
upstreamProxyManager.destroy();
|
|
100
|
+
void mavenAffinityIndex.destroy();
|
|
84
101
|
};
|
|
85
102
|
|
|
86
103
|
process.on("SIGINT", shutdown);
|
|
@@ -4,6 +4,7 @@ import https from "node:https";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { getCacheFilePath } from "../cache/cache-path.js";
|
|
6
6
|
import { detectPackageEcosystem } from "../common/ecosystem.js";
|
|
7
|
+
import { parseMavenReleaseCanonical } from "../common/maven-canonical.js";
|
|
7
8
|
|
|
8
9
|
const LOCAL_FS_ERROR_CODES = new Set([
|
|
9
10
|
"EACCES",
|
|
@@ -158,7 +159,13 @@ function forwardDirectRequest(req, res, urlObj, timeoutMs, upstreamProxyManager
|
|
|
158
159
|
req.pipe(upstreamReq);
|
|
159
160
|
}
|
|
160
161
|
|
|
161
|
-
export function createHttpRequestHandler({
|
|
162
|
+
export function createHttpRequestHandler({
|
|
163
|
+
config,
|
|
164
|
+
downloader,
|
|
165
|
+
upstreamProxyManager = null,
|
|
166
|
+
matchesDomain,
|
|
167
|
+
mavenAffinityIndex = null,
|
|
168
|
+
}) {
|
|
162
169
|
return async function handleHttpRequestPath(req, res, forcedProtocol = null) {
|
|
163
170
|
let urlObj;
|
|
164
171
|
try {
|
|
@@ -176,12 +183,18 @@ export function createHttpRequestHandler({ config, downloader, upstreamProxyMana
|
|
|
176
183
|
}
|
|
177
184
|
|
|
178
185
|
let cachePath;
|
|
186
|
+
let ecosystem;
|
|
187
|
+
let canonical = null;
|
|
179
188
|
try {
|
|
180
|
-
|
|
189
|
+
ecosystem = detectPackageEcosystem(urlObj, config, matchesDomain);
|
|
181
190
|
cachePath = getCacheFilePath(config.cacheDir, urlObj, {
|
|
182
191
|
ecosystem,
|
|
183
192
|
includeHost: ecosystem !== "maven",
|
|
184
193
|
});
|
|
194
|
+
|
|
195
|
+
if (ecosystem === "maven" && mavenAffinityIndex?.enabled) {
|
|
196
|
+
canonical = parseMavenReleaseCanonical(urlObj);
|
|
197
|
+
}
|
|
185
198
|
} catch (error) {
|
|
186
199
|
const message = `Invalid cache path: ${error.message}`;
|
|
187
200
|
sendErrorText(res, 400, message, "proxy");
|
|
@@ -194,12 +207,49 @@ export function createHttpRequestHandler({ config, downloader, upstreamProxyMana
|
|
|
194
207
|
return;
|
|
195
208
|
}
|
|
196
209
|
|
|
210
|
+
if (canonical && mavenAffinityIndex) {
|
|
211
|
+
const preferredPath = await mavenAffinityIndex.resolvePreferredCachePath(canonical.canonicalKey);
|
|
212
|
+
if (preferredPath) {
|
|
213
|
+
console.log(`[proxy] affinity hit canonical=${canonical.canonicalKey} host=${urlObj.hostname}`);
|
|
214
|
+
await serveFile(res, req, preferredPath);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (mavenAffinityIndex.shouldSkipRequest(canonical.canonicalKey, urlObj)) {
|
|
219
|
+
console.log(`[proxy] affinity negative skip canonical=${canonical.canonicalKey} host=${urlObj.hostname}`);
|
|
220
|
+
sendText(res, 404, "Not Found");
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
197
225
|
try {
|
|
198
226
|
await fs.promises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
199
227
|
await downloader.ensureCached(urlObj, cachePath, req.headers);
|
|
228
|
+
|
|
229
|
+
if (canonical && mavenAffinityIndex) {
|
|
230
|
+
mavenAffinityIndex.recordSuccess({
|
|
231
|
+
canonicalKey: canonical.canonicalKey,
|
|
232
|
+
host: urlObj.hostname,
|
|
233
|
+
cachePath,
|
|
234
|
+
fileName: canonical.fileName,
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
|
|
200
238
|
res.setHeader("x-cache", "MISS");
|
|
201
239
|
await serveFile(res, req, cachePath);
|
|
202
240
|
} catch (error) {
|
|
241
|
+
if (
|
|
242
|
+
canonical &&
|
|
243
|
+
mavenAffinityIndex &&
|
|
244
|
+
(error.statusCode === 404 || error.statusCode === 410)
|
|
245
|
+
) {
|
|
246
|
+
mavenAffinityIndex.recordNegative({
|
|
247
|
+
canonicalKey: canonical.canonicalKey,
|
|
248
|
+
urlObj,
|
|
249
|
+
statusCode: error.statusCode,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
|
|
203
253
|
if (isLocalFsWriteError(error)) {
|
|
204
254
|
if (!error.statusCode) {
|
|
205
255
|
error.statusCode = 500;
|
|
@@ -12,12 +12,20 @@ function sendErrorText(res, statusCode, message) {
|
|
|
12
12
|
sendText(res, statusCode, message);
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
export function startProxyServer(
|
|
15
|
+
export function startProxyServer(
|
|
16
|
+
config,
|
|
17
|
+
certManager,
|
|
18
|
+
downloader,
|
|
19
|
+
matchesDomain,
|
|
20
|
+
upstreamProxyManager = null,
|
|
21
|
+
mavenAffinityIndex = null,
|
|
22
|
+
) {
|
|
16
23
|
const handleHttpRequestPath = createHttpRequestHandler({
|
|
17
24
|
config,
|
|
18
25
|
downloader,
|
|
19
26
|
upstreamProxyManager,
|
|
20
27
|
matchesDomain,
|
|
28
|
+
mavenAffinityIndex,
|
|
21
29
|
});
|
|
22
30
|
const mitmHttpServer = createMitmHttpServer(handleHttpRequestPath);
|
|
23
31
|
|
|
@@ -1,7 +1,23 @@
|
|
|
1
|
+
import http from "node:http";
|
|
2
|
+
import https from "node:https";
|
|
1
3
|
import net from "node:net";
|
|
2
4
|
import tls from "node:tls";
|
|
3
5
|
import { ProxyAgent } from "proxy-agent";
|
|
4
6
|
|
|
7
|
+
function toPositiveInt(value, fallback) {
|
|
8
|
+
const parsed = Number.parseInt(value, 10);
|
|
9
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function buildAgentOptions(config) {
|
|
13
|
+
return {
|
|
14
|
+
keepAlive: Boolean(config.outboundKeepAlive),
|
|
15
|
+
keepAliveMsecs: toPositiveInt(config.outboundKeepAliveMsecs, 1000),
|
|
16
|
+
maxSockets: toPositiveInt(config.outboundMaxSockets, 64),
|
|
17
|
+
maxFreeSockets: toPositiveInt(config.outboundMaxFreeSockets, 16),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
5
21
|
function normalizeHostname(hostname) {
|
|
6
22
|
return String(hostname || "")
|
|
7
23
|
.trim()
|
|
@@ -64,6 +80,12 @@ export class UpstreamProxyManager {
|
|
|
64
80
|
this.config = config;
|
|
65
81
|
this.matchesDomain = matchesDomain;
|
|
66
82
|
this.agentCache = new Map();
|
|
83
|
+
this.directHttpAgent = new http.Agent(buildAgentOptions(config));
|
|
84
|
+
this.directHttpsAgent = new https.Agent(buildAgentOptions(config));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
getDirectAgentForProtocol(protocol) {
|
|
88
|
+
return protocol === "https:" ? this.directHttpsAgent : this.directHttpAgent;
|
|
67
89
|
}
|
|
68
90
|
|
|
69
91
|
shouldBypass(hostname) {
|
|
@@ -111,17 +133,21 @@ export class UpstreamProxyManager {
|
|
|
111
133
|
}
|
|
112
134
|
|
|
113
135
|
getAgentForUrl(urlObj) {
|
|
114
|
-
const
|
|
136
|
+
const protocol = urlObj?.protocol === "https:" ? "https:" : "http:";
|
|
137
|
+
const hostname = String(urlObj?.hostname || "");
|
|
138
|
+
const proxyUrl = this.getProxyUrlFor(protocol, hostname);
|
|
139
|
+
|
|
115
140
|
if (!proxyUrl) {
|
|
116
|
-
return
|
|
141
|
+
return this.getDirectAgentForProtocol(protocol);
|
|
117
142
|
}
|
|
118
143
|
|
|
119
|
-
const cacheKey =
|
|
144
|
+
const cacheKey = `proxy:${proxyUrl}`;
|
|
120
145
|
if (!this.agentCache.has(cacheKey)) {
|
|
121
146
|
// proxy-agent v6 expects resolver-style options for deterministic proxy routing.
|
|
122
147
|
this.agentCache.set(
|
|
123
148
|
cacheKey,
|
|
124
149
|
new ProxyAgent({
|
|
150
|
+
...buildAgentOptions(this.config),
|
|
125
151
|
getProxyForUrl: () => proxyUrl,
|
|
126
152
|
}),
|
|
127
153
|
);
|
|
@@ -134,6 +160,18 @@ export class UpstreamProxyManager {
|
|
|
134
160
|
return Boolean(this.getProxyUrlFor(protocol, hostname));
|
|
135
161
|
}
|
|
136
162
|
|
|
163
|
+
destroy() {
|
|
164
|
+
for (const agent of this.agentCache.values()) {
|
|
165
|
+
if (typeof agent?.destroy === "function") {
|
|
166
|
+
agent.destroy();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
this.agentCache.clear();
|
|
171
|
+
this.directHttpAgent.destroy();
|
|
172
|
+
this.directHttpsAgent.destroy();
|
|
173
|
+
}
|
|
174
|
+
|
|
137
175
|
async createConnectTunnel(targetHost, targetPort, timeoutMs) {
|
|
138
176
|
const proxyUrlText = this.getProxyUrlFor("https:", targetHost);
|
|
139
177
|
if (!proxyUrlText) {
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import { DailyLogFile } from "./daily-log-file.js";
|
|
2
|
-
|
|
3
|
-
export class DownloadLogWriter {
|
|
4
|
-
constructor(logDir, retentionDays = 7) {
|
|
5
|
-
this.logFile = new DailyLogFile({
|
|
6
|
-
logDir,
|
|
7
|
-
filePrefix: "download",
|
|
8
|
-
retentionDays,
|
|
9
|
-
});
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
async append(event, url, details = {}) {
|
|
13
|
-
const record = {
|
|
14
|
-
time: new Date().toISOString(),
|
|
15
|
-
event,
|
|
16
|
-
url,
|
|
17
|
-
...details,
|
|
18
|
-
};
|
|
19
|
-
await this.logFile.appendLine(JSON.stringify(record));
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
write(event, url, details = {}) {
|
|
23
|
-
this.append(event, url, details).catch((error) => {
|
|
24
|
-
console.warn(`[downloader] write download log failed: ${error.message}`);
|
|
25
|
-
});
|
|
26
|
-
}
|
|
27
|
-
}
|