@lincoln504/pi-research 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cli.mjs +1287 -540
- package/dist/thread-worker.mjs +30 -11
- package/docs/AGENT-SKILL.md +11 -4
- package/docs/ARCHITECTURE.md +19 -9
- package/docs/CHANGELOG.md +81 -1
- package/docs/CONFIGURATION.md +9 -7
- package/docs/KNOWLEDGE-STORE.md +4 -3
- package/docs/PI-EXTENSION.md +1 -1
- package/docs/SDK.md +55 -0
- package/docs/THIRD-PARTY-NOTICES.md +1 -1
- package/package.json +5 -4
- package/scripts/cleanup.cjs +50 -10
- package/scripts/prepare.cjs +15 -5
- package/scripts/setup.cjs +24 -4
- package/skills/pi-research/README.md +9 -2
- package/skills/pi-research/SKILL.md +6 -3
- package/skills/pi-research/references/configuration.md +6 -2
- package/skills/pi-research/scripts/run.mjs +24 -2
- package/src/cli.ts +149 -21
- package/src/config.ts +56 -15
- package/src/constants.ts +31 -0
- package/src/core/interfaces/knowledge-interfaces.ts +6 -1
- package/src/core/interfaces/state-manager-interfaces.ts +1 -1
- package/src/core/pi-version.ts +136 -0
- package/src/core/service-registry.ts +117 -8
- package/src/index.ts +11 -18
- package/src/infrastructure/browser/browser-client.ts +16 -4
- package/src/infrastructure/browser/browser-error-utils.ts +6 -1
- package/src/infrastructure/browser/browser-server.ts +17 -5
- package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
- package/src/infrastructure/browser/cleanup-utils.ts +45 -7
- package/src/infrastructure/browser/client-agent.ts +4 -3
- package/src/infrastructure/browser/priority-task-queue.ts +10 -3
- package/src/infrastructure/browser/scheduler-factory.ts +26 -1
- package/src/infrastructure/browser/task-execution-service.ts +92 -28
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
- package/src/infrastructure/browser/thread-worker.mjs +30 -11
- package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
- package/src/infrastructure/embedding/embedding-client.ts +22 -8
- package/src/infrastructure/embedding/embedding-factory.ts +28 -11
- package/src/infrastructure/embedding/embedding-server.ts +65 -4
- package/src/infrastructure/file-lock-service.ts +223 -26
- package/src/infrastructure/process-lifecycle-service.ts +108 -31
- package/src/infrastructure/state/state-backup-manager.ts +5 -7
- package/src/infrastructure/state/state-manager-service.ts +1 -1
- package/src/infrastructure/state/state-manager.ts +19 -20
- package/src/infrastructure/types/state-types.ts +9 -0
- package/src/knowledge/embedder-init.ts +1 -1
- package/src/knowledge/embedder.ts +52 -3
- package/src/knowledge/index.ts +1 -1
- package/src/knowledge/store.ts +35 -9
- package/src/knowledge/webgpu-viability.ts +9 -4
- package/src/knowledge/writer-queue.ts +88 -46
- package/src/orchestration/deep-research-orchestrator.ts +13 -2
- package/src/orchestration/quick-research-orchestrator.ts +2 -2
- package/src/orchestration/research-orchestration-service.ts +2 -4
- package/src/orchestration/research-synthesis-service.ts +33 -11
- package/src/security/cisa-kev.ts +10 -3
- package/src/security/github-advisories.ts +18 -7
- package/src/security/nvd-client.ts +17 -6
- package/src/security/osv-client.ts +14 -4
- package/src/skill-install/skill-installer.ts +135 -9
- package/src/stackexchange/rest-client.ts +6 -2
- package/src/tools/research-knowledge-search.ts +15 -0
- package/src/tools/search.ts +6 -1
- package/src/utils/atomic-replace.ts +101 -0
- package/src/utils/citation-utils.ts +3 -1
- package/src/utils/http-body.ts +176 -0
- package/src/web-research/browser-search.ts +50 -4
- package/src/web-research/search.ts +20 -5
- package/src/web-research/types.ts +17 -4
- package/src/web-research/web-scraper.ts +7 -50
- package/src/youtube/transcript-client.ts +5 -1
package/dist/cli.mjs
CHANGED
|
@@ -1017,11 +1017,12 @@ function disposeAllServices(container = globalServiceContainer) {
|
|
|
1017
1017
|
function resetServiceContainer(container = globalServiceContainer) {
|
|
1018
1018
|
return container.reset();
|
|
1019
1019
|
}
|
|
1020
|
-
var initializationContext, ServiceContainer, globalServiceContainer;
|
|
1020
|
+
var DISPOSE_INFLIGHT_INIT_TIMEOUT_MS, initializationContext, ServiceContainer, globalServiceContainer;
|
|
1021
1021
|
var init_service_registry = __esm({
|
|
1022
1022
|
"src/core/service-registry.ts"() {
|
|
1023
1023
|
"use strict";
|
|
1024
1024
|
init_logger();
|
|
1025
|
+
DISPOSE_INFLIGHT_INIT_TIMEOUT_MS = 5e3;
|
|
1025
1026
|
initializationContext = new AsyncLocalStorage4();
|
|
1026
1027
|
ServiceContainer = class {
|
|
1027
1028
|
services = /* @__PURE__ */ new Map();
|
|
@@ -1113,11 +1114,26 @@ var init_service_registry = __esm({
|
|
|
1113
1114
|
throw new Error(`Service '${name}' is not registered`);
|
|
1114
1115
|
}
|
|
1115
1116
|
if (registration.instance) {
|
|
1117
|
+
if (registration.initializationPromise && caller === void 0) {
|
|
1118
|
+
await registration.initializationPromise.catch(() => {
|
|
1119
|
+
});
|
|
1120
|
+
if (!registration.instance) {
|
|
1121
|
+
return this.get(name, ctx);
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1116
1124
|
if (ctx && registration.instance.initialize) {
|
|
1117
|
-
|
|
1125
|
+
const reinit = initializationContext.run(name, async () => {
|
|
1118
1126
|
await registration.instance.initialize(ctx);
|
|
1119
1127
|
return registration.instance;
|
|
1120
1128
|
});
|
|
1129
|
+
registration.initializationPromise = reinit;
|
|
1130
|
+
const clearIfCurrent = () => {
|
|
1131
|
+
if (registration.initializationPromise === reinit) {
|
|
1132
|
+
registration.initializationPromise = null;
|
|
1133
|
+
}
|
|
1134
|
+
};
|
|
1135
|
+
reinit.then(clearIfCurrent, clearIfCurrent);
|
|
1136
|
+
return reinit;
|
|
1121
1137
|
}
|
|
1122
1138
|
return registration.instance;
|
|
1123
1139
|
}
|
|
@@ -1137,7 +1153,21 @@ var init_service_registry = __esm({
|
|
|
1137
1153
|
}
|
|
1138
1154
|
}
|
|
1139
1155
|
/**
|
|
1140
|
-
* Get a service instance synchronously (returns null if not
|
|
1156
|
+
* Get a service instance synchronously (returns null if not registered or not
|
|
1157
|
+
* yet constructed).
|
|
1158
|
+
*
|
|
1159
|
+
* Deliberately returns instances in NON-settled lifecycles too. This is a
|
|
1160
|
+
* lifecycle *inspector* as much as an accessor: the health check resolves the
|
|
1161
|
+
* knowledge store through it precisely so it can report `initializing` /
|
|
1162
|
+
* `disabled` / `disposed (not running)` WITHOUT forcing initialization (forcing
|
|
1163
|
+
* it was a past cause of 45 s CI stalls). Filtering those states out here would
|
|
1164
|
+
* make every one of them indistinguishable from "absent".
|
|
1165
|
+
*
|
|
1166
|
+
* The trade-off: because `_initializeService` publishes the instance before
|
|
1167
|
+
* awaiting its `initialize()` (to break dependency cycles), a caller that
|
|
1168
|
+
* immediately *uses* the result can catch a half-built service. Callers that
|
|
1169
|
+
* need a ready service must use the async `get()`, which waits — see the note
|
|
1170
|
+
* there. Callers that only inspect `lifecycle` are safe by construction.
|
|
1141
1171
|
*/
|
|
1142
1172
|
tryGet(name) {
|
|
1143
1173
|
if (this.isDisposing) {
|
|
@@ -1171,6 +1201,10 @@ var init_service_registry = __esm({
|
|
|
1171
1201
|
if (!registration) {
|
|
1172
1202
|
throw new Error(`Service '${name}' is not registered`);
|
|
1173
1203
|
}
|
|
1204
|
+
if (registration.initializationPromise) {
|
|
1205
|
+
await registration.initializationPromise.catch(() => {
|
|
1206
|
+
});
|
|
1207
|
+
}
|
|
1174
1208
|
if (registration.instance && registration.instance.dispose) {
|
|
1175
1209
|
await registration.instance.dispose().catch((err) => {
|
|
1176
1210
|
logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
|
|
@@ -1190,6 +1224,10 @@ var init_service_registry = __esm({
|
|
|
1190
1224
|
if (!registration) {
|
|
1191
1225
|
throw new Error(`Service '${name}' is not registered`);
|
|
1192
1226
|
}
|
|
1227
|
+
if (registration.initializationPromise) {
|
|
1228
|
+
await registration.initializationPromise.catch(() => {
|
|
1229
|
+
});
|
|
1230
|
+
}
|
|
1193
1231
|
if (registration.instance && registration.instance.dispose) {
|
|
1194
1232
|
await registration.instance.dispose().catch((err) => {
|
|
1195
1233
|
logger.warn(`[ServiceContainer] Error disposing old service '${name}':`, err);
|
|
@@ -1219,6 +1257,24 @@ var init_service_registry = __esm({
|
|
|
1219
1257
|
logger.log("[ServiceContainer] Disposing all services (DAG-ordered teardown)...");
|
|
1220
1258
|
}
|
|
1221
1259
|
try {
|
|
1260
|
+
const inFlight2 = Array.from(this.services.values()).map((r) => r.initializationPromise).filter((p) => p != null);
|
|
1261
|
+
if (inFlight2.length > 0) {
|
|
1262
|
+
if (this.defaultOptions.enableLogging) {
|
|
1263
|
+
logger.debug(`[ServiceContainer] Waiting for ${inFlight2.length} in-flight initialization(s) before teardown`);
|
|
1264
|
+
}
|
|
1265
|
+
let settleTimer;
|
|
1266
|
+
await Promise.race([
|
|
1267
|
+
Promise.allSettled(inFlight2),
|
|
1268
|
+
new Promise((resolve8) => {
|
|
1269
|
+
settleTimer = setTimeout(() => {
|
|
1270
|
+
logger.warn("[ServiceContainer] Timed out waiting for in-flight service initialization; proceeding with teardown");
|
|
1271
|
+
resolve8();
|
|
1272
|
+
}, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
|
|
1273
|
+
settleTimer.unref?.();
|
|
1274
|
+
})
|
|
1275
|
+
]);
|
|
1276
|
+
if (settleTimer) clearTimeout(settleTimer);
|
|
1277
|
+
}
|
|
1222
1278
|
const activeServices = Array.from(this.services.keys()).filter(
|
|
1223
1279
|
(name) => this.services.get(name)?.instance !== null
|
|
1224
1280
|
);
|
|
@@ -1294,12 +1350,12 @@ var init_service_registry = __esm({
|
|
|
1294
1350
|
if (!registration) {
|
|
1295
1351
|
return null;
|
|
1296
1352
|
}
|
|
1297
|
-
if (registration.instance) {
|
|
1298
|
-
return "initialized" /* INITIALIZED */;
|
|
1299
|
-
}
|
|
1300
1353
|
if (registration.initializationPromise) {
|
|
1301
1354
|
return "initializing" /* INITIALIZING */;
|
|
1302
1355
|
}
|
|
1356
|
+
if (registration.instance) {
|
|
1357
|
+
return "initialized" /* INITIALIZED */;
|
|
1358
|
+
}
|
|
1303
1359
|
return "uninitialized" /* UNINITIALIZED */;
|
|
1304
1360
|
}
|
|
1305
1361
|
/**
|
|
@@ -1756,11 +1812,11 @@ function stripTrailingLlmPunctuation(input) {
|
|
|
1756
1812
|
continue;
|
|
1757
1813
|
}
|
|
1758
1814
|
if (last === ")" || last === "]" || last === "}") {
|
|
1759
|
-
const
|
|
1815
|
+
const open5 = last === ")" ? "(" : last === "]" ? "[" : "{";
|
|
1760
1816
|
let opens = 0;
|
|
1761
1817
|
let closes = 0;
|
|
1762
1818
|
for (const ch of s) {
|
|
1763
|
-
if (ch ===
|
|
1819
|
+
if (ch === open5) opens++;
|
|
1764
1820
|
else if (ch === last) closes++;
|
|
1765
1821
|
}
|
|
1766
1822
|
if (closes > opens) {
|
|
@@ -2043,8 +2099,69 @@ var init_host_config = __esm({
|
|
|
2043
2099
|
}
|
|
2044
2100
|
});
|
|
2045
2101
|
|
|
2102
|
+
// src/utils/atomic-replace.ts
|
|
2103
|
+
import * as fs4 from "node:fs/promises";
|
|
2104
|
+
import * as fsSync from "node:fs";
|
|
2105
|
+
function isRetryable(err) {
|
|
2106
|
+
const code = err?.code;
|
|
2107
|
+
return code !== void 0 && RETRYABLE_CODES.has(code);
|
|
2108
|
+
}
|
|
2109
|
+
async function replaceFile(tmpPath, destPath) {
|
|
2110
|
+
let lastError;
|
|
2111
|
+
for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
|
|
2112
|
+
try {
|
|
2113
|
+
await fs4.rename(tmpPath, destPath);
|
|
2114
|
+
return "renamed";
|
|
2115
|
+
} catch (err) {
|
|
2116
|
+
lastError = err;
|
|
2117
|
+
if (process.platform !== "win32" || !isRetryable(err)) throw err;
|
|
2118
|
+
if (attempt < MAX_RENAME_ATTEMPTS - 1) {
|
|
2119
|
+
await new Promise((r) => setTimeout(r, RENAME_RETRY_DELAY_MS * (attempt + 1)));
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
try {
|
|
2124
|
+
await fs4.copyFile(tmpPath, destPath);
|
|
2125
|
+
return "copied";
|
|
2126
|
+
} catch {
|
|
2127
|
+
throw lastError;
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
function replaceFileSync(tmpPath, destPath) {
|
|
2131
|
+
let lastError;
|
|
2132
|
+
for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
|
|
2133
|
+
try {
|
|
2134
|
+
fsSync.renameSync(tmpPath, destPath);
|
|
2135
|
+
return "renamed";
|
|
2136
|
+
} catch (err) {
|
|
2137
|
+
lastError = err;
|
|
2138
|
+
if (process.platform !== "win32" || !isRetryable(err)) throw err;
|
|
2139
|
+
if (attempt < MAX_RENAME_ATTEMPTS - 1) {
|
|
2140
|
+
const until = Date.now() + RENAME_RETRY_DELAY_MS * (attempt + 1);
|
|
2141
|
+
while (Date.now() < until) {
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
try {
|
|
2147
|
+
fsSync.copyFileSync(tmpPath, destPath);
|
|
2148
|
+
return "copied";
|
|
2149
|
+
} catch {
|
|
2150
|
+
throw lastError;
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
var RETRYABLE_CODES, MAX_RENAME_ATTEMPTS, RENAME_RETRY_DELAY_MS;
|
|
2154
|
+
var init_atomic_replace = __esm({
|
|
2155
|
+
"src/utils/atomic-replace.ts"() {
|
|
2156
|
+
"use strict";
|
|
2157
|
+
RETRYABLE_CODES = /* @__PURE__ */ new Set(["EPERM", "EBUSY", "EACCES"]);
|
|
2158
|
+
MAX_RENAME_ATTEMPTS = 5;
|
|
2159
|
+
RENAME_RETRY_DELAY_MS = 25;
|
|
2160
|
+
}
|
|
2161
|
+
});
|
|
2162
|
+
|
|
2046
2163
|
// src/config.ts
|
|
2047
|
-
import * as
|
|
2164
|
+
import * as fs5 from "node:fs";
|
|
2048
2165
|
import * as path5 from "node:path";
|
|
2049
2166
|
import * as os2 from "node:os";
|
|
2050
2167
|
import { Type } from "typebox";
|
|
@@ -2059,8 +2176,8 @@ function getProjectSettingsRegistryPath() {
|
|
|
2059
2176
|
function loadProjectSettingsRegistry() {
|
|
2060
2177
|
const registryPath = getProjectSettingsRegistryPath();
|
|
2061
2178
|
try {
|
|
2062
|
-
if (
|
|
2063
|
-
const content =
|
|
2179
|
+
if (fs5.existsSync(registryPath)) {
|
|
2180
|
+
const content = fs5.readFileSync(registryPath, "utf-8");
|
|
2064
2181
|
return JSON.parse(content);
|
|
2065
2182
|
}
|
|
2066
2183
|
} catch (err) {
|
|
@@ -2081,8 +2198,8 @@ function sleepSync(ms) {
|
|
|
2081
2198
|
function updateProjectSettingsRegistry(mutate) {
|
|
2082
2199
|
const registryPath = getProjectSettingsRegistryPath();
|
|
2083
2200
|
const dir = path5.dirname(registryPath);
|
|
2084
|
-
if (!
|
|
2085
|
-
|
|
2201
|
+
if (!fs5.existsSync(dir)) {
|
|
2202
|
+
fs5.mkdirSync(dir, { recursive: true, mode: 448 });
|
|
2086
2203
|
}
|
|
2087
2204
|
const lockPath = `${registryPath}.lock`;
|
|
2088
2205
|
let lockFd = null;
|
|
@@ -2090,14 +2207,14 @@ function updateProjectSettingsRegistry(mutate) {
|
|
|
2090
2207
|
try {
|
|
2091
2208
|
for (let i = 0; i < maxRetries; i++) {
|
|
2092
2209
|
try {
|
|
2093
|
-
lockFd =
|
|
2210
|
+
lockFd = fs5.openSync(lockPath, "wx", 384);
|
|
2094
2211
|
break;
|
|
2095
2212
|
} catch (err) {
|
|
2096
2213
|
if (err.code === "EEXIST") {
|
|
2097
2214
|
try {
|
|
2098
|
-
const stats =
|
|
2215
|
+
const stats = fs5.statSync(lockPath);
|
|
2099
2216
|
if (Date.now() - stats.mtimeMs > 3e4) {
|
|
2100
|
-
|
|
2217
|
+
fs5.unlinkSync(lockPath);
|
|
2101
2218
|
continue;
|
|
2102
2219
|
}
|
|
2103
2220
|
} catch {
|
|
@@ -2112,9 +2229,9 @@ function updateProjectSettingsRegistry(mutate) {
|
|
|
2112
2229
|
throw new Error(`Failed to acquire lock for project settings registry after ${maxRetries} retries. Aborting to prevent data corruption.`);
|
|
2113
2230
|
}
|
|
2114
2231
|
let registry = {};
|
|
2115
|
-
if (
|
|
2232
|
+
if (fs5.existsSync(registryPath)) {
|
|
2116
2233
|
try {
|
|
2117
|
-
registry = JSON.parse(
|
|
2234
|
+
registry = JSON.parse(fs5.readFileSync(registryPath, "utf-8"));
|
|
2118
2235
|
} catch (readErr) {
|
|
2119
2236
|
if (readErr?.code === "ENOENT") {
|
|
2120
2237
|
registry = {};
|
|
@@ -2129,15 +2246,15 @@ function updateProjectSettingsRegistry(mutate) {
|
|
|
2129
2246
|
mutate(registry);
|
|
2130
2247
|
const tmpPath = `${registryPath}.tmp.${process.pid}.${Date.now()}`;
|
|
2131
2248
|
try {
|
|
2132
|
-
|
|
2133
|
-
|
|
2249
|
+
fs5.writeFileSync(tmpPath, JSON.stringify(registry, null, 2), { encoding: "utf-8", mode: 384 });
|
|
2250
|
+
fs5.renameSync(tmpPath, registryPath);
|
|
2134
2251
|
try {
|
|
2135
|
-
|
|
2252
|
+
fs5.chmodSync(registryPath, 384);
|
|
2136
2253
|
} catch {
|
|
2137
2254
|
}
|
|
2138
2255
|
} catch (writeErr) {
|
|
2139
2256
|
try {
|
|
2140
|
-
|
|
2257
|
+
fs5.unlinkSync(tmpPath);
|
|
2141
2258
|
} catch {
|
|
2142
2259
|
}
|
|
2143
2260
|
throw writeErr;
|
|
@@ -2148,11 +2265,11 @@ function updateProjectSettingsRegistry(mutate) {
|
|
|
2148
2265
|
} finally {
|
|
2149
2266
|
if (lockFd !== null) {
|
|
2150
2267
|
try {
|
|
2151
|
-
|
|
2268
|
+
fs5.closeSync(lockFd);
|
|
2152
2269
|
} catch {
|
|
2153
2270
|
}
|
|
2154
2271
|
try {
|
|
2155
|
-
|
|
2272
|
+
fs5.unlinkSync(lockPath);
|
|
2156
2273
|
} catch {
|
|
2157
2274
|
}
|
|
2158
2275
|
}
|
|
@@ -2182,7 +2299,7 @@ function describeKnowledgeStoreMode(cwd = process.cwd(), iface) {
|
|
|
2182
2299
|
const KEY = "PI_RESEARCH_KNOWLEDGE_STORE_MODE";
|
|
2183
2300
|
const fileHasKey = (p) => {
|
|
2184
2301
|
try {
|
|
2185
|
-
return
|
|
2302
|
+
return fs5.existsSync(p) && parseDotEnv(fs5.readFileSync(p, "utf-8"))[KEY] !== void 0;
|
|
2186
2303
|
} catch {
|
|
2187
2304
|
return false;
|
|
2188
2305
|
}
|
|
@@ -2194,7 +2311,7 @@ function describeKnowledgeStoreMode(cwd = process.cwd(), iface) {
|
|
|
2194
2311
|
let registryHasKey = false;
|
|
2195
2312
|
try {
|
|
2196
2313
|
const registry = loadProjectSettingsRegistry();
|
|
2197
|
-
registryHasKey = registry
|
|
2314
|
+
registryHasKey = findRegistryEntry(registry, normalizeWorkspacePath(cwd))?.[KEY] !== void 0;
|
|
2198
2315
|
} catch {
|
|
2199
2316
|
}
|
|
2200
2317
|
if (registryHasKey) origin = "this directory (project settings)";
|
|
@@ -2220,14 +2337,27 @@ function parseDotEnv(content) {
|
|
|
2220
2337
|
}
|
|
2221
2338
|
return out;
|
|
2222
2339
|
}
|
|
2340
|
+
function findRegistryKey(registry, normalizedCwd) {
|
|
2341
|
+
if (registry[normalizedCwd] !== void 0) return normalizedCwd;
|
|
2342
|
+
if (process.platform !== "win32" && process.platform !== "darwin") return void 0;
|
|
2343
|
+
const lowered = normalizedCwd.toLowerCase();
|
|
2344
|
+
for (const key of Object.keys(registry)) {
|
|
2345
|
+
if (key.toLowerCase() === lowered) return key;
|
|
2346
|
+
}
|
|
2347
|
+
return void 0;
|
|
2348
|
+
}
|
|
2349
|
+
function findRegistryEntry(registry, normalizedCwd) {
|
|
2350
|
+
const key = findRegistryKey(registry, normalizedCwd);
|
|
2351
|
+
return key === void 0 ? void 0 : registry[key];
|
|
2352
|
+
}
|
|
2223
2353
|
function loadEnvFiles(cwd, iface) {
|
|
2224
2354
|
const merged = {};
|
|
2225
2355
|
const globalPath = getGlobalEnvFilePath();
|
|
2226
2356
|
const registry = loadProjectSettingsRegistry();
|
|
2227
2357
|
const normalizedCwd = normalizeWorkspacePath(cwd);
|
|
2228
|
-
const homeRegistry = registry
|
|
2358
|
+
const homeRegistry = findRegistryEntry(registry, normalizeWorkspacePath(os2.homedir()));
|
|
2229
2359
|
try {
|
|
2230
|
-
if (!
|
|
2360
|
+
if (!fs5.existsSync(globalPath) && homeRegistry && Object.keys(homeRegistry).length > 0) {
|
|
2231
2361
|
logger.info("[config] config.env missing. Initializing from user settings in central registry...");
|
|
2232
2362
|
const migrated = {};
|
|
2233
2363
|
for (const key of USER_MIGRATION_KEYS) {
|
|
@@ -2242,8 +2372,8 @@ function loadEnvFiles(cwd, iface) {
|
|
|
2242
2372
|
logger.warn("[config] Failed one-time config.env migration:", err);
|
|
2243
2373
|
}
|
|
2244
2374
|
try {
|
|
2245
|
-
if (
|
|
2246
|
-
Object.assign(merged, parseDotEnv(
|
|
2375
|
+
if (fs5.existsSync(globalPath)) {
|
|
2376
|
+
Object.assign(merged, parseDotEnv(fs5.readFileSync(globalPath, "utf-8")));
|
|
2247
2377
|
}
|
|
2248
2378
|
} catch (err) {
|
|
2249
2379
|
logger.warn("[config] Failed to read global env file:", err);
|
|
@@ -2251,8 +2381,8 @@ function loadEnvFiles(cwd, iface) {
|
|
|
2251
2381
|
if (iface) {
|
|
2252
2382
|
const ifacePath = getInterfaceEnvFilePath(iface);
|
|
2253
2383
|
try {
|
|
2254
|
-
if (
|
|
2255
|
-
Object.assign(merged, parseDotEnv(
|
|
2384
|
+
if (fs5.existsSync(ifacePath)) {
|
|
2385
|
+
Object.assign(merged, parseDotEnv(fs5.readFileSync(ifacePath, "utf-8")));
|
|
2256
2386
|
}
|
|
2257
2387
|
} catch (err) {
|
|
2258
2388
|
logger.warn(`[config] Failed to read interface config ${ifacePath}:`, err);
|
|
@@ -2260,30 +2390,33 @@ function loadEnvFiles(cwd, iface) {
|
|
|
2260
2390
|
}
|
|
2261
2391
|
const legacyPath = getLocalEnvFilePath(cwd);
|
|
2262
2392
|
let legacyEnv = {};
|
|
2263
|
-
if (
|
|
2393
|
+
if (fs5.existsSync(legacyPath)) {
|
|
2264
2394
|
try {
|
|
2265
2395
|
logger.warn(`[config] WARNING: .pi-research.env files are deprecated. Settings will be auto-migrated to the centralized registry. Remove ${legacyPath} after migration.`);
|
|
2266
|
-
legacyEnv = parseDotEnv(
|
|
2396
|
+
legacyEnv = parseDotEnv(fs5.readFileSync(legacyPath, "utf-8"));
|
|
2267
2397
|
Object.assign(merged, legacyEnv);
|
|
2268
2398
|
const legacyLocal = Object.fromEntries(
|
|
2269
2399
|
Object.entries(legacyEnv).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
|
|
2270
2400
|
);
|
|
2271
2401
|
const existingLocal = Object.fromEntries(
|
|
2272
|
-
Object.entries(registry
|
|
2402
|
+
Object.entries(findRegistryEntry(registry, normalizedCwd) ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
|
|
2273
2403
|
);
|
|
2274
2404
|
if (Object.keys(legacyLocal).length > 0 && JSON.stringify(existingLocal) !== JSON.stringify(legacyLocal)) {
|
|
2275
2405
|
logger.info(`[config] Migrating legacy .pi-research.env settings from ${cwd} to central registry...`);
|
|
2276
|
-
|
|
2406
|
+
const mergeKey = findRegistryKey(registry, normalizedCwd) ?? normalizedCwd;
|
|
2407
|
+
registry[mergeKey] = { ...registry[mergeKey], ...legacyLocal };
|
|
2277
2408
|
updateProjectSettingsRegistry((disk) => {
|
|
2278
|
-
|
|
2409
|
+
const diskKey = findRegistryKey(disk, normalizedCwd) ?? normalizedCwd;
|
|
2410
|
+
disk[diskKey] = { ...disk[diskKey], ...legacyLocal };
|
|
2279
2411
|
});
|
|
2280
2412
|
}
|
|
2281
2413
|
} catch (err) {
|
|
2282
2414
|
logger.warn(`[config] Failed to load legacy settings for ${cwd}:`, err);
|
|
2283
2415
|
}
|
|
2284
2416
|
}
|
|
2285
|
-
|
|
2286
|
-
|
|
2417
|
+
const cwdEntry = findRegistryEntry(registry, normalizedCwd);
|
|
2418
|
+
if (cwdEntry) {
|
|
2419
|
+
for (const [key, val] of Object.entries(cwdEntry)) {
|
|
2287
2420
|
if (LOCAL_SCOPE_KEYS.has(key)) {
|
|
2288
2421
|
if (legacyEnv[key] !== void 0 && legacyEnv[key] !== val) {
|
|
2289
2422
|
logger.warn(`[config] Config divergence for ${key} in ${cwd}: registry value differs from legacy .pi-research.env. Registry wins.`);
|
|
@@ -2293,7 +2426,7 @@ function loadEnvFiles(cwd, iface) {
|
|
|
2293
2426
|
logger.warn(`[config] Ignoring stale user-scoped ${key} in the project registry (config.env wins). Re-save your user settings to drop it.`);
|
|
2294
2427
|
}
|
|
2295
2428
|
}
|
|
2296
|
-
} else if (Object.keys(merged).length === 0 && !
|
|
2429
|
+
} else if (Object.keys(merged).length === 0 && !fs5.existsSync(legacyPath) && !fs5.existsSync(globalPath)) {
|
|
2297
2430
|
logger.warn(`[config] No configuration found for workspace: ${cwd}. Using code defaults. Run /research-config to configure.`);
|
|
2298
2431
|
}
|
|
2299
2432
|
warnLegacyEnvVarsOnce(merged);
|
|
@@ -2357,7 +2490,8 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
|
|
|
2357
2490
|
if (scope === "local") {
|
|
2358
2491
|
const normalizedCwd = normalizeWorkspacePath(cwd);
|
|
2359
2492
|
updateProjectSettingsRegistry((registry) => {
|
|
2360
|
-
|
|
2493
|
+
const key = findRegistryKey(registry, normalizedCwd) ?? normalizedCwd;
|
|
2494
|
+
registry[key] = { ...registry[key] ?? {}, ...newValues };
|
|
2361
2495
|
});
|
|
2362
2496
|
logger.debug(`[config] Saved project settings for ${normalizedCwd} to central registry.`);
|
|
2363
2497
|
return;
|
|
@@ -2365,22 +2499,22 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
|
|
|
2365
2499
|
const p = getGlobalEnvFilePath();
|
|
2366
2500
|
const lockPath = `${p}.lock`;
|
|
2367
2501
|
const dir = path5.dirname(p);
|
|
2368
|
-
if (!
|
|
2369
|
-
|
|
2502
|
+
if (!fs5.existsSync(dir)) {
|
|
2503
|
+
fs5.mkdirSync(dir, { recursive: true, mode: 448 });
|
|
2370
2504
|
}
|
|
2371
2505
|
let lockFd = null;
|
|
2372
2506
|
const lockRetryDelay = 50;
|
|
2373
2507
|
const lockMaxRetries = 100;
|
|
2374
2508
|
for (let i = 0; i < lockMaxRetries; i++) {
|
|
2375
2509
|
try {
|
|
2376
|
-
lockFd =
|
|
2510
|
+
lockFd = fs5.openSync(lockPath, "wx", 384);
|
|
2377
2511
|
break;
|
|
2378
2512
|
} catch (err) {
|
|
2379
2513
|
if (err.code === "EEXIST") {
|
|
2380
2514
|
try {
|
|
2381
|
-
const stats =
|
|
2515
|
+
const stats = fs5.statSync(lockPath);
|
|
2382
2516
|
if (Date.now() - stats.mtimeMs > 3e4) {
|
|
2383
|
-
|
|
2517
|
+
fs5.unlinkSync(lockPath);
|
|
2384
2518
|
continue;
|
|
2385
2519
|
}
|
|
2386
2520
|
} catch {
|
|
@@ -2397,8 +2531,8 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
|
|
|
2397
2531
|
}
|
|
2398
2532
|
try {
|
|
2399
2533
|
let lines = [];
|
|
2400
|
-
if (
|
|
2401
|
-
lines =
|
|
2534
|
+
if (fs5.existsSync(p)) {
|
|
2535
|
+
lines = fs5.readFileSync(p, "utf-8").split("\n");
|
|
2402
2536
|
} else {
|
|
2403
2537
|
lines = [
|
|
2404
2538
|
`# pi-research global configuration`,
|
|
@@ -2437,35 +2571,32 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
|
|
|
2437
2571
|
}
|
|
2438
2572
|
}
|
|
2439
2573
|
const tmpPath = `${p}.tmp.${process.pid}.${Date.now()}`;
|
|
2440
|
-
|
|
2574
|
+
fs5.writeFileSync(tmpPath, outLines.join("\n"), { encoding: "utf-8", mode: 384 });
|
|
2441
2575
|
try {
|
|
2442
|
-
|
|
2443
|
-
} catch (renameErr) {
|
|
2444
|
-
if (process.platform === "win32") {
|
|
2445
|
-
fs4.copyFileSync(tmpPath, p);
|
|
2576
|
+
if (replaceFileSync(tmpPath, p) === "copied") {
|
|
2446
2577
|
try {
|
|
2447
|
-
|
|
2578
|
+
fs5.unlinkSync(tmpPath);
|
|
2448
2579
|
} catch {
|
|
2449
2580
|
}
|
|
2450
|
-
} else {
|
|
2451
|
-
try {
|
|
2452
|
-
fs4.unlinkSync(tmpPath);
|
|
2453
|
-
} catch {
|
|
2454
|
-
}
|
|
2455
|
-
throw renameErr;
|
|
2456
2581
|
}
|
|
2582
|
+
} catch (renameErr) {
|
|
2583
|
+
try {
|
|
2584
|
+
fs5.unlinkSync(tmpPath);
|
|
2585
|
+
} catch {
|
|
2586
|
+
}
|
|
2587
|
+
throw renameErr;
|
|
2457
2588
|
}
|
|
2458
2589
|
try {
|
|
2459
|
-
|
|
2590
|
+
fs5.chmodSync(p, 384);
|
|
2460
2591
|
} catch {
|
|
2461
2592
|
}
|
|
2462
2593
|
} catch (err) {
|
|
2463
2594
|
logger.error(`[config] Failed to write config to ${p}:`, err);
|
|
2464
2595
|
throw err;
|
|
2465
2596
|
} finally {
|
|
2466
|
-
|
|
2597
|
+
fs5.closeSync(lockFd);
|
|
2467
2598
|
try {
|
|
2468
|
-
|
|
2599
|
+
fs5.unlinkSync(lockPath);
|
|
2469
2600
|
} catch {
|
|
2470
2601
|
}
|
|
2471
2602
|
}
|
|
@@ -2592,6 +2723,7 @@ var init_config = __esm({
|
|
|
2592
2723
|
init_logger();
|
|
2593
2724
|
init_host_config();
|
|
2594
2725
|
init_text_utils();
|
|
2726
|
+
init_atomic_replace();
|
|
2595
2727
|
ConfigSchema = Type.Object({
|
|
2596
2728
|
/** Per-researcher timeout in milliseconds (default: 300000, range: 3-30 min) */
|
|
2597
2729
|
RESEARCHER_TIMEOUT_MS: Type.Number({ minimum: 18e4, maximum: 18e5, default: 3e5 }),
|
|
@@ -2811,7 +2943,11 @@ function getMaxScrapeBatches(config) {
|
|
|
2811
2943
|
return 2;
|
|
2812
2944
|
}
|
|
2813
2945
|
}
|
|
2814
|
-
|
|
2946
|
+
function resolveExcludedTools(callerExclusions, configDisabled = []) {
|
|
2947
|
+
const base = callerExclusions && callerExclusions.length > 0 ? callerExclusions : DEFAULT_EXCLUDED_TOOLS;
|
|
2948
|
+
return [.../* @__PURE__ */ new Set([...base, ...configDisabled])];
|
|
2949
|
+
}
|
|
2950
|
+
var REQUEST_DELAY_MS_NVD, REQUEST_DELAY_MS_OTHER, MAX_GATHERING_CALLS, MAX_GREP_CALLS, MAX_SCRAPE_URLS, MAX_TEAM_SIZE_LEVEL_1, MAX_TEAM_SIZE_LEVEL_2, MAX_TEAM_SIZE_LEVEL_3, MAX_ROUNDS_LEVEL_1, MAX_ROUNDS_LEVEL_2, MAX_ROUNDS_LEVEL_3, MAX_EXTRA_ROUNDS_WITH_STEERING, OSV_TIMEOUT_MS, WEBGPU_PROBE_TIMEOUT_MS, MAX_QUERY_LENGTH, MAX_FILENAME_QUERY_LENGTH, MIN_QUERY_LENGTH, MAX_EXPORT_RETRIES, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS, DEFAULT_MODEL_CONTEXT_WINDOW, BATCH_2_DEFAULT_CONCURRENCY, MAX_SCRAPE_CONTENT_CHARS_PER_DOC, RESEARCHER_LAUNCH_DELAY_MS, MAX_QUERIES_PER_RESEARCHER_LEVEL_1, MAX_QUERIES_PER_RESEARCHER_LEVEL_2, MAX_QUERIES_PER_RESEARCHER_LEVEL_3, DEFAULT_EXCLUDED_TOOLS;
|
|
2815
2951
|
var init_constants = __esm({
|
|
2816
2952
|
"src/constants.ts"() {
|
|
2817
2953
|
"use strict";
|
|
@@ -2844,6 +2980,7 @@ var init_constants = __esm({
|
|
|
2844
2980
|
MAX_QUERIES_PER_RESEARCHER_LEVEL_1 = 10;
|
|
2845
2981
|
MAX_QUERIES_PER_RESEARCHER_LEVEL_2 = 15;
|
|
2846
2982
|
MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
|
|
2983
|
+
DEFAULT_EXCLUDED_TOOLS = ["grep"];
|
|
2847
2984
|
}
|
|
2848
2985
|
});
|
|
2849
2986
|
|
|
@@ -2870,8 +3007,8 @@ function findMatchingBracket(text, start) {
|
|
|
2870
3007
|
let depth = 0;
|
|
2871
3008
|
let inString = false;
|
|
2872
3009
|
let escaped = false;
|
|
2873
|
-
const
|
|
2874
|
-
const close =
|
|
3010
|
+
const open5 = text[start];
|
|
3011
|
+
const close = open5 === "{" ? "}" : "]";
|
|
2875
3012
|
for (let i = start; i < text.length; i++) {
|
|
2876
3013
|
const ch = text[i];
|
|
2877
3014
|
if (escaped) {
|
|
@@ -2887,7 +3024,7 @@ function findMatchingBracket(text, start) {
|
|
|
2887
3024
|
continue;
|
|
2888
3025
|
}
|
|
2889
3026
|
if (inString) continue;
|
|
2890
|
-
if (ch ===
|
|
3027
|
+
if (ch === open5) depth++;
|
|
2891
3028
|
else if (ch === close) {
|
|
2892
3029
|
depth--;
|
|
2893
3030
|
if (depth === 0) return { index: i, inString: false };
|
|
@@ -3227,7 +3364,7 @@ var init_agentic_repair = __esm({
|
|
|
3227
3364
|
import { ModelRegistry, ModelRuntime, getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
3228
3365
|
import { InMemoryCredentialStore } from "@earendil-works/pi-ai";
|
|
3229
3366
|
import * as path6 from "node:path";
|
|
3230
|
-
import * as
|
|
3367
|
+
import * as fs6 from "node:fs";
|
|
3231
3368
|
function safeGetAll(registry) {
|
|
3232
3369
|
if (registry && typeof registry.getAll === "function") {
|
|
3233
3370
|
try {
|
|
@@ -3271,8 +3408,8 @@ async function buildModelRegistry(apiKey, provider) {
|
|
|
3271
3408
|
const agentDir = getAgentDir();
|
|
3272
3409
|
const modelsJsonPath = path6.join(agentDir, "models.json");
|
|
3273
3410
|
const authPath = path6.join(agentDir, "auth.json");
|
|
3274
|
-
const hasModels =
|
|
3275
|
-
const hasAuth =
|
|
3411
|
+
const hasModels = fs6.existsSync(modelsJsonPath);
|
|
3412
|
+
const hasAuth = fs6.existsSync(authPath);
|
|
3276
3413
|
const runtime = await ModelRuntime.create({
|
|
3277
3414
|
authPath: hasAuth ? authPath : void 0,
|
|
3278
3415
|
credentials: hasAuth ? void 0 : new InMemoryCredentialStore(),
|
|
@@ -3384,9 +3521,9 @@ ${credentialGuidance()}
|
|
|
3384
3521
|
}
|
|
3385
3522
|
function readModelsJsonProviderOrder() {
|
|
3386
3523
|
const modelsJsonPath = path6.join(getAgentDir(), "models.json");
|
|
3387
|
-
if (!
|
|
3524
|
+
if (!fs6.existsSync(modelsJsonPath)) return [];
|
|
3388
3525
|
try {
|
|
3389
|
-
const raw = JSON.parse(
|
|
3526
|
+
const raw = JSON.parse(fs6.readFileSync(modelsJsonPath, "utf-8"));
|
|
3390
3527
|
return Object.keys(raw?.providers ?? {});
|
|
3391
3528
|
} catch (err) {
|
|
3392
3529
|
logger.warn(`[ModelRegistry] Could not read provider order from models.json: ${err}`);
|
|
@@ -3592,11 +3729,11 @@ var init_retry_utils = __esm({
|
|
|
3592
3729
|
});
|
|
3593
3730
|
|
|
3594
3731
|
// src/infrastructure/file-lock-service.ts
|
|
3595
|
-
import * as
|
|
3732
|
+
import * as fs7 from "node:fs/promises";
|
|
3596
3733
|
import * as crypto from "node:crypto";
|
|
3597
3734
|
import * as pathmod from "node:path";
|
|
3598
3735
|
import { AsyncLocalStorage as AsyncLocalStorage6 } from "node:async_hooks";
|
|
3599
|
-
var lockContext, FileLockService;
|
|
3736
|
+
var lockContext, CLEANUP_DRAIN_TIMEOUT_MS, TRASH_SWEEP_AGE_MS, FileLockService;
|
|
3600
3737
|
var init_file_lock_service = __esm({
|
|
3601
3738
|
"src/infrastructure/file-lock-service.ts"() {
|
|
3602
3739
|
"use strict";
|
|
@@ -3605,6 +3742,8 @@ var init_file_lock_service = __esm({
|
|
|
3605
3742
|
init_service_registry();
|
|
3606
3743
|
init_service_names();
|
|
3607
3744
|
lockContext = new AsyncLocalStorage6();
|
|
3745
|
+
CLEANUP_DRAIN_TIMEOUT_MS = 5e3;
|
|
3746
|
+
TRASH_SWEEP_AGE_MS = 10 * 6e4;
|
|
3608
3747
|
FileLockService = class {
|
|
3609
3748
|
name = ServiceNames.FILE_LOCK_SERVICE;
|
|
3610
3749
|
lifecycle = "uninitialized" /* UNINITIALIZED */;
|
|
@@ -3627,6 +3766,9 @@ var init_file_lock_service = __esm({
|
|
|
3627
3766
|
// PID reuse. Null when unavailable (no processLifecycle, or lookup failed) —
|
|
3628
3767
|
// then lock content omits startTime and readers treat us as a legacy owner.
|
|
3629
3768
|
ownStartTime = null;
|
|
3769
|
+
// Set by cleanup(). A retired instance no longer manages the lock file, so
|
|
3770
|
+
// acquiring through it would hand out exclusivity it cannot honour.
|
|
3771
|
+
_retired = false;
|
|
3630
3772
|
constructor(options) {
|
|
3631
3773
|
this.lockFilePath = options.lockFilePath;
|
|
3632
3774
|
this.lockTimeout = options.lockTimeout ?? 2e4;
|
|
@@ -3674,7 +3816,7 @@ var init_file_lock_service = __esm({
|
|
|
3674
3816
|
this.lifecycle = "initializing" /* INITIALIZING */;
|
|
3675
3817
|
try {
|
|
3676
3818
|
const lockDir = pathmod.dirname(this.lockFilePath);
|
|
3677
|
-
await
|
|
3819
|
+
await fs7.mkdir(lockDir, { recursive: true, mode: 448 });
|
|
3678
3820
|
} catch (err) {
|
|
3679
3821
|
logger.warn(`[FileLockService] Failed to create lock directory: ${err instanceof Error ? err.message : String(err)}`);
|
|
3680
3822
|
}
|
|
@@ -3694,15 +3836,104 @@ var init_file_lock_service = __esm({
|
|
|
3694
3836
|
await this.cleanup();
|
|
3695
3837
|
this.lifecycle = "disposed" /* DISPOSED */;
|
|
3696
3838
|
}
|
|
3839
|
+
/**
|
|
3840
|
+
* Put a lock file back after a trash-and-verify decided not to reclaim it.
|
|
3841
|
+
*
|
|
3842
|
+
* The old form linked it back inside a swallowing catch and then unlinked the
|
|
3843
|
+
* trash copy unconditionally — so any link failure other than "a peer already
|
|
3844
|
+
* re-created it" DESTROYED the lock this branch exists to preserve. Distinguish
|
|
3845
|
+
* the two: if the path is occupied again the peer's lock wins and ours is dropped;
|
|
3846
|
+
* otherwise move ours back rather than deleting it.
|
|
3847
|
+
*/
|
|
3848
|
+
async _restoreLockFile(trashPath) {
|
|
3849
|
+
try {
|
|
3850
|
+
await fs7.link(trashPath, this.lockFilePath);
|
|
3851
|
+
await fs7.unlink(trashPath).catch(() => {
|
|
3852
|
+
});
|
|
3853
|
+
return;
|
|
3854
|
+
} catch {
|
|
3855
|
+
}
|
|
3856
|
+
const peerHoldsIt = await fs7.access(this.lockFilePath).then(() => true).catch(() => false);
|
|
3857
|
+
if (peerHoldsIt) {
|
|
3858
|
+
await fs7.unlink(trashPath).catch(() => {
|
|
3859
|
+
});
|
|
3860
|
+
return;
|
|
3861
|
+
}
|
|
3862
|
+
try {
|
|
3863
|
+
const content = await fs7.readFile(trashPath, "utf-8");
|
|
3864
|
+
const handle = await fs7.open(this.lockFilePath, "wx", 384);
|
|
3865
|
+
try {
|
|
3866
|
+
await handle.write(content);
|
|
3867
|
+
await handle.sync();
|
|
3868
|
+
} finally {
|
|
3869
|
+
await handle.close().catch(() => {
|
|
3870
|
+
});
|
|
3871
|
+
}
|
|
3872
|
+
await fs7.unlink(trashPath).catch(() => {
|
|
3873
|
+
});
|
|
3874
|
+
} catch {
|
|
3875
|
+
const occupied = await fs7.access(this.lockFilePath).then(() => true).catch(() => false);
|
|
3876
|
+
if (occupied) await fs7.unlink(trashPath).catch(() => {
|
|
3877
|
+
});
|
|
3878
|
+
}
|
|
3879
|
+
}
|
|
3880
|
+
/**
|
|
3881
|
+
* Hand our turn back to the next caller in the FIFO queue.
|
|
3882
|
+
*
|
|
3883
|
+
* Every path that stops holding the lock must run this, including the ones that
|
|
3884
|
+
* find the handle already gone. `acquireLock` parks on `await previous` with no
|
|
3885
|
+
* timeout of its own — the lockTimeout only starts counting once the turn has
|
|
3886
|
+
* been granted — so a turn that is never resolved is not a slow acquire, it is a
|
|
3887
|
+
* permanent one.
|
|
3888
|
+
*/
|
|
3889
|
+
_releaseTurn() {
|
|
3890
|
+
const resolve8 = this.resolveTurn;
|
|
3891
|
+
this.resolveTurn = null;
|
|
3892
|
+
if (resolve8) resolve8();
|
|
3893
|
+
}
|
|
3894
|
+
/**
|
|
3895
|
+
* Wait, bounded, for an in-flight critical section to finish.
|
|
3896
|
+
*
|
|
3897
|
+
* Yanking a held lock is not a clean teardown: unlinking the file while the
|
|
3898
|
+
* holder is still mid read-modify-write drops cross-process exclusion, and a peer
|
|
3899
|
+
* that acquires in that window loses the holder's update — precisely the hazard
|
|
3900
|
+
* the lock exists to prevent. Since every caller of {@link cleanup} is a shutdown
|
|
3901
|
+
* path, waiting briefly for a normal release is both safe and correct.
|
|
3902
|
+
*
|
|
3903
|
+
* Bounded because shutdown must make progress regardless: if the holder is wedged
|
|
3904
|
+
* (or is itself the caller, which would otherwise deadlock), we give up and force
|
|
3905
|
+
* the teardown, which is the pre-existing behaviour.
|
|
3906
|
+
*/
|
|
3907
|
+
async _drainHeldLock(timeoutMs) {
|
|
3908
|
+
const deadline = Date.now() + timeoutMs;
|
|
3909
|
+
while (this.lockHandle !== null && Date.now() < deadline) {
|
|
3910
|
+
await this.sleep(Math.min(25, this.lockRetryDelay));
|
|
3911
|
+
}
|
|
3912
|
+
return this.lockHandle === null;
|
|
3913
|
+
}
|
|
3697
3914
|
/**
|
|
3698
3915
|
* Clean up any stale lock files on initialization.
|
|
3699
3916
|
* This handles cases where locks weren't released due to crashes.
|
|
3700
3917
|
*/
|
|
3701
3918
|
async cleanupStaleLocksOnStartup() {
|
|
3702
3919
|
try {
|
|
3703
|
-
const
|
|
3920
|
+
const dir = pathmod.dirname(this.lockFilePath);
|
|
3921
|
+
const prefix = `${pathmod.basename(this.lockFilePath)}.trash.`;
|
|
3922
|
+
for (const name of await fs7.readdir(dir)) {
|
|
3923
|
+
if (!name.startsWith(prefix)) continue;
|
|
3924
|
+
const p = pathmod.join(dir, name);
|
|
3925
|
+
const st = await fs7.stat(p).catch(() => null);
|
|
3926
|
+
if (st && Date.now() - st.mtimeMs > TRASH_SWEEP_AGE_MS) {
|
|
3927
|
+
await fs7.unlink(p).catch(() => {
|
|
3928
|
+
});
|
|
3929
|
+
}
|
|
3930
|
+
}
|
|
3931
|
+
} catch {
|
|
3932
|
+
}
|
|
3933
|
+
try {
|
|
3934
|
+
const rawContent = await fs7.readFile(this.lockFilePath, "utf-8").catch(() => null);
|
|
3704
3935
|
if (rawContent === null) return;
|
|
3705
|
-
const stats = await
|
|
3936
|
+
const stats = await fs7.stat(this.lockFilePath);
|
|
3706
3937
|
const lockAge = Date.now() - stats.mtimeMs;
|
|
3707
3938
|
const parsed = this._parseLockContent(rawContent);
|
|
3708
3939
|
const memo = /* @__PURE__ */ new Map();
|
|
@@ -3713,21 +3944,21 @@ var init_file_lock_service = __esm({
|
|
|
3713
3944
|
);
|
|
3714
3945
|
const expectedUuid = parsed?.uuid ?? "";
|
|
3715
3946
|
const trashPath = `${this.lockFilePath}.trash.${crypto.randomBytes(8).toString("hex")}`;
|
|
3947
|
+
let renamed = false;
|
|
3716
3948
|
try {
|
|
3717
|
-
await
|
|
3718
|
-
|
|
3949
|
+
await fs7.rename(this.lockFilePath, trashPath);
|
|
3950
|
+
renamed = true;
|
|
3951
|
+
const trashContent = await fs7.readFile(trashPath, "utf-8");
|
|
3719
3952
|
const trashParsed = this._parseLockContent(trashContent);
|
|
3720
3953
|
if ((trashParsed?.uuid ?? "") !== expectedUuid) {
|
|
3721
|
-
|
|
3722
|
-
await fs6.link(trashPath, this.lockFilePath);
|
|
3723
|
-
} catch {
|
|
3724
|
-
}
|
|
3725
|
-
await fs6.unlink(trashPath);
|
|
3954
|
+
await this._restoreLockFile(trashPath);
|
|
3726
3955
|
return;
|
|
3727
3956
|
}
|
|
3728
|
-
await
|
|
3957
|
+
await fs7.unlink(trashPath).catch(() => {
|
|
3958
|
+
});
|
|
3729
3959
|
logger.log("[FileLockService] Stale lock removed");
|
|
3730
3960
|
} catch {
|
|
3961
|
+
if (renamed) await this._restoreLockFile(trashPath);
|
|
3731
3962
|
}
|
|
3732
3963
|
}
|
|
3733
3964
|
} catch (error) {
|
|
@@ -3745,6 +3976,9 @@ var init_file_lock_service = __esm({
|
|
|
3745
3976
|
* @throws Error if unable to acquire lock within timeout
|
|
3746
3977
|
*/
|
|
3747
3978
|
async acquireLock() {
|
|
3979
|
+
if (this._retired) {
|
|
3980
|
+
throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service has been disposed`);
|
|
3981
|
+
}
|
|
3748
3982
|
const heldLocks = lockContext.getStore();
|
|
3749
3983
|
const lockKey = `${this.lockFilePath}:${this.lockUuid}`;
|
|
3750
3984
|
if (heldLocks?.has(lockKey)) {
|
|
@@ -3768,17 +4002,27 @@ var init_file_lock_service = __esm({
|
|
|
3768
4002
|
if (myResolve) myResolve();
|
|
3769
4003
|
return;
|
|
3770
4004
|
}
|
|
4005
|
+
if (this._retired) {
|
|
4006
|
+
myResolve();
|
|
4007
|
+
throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service was disposed while queued`);
|
|
4008
|
+
}
|
|
3771
4009
|
this.resolveTurn = myResolve;
|
|
3772
4010
|
const startTime = Date.now();
|
|
3773
4011
|
let contentionCount = 0;
|
|
3774
4012
|
const contendedOwnerAliveMemo = /* @__PURE__ */ new Map();
|
|
3775
4013
|
try {
|
|
3776
4014
|
for (let _attempt = 0; _attempt < this.lockRetries; _attempt++) {
|
|
4015
|
+
if (this._retired) {
|
|
4016
|
+
throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service was disposed during acquisition`);
|
|
4017
|
+
}
|
|
3777
4018
|
let handle = null;
|
|
3778
4019
|
try {
|
|
3779
|
-
handle = await
|
|
4020
|
+
handle = await fs7.open(this.lockFilePath, "wx", 384);
|
|
3780
4021
|
await handle.write(this._makeLockContent());
|
|
3781
4022
|
await handle.sync();
|
|
4023
|
+
if (this._retired) {
|
|
4024
|
+
throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service was disposed during acquisition`);
|
|
4025
|
+
}
|
|
3782
4026
|
this.lockHandle = handle;
|
|
3783
4027
|
this.lockCount = 1;
|
|
3784
4028
|
if (heldLocks) {
|
|
@@ -3800,7 +4044,7 @@ var init_file_lock_service = __esm({
|
|
|
3800
4044
|
} catch {
|
|
3801
4045
|
}
|
|
3802
4046
|
try {
|
|
3803
|
-
await
|
|
4047
|
+
await fs7.unlink(this.lockFilePath);
|
|
3804
4048
|
} catch {
|
|
3805
4049
|
}
|
|
3806
4050
|
}
|
|
@@ -3809,45 +4053,44 @@ var init_file_lock_service = __esm({
|
|
|
3809
4053
|
if (errnoError.code === "EEXIST") {
|
|
3810
4054
|
contentionCount++;
|
|
3811
4055
|
try {
|
|
3812
|
-
const rawContent = await
|
|
4056
|
+
const rawContent = await fs7.readFile(this.lockFilePath, "utf-8");
|
|
3813
4057
|
const parsed = this._parseLockContent(rawContent);
|
|
3814
4058
|
const lockUuid = parsed?.uuid ?? "";
|
|
3815
|
-
const stats = await
|
|
4059
|
+
const stats = await fs7.stat(this.lockFilePath);
|
|
3816
4060
|
const lockAge = Date.now() - stats.mtimeMs;
|
|
3817
4061
|
if (lockUuid === this.lockUuid) {
|
|
3818
4062
|
try {
|
|
3819
|
-
await
|
|
4063
|
+
await fs7.unlink(this.lockFilePath);
|
|
3820
4064
|
} catch {
|
|
3821
4065
|
}
|
|
3822
4066
|
continue;
|
|
3823
4067
|
}
|
|
3824
4068
|
if (await this._shouldReclaim(parsed?.pid ?? null, parsed?.startTime ?? null, lockAge, contendedOwnerAliveMemo)) {
|
|
3825
4069
|
const trashPath = `${this.lockFilePath}.trash.${crypto.randomBytes(8).toString("hex")}`;
|
|
4070
|
+
let renamed = false;
|
|
3826
4071
|
try {
|
|
3827
|
-
await
|
|
3828
|
-
|
|
4072
|
+
await fs7.rename(this.lockFilePath, trashPath);
|
|
4073
|
+
renamed = true;
|
|
4074
|
+
const trashContent = await fs7.readFile(trashPath, "utf-8");
|
|
3829
4075
|
const trashParsed = this._parseLockContent(trashContent);
|
|
3830
4076
|
if ((trashParsed?.uuid ?? "") !== lockUuid) {
|
|
3831
|
-
|
|
3832
|
-
await fs6.link(trashPath, this.lockFilePath);
|
|
3833
|
-
} catch {
|
|
3834
|
-
}
|
|
3835
|
-
await fs6.unlink(trashPath);
|
|
4077
|
+
await this._restoreLockFile(trashPath);
|
|
3836
4078
|
continue;
|
|
3837
4079
|
}
|
|
3838
|
-
await
|
|
4080
|
+
await fs7.unlink(trashPath).catch(() => {
|
|
4081
|
+
});
|
|
3839
4082
|
} catch {
|
|
4083
|
+
if (renamed) await this._restoreLockFile(trashPath);
|
|
3840
4084
|
}
|
|
3841
4085
|
continue;
|
|
3842
4086
|
}
|
|
3843
|
-
} catch (
|
|
3844
|
-
const
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
}
|
|
4087
|
+
} catch (inspectError) {
|
|
4088
|
+
const code = inspectError?.code;
|
|
4089
|
+
if (code === "ENOENT") continue;
|
|
4090
|
+
logger.warn(
|
|
4091
|
+
`[FileLockService] Could not inspect contended lock at ${this.lockFilePath} (${code ?? "unknown"}); backing off rather than reclaiming it.`
|
|
4092
|
+
);
|
|
4093
|
+
metrics.increment("state_lock_inspect_failed_total", 1, { code: code ?? "unknown" });
|
|
3851
4094
|
}
|
|
3852
4095
|
if (Date.now() - startTime >= this.lockTimeout) {
|
|
3853
4096
|
const duration = Date.now() - startTime;
|
|
@@ -3868,10 +4111,7 @@ var init_file_lock_service = __esm({
|
|
|
3868
4111
|
metrics.increment("state_lock_acquire_total", 1, { status: "failed" });
|
|
3869
4112
|
throw new Error(`Failed to acquire lock after ${this.lockRetries} retries`);
|
|
3870
4113
|
} catch (err) {
|
|
3871
|
-
|
|
3872
|
-
this.resolveTurn();
|
|
3873
|
-
this.resolveTurn = null;
|
|
3874
|
-
}
|
|
4114
|
+
this._releaseTurn();
|
|
3875
4115
|
throw err;
|
|
3876
4116
|
}
|
|
3877
4117
|
}
|
|
@@ -3880,6 +4120,7 @@ var init_file_lock_service = __esm({
|
|
|
3880
4120
|
*/
|
|
3881
4121
|
async releaseLock() {
|
|
3882
4122
|
if (this.lockHandle === null) {
|
|
4123
|
+
this._releaseTurn();
|
|
3883
4124
|
return;
|
|
3884
4125
|
}
|
|
3885
4126
|
this.lockCount--;
|
|
@@ -3887,10 +4128,10 @@ var init_file_lock_service = __esm({
|
|
|
3887
4128
|
return;
|
|
3888
4129
|
}
|
|
3889
4130
|
try {
|
|
3890
|
-
const content = await
|
|
4131
|
+
const content = await fs7.readFile(this.lockFilePath, "utf8").catch(() => "");
|
|
3891
4132
|
const parsed = this._parseLockContent(content);
|
|
3892
4133
|
if (parsed?.uuid === this.lockUuid) {
|
|
3893
|
-
await
|
|
4134
|
+
await fs7.unlink(this.lockFilePath).catch(() => {
|
|
3894
4135
|
});
|
|
3895
4136
|
}
|
|
3896
4137
|
await this.lockHandle.close().catch(() => {
|
|
@@ -3903,19 +4144,20 @@ var init_file_lock_service = __esm({
|
|
|
3903
4144
|
}
|
|
3904
4145
|
metrics.setGauge("state_lock_held", 0);
|
|
3905
4146
|
} finally {
|
|
3906
|
-
|
|
3907
|
-
this.resolveTurn = null;
|
|
3908
|
-
if (resolve8) {
|
|
3909
|
-
resolve8();
|
|
3910
|
-
}
|
|
4147
|
+
this._releaseTurn();
|
|
3911
4148
|
}
|
|
3912
4149
|
}
|
|
3913
4150
|
/**
|
|
3914
4151
|
* Execute a function within the scope of a filesystem lock.
|
|
3915
4152
|
* This is the preferred way to use the lock service.
|
|
3916
4153
|
* Safe re-entrancy is supported within the same async flow.
|
|
4154
|
+
*
|
|
4155
|
+
* The acquire timeout is the one given to the constructor (`lockTimeout`). This
|
|
4156
|
+
* used to take a second `_timeout` argument that was accepted and then ignored
|
|
4157
|
+
* entirely — a caller passing 5000 silently got the constructed 20000 — so it is
|
|
4158
|
+
* gone rather than left to mislead. No caller was passing it.
|
|
3917
4159
|
*/
|
|
3918
|
-
async withLock(callback
|
|
4160
|
+
async withLock(callback) {
|
|
3919
4161
|
const heldLocks = lockContext.getStore();
|
|
3920
4162
|
const lockKey = `${this.lockFilePath}:${this.lockUuid}`;
|
|
3921
4163
|
if (heldLocks?.has(lockKey)) {
|
|
@@ -3931,15 +4173,29 @@ var init_file_lock_service = __esm({
|
|
|
3931
4173
|
});
|
|
3932
4174
|
}
|
|
3933
4175
|
/**
|
|
3934
|
-
*
|
|
4176
|
+
* Clean up any lock file owned by this instance, and retire the instance.
|
|
4177
|
+
*
|
|
4178
|
+
* Called only from shutdown paths (`dispose`, `StateManager.cleanup`). It first
|
|
4179
|
+
* gives an in-flight critical section a bounded chance to release normally, then
|
|
4180
|
+
* forces the teardown if it has not. After this the instance is retired:
|
|
4181
|
+
* `acquireLock` fails fast rather than blocking on a queue nothing will ever
|
|
4182
|
+
* drain, which is what makes forcing the teardown safe to do at all.
|
|
3935
4183
|
*/
|
|
3936
4184
|
async cleanup() {
|
|
4185
|
+
const drained = await this._drainHeldLock(CLEANUP_DRAIN_TIMEOUT_MS);
|
|
4186
|
+
if (!drained) {
|
|
4187
|
+
logger.warn(
|
|
4188
|
+
`[FileLockService] Lock at ${this.lockFilePath} still held after ${CLEANUP_DRAIN_TIMEOUT_MS}ms; forcing teardown. A concurrent writer in another process may observe the lock as free.`
|
|
4189
|
+
);
|
|
4190
|
+
metrics.increment("state_lock_forced_teardown_total", 1);
|
|
4191
|
+
}
|
|
4192
|
+
this._retired = true;
|
|
3937
4193
|
if (this.lockHandle) {
|
|
3938
4194
|
try {
|
|
3939
|
-
const content = await
|
|
4195
|
+
const content = await fs7.readFile(this.lockFilePath, "utf8").catch(() => "");
|
|
3940
4196
|
const parsed = this._parseLockContent(content);
|
|
3941
4197
|
if (parsed?.uuid === this.lockUuid) {
|
|
3942
|
-
await
|
|
4198
|
+
await fs7.unlink(this.lockFilePath).catch(() => {
|
|
3943
4199
|
});
|
|
3944
4200
|
}
|
|
3945
4201
|
} catch {
|
|
@@ -3951,6 +4207,7 @@ var init_file_lock_service = __esm({
|
|
|
3951
4207
|
this.lockHandle = null;
|
|
3952
4208
|
}
|
|
3953
4209
|
this.lockCount = 0;
|
|
4210
|
+
this._releaseTurn();
|
|
3954
4211
|
}
|
|
3955
4212
|
/**
|
|
3956
4213
|
* Check if the current instance holds the lock
|
|
@@ -4035,6 +4292,93 @@ var init_file_lock_service = __esm({
|
|
|
4035
4292
|
}
|
|
4036
4293
|
});
|
|
4037
4294
|
|
|
4295
|
+
// src/utils/http-body.ts
|
|
4296
|
+
async function readBodyCapped(response, maxBytes) {
|
|
4297
|
+
const body = response.body;
|
|
4298
|
+
if (!body || typeof body.getReader !== "function") {
|
|
4299
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
4300
|
+
}
|
|
4301
|
+
const reader = body.getReader();
|
|
4302
|
+
const chunks = [];
|
|
4303
|
+
let total = 0;
|
|
4304
|
+
try {
|
|
4305
|
+
for (; ; ) {
|
|
4306
|
+
const { done, value } = await reader.read();
|
|
4307
|
+
if (done) break;
|
|
4308
|
+
if (value) {
|
|
4309
|
+
total += value.byteLength;
|
|
4310
|
+
if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
|
|
4311
|
+
chunks.push(value);
|
|
4312
|
+
}
|
|
4313
|
+
}
|
|
4314
|
+
} finally {
|
|
4315
|
+
try {
|
|
4316
|
+
await reader.cancel();
|
|
4317
|
+
} catch {
|
|
4318
|
+
}
|
|
4319
|
+
}
|
|
4320
|
+
const out = new Uint8Array(total);
|
|
4321
|
+
let offset = 0;
|
|
4322
|
+
for (const c of chunks) {
|
|
4323
|
+
out.set(c, offset);
|
|
4324
|
+
offset += c.byteLength;
|
|
4325
|
+
}
|
|
4326
|
+
return out;
|
|
4327
|
+
}
|
|
4328
|
+
function hasStreamBody(response) {
|
|
4329
|
+
const body = response.body;
|
|
4330
|
+
return !!body && typeof body.getReader === "function";
|
|
4331
|
+
}
|
|
4332
|
+
async function readTextCapped(response, maxBytes = MAX_JSON_BODY_BYTES) {
|
|
4333
|
+
if (hasStreamBody(response)) {
|
|
4334
|
+
return new TextDecoder("utf-8").decode(await readBodyCapped(response, maxBytes));
|
|
4335
|
+
}
|
|
4336
|
+
if (typeof response.arrayBuffer === "function") {
|
|
4337
|
+
return new TextDecoder("utf-8").decode(new Uint8Array(await response.arrayBuffer()));
|
|
4338
|
+
}
|
|
4339
|
+
return response.text();
|
|
4340
|
+
}
|
|
4341
|
+
async function readJsonCapped(response, maxBytes = MAX_JSON_BODY_BYTES) {
|
|
4342
|
+
if (hasStreamBody(response) || typeof response.arrayBuffer === "function" || typeof response.text === "function") {
|
|
4343
|
+
return JSON.parse(await readTextCapped(response, maxBytes));
|
|
4344
|
+
}
|
|
4345
|
+
return response.json();
|
|
4346
|
+
}
|
|
4347
|
+
var BodyTooLargeError, MAX_JSON_BODY_BYTES, MAX_BULK_JSON_BODY_BYTES, Utf8Body;
|
|
4348
|
+
var init_http_body = __esm({
|
|
4349
|
+
"src/utils/http-body.ts"() {
|
|
4350
|
+
"use strict";
|
|
4351
|
+
BodyTooLargeError = class extends Error {
|
|
4352
|
+
constructor(bytesRead, maxBytes) {
|
|
4353
|
+
super(`Response body exceeded ${maxBytes} byte cap`);
|
|
4354
|
+
this.bytesRead = bytesRead;
|
|
4355
|
+
this.maxBytes = maxBytes;
|
|
4356
|
+
this.name = "BodyTooLargeError";
|
|
4357
|
+
}
|
|
4358
|
+
bytesRead;
|
|
4359
|
+
maxBytes;
|
|
4360
|
+
};
|
|
4361
|
+
MAX_JSON_BODY_BYTES = 32 * 1024 * 1024;
|
|
4362
|
+
MAX_BULK_JSON_BODY_BYTES = 96 * 1024 * 1024;
|
|
4363
|
+
Utf8Body = class {
|
|
4364
|
+
chunks = [];
|
|
4365
|
+
bytes = 0;
|
|
4366
|
+
/** Total bytes accumulated so far. */
|
|
4367
|
+
get byteLength() {
|
|
4368
|
+
return this.bytes;
|
|
4369
|
+
}
|
|
4370
|
+
push(chunk) {
|
|
4371
|
+
this.chunks.push(chunk);
|
|
4372
|
+
this.bytes += chunk.byteLength;
|
|
4373
|
+
}
|
|
4374
|
+
/** Concatenate and decode. Safe to call once the stream has ended. */
|
|
4375
|
+
toString() {
|
|
4376
|
+
return Buffer.concat(this.chunks).toString("utf8");
|
|
4377
|
+
}
|
|
4378
|
+
};
|
|
4379
|
+
}
|
|
4380
|
+
});
|
|
4381
|
+
|
|
4038
4382
|
// src/utils/circuit-breaker.ts
|
|
4039
4383
|
var CircuitBreaker;
|
|
4040
4384
|
var init_circuit_breaker = __esm({
|
|
@@ -4599,7 +4943,7 @@ var init_embedder_utils = __esm({
|
|
|
4599
4943
|
});
|
|
4600
4944
|
|
|
4601
4945
|
// src/knowledge/store.ts
|
|
4602
|
-
import * as
|
|
4946
|
+
import * as fs13 from "node:fs";
|
|
4603
4947
|
import * as fsPromises from "node:fs/promises";
|
|
4604
4948
|
import * as path13 from "node:path";
|
|
4605
4949
|
function isLanceCommitConflict(err) {
|
|
@@ -4673,7 +5017,7 @@ var init_store = __esm({
|
|
|
4673
5017
|
}
|
|
4674
5018
|
async loadManifest() {
|
|
4675
5019
|
try {
|
|
4676
|
-
if (
|
|
5020
|
+
if (fs13.existsSync(this.manifestPath)) {
|
|
4677
5021
|
const content = await fsPromises.readFile(this.manifestPath, "utf-8");
|
|
4678
5022
|
const manifest = JSON.parse(content);
|
|
4679
5023
|
if (manifest.activeTableName) {
|
|
@@ -4695,7 +5039,7 @@ var init_store = __esm({
|
|
|
4695
5039
|
return true;
|
|
4696
5040
|
} catch (err) {
|
|
4697
5041
|
logger.warn("[store] Failed to save manifest:", err);
|
|
4698
|
-
if (
|
|
5042
|
+
if (fs13.existsSync(tempPath)) {
|
|
4699
5043
|
try {
|
|
4700
5044
|
await fsPromises.unlink(tempPath);
|
|
4701
5045
|
} catch {
|
|
@@ -4723,8 +5067,8 @@ var init_store = __esm({
|
|
|
4723
5067
|
async openInternal() {
|
|
4724
5068
|
if (this.db) return;
|
|
4725
5069
|
try {
|
|
4726
|
-
if (!
|
|
4727
|
-
|
|
5070
|
+
if (!fs13.existsSync(this.options.dbDir)) {
|
|
5071
|
+
fs13.mkdirSync(this.options.dbDir, { recursive: true });
|
|
4728
5072
|
}
|
|
4729
5073
|
this.db = await (await getLancedb()).connect(this.options.dbDir);
|
|
4730
5074
|
const tableNames = await this.db.tableNames();
|
|
@@ -4827,7 +5171,7 @@ var init_store = __esm({
|
|
|
4827
5171
|
const oldDir = path13.join(this.options.dbDir, `${this.tableName}.lance`);
|
|
4828
5172
|
const backupDir = path13.join(this.options.dbDir, `${backupTableName}.lance`);
|
|
4829
5173
|
try {
|
|
4830
|
-
if (
|
|
5174
|
+
if (fs13.existsSync(oldDir)) {
|
|
4831
5175
|
await fsPromises.rename(oldDir, backupDir);
|
|
4832
5176
|
logger.info(`[store] Successfully backed up database directory to ${backupDir}`);
|
|
4833
5177
|
}
|
|
@@ -5111,10 +5455,22 @@ var init_store = __esm({
|
|
|
5111
5455
|
}
|
|
5112
5456
|
}
|
|
5113
5457
|
}
|
|
5114
|
-
|
|
5115
|
-
|
|
5458
|
+
/**
|
|
5459
|
+
* @param olderThan When set, only rows with `timestamp < olderThan` are removed.
|
|
5460
|
+
* This is what lets a refresh write the replacement chunks FIRST and prune the
|
|
5461
|
+
* superseded ones after, instead of deleting first and losing the entry
|
|
5462
|
+
* outright if the insert then fails. `timestamp` is used as the discriminator
|
|
5463
|
+
* because it is a real (indexed) column — `contentHash` lives inside the
|
|
5464
|
+
* JSON-encoded `metadata` blob and cannot be filtered on.
|
|
5465
|
+
*/
|
|
5466
|
+
async deleteByUrlAndType(url, ingestionType, olderThan) {
|
|
5467
|
+
if (this.isClosing) {
|
|
5468
|
+
logger.warn(`[store] Prune of ${ingestionType} chunks for ${url} skipped \u2014 store is closing; superseded duplicates remain until the next refresh.`);
|
|
5469
|
+
metrics.increment("knowledge_store_delete_total", 1, { operation: "by_url_and_type", status: "skipped_closing" });
|
|
5470
|
+
}
|
|
5471
|
+
return this.trackOperation("deleteByUrlAndType", void 0, () => this.deleteByUrlAndTypeInner(url, ingestionType, olderThan));
|
|
5116
5472
|
}
|
|
5117
|
-
async deleteByUrlAndTypeInner(url, ingestionType) {
|
|
5473
|
+
async deleteByUrlAndTypeInner(url, ingestionType, olderThan) {
|
|
5118
5474
|
let retryCount = 0;
|
|
5119
5475
|
const MAX_RETRIES = 5;
|
|
5120
5476
|
const BASE_DELAY = 100;
|
|
@@ -5125,11 +5481,12 @@ var init_store = __esm({
|
|
|
5125
5481
|
try {
|
|
5126
5482
|
const escapedUrl = url.replace(/'/g, "''");
|
|
5127
5483
|
const escapedType = ingestionType.replace(/'/g, "''");
|
|
5128
|
-
|
|
5484
|
+
const ageClause = Number.isFinite(olderThan) ? ` AND timestamp < ${Math.floor(olderThan)}` : "";
|
|
5485
|
+
await table.delete(`url = '${escapedUrl}' AND ingestion_type = '${escapedType}' AND (${scopeFilter})${ageClause}`);
|
|
5129
5486
|
const duration = Date.now() - startTime;
|
|
5130
5487
|
metrics.observe("knowledge_store_delete_duration_ms", duration, { operation: "by_url_and_type" });
|
|
5131
5488
|
metrics.increment("knowledge_store_delete_total", 1, { operation: "by_url_and_type", status: "success" });
|
|
5132
|
-
logger.log(`[store] Deleted ${ingestionType} chunks for ${url} (scoped)`);
|
|
5489
|
+
logger.log(`[store] Deleted ${ingestionType} chunks for ${url} (scoped)${ageClause ? " older than the replacement" : ""}`);
|
|
5133
5490
|
return;
|
|
5134
5491
|
} catch (err) {
|
|
5135
5492
|
if (isLanceCommitConflict(err)) {
|
|
@@ -5182,8 +5539,8 @@ var init_store = __esm({
|
|
|
5182
5539
|
});
|
|
5183
5540
|
try {
|
|
5184
5541
|
const dir = path13.dirname(outputPath);
|
|
5185
|
-
await
|
|
5186
|
-
await
|
|
5542
|
+
await fs13.promises.mkdir(dir, { recursive: true });
|
|
5543
|
+
await fs13.promises.writeFile(outputPath, JSON.stringify(exportData), "utf-8");
|
|
5187
5544
|
logger.info(`[store] Successfully exported ${exportData.length} entries to ${outputPath}`);
|
|
5188
5545
|
} catch (err) {
|
|
5189
5546
|
logger.error(`[store] Failed to export knowledge store for web:`, err);
|
|
@@ -5200,7 +5557,7 @@ var init_store = __esm({
|
|
|
5200
5557
|
const scopeFilter = this.getScopeFilter();
|
|
5201
5558
|
const startTime = Date.now();
|
|
5202
5559
|
const escapedUrl = url.replace(/'/g, "''");
|
|
5203
|
-
const results = await table.query().where(`url = '${escapedUrl}' AND ingestion_type = 'synthesis-description' AND (${scopeFilter})`).limit(
|
|
5560
|
+
const results = await table.query().where(`url = '${escapedUrl}' AND ingestion_type = 'synthesis-description' AND (${scopeFilter})`).limit(16).toArray();
|
|
5204
5561
|
const duration = Date.now() - startTime;
|
|
5205
5562
|
metrics.observe("knowledge_store_query_duration_ms", duration, { operation: "rebuild_document" });
|
|
5206
5563
|
metrics.increment("knowledge_store_query_total", 1, { operation: "rebuild_document" });
|
|
@@ -5208,6 +5565,7 @@ var init_store = __esm({
|
|
|
5208
5565
|
metrics.increment("knowledge_store_cache_hits_total", 1, { status: "miss" });
|
|
5209
5566
|
return null;
|
|
5210
5567
|
}
|
|
5568
|
+
results.sort((a, b) => Number(b.timestamp) - Number(a.timestamp));
|
|
5211
5569
|
const r = results[0];
|
|
5212
5570
|
const rawAgeMs = Date.now() - Number(r.timestamp);
|
|
5213
5571
|
const hasAge = Number.isFinite(rawAgeMs);
|
|
@@ -5363,14 +5721,14 @@ var init_store = __esm({
|
|
|
5363
5721
|
for (const name of migrationDirs) {
|
|
5364
5722
|
const fullPath = path13.join(this.options.dbDir, name);
|
|
5365
5723
|
logger.info(`[store] Removing orphaned migration temp directory: ${name}`);
|
|
5366
|
-
await fsPromises.rm(fullPath, { recursive: true, force: true });
|
|
5724
|
+
await fsPromises.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
5367
5725
|
}
|
|
5368
5726
|
if (backupDirs.length > 1) {
|
|
5369
5727
|
backupDirs.sort();
|
|
5370
5728
|
for (const name of backupDirs.slice(0, -1)) {
|
|
5371
5729
|
const fullPath = path13.join(this.options.dbDir, name);
|
|
5372
5730
|
logger.info(`[store] Removing superseded backup directory: ${name}`);
|
|
5373
|
-
await fsPromises.rm(fullPath, { recursive: true, force: true });
|
|
5731
|
+
await fsPromises.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
5374
5732
|
}
|
|
5375
5733
|
}
|
|
5376
5734
|
} catch (err) {
|
|
@@ -5552,44 +5910,47 @@ var init_writer_queue = __esm({
|
|
|
5552
5910
|
async process() {
|
|
5553
5911
|
if (this.processing) return;
|
|
5554
5912
|
this.processing = true;
|
|
5555
|
-
|
|
5556
|
-
|
|
5557
|
-
|
|
5558
|
-
const urlKey = normalizeUrl(item.url);
|
|
5559
|
-
const prev = this.inflightByUrl.get(urlKey) ?? Promise.resolve();
|
|
5560
|
-
const inflight = prev.then(() => this._ingestInner(item));
|
|
5561
|
-
this.inflightByUrl.set(urlKey, inflight);
|
|
5913
|
+
try {
|
|
5914
|
+
while (this.queue.length > 0) {
|
|
5915
|
+
const item = this.queue.shift();
|
|
5562
5916
|
try {
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
5567
|
-
}
|
|
5568
|
-
}
|
|
5569
|
-
} catch (err) {
|
|
5570
|
-
if (isEmbedderUnreachable(err)) {
|
|
5571
|
-
logger.warn(`[writer-queue] Embedder unreachable for ${item.url}, retrying once after 2s...`);
|
|
5572
|
-
await new Promise((r) => setTimeout(r, 2e3));
|
|
5917
|
+
const urlKey = normalizeUrl(item.url);
|
|
5918
|
+
const prev = this.inflightByUrl.get(urlKey) ?? Promise.resolve();
|
|
5919
|
+
const inflight = prev.then(() => this._ingestInner(item));
|
|
5920
|
+
this.inflightByUrl.set(urlKey, inflight);
|
|
5573
5921
|
try {
|
|
5574
|
-
await
|
|
5575
|
-
}
|
|
5576
|
-
|
|
5577
|
-
|
|
5922
|
+
await inflight;
|
|
5923
|
+
} finally {
|
|
5924
|
+
if (this.inflightByUrl.get(urlKey) === inflight) {
|
|
5925
|
+
this.inflightByUrl.delete(urlKey);
|
|
5926
|
+
}
|
|
5927
|
+
}
|
|
5928
|
+
} catch (err) {
|
|
5929
|
+
if (isEmbedderUnreachable(err)) {
|
|
5930
|
+
logger.warn(`[writer-queue] Embedder unreachable for ${item.url}, retrying once after 2s...`);
|
|
5931
|
+
await new Promise((r) => setTimeout(r, 2e3));
|
|
5932
|
+
try {
|
|
5933
|
+
await this.ingest(item);
|
|
5934
|
+
} catch (retryErr) {
|
|
5935
|
+
logger.error(`[writer-queue] Retry failed for ${item.url}, dropping:`, retryErr);
|
|
5936
|
+
metrics.increment("knowledge_ingest_dropped_total", 1, { reason: "embedder_unreachable" });
|
|
5937
|
+
}
|
|
5938
|
+
} else if (isNoSpace(err)) {
|
|
5939
|
+
logger.error(`[writer-queue] ENOSPC: disk full \u2014 dropping ${item.url}. Free up disk space to resume knowledge ingestion.`);
|
|
5940
|
+
metrics.increment("knowledge_ingest_dropped_total", 1, { reason: "enospc" });
|
|
5941
|
+
} else {
|
|
5942
|
+
logger.error(`[writer-queue] Failed to ingest ${item.url}:`, err);
|
|
5943
|
+
metrics.increment("knowledge_ingest_dropped_total", 1, { reason: "ingest_error" });
|
|
5578
5944
|
}
|
|
5579
|
-
} else if (isNoSpace(err)) {
|
|
5580
|
-
logger.error(`[writer-queue] ENOSPC: disk full \u2014 dropping ${item.url}. Free up disk space to resume knowledge ingestion.`);
|
|
5581
|
-
metrics.increment("knowledge_ingest_dropped_total", 1, { reason: "enospc" });
|
|
5582
|
-
} else {
|
|
5583
|
-
logger.error(`[writer-queue] Failed to ingest ${item.url}:`, err);
|
|
5584
|
-
metrics.increment("knowledge_ingest_dropped_total", 1, { reason: "ingest_error" });
|
|
5585
5945
|
}
|
|
5586
5946
|
}
|
|
5587
|
-
}
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5947
|
+
} finally {
|
|
5948
|
+
this.processing = false;
|
|
5949
|
+
const resolvers = [...this.drainResolvers];
|
|
5950
|
+
this.drainResolvers = [];
|
|
5951
|
+
for (const resolve8 of resolvers) {
|
|
5952
|
+
resolve8();
|
|
5953
|
+
}
|
|
5593
5954
|
}
|
|
5594
5955
|
}
|
|
5595
5956
|
async ingest(item) {
|
|
@@ -5630,15 +5991,23 @@ var init_writer_queue = __esm({
|
|
|
5630
5991
|
};
|
|
5631
5992
|
const existing = await this.options.store.findByUrl(normalizedUrl);
|
|
5632
5993
|
const sameType = existing.filter((c) => c.metadata["ingestionType"] === incomingType);
|
|
5633
|
-
if (sameType.length > 0 && sameType[0].metadata["contentHash"] === hash) {
|
|
5634
|
-
logger.log(`[writer-queue] Skipping ${normalizedUrl} (${incomingType}) \u2014 content unchanged.`);
|
|
5635
|
-
return;
|
|
5636
|
-
}
|
|
5637
5994
|
if (sameType.length > 0) {
|
|
5638
|
-
|
|
5995
|
+
const newestTs = Math.max(...sameType.map((d) => Number(d.timestamp) || 0));
|
|
5996
|
+
const newest = sameType.find((d) => (Number(d.timestamp) || 0) === newestTs);
|
|
5997
|
+
if (newest.metadata["contentHash"] === hash) {
|
|
5998
|
+
if (sameType.some((d) => (Number(d.timestamp) || 0) < newestTs)) {
|
|
5999
|
+
await this.options.store.deleteByUrlAndType(normalizedUrl, incomingType, newestTs).catch((err) => {
|
|
6000
|
+
logger.warn(`[writer-queue] Sweeping superseded duplicates for ${normalizedUrl} (${incomingType}) failed:`, err);
|
|
6001
|
+
metrics.increment("knowledge_ingest_stale_prune_failed_total", 1);
|
|
6002
|
+
});
|
|
6003
|
+
}
|
|
6004
|
+
logger.log(`[writer-queue] Skipping ${normalizedUrl} (${incomingType}) \u2014 content unchanged.`);
|
|
6005
|
+
return;
|
|
6006
|
+
}
|
|
5639
6007
|
}
|
|
5640
6008
|
const rawChunks = this.options.chunker ? this.options.chunker.chunk(item.markdown) : [{ text: item.markdown, actual_overlap: 0 }];
|
|
5641
6009
|
if (rawChunks.length === 0) return;
|
|
6010
|
+
const insertedAt = Date.now();
|
|
5642
6011
|
const docs = rawChunks.map((chunk, i) => ({
|
|
5643
6012
|
url: normalizedUrl,
|
|
5644
6013
|
text: chunk.text,
|
|
@@ -5653,9 +6022,15 @@ var init_writer_queue = __esm({
|
|
|
5653
6022
|
chunkIndex: i,
|
|
5654
6023
|
totalChunks: rawChunks.length
|
|
5655
6024
|
},
|
|
5656
|
-
timestamp:
|
|
6025
|
+
timestamp: insertedAt
|
|
5657
6026
|
}));
|
|
5658
6027
|
await this.options.store.addDocuments(docs);
|
|
6028
|
+
if (sameType.length > 0) {
|
|
6029
|
+
await this.options.store.deleteByUrlAndType(normalizedUrl, incomingType, insertedAt).catch((err) => {
|
|
6030
|
+
logger.warn(`[writer-queue] Replacement for ${normalizedUrl} (${incomingType}) is stored, but pruning the superseded chunks failed; duplicates remain until the next refresh:`, err);
|
|
6031
|
+
metrics.increment("knowledge_ingest_stale_prune_failed_total", 1);
|
|
6032
|
+
});
|
|
6033
|
+
}
|
|
5659
6034
|
}
|
|
5660
6035
|
async drain(timeoutMs) {
|
|
5661
6036
|
return new Promise((resolve8) => {
|
|
@@ -5923,7 +6298,7 @@ var init_model_config = __esm({
|
|
|
5923
6298
|
});
|
|
5924
6299
|
|
|
5925
6300
|
// src/knowledge/index.ts
|
|
5926
|
-
import * as
|
|
6301
|
+
import * as fs14 from "node:fs";
|
|
5927
6302
|
function getMigrationStrategy(config) {
|
|
5928
6303
|
const strategy = config.MIGRATION_STRATEGY;
|
|
5929
6304
|
if (!strategy) return void 0;
|
|
@@ -5985,10 +6360,10 @@ async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory,
|
|
|
5985
6360
|
}
|
|
5986
6361
|
async function forceDeleteKnowledgeStore(config, workspace) {
|
|
5987
6362
|
const dbDir = getDbDir(config, workspace);
|
|
5988
|
-
if (
|
|
6363
|
+
if (fs14.existsSync(dbDir)) {
|
|
5989
6364
|
try {
|
|
5990
|
-
|
|
5991
|
-
|
|
6365
|
+
fs14.rmSync(dbDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
6366
|
+
fs14.mkdirSync(dbDir, { recursive: true });
|
|
5992
6367
|
logger.info(`[knowledge] Knowledge store at ${dbDir} forcefully deleted and recreated.`);
|
|
5993
6368
|
} catch (err) {
|
|
5994
6369
|
logger.error(`[knowledge] Failed to forcefully delete directory ${dbDir}:`, err);
|
|
@@ -6123,11 +6498,11 @@ async function isModelCached(model) {
|
|
|
6123
6498
|
const env = await getHFEnv();
|
|
6124
6499
|
const cacheDir = env.cacheDir;
|
|
6125
6500
|
if (!cacheDir) return false;
|
|
6126
|
-
const { access:
|
|
6501
|
+
const { access: access3, readdir: readdir6, readFile: readFile7, stat: stat5, rm: rm3 } = await import("node:fs/promises");
|
|
6127
6502
|
const path19 = await import("node:path");
|
|
6128
6503
|
const modelDir = path19.default.join(cacheDir, model);
|
|
6129
6504
|
const onnxDir = path19.default.join(modelDir, "onnx");
|
|
6130
|
-
await
|
|
6505
|
+
await access3(path19.default.join(onnxDir, "model.onnx"));
|
|
6131
6506
|
try {
|
|
6132
6507
|
const cfgRaw = await readFile7(path19.default.join(modelDir, "config.json"), "utf-8");
|
|
6133
6508
|
JSON.parse(cfgRaw);
|
|
@@ -6144,7 +6519,7 @@ async function isModelCached(model) {
|
|
|
6144
6519
|
}
|
|
6145
6520
|
}
|
|
6146
6521
|
if (!hasTokenizer) return false;
|
|
6147
|
-
const entries = await
|
|
6522
|
+
const entries = await readdir6(onnxDir).catch(() => []);
|
|
6148
6523
|
const hasExternalWeights = entries.includes("model.onnx_data");
|
|
6149
6524
|
if (hasExternalWeights) {
|
|
6150
6525
|
const dataStat = await stat5(path19.default.join(onnxDir, "model.onnx_data")).catch(() => null);
|
|
@@ -6185,7 +6560,7 @@ async function purgeModelCache(model) {
|
|
|
6185
6560
|
logger.warn(`[embedder] Refusing to purge unsafe model-cache path for model id ${JSON.stringify(model)}`);
|
|
6186
6561
|
return;
|
|
6187
6562
|
}
|
|
6188
|
-
await rm3(modelDir, { recursive: true, force: true });
|
|
6563
|
+
await rm3(modelDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
6189
6564
|
logger.warn(`[embedder] Purged corrupt model cache at ${modelDir}`);
|
|
6190
6565
|
} catch (err) {
|
|
6191
6566
|
logger.warn("[embedder] Failed to purge model cache:", err);
|
|
@@ -6385,19 +6760,21 @@ function runProbe(model, cacheDir, timeoutMs) {
|
|
|
6385
6760
|
finish(false);
|
|
6386
6761
|
}, timeoutMs);
|
|
6387
6762
|
if (typeof timer.unref === "function") timer.unref();
|
|
6388
|
-
|
|
6389
|
-
|
|
6763
|
+
const stdoutChunks = [];
|
|
6764
|
+
const stderrChunks = [];
|
|
6390
6765
|
child.stdout?.on("data", (d) => {
|
|
6391
|
-
|
|
6766
|
+
stdoutChunks.push(Buffer.from(d));
|
|
6392
6767
|
});
|
|
6393
6768
|
child.stderr?.on("data", (d) => {
|
|
6394
|
-
|
|
6769
|
+
stderrChunks.push(Buffer.from(d));
|
|
6395
6770
|
});
|
|
6396
6771
|
child.on("error", (err) => {
|
|
6397
6772
|
logger.warn("[embedder] WebGPU probe process error \u2014 defaulting to CPU:", err);
|
|
6398
6773
|
finish(false);
|
|
6399
6774
|
});
|
|
6400
6775
|
child.on("close", (code, signal) => {
|
|
6776
|
+
const stdout = Buffer.concat(stdoutChunks).toString("utf8");
|
|
6777
|
+
const stderr = Buffer.concat(stderrChunks).toString("utf8");
|
|
6401
6778
|
if (stdout.includes("PROBE_OK")) {
|
|
6402
6779
|
finish(true);
|
|
6403
6780
|
return;
|
|
@@ -6490,6 +6867,14 @@ var init_embedder = __esm({
|
|
|
6490
6867
|
idleTimer = null;
|
|
6491
6868
|
IDLE_TIMEOUT_MS = 60 * 1e3;
|
|
6492
6869
|
activeEmbeddings = 0;
|
|
6870
|
+
/**
|
|
6871
|
+
* Why the in-flight dispose was started. An IDLE dispose is a memory-reclaim
|
|
6872
|
+
* pause that a later embed is expected to undo, so initialize() waits it out and
|
|
6873
|
+
* re-initializes. A TERMINAL dispose (shutdown, config change, explicit
|
|
6874
|
+
* disposal) must never be undone — re-initializing there would resurrect the
|
|
6875
|
+
* ONNX pipeline during teardown. Only meaningful while state === 'disposing'.
|
|
6876
|
+
*/
|
|
6877
|
+
disposeReason = "terminal";
|
|
6493
6878
|
constructor(options) {
|
|
6494
6879
|
this.model = options.model;
|
|
6495
6880
|
this.poolingMode = options.pooling ?? "mean";
|
|
@@ -6523,7 +6908,7 @@ var init_embedder = __esm({
|
|
|
6523
6908
|
}
|
|
6524
6909
|
if (this.state === "ready") {
|
|
6525
6910
|
logger.info(`[embedder] Idle timeout reached (${this.IDLE_TIMEOUT_MS}ms), releasing GPU memory...`);
|
|
6526
|
-
this.dispose().catch((err) => logger.warn("[embedder] Failed to dispose on idle:", err));
|
|
6911
|
+
this.dispose("idle").catch((err) => logger.warn("[embedder] Failed to dispose on idle:", err));
|
|
6527
6912
|
}
|
|
6528
6913
|
}, this.IDLE_TIMEOUT_MS);
|
|
6529
6914
|
if (this.idleTimer) safeUnref(this.idleTimer);
|
|
@@ -6537,7 +6922,20 @@ var init_embedder = __esm({
|
|
|
6537
6922
|
async initialize() {
|
|
6538
6923
|
if (this.state === "ready") return;
|
|
6539
6924
|
if (this.state === "disposing") {
|
|
6540
|
-
|
|
6925
|
+
if (this.disposeReason === "idle") {
|
|
6926
|
+
logger.debug("[embedder] initialize() raced an idle dispose \u2014 waiting for it to finish, then re-initializing.");
|
|
6927
|
+
await this.disposePromise?.catch(() => {
|
|
6928
|
+
});
|
|
6929
|
+
if (this.state === "ready") return;
|
|
6930
|
+
if (this.disposeReason === "terminal") {
|
|
6931
|
+
throw new Error("Cannot initialize while disposing");
|
|
6932
|
+
}
|
|
6933
|
+
if (this.state === "disposing") {
|
|
6934
|
+
throw new Error("Cannot initialize while disposing");
|
|
6935
|
+
}
|
|
6936
|
+
} else {
|
|
6937
|
+
throw new Error("Cannot initialize while disposing");
|
|
6938
|
+
}
|
|
6541
6939
|
}
|
|
6542
6940
|
if (this.initializingPromise) {
|
|
6543
6941
|
return this.initializingPromise;
|
|
@@ -6859,12 +7257,21 @@ var init_embedder = __esm({
|
|
|
6859
7257
|
this.recoveryPromise = null;
|
|
6860
7258
|
}
|
|
6861
7259
|
}
|
|
6862
|
-
|
|
7260
|
+
/**
|
|
7261
|
+
* @param reason `'idle'` for the memory-reclaim pause driven by the idle timer —
|
|
7262
|
+
* a later embed is allowed to revive the embedder. Anything else is terminal
|
|
7263
|
+
* (shutdown, config change, explicit disposal) and must never be revived, so
|
|
7264
|
+
* the default is deliberately the safe one: callers that do not opt in get
|
|
7265
|
+
* terminal semantics.
|
|
7266
|
+
*/
|
|
7267
|
+
async dispose(reason = "terminal") {
|
|
6863
7268
|
if (this.state === "idle") return;
|
|
6864
7269
|
this.stopIdleTimer();
|
|
6865
7270
|
if (this.state === "disposing" && this.disposePromise) {
|
|
7271
|
+
if (reason === "terminal") this.disposeReason = "terminal";
|
|
6866
7272
|
return this.disposePromise;
|
|
6867
7273
|
}
|
|
7274
|
+
this.disposeReason = reason;
|
|
6868
7275
|
this.state = "disposing";
|
|
6869
7276
|
this.disposePromise = (async () => {
|
|
6870
7277
|
const maxWaitMs = 5e3;
|
|
@@ -6909,8 +7316,21 @@ var init_embedder = __esm({
|
|
|
6909
7316
|
|
|
6910
7317
|
// src/infrastructure/embedding/embedding-server.ts
|
|
6911
7318
|
import * as http4 from "node:http";
|
|
7319
|
+
import * as crypto7 from "node:crypto";
|
|
6912
7320
|
import * as path15 from "node:path";
|
|
6913
|
-
|
|
7321
|
+
function getEmbeddingServerAuthSecret() {
|
|
7322
|
+
if (!_embeddingAuthSecret) {
|
|
7323
|
+
_embeddingAuthSecret = crypto7.randomBytes(32).toString("hex");
|
|
7324
|
+
}
|
|
7325
|
+
return _embeddingAuthSecret;
|
|
7326
|
+
}
|
|
7327
|
+
function isAuthorized(presented) {
|
|
7328
|
+
const header = Array.isArray(presented) ? presented[0] : presented;
|
|
7329
|
+
const expected = Buffer.from(getEmbeddingServerAuthSecret(), "utf8");
|
|
7330
|
+
const actual = Buffer.from(header ?? "", "utf8");
|
|
7331
|
+
return actual.length === expected.length && crypto7.timingSafeEqual(actual, expected);
|
|
7332
|
+
}
|
|
7333
|
+
var _embeddingAuthSecret, SerialQueue, EmbeddingServer;
|
|
6914
7334
|
var init_embedding_server = __esm({
|
|
6915
7335
|
"src/infrastructure/embedding/embedding-server.ts"() {
|
|
6916
7336
|
"use strict";
|
|
@@ -6918,6 +7338,8 @@ var init_embedding_server = __esm({
|
|
|
6918
7338
|
init_stdio_capture();
|
|
6919
7339
|
init_log_utils();
|
|
6920
7340
|
init_disk_space_checker();
|
|
7341
|
+
init_http_body();
|
|
7342
|
+
_embeddingAuthSecret = null;
|
|
6921
7343
|
SerialQueue = class {
|
|
6922
7344
|
running = false;
|
|
6923
7345
|
tasks = [];
|
|
@@ -7220,12 +7642,18 @@ var init_embedding_server = __esm({
|
|
|
7220
7642
|
}
|
|
7221
7643
|
// ---- HTTP request handler ----
|
|
7222
7644
|
async handleRequest(req, res) {
|
|
7645
|
+
if (!isAuthorized(req.headers["x-embedding-auth"])) {
|
|
7646
|
+
res.writeHead(403, { "Content-Type": "application/json" });
|
|
7647
|
+
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
7648
|
+
req.resume();
|
|
7649
|
+
return;
|
|
7650
|
+
}
|
|
7223
7651
|
const MAX_BODY_SIZE = 50 * 1024 * 1024;
|
|
7224
|
-
|
|
7652
|
+
const collected = new Utf8Body();
|
|
7225
7653
|
let oversized = false;
|
|
7226
7654
|
req.on("data", (chunk) => {
|
|
7227
|
-
|
|
7228
|
-
if (
|
|
7655
|
+
collected.push(chunk);
|
|
7656
|
+
if (collected.byteLength > MAX_BODY_SIZE) {
|
|
7229
7657
|
oversized = true;
|
|
7230
7658
|
req.destroy(new Error("Payload too large"));
|
|
7231
7659
|
}
|
|
@@ -7249,6 +7677,7 @@ var init_embedding_server = __esm({
|
|
|
7249
7677
|
}
|
|
7250
7678
|
finish();
|
|
7251
7679
|
});
|
|
7680
|
+
req.on("close", finish);
|
|
7252
7681
|
req.on("end", async () => {
|
|
7253
7682
|
try {
|
|
7254
7683
|
if (req.method === "GET" && req.url === "/health") {
|
|
@@ -7268,7 +7697,7 @@ var init_embedding_server = __esm({
|
|
|
7268
7697
|
finish();
|
|
7269
7698
|
return;
|
|
7270
7699
|
}
|
|
7271
|
-
const data = JSON.parse(
|
|
7700
|
+
const data = JSON.parse(collected.toString());
|
|
7272
7701
|
switch (req.url) {
|
|
7273
7702
|
case "/embed": {
|
|
7274
7703
|
this.assertServing();
|
|
@@ -7318,17 +7747,24 @@ var init_embedding_client = __esm({
|
|
|
7318
7747
|
"src/infrastructure/embedding/embedding-client.ts"() {
|
|
7319
7748
|
"use strict";
|
|
7320
7749
|
init_logger();
|
|
7750
|
+
init_http_body();
|
|
7321
7751
|
EmbeddingClient = class {
|
|
7322
|
-
constructor(port, _device = "unknown") {
|
|
7752
|
+
constructor(port, _device = "unknown", authSecret) {
|
|
7323
7753
|
this.port = port;
|
|
7324
7754
|
this._device = _device;
|
|
7755
|
+
this.authSecret = authSecret;
|
|
7325
7756
|
logger.log(`[EmbeddingClient] Connecting to embedding server at http://127.0.0.1:${port}`);
|
|
7326
7757
|
}
|
|
7327
7758
|
port;
|
|
7328
7759
|
_device;
|
|
7760
|
+
authSecret;
|
|
7329
7761
|
// Public so store.ts can set the dimension from existing table schema
|
|
7330
7762
|
// before the embedder has been fully warmed up (see Embedder.setDimension).
|
|
7331
7763
|
dimension = null;
|
|
7764
|
+
/** Auth header for the leader, or nothing when connecting to a pre-auth leader. */
|
|
7765
|
+
authHeaders() {
|
|
7766
|
+
return this.authSecret ? { "x-embedding-auth": this.authSecret } : {};
|
|
7767
|
+
}
|
|
7332
7768
|
// ---- IEmbedder ----
|
|
7333
7769
|
getDevice() {
|
|
7334
7770
|
return this._device;
|
|
@@ -7382,26 +7818,27 @@ var init_embedding_client = __esm({
|
|
|
7382
7818
|
hostname: "127.0.0.1",
|
|
7383
7819
|
port: this.port,
|
|
7384
7820
|
path: path19,
|
|
7385
|
-
method: "GET"
|
|
7821
|
+
method: "GET",
|
|
7822
|
+
headers: this.authHeaders()
|
|
7386
7823
|
},
|
|
7387
7824
|
(res) => {
|
|
7388
|
-
|
|
7825
|
+
const collected = new Utf8Body();
|
|
7389
7826
|
res.on("data", (chunk) => {
|
|
7390
|
-
|
|
7827
|
+
collected.push(chunk);
|
|
7391
7828
|
});
|
|
7392
7829
|
res.on("end", () => {
|
|
7393
7830
|
if (resolved) return;
|
|
7394
7831
|
resolved = true;
|
|
7395
7832
|
clearTimeout(timer);
|
|
7396
7833
|
try {
|
|
7397
|
-
const parsed = JSON.parse(
|
|
7834
|
+
const parsed = JSON.parse(collected.toString());
|
|
7398
7835
|
if (res.statusCode !== 200) {
|
|
7399
7836
|
reject(new Error(`[EmbeddingClient] GET ${path19} returned HTTP ${res.statusCode}`));
|
|
7400
7837
|
} else {
|
|
7401
7838
|
resolve8(parsed);
|
|
7402
7839
|
}
|
|
7403
7840
|
} catch (_e) {
|
|
7404
|
-
reject(new Error(`[EmbeddingClient] Failed to parse GET ${path19} response: ${
|
|
7841
|
+
reject(new Error(`[EmbeddingClient] Failed to parse GET ${path19} response: ${collected.toString()}`));
|
|
7405
7842
|
}
|
|
7406
7843
|
});
|
|
7407
7844
|
res.on("error", (err) => {
|
|
@@ -7443,20 +7880,21 @@ var init_embedding_client = __esm({
|
|
|
7443
7880
|
method: "POST",
|
|
7444
7881
|
headers: {
|
|
7445
7882
|
"Content-Type": "application/json",
|
|
7446
|
-
"Content-Length": Buffer.byteLength(body)
|
|
7883
|
+
"Content-Length": Buffer.byteLength(body),
|
|
7884
|
+
...this.authHeaders()
|
|
7447
7885
|
}
|
|
7448
7886
|
},
|
|
7449
7887
|
(res) => {
|
|
7450
|
-
|
|
7888
|
+
const collected = new Utf8Body();
|
|
7451
7889
|
res.on("data", (chunk) => {
|
|
7452
|
-
|
|
7890
|
+
collected.push(chunk);
|
|
7453
7891
|
});
|
|
7454
7892
|
res.on("end", () => {
|
|
7455
7893
|
if (resolved) return;
|
|
7456
7894
|
resolved = true;
|
|
7457
7895
|
clearTimeout(timer);
|
|
7458
7896
|
try {
|
|
7459
|
-
const parsed = JSON.parse(
|
|
7897
|
+
const parsed = JSON.parse(collected.toString());
|
|
7460
7898
|
if (res.statusCode !== 200) {
|
|
7461
7899
|
const msg2 = parsed?.["error"];
|
|
7462
7900
|
reject(new Error(typeof msg2 === "string" ? msg2 : `HTTP ${res.statusCode}`));
|
|
@@ -7464,7 +7902,7 @@ var init_embedding_client = __esm({
|
|
|
7464
7902
|
resolve8(parsed);
|
|
7465
7903
|
}
|
|
7466
7904
|
} catch (_e) {
|
|
7467
|
-
reject(new Error(`[EmbeddingClient] Failed to parse POST ${path19} response: ${
|
|
7905
|
+
reject(new Error(`[EmbeddingClient] Failed to parse POST ${path19} response: ${collected.toString()}`));
|
|
7468
7906
|
}
|
|
7469
7907
|
});
|
|
7470
7908
|
res.on("error", (err) => {
|
|
@@ -7490,7 +7928,7 @@ var init_embedding_client = __esm({
|
|
|
7490
7928
|
});
|
|
7491
7929
|
|
|
7492
7930
|
// src/infrastructure/embedding/embedding-factory.ts
|
|
7493
|
-
import * as
|
|
7931
|
+
import * as crypto8 from "node:crypto";
|
|
7494
7932
|
import * as net2 from "node:net";
|
|
7495
7933
|
async function isPortListening2(port, timeoutMs = 2e3) {
|
|
7496
7934
|
return new Promise((resolve8) => {
|
|
@@ -7535,13 +7973,14 @@ async function getEmbedder(config, _attempt = 0) {
|
|
|
7535
7973
|
return _embeddingInstance;
|
|
7536
7974
|
}
|
|
7537
7975
|
logger.info(`[EmbeddingFactory] Configuration change detected (${_cachedModel} on ${_cachedDevice} -> ${cfg.EMBEDDING_MODEL} on ${cfg.EMBEDDING_DEVICE}). Disposing stale instance.`);
|
|
7538
|
-
|
|
7976
|
+
const stale = _embeddingInstance;
|
|
7539
7977
|
_embeddingInstance = null;
|
|
7978
|
+
await stale.dispose?.();
|
|
7540
7979
|
}
|
|
7541
|
-
if (_embeddingInitPromise) return _embeddingInitPromise;
|
|
7980
|
+
if (_embeddingInitPromise && _attempt === 0) return _embeddingInitPromise;
|
|
7542
7981
|
let p;
|
|
7543
7982
|
const init = async () => {
|
|
7544
|
-
const serverId =
|
|
7983
|
+
const serverId = crypto8.randomUUID();
|
|
7545
7984
|
const stateManager = await getService(ServiceNames.STATE_MANAGER);
|
|
7546
7985
|
const processLifecycle = await getService(ServiceNames.PROCESS_LIFECYCLE);
|
|
7547
7986
|
const myStartTime = await processLifecycle.getCurrentProcessStartTime();
|
|
@@ -7563,6 +8002,9 @@ async function getEmbedder(config, _attempt = 0) {
|
|
|
7563
8002
|
pid: process.pid,
|
|
7564
8003
|
serverId,
|
|
7565
8004
|
model: cfg.EMBEDDING_MODEL,
|
|
8005
|
+
// Published with the claim, not with the port, so a follower that adopts
|
|
8006
|
+
// this leader always has the secret available whenever the port is.
|
|
8007
|
+
authSecret: getEmbeddingServerAuthSecret(),
|
|
7566
8008
|
...myStartTime !== null ? { startTime: myStartTime } : {}
|
|
7567
8009
|
};
|
|
7568
8010
|
iAmCandidate = true;
|
|
@@ -7603,7 +8045,7 @@ async function getEmbedder(config, _attempt = 0) {
|
|
|
7603
8045
|
const portOk = await isPortListening2(info.port);
|
|
7604
8046
|
if (portOk) {
|
|
7605
8047
|
logger.info(`[EmbeddingFactory] Connecting to embedding server on port ${info.port}`);
|
|
7606
|
-
const client = new EmbeddingClient(info.port);
|
|
8048
|
+
const client = new EmbeddingClient(info.port, cfg.EMBEDDING_DEVICE, info.authSecret);
|
|
7607
8049
|
await client.fetchHealth();
|
|
7608
8050
|
_embeddingInstance = client;
|
|
7609
8051
|
_cachedModel = cfg.EMBEDDING_MODEL;
|
|
@@ -7619,7 +8061,6 @@ async function getEmbedder(config, _attempt = 0) {
|
|
|
7619
8061
|
}
|
|
7620
8062
|
}
|
|
7621
8063
|
}
|
|
7622
|
-
_embeddingInitPromise = null;
|
|
7623
8064
|
if (_attempt + 1 >= MAX_LEADER_WAIT_RETRIES) {
|
|
7624
8065
|
throw new Error(
|
|
7625
8066
|
`[EmbeddingFactory] Could not obtain an embedding server after ${MAX_LEADER_WAIT_RETRIES} attempts (no process became leader; model init may be failing repeatedly).`
|
|
@@ -7654,11 +8095,10 @@ async function getEmbedder(config, _attempt = 0) {
|
|
|
7654
8095
|
};
|
|
7655
8096
|
p = init();
|
|
7656
8097
|
_embeddingInitPromise = p;
|
|
7657
|
-
|
|
7658
|
-
_embeddingInitPromise = null;
|
|
7659
|
-
}
|
|
7660
|
-
|
|
7661
|
-
});
|
|
8098
|
+
const clearIfCurrent = () => {
|
|
8099
|
+
if (_embeddingInitPromise === p) _embeddingInitPromise = null;
|
|
8100
|
+
};
|
|
8101
|
+
p.then(clearIfCurrent).catch(clearIfCurrent);
|
|
7662
8102
|
return p;
|
|
7663
8103
|
}
|
|
7664
8104
|
async function clearEmbeddingInstance() {
|
|
@@ -8686,6 +9126,13 @@ function createResearchKnowledgeSearchTool(iface) {
|
|
|
8686
9126
|
return buildSteeringResult(result, urls);
|
|
8687
9127
|
} catch (error) {
|
|
8688
9128
|
const durationMs = Date.now() - startTime;
|
|
9129
|
+
const msg2 = error instanceof Error ? error.message : String(error);
|
|
9130
|
+
if (signal?.aborted || error instanceof Error && error.name === "AbortError" || /^(aborted|research (aborted|cancelled))$/i.test(msg2)) {
|
|
9131
|
+
metrics.observe("research_knowledge_search_duration_ms", durationMs, { status: "cancelled" });
|
|
9132
|
+
metrics.increment("research_knowledge_search_total", 1, { status: "cancelled" });
|
|
9133
|
+
logger.debug("[research-knowledge-search] Cancelled mid-search \u2014 propagating rather than reporting an empty store.");
|
|
9134
|
+
throw error;
|
|
9135
|
+
}
|
|
8689
9136
|
metrics.observe("research_knowledge_search_duration_ms", durationMs, { status: "error" });
|
|
8690
9137
|
metrics.increment("research_knowledge_search_total", 1, { status: "error" });
|
|
8691
9138
|
logger.error("[research-knowledge-search] Tool execution failed:", error);
|
|
@@ -8756,10 +9203,19 @@ __export(skill_installer_exports, {
|
|
|
8756
9203
|
skillUninstallCandidates: () => skillUninstallCandidates,
|
|
8757
9204
|
uninstallSkill: () => uninstallSkill
|
|
8758
9205
|
});
|
|
8759
|
-
import * as
|
|
9206
|
+
import * as crypto10 from "node:crypto";
|
|
9207
|
+
import * as fs16 from "node:fs";
|
|
8760
9208
|
import * as os6 from "node:os";
|
|
8761
9209
|
import * as path17 from "node:path";
|
|
8762
9210
|
import { fileURLToPath as fileURLToPath4 } from "node:url";
|
|
9211
|
+
function readSkillVersion(sourceDir) {
|
|
9212
|
+
try {
|
|
9213
|
+
const md = fs16.readFileSync(path17.join(sourceDir, "SKILL.md"), "utf-8");
|
|
9214
|
+
return /"version"\s*:\s*"([^"]+)"/.exec(md)?.[1] ?? null;
|
|
9215
|
+
} catch {
|
|
9216
|
+
return null;
|
|
9217
|
+
}
|
|
9218
|
+
}
|
|
8763
9219
|
function homeDir(opts) {
|
|
8764
9220
|
return opts?.home ?? os6.homedir();
|
|
8765
9221
|
}
|
|
@@ -8786,7 +9242,7 @@ function resolveSkillSourceDir(explicit) {
|
|
|
8786
9242
|
}
|
|
8787
9243
|
for (const c of candidates) {
|
|
8788
9244
|
try {
|
|
8789
|
-
if (
|
|
9245
|
+
if (fs16.existsSync(path17.join(c, "SKILL.md"))) return path17.resolve(c);
|
|
8790
9246
|
} catch {
|
|
8791
9247
|
}
|
|
8792
9248
|
}
|
|
@@ -8798,8 +9254,8 @@ function getManifestPath(opts) {
|
|
|
8798
9254
|
function readManifest(opts) {
|
|
8799
9255
|
const p = getManifestPath(opts);
|
|
8800
9256
|
try {
|
|
8801
|
-
if (
|
|
8802
|
-
const parsed = JSON.parse(
|
|
9257
|
+
if (fs16.existsSync(p)) {
|
|
9258
|
+
const parsed = JSON.parse(fs16.readFileSync(p, "utf-8"));
|
|
8803
9259
|
if (parsed && Array.isArray(parsed.entries)) {
|
|
8804
9260
|
return { version: 1, package: PACKAGE_NAME, entries: parsed.entries };
|
|
8805
9261
|
}
|
|
@@ -8810,10 +9266,10 @@ function readManifest(opts) {
|
|
|
8810
9266
|
}
|
|
8811
9267
|
function writeManifest(manifest, opts) {
|
|
8812
9268
|
const p = getManifestPath(opts);
|
|
8813
|
-
|
|
9269
|
+
fs16.mkdirSync(path17.dirname(p), { recursive: true });
|
|
8814
9270
|
const tmp = `${p}.tmp`;
|
|
8815
|
-
|
|
8816
|
-
|
|
9271
|
+
fs16.writeFileSync(tmp, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
|
|
9272
|
+
fs16.renameSync(tmp, p);
|
|
8817
9273
|
}
|
|
8818
9274
|
function upsertEntry(manifest, entry) {
|
|
8819
9275
|
const idx = manifest.entries.findIndex((e) => e.path === entry.path);
|
|
@@ -8822,9 +9278,9 @@ function upsertEntry(manifest, entry) {
|
|
|
8822
9278
|
}
|
|
8823
9279
|
function isOwnedSymlink(targetPath, expectedSource) {
|
|
8824
9280
|
try {
|
|
8825
|
-
const st =
|
|
9281
|
+
const st = fs16.lstatSync(targetPath);
|
|
8826
9282
|
if (!st.isSymbolicLink()) return false;
|
|
8827
|
-
const dest = path17.resolve(path17.dirname(targetPath),
|
|
9283
|
+
const dest = path17.resolve(path17.dirname(targetPath), fs16.readlinkSync(targetPath));
|
|
8828
9284
|
if (expectedSource && path17.resolve(expectedSource) === dest) return true;
|
|
8829
9285
|
return /[/\\]pi-research[/\\]skills[/\\]pi-research$/.test(dest);
|
|
8830
9286
|
} catch {
|
|
@@ -8833,7 +9289,7 @@ function isOwnedSymlink(targetPath, expectedSource) {
|
|
|
8833
9289
|
}
|
|
8834
9290
|
function isOwnedCopy(targetPath) {
|
|
8835
9291
|
try {
|
|
8836
|
-
const md =
|
|
9292
|
+
const md = fs16.readFileSync(path17.join(targetPath, "SKILL.md"), "utf-8");
|
|
8837
9293
|
return md.includes(PACKAGE_NAME);
|
|
8838
9294
|
} catch {
|
|
8839
9295
|
return false;
|
|
@@ -8846,12 +9302,12 @@ function detectHarnesses(opts) {
|
|
|
8846
9302
|
const absSkillPath = path17.join(absSkillsDir, SKILL_NAME);
|
|
8847
9303
|
let present = false;
|
|
8848
9304
|
try {
|
|
8849
|
-
present =
|
|
9305
|
+
present = fs16.existsSync(path17.join(home, h.baseDir));
|
|
8850
9306
|
} catch {
|
|
8851
9307
|
}
|
|
8852
9308
|
let installed = "none";
|
|
8853
9309
|
try {
|
|
8854
|
-
if (
|
|
9310
|
+
if (fs16.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
|
|
8855
9311
|
if (isOwnedSymlink(absSkillPath)) installed = "owned-symlink";
|
|
8856
9312
|
else if (isOwnedCopy(absSkillPath)) installed = "owned-copy";
|
|
8857
9313
|
else installed = "foreign";
|
|
@@ -8863,7 +9319,7 @@ function detectHarnesses(opts) {
|
|
|
8863
9319
|
}
|
|
8864
9320
|
function isSymlinkPresent(p) {
|
|
8865
9321
|
try {
|
|
8866
|
-
return
|
|
9322
|
+
return fs16.lstatSync(p).isSymbolicLink();
|
|
8867
9323
|
} catch {
|
|
8868
9324
|
return false;
|
|
8869
9325
|
}
|
|
@@ -8880,6 +9336,7 @@ function installSkill(toolIds, opts = {}) {
|
|
|
8880
9336
|
const manifest = readManifest(opts);
|
|
8881
9337
|
const results = [];
|
|
8882
9338
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9339
|
+
const skillVersion = readSkillVersion(source);
|
|
8883
9340
|
for (const id of toolIds) {
|
|
8884
9341
|
const def = HARNESSES.find((h) => h.id === id);
|
|
8885
9342
|
if (!def) {
|
|
@@ -8891,13 +9348,17 @@ function installSkill(toolIds, opts = {}) {
|
|
|
8891
9348
|
const type = opts.copy ? "copy" : "symlink";
|
|
8892
9349
|
if (isOwnedSymlink(absSkillPath, source) || isOwnedCopy(absSkillPath)) {
|
|
8893
9350
|
try {
|
|
8894
|
-
const lst =
|
|
9351
|
+
const lst = fs16.lstatSync(absSkillPath);
|
|
8895
9352
|
if (lst.isSymbolicLink()) {
|
|
8896
|
-
const dest = path17.resolve(path17.dirname(absSkillPath),
|
|
9353
|
+
const dest = path17.resolve(path17.dirname(absSkillPath), fs16.readlinkSync(absSkillPath));
|
|
8897
9354
|
if (dest !== path17.resolve(source)) {
|
|
8898
|
-
|
|
8899
|
-
|
|
8900
|
-
|
|
9355
|
+
if (opts.dryRun) {
|
|
9356
|
+
results.push({ tool: id, path: absSkillPath, type: "symlink", status: "planned", message: "would re-point stale symlink to current source" });
|
|
9357
|
+
continue;
|
|
9358
|
+
}
|
|
9359
|
+
fs16.unlinkSync(absSkillPath);
|
|
9360
|
+
fs16.symlinkSync(source, absSkillPath, process.platform === "win32" ? "junction" : "dir");
|
|
9361
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: "symlink", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
|
|
8901
9362
|
results.push({ tool: id, path: absSkillPath, type: "symlink", status: "installed", message: "re-pointed stale symlink to current source" });
|
|
8902
9363
|
continue;
|
|
8903
9364
|
}
|
|
@@ -8905,10 +9366,10 @@ function installSkill(toolIds, opts = {}) {
|
|
|
8905
9366
|
} catch {
|
|
8906
9367
|
}
|
|
8907
9368
|
results.push({ tool: id, path: absSkillPath, type, status: "already-installed" });
|
|
8908
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? "symlink" : "copy", source, createdAt: now });
|
|
9369
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? "symlink" : "copy", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
|
|
8909
9370
|
continue;
|
|
8910
9371
|
}
|
|
8911
|
-
if (
|
|
9372
|
+
if (fs16.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
|
|
8912
9373
|
results.push({ tool: id, path: absSkillPath, type, status: "skipped-foreign", message: 'a non-pi-research "research" skill already exists here; left untouched' });
|
|
8913
9374
|
continue;
|
|
8914
9375
|
}
|
|
@@ -8917,20 +9378,20 @@ function installSkill(toolIds, opts = {}) {
|
|
|
8917
9378
|
continue;
|
|
8918
9379
|
}
|
|
8919
9380
|
try {
|
|
8920
|
-
|
|
9381
|
+
fs16.mkdirSync(absSkillsDir, { recursive: true });
|
|
8921
9382
|
if (type === "symlink") {
|
|
8922
9383
|
try {
|
|
8923
|
-
|
|
9384
|
+
fs16.symlinkSync(source, absSkillPath, process.platform === "win32" ? "junction" : "dir");
|
|
8924
9385
|
} catch (err) {
|
|
8925
9386
|
copyDir(source, absSkillPath);
|
|
8926
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type: "copy", source, createdAt: now });
|
|
9387
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: "copy", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
|
|
8927
9388
|
results.push({ tool: id, path: absSkillPath, type: "copy", status: "installed", message: `symlink unavailable (${err?.code ?? err?.message}); copied instead` });
|
|
8928
9389
|
continue;
|
|
8929
9390
|
}
|
|
8930
9391
|
} else {
|
|
8931
9392
|
copyDir(source, absSkillPath);
|
|
8932
9393
|
}
|
|
8933
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now });
|
|
9394
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
|
|
8934
9395
|
results.push({ tool: id, path: absSkillPath, type, status: "installed" });
|
|
8935
9396
|
} catch (err) {
|
|
8936
9397
|
results.push({ tool: id, path: absSkillPath, type, status: "error", message: err?.message ?? String(err) });
|
|
@@ -8957,7 +9418,7 @@ function uninstallSkill(toolIds, opts = {}) {
|
|
|
8957
9418
|
continue;
|
|
8958
9419
|
}
|
|
8959
9420
|
const owned = isOwnedSymlink(skillPath) || isOwnedCopy(skillPath);
|
|
8960
|
-
const present =
|
|
9421
|
+
const present = fs16.existsSync(skillPath) || isSymlinkPresent(skillPath);
|
|
8961
9422
|
if (!present) {
|
|
8962
9423
|
results.push({ tool, path: skillPath, status: "not-present" });
|
|
8963
9424
|
continue;
|
|
@@ -8975,8 +9436,8 @@ function uninstallSkill(toolIds, opts = {}) {
|
|
|
8975
9436
|
continue;
|
|
8976
9437
|
}
|
|
8977
9438
|
try {
|
|
8978
|
-
if (isSymlinkPresent(skillPath))
|
|
8979
|
-
else
|
|
9439
|
+
if (isSymlinkPresent(skillPath)) fs16.unlinkSync(skillPath);
|
|
9440
|
+
else fs16.rmSync(skillPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
8980
9441
|
results.push({ tool, path: skillPath, status: "removed" });
|
|
8981
9442
|
} catch (err) {
|
|
8982
9443
|
results.push({ tool, path: skillPath, status: "error", message: err?.message ?? String(err) });
|
|
@@ -8989,7 +9450,7 @@ function uninstallSkill(toolIds, opts = {}) {
|
|
|
8989
9450
|
writeManifest({ version: 1, package: PACKAGE_NAME, entries: keep }, opts);
|
|
8990
9451
|
} else {
|
|
8991
9452
|
try {
|
|
8992
|
-
|
|
9453
|
+
fs16.rmSync(getManifestPath(opts));
|
|
8993
9454
|
} catch {
|
|
8994
9455
|
}
|
|
8995
9456
|
}
|
|
@@ -8997,7 +9458,7 @@ function uninstallSkill(toolIds, opts = {}) {
|
|
|
8997
9458
|
return results;
|
|
8998
9459
|
}
|
|
8999
9460
|
function reconcileSkillInstalls(opts = {}) {
|
|
9000
|
-
const result = { pruned: [], repointed: [] };
|
|
9461
|
+
const result = { pruned: [], repointed: [], refreshed: [] };
|
|
9001
9462
|
const manifest = readManifest(opts);
|
|
9002
9463
|
if (manifest.entries.length === 0) return result;
|
|
9003
9464
|
let source;
|
|
@@ -9006,10 +9467,68 @@ function reconcileSkillInstalls(opts = {}) {
|
|
|
9006
9467
|
} catch {
|
|
9007
9468
|
source = null;
|
|
9008
9469
|
}
|
|
9470
|
+
const currentVersion = source !== null ? readSkillVersion(source) : null;
|
|
9009
9471
|
const keep = [];
|
|
9010
9472
|
for (const e of manifest.entries) {
|
|
9011
9473
|
if (e.type !== "symlink") {
|
|
9012
|
-
|
|
9474
|
+
const stale2 = currentVersion !== null && e.version !== currentVersion;
|
|
9475
|
+
if (stale2 && source !== null && fs16.existsSync(e.path) && isOwnedCopy(e.path)) {
|
|
9476
|
+
const dir = path17.dirname(e.path);
|
|
9477
|
+
const base = path17.basename(e.path);
|
|
9478
|
+
try {
|
|
9479
|
+
for (const name of fs16.readdirSync(dir)) {
|
|
9480
|
+
if (name.startsWith(`.${base}.new-`) || name.startsWith(`.${base}.old-`)) {
|
|
9481
|
+
fs16.rmSync(path17.join(dir, name), { recursive: true, force: true });
|
|
9482
|
+
}
|
|
9483
|
+
}
|
|
9484
|
+
} catch {
|
|
9485
|
+
}
|
|
9486
|
+
const suffix = crypto10.randomBytes(4).toString("hex");
|
|
9487
|
+
const fresh = path17.join(dir, `.${base}.new-${suffix}`);
|
|
9488
|
+
const retired = path17.join(dir, `.${base}.old-${suffix}`);
|
|
9489
|
+
try {
|
|
9490
|
+
copyDir(source, fresh);
|
|
9491
|
+
} catch {
|
|
9492
|
+
try {
|
|
9493
|
+
fs16.rmSync(fresh, { recursive: true, force: true });
|
|
9494
|
+
} catch {
|
|
9495
|
+
}
|
|
9496
|
+
keep.push(e);
|
|
9497
|
+
continue;
|
|
9498
|
+
}
|
|
9499
|
+
try {
|
|
9500
|
+
fs16.renameSync(e.path, retired);
|
|
9501
|
+
} catch {
|
|
9502
|
+
try {
|
|
9503
|
+
fs16.rmSync(fresh, { recursive: true, force: true });
|
|
9504
|
+
} catch {
|
|
9505
|
+
}
|
|
9506
|
+
keep.push(e);
|
|
9507
|
+
continue;
|
|
9508
|
+
}
|
|
9509
|
+
try {
|
|
9510
|
+
fs16.renameSync(fresh, e.path);
|
|
9511
|
+
} catch {
|
|
9512
|
+
try {
|
|
9513
|
+
fs16.renameSync(retired, e.path);
|
|
9514
|
+
} catch {
|
|
9515
|
+
}
|
|
9516
|
+
try {
|
|
9517
|
+
fs16.rmSync(fresh, { recursive: true, force: true });
|
|
9518
|
+
} catch {
|
|
9519
|
+
}
|
|
9520
|
+
keep.push(e);
|
|
9521
|
+
continue;
|
|
9522
|
+
}
|
|
9523
|
+
try {
|
|
9524
|
+
fs16.rmSync(retired, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
9525
|
+
} catch {
|
|
9526
|
+
}
|
|
9527
|
+
keep.push({ ...e, source, version: currentVersion });
|
|
9528
|
+
result.refreshed.push(e.path);
|
|
9529
|
+
} else {
|
|
9530
|
+
keep.push(e);
|
|
9531
|
+
}
|
|
9013
9532
|
continue;
|
|
9014
9533
|
}
|
|
9015
9534
|
if (!isSymlinkPresent(e.path)) {
|
|
@@ -9020,18 +9539,18 @@ function reconcileSkillInstalls(opts = {}) {
|
|
|
9020
9539
|
keep.push(e);
|
|
9021
9540
|
continue;
|
|
9022
9541
|
}
|
|
9023
|
-
const targetLive =
|
|
9542
|
+
const targetLive = fs16.existsSync(e.path);
|
|
9024
9543
|
let dest = null;
|
|
9025
9544
|
try {
|
|
9026
|
-
dest = path17.resolve(path17.dirname(e.path),
|
|
9545
|
+
dest = path17.resolve(path17.dirname(e.path), fs16.readlinkSync(e.path));
|
|
9027
9546
|
} catch {
|
|
9028
9547
|
}
|
|
9029
9548
|
const stale = source !== null && dest !== null && dest !== path17.resolve(source);
|
|
9030
9549
|
if (!targetLive || stale) {
|
|
9031
9550
|
if (source !== null) {
|
|
9032
9551
|
try {
|
|
9033
|
-
|
|
9034
|
-
|
|
9552
|
+
fs16.unlinkSync(e.path);
|
|
9553
|
+
fs16.symlinkSync(source, e.path, process.platform === "win32" ? "junction" : "dir");
|
|
9035
9554
|
keep.push({ ...e, type: "symlink", source });
|
|
9036
9555
|
result.repointed.push(e.path);
|
|
9037
9556
|
} catch {
|
|
@@ -9039,7 +9558,7 @@ function reconcileSkillInstalls(opts = {}) {
|
|
|
9039
9558
|
}
|
|
9040
9559
|
} else if (!targetLive) {
|
|
9041
9560
|
try {
|
|
9042
|
-
|
|
9561
|
+
fs16.unlinkSync(e.path);
|
|
9043
9562
|
result.pruned.push(e.path);
|
|
9044
9563
|
} catch {
|
|
9045
9564
|
keep.push(e);
|
|
@@ -9054,14 +9573,22 @@ function reconcileSkillInstalls(opts = {}) {
|
|
|
9054
9573
|
if (keep.length > 0) writeManifest({ version: 1, package: PACKAGE_NAME, entries: keep }, opts);
|
|
9055
9574
|
else {
|
|
9056
9575
|
try {
|
|
9057
|
-
|
|
9576
|
+
fs16.rmSync(getManifestPath(opts));
|
|
9058
9577
|
} catch {
|
|
9059
9578
|
}
|
|
9060
9579
|
}
|
|
9061
9580
|
return result;
|
|
9062
9581
|
}
|
|
9063
9582
|
function copyDir(src, dest) {
|
|
9064
|
-
|
|
9583
|
+
const st = fs16.statSync(src);
|
|
9584
|
+
if (st.isDirectory()) {
|
|
9585
|
+
fs16.mkdirSync(dest, { recursive: true });
|
|
9586
|
+
for (const name of fs16.readdirSync(src)) {
|
|
9587
|
+
copyDir(path17.join(src, name), path17.join(dest, name));
|
|
9588
|
+
}
|
|
9589
|
+
} else {
|
|
9590
|
+
fs16.copyFileSync(src, dest);
|
|
9591
|
+
}
|
|
9065
9592
|
}
|
|
9066
9593
|
var HARNESSES, SKILL_NAME, PACKAGE_NAME, SKILL_AGENT_TARGETS, isTarget;
|
|
9067
9594
|
var init_skill_installer = __esm({
|
|
@@ -9083,7 +9610,8 @@ var init_skill_installer = __esm({
|
|
|
9083
9610
|
});
|
|
9084
9611
|
|
|
9085
9612
|
// src/cli.ts
|
|
9086
|
-
import * as
|
|
9613
|
+
import * as fs17 from "node:fs";
|
|
9614
|
+
import * as os7 from "node:os";
|
|
9087
9615
|
import * as path18 from "node:path";
|
|
9088
9616
|
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
9089
9617
|
|
|
@@ -9635,19 +10163,19 @@ function rewriteCitationMarkers(text, remap, isDangling) {
|
|
|
9635
10163
|
}
|
|
9636
10164
|
function fencedCodeMask(text) {
|
|
9637
10165
|
const ranges = [];
|
|
9638
|
-
let
|
|
10166
|
+
let open5 = -1;
|
|
9639
10167
|
let pos = 0;
|
|
9640
10168
|
for (const line of text.split("\n")) {
|
|
9641
10169
|
if (/^\s{0,3}(```|~~~)/.test(line)) {
|
|
9642
|
-
if (
|
|
10170
|
+
if (open5 < 0) open5 = pos;
|
|
9643
10171
|
else {
|
|
9644
|
-
ranges.push([
|
|
9645
|
-
|
|
10172
|
+
ranges.push([open5, pos + line.length]);
|
|
10173
|
+
open5 = -1;
|
|
9646
10174
|
}
|
|
9647
10175
|
}
|
|
9648
10176
|
pos += line.length + 1;
|
|
9649
10177
|
}
|
|
9650
|
-
if (
|
|
10178
|
+
if (open5 >= 0) ranges.push([open5, text.length]);
|
|
9651
10179
|
return (offset) => ranges.some(([start, end]) => offset >= start && offset < end);
|
|
9652
10180
|
}
|
|
9653
10181
|
function inInlineCodeSpan(text, offset) {
|
|
@@ -10543,11 +11071,11 @@ import * as path7 from "node:path";
|
|
|
10543
11071
|
init_config();
|
|
10544
11072
|
init_logger();
|
|
10545
11073
|
import * as crypto2 from "node:crypto";
|
|
10546
|
-
import { join as
|
|
11074
|
+
import { join as join7 } from "node:path";
|
|
10547
11075
|
import { existsSync as existsSync5, mkdirSync as mkdirSync3 } from "node:fs";
|
|
10548
11076
|
import { platform, homedir as homedir2 } from "node:os";
|
|
10549
11077
|
function getWindowsCamoufoxDir() {
|
|
10550
|
-
return
|
|
11078
|
+
return join7(homedir2(), "AppData", "Local", "camoufox", "camoufox", "Cache");
|
|
10551
11079
|
}
|
|
10552
11080
|
function getBrowserCacheDir() {
|
|
10553
11081
|
if (process.env["PLAYWRIGHT_BROWSERS_PATH"]) {
|
|
@@ -10557,10 +11085,10 @@ function getBrowserCacheDir() {
|
|
|
10557
11085
|
if (osPlatform === "win32") {
|
|
10558
11086
|
return getWindowsCamoufoxDir();
|
|
10559
11087
|
} else if (osPlatform === "darwin") {
|
|
10560
|
-
return
|
|
11088
|
+
return join7(homedir2(), "Library", "Caches", "camoufox");
|
|
10561
11089
|
} else {
|
|
10562
|
-
const cacheHome = process.env["XDG_CACHE_HOME"] ||
|
|
10563
|
-
return
|
|
11090
|
+
const cacheHome = process.env["XDG_CACHE_HOME"] || join7(homedir2(), ".cache");
|
|
11091
|
+
return join7(cacheHome, "camoufox");
|
|
10564
11092
|
}
|
|
10565
11093
|
}
|
|
10566
11094
|
function getBrowserProfileDir(config) {
|
|
@@ -10569,8 +11097,8 @@ function getBrowserProfileDir(config) {
|
|
|
10569
11097
|
if (configured && configured.trim().length > 0) {
|
|
10570
11098
|
dir = configured;
|
|
10571
11099
|
} else {
|
|
10572
|
-
const cacheHome = process.env["XDG_CACHE_HOME"] ||
|
|
10573
|
-
dir =
|
|
11100
|
+
const cacheHome = process.env["XDG_CACHE_HOME"] || join7(homedir2(), ".cache");
|
|
11101
|
+
dir = join7(cacheHome, "pi-research", "profiles");
|
|
10574
11102
|
}
|
|
10575
11103
|
if (!existsSync5(dir)) {
|
|
10576
11104
|
try {
|
|
@@ -10623,10 +11151,10 @@ function getCamoufoxBinaryPath() {
|
|
|
10623
11151
|
if (osPlatform === "win32") {
|
|
10624
11152
|
return getWindowsCamoufoxDir();
|
|
10625
11153
|
} else if (osPlatform === "darwin") {
|
|
10626
|
-
return
|
|
11154
|
+
return join7(homedir2(), "Library", "Caches", "camoufox");
|
|
10627
11155
|
} else {
|
|
10628
|
-
const cacheHome = process.env["XDG_CACHE_HOME"] ||
|
|
10629
|
-
return
|
|
11156
|
+
const cacheHome = process.env["XDG_CACHE_HOME"] || join7(homedir2(), ".cache");
|
|
11157
|
+
return join7(cacheHome, "camoufox");
|
|
10630
11158
|
}
|
|
10631
11159
|
}
|
|
10632
11160
|
function generateSchedulerVersion(config) {
|
|
@@ -10664,6 +11192,7 @@ function isFullMockMode() {
|
|
|
10664
11192
|
// src/infrastructure/browser/browser-client.ts
|
|
10665
11193
|
init_logger();
|
|
10666
11194
|
init_error_tracker();
|
|
11195
|
+
init_http_body();
|
|
10667
11196
|
import * as http2 from "node:http";
|
|
10668
11197
|
|
|
10669
11198
|
// src/infrastructure/browser/client-agent.ts
|
|
@@ -10673,7 +11202,7 @@ var clientAgent = new http.Agent({
|
|
|
10673
11202
|
maxSockets: 100,
|
|
10674
11203
|
// Allow up to 100 concurrent requests to the leader
|
|
10675
11204
|
maxFreeSockets: 10,
|
|
10676
|
-
//
|
|
11205
|
+
// 180s (3x the client timeout) to handle slow browser responses
|
|
10677
11206
|
// and prevent "socket hang up" errors during peak load
|
|
10678
11207
|
timeout: 18e4
|
|
10679
11208
|
});
|
|
@@ -10690,6 +11219,12 @@ var BrowserClient = class {
|
|
|
10690
11219
|
}
|
|
10691
11220
|
port;
|
|
10692
11221
|
authSecret;
|
|
11222
|
+
/** The leader port this client was built for — lets the herd-guard cooldown
|
|
11223
|
+
* verify the cached handle still targets the REGISTERED leader, not a
|
|
11224
|
+
* predecessor that died inside the cooldown window. */
|
|
11225
|
+
get targetPort() {
|
|
11226
|
+
return this.port;
|
|
11227
|
+
}
|
|
10693
11228
|
async request(path19, data, signal) {
|
|
10694
11229
|
const start = Date.now();
|
|
10695
11230
|
const operation = path19.includes("/search") ? "search" : path19.includes("/scrape") ? "browser-task" : path19.includes("/healthcheck") ? "healthcheck" : "network";
|
|
@@ -10740,8 +11275,8 @@ var BrowserClient = class {
|
|
|
10740
11275
|
"X-Browser-Auth": this.authSecret
|
|
10741
11276
|
}
|
|
10742
11277
|
}, (res) => {
|
|
10743
|
-
|
|
10744
|
-
res.on("data", (chunk) =>
|
|
11278
|
+
const collected = new Utf8Body();
|
|
11279
|
+
res.on("data", (chunk) => collected.push(chunk));
|
|
10745
11280
|
res.on("end", () => {
|
|
10746
11281
|
if (resolved) return;
|
|
10747
11282
|
resolved = true;
|
|
@@ -10749,7 +11284,7 @@ var BrowserClient = class {
|
|
|
10749
11284
|
abortCleanup?.();
|
|
10750
11285
|
const duration = Date.now() - start;
|
|
10751
11286
|
try {
|
|
10752
|
-
const parsed = JSON.parse(
|
|
11287
|
+
const parsed = JSON.parse(collected.toString());
|
|
10753
11288
|
if (res.statusCode === 503 && parsed?.error === "draining") {
|
|
10754
11289
|
const error = new Error(
|
|
10755
11290
|
`Browser pool leader is draining (${parsed.reason || "leadership-lost"}) \u2014 re-elect and retry`
|
|
@@ -10774,7 +11309,8 @@ var BrowserClient = class {
|
|
|
10774
11309
|
resolve8(parsed);
|
|
10775
11310
|
}
|
|
10776
11311
|
} catch (_e) {
|
|
10777
|
-
const
|
|
11312
|
+
const raw = collected.toString();
|
|
11313
|
+
const preview = raw.length > 200 ? raw.slice(0, 200) + "..." : raw;
|
|
10778
11314
|
const error = new Error(`Failed to parse response (status ${res.statusCode}): ${preview}`);
|
|
10779
11315
|
errorTracker.trackError(error, {
|
|
10780
11316
|
component: "browser-manager",
|
|
@@ -10856,6 +11392,7 @@ init_service_interfaces();
|
|
|
10856
11392
|
|
|
10857
11393
|
// src/infrastructure/browser/browser-server.ts
|
|
10858
11394
|
init_logger();
|
|
11395
|
+
init_http_body();
|
|
10859
11396
|
import * as http3 from "node:http";
|
|
10860
11397
|
import * as crypto3 from "node:crypto";
|
|
10861
11398
|
|
|
@@ -10898,7 +11435,11 @@ function isPoolShutdownError(error) {
|
|
|
10898
11435
|
// classifier matched it, so the first attempt rethrew instead of retrying —
|
|
10899
11436
|
// and auto-recovery fires only after 3 consecutive worker failures, i.e.
|
|
10900
11437
|
// exactly when a burst of tasks is in flight and would all die together.
|
|
10901
|
-
err.message.includes("Worker pool is being reset"))
|
|
11438
|
+
err.message.includes("Worker pool is being reset") || // poolifier's wording once destroy() has COMPLETED (vs "destroying pool"
|
|
11439
|
+
// while it is in progress). A queued closure holding a pool captured before
|
|
11440
|
+
// an auto-recovery destroy dispatches into exactly this window; it is the
|
|
11441
|
+
// same transient swap the "being reset" string covers, not a hard fault.
|
|
11442
|
+
err.message.includes("Cannot execute a task on not started pool"));
|
|
10902
11443
|
}
|
|
10903
11444
|
var sessionCircuitBreakers = /* @__PURE__ */ new Map();
|
|
10904
11445
|
var MAX_SESSION_BREAKERS = 256;
|
|
@@ -11006,11 +11547,13 @@ var BrowserServer = class {
|
|
|
11006
11547
|
const expected = Buffer.from(getOrCreateAuthSecret(), "utf8");
|
|
11007
11548
|
const actual = Buffer.from(authHeader ?? "", "utf8");
|
|
11008
11549
|
if (actual.length !== expected.length || !crypto3.timingSafeEqual(actual, expected)) {
|
|
11550
|
+
req.resume();
|
|
11009
11551
|
res.writeHead(403, { "Content-Type": "application/json" });
|
|
11010
11552
|
res.end(JSON.stringify({ error: "Unauthorized" }));
|
|
11011
11553
|
return;
|
|
11012
11554
|
}
|
|
11013
11555
|
if (this.draining) {
|
|
11556
|
+
req.resume();
|
|
11014
11557
|
res.writeHead(503, {
|
|
11015
11558
|
"Content-Type": "application/json",
|
|
11016
11559
|
"Connection": "close"
|
|
@@ -11019,16 +11562,17 @@ var BrowserServer = class {
|
|
|
11019
11562
|
return;
|
|
11020
11563
|
}
|
|
11021
11564
|
if (req.method !== "POST") {
|
|
11565
|
+
req.resume();
|
|
11022
11566
|
res.writeHead(405);
|
|
11023
11567
|
res.end("Method Not Allowed");
|
|
11024
11568
|
return;
|
|
11025
11569
|
}
|
|
11026
|
-
|
|
11570
|
+
const collected = new Utf8Body();
|
|
11027
11571
|
let oversized = false;
|
|
11028
11572
|
const MAX_BODY_SIZE = 10 * 1024 * 1024;
|
|
11029
11573
|
req.on("data", (chunk) => {
|
|
11030
|
-
|
|
11031
|
-
if (
|
|
11574
|
+
collected.push(chunk);
|
|
11575
|
+
if (collected.byteLength > MAX_BODY_SIZE) {
|
|
11032
11576
|
oversized = true;
|
|
11033
11577
|
req.destroy(new Error("Payload too large"));
|
|
11034
11578
|
}
|
|
@@ -11045,7 +11589,7 @@ var BrowserServer = class {
|
|
|
11045
11589
|
});
|
|
11046
11590
|
req.on("end", async () => {
|
|
11047
11591
|
try {
|
|
11048
|
-
const data = JSON.parse(
|
|
11592
|
+
const data = JSON.parse(collected.toString());
|
|
11049
11593
|
let result;
|
|
11050
11594
|
switch (req.url) {
|
|
11051
11595
|
case "/search":
|
|
@@ -11424,10 +11968,10 @@ var PriorityTaskQueue = class {
|
|
|
11424
11968
|
this.activeCount++;
|
|
11425
11969
|
logger.debug(`[PriorityQueue] Starting task: ${task.type}. Active: ${this.activeCount}/${this.maxTotalConcurrency}`);
|
|
11426
11970
|
let onAbort;
|
|
11427
|
-
const fnPromise = task.fn();
|
|
11428
|
-
fnPromise.catch(() => {
|
|
11429
|
-
});
|
|
11430
11971
|
try {
|
|
11972
|
+
const fnPromise = task.fn();
|
|
11973
|
+
fnPromise.catch(() => {
|
|
11974
|
+
});
|
|
11431
11975
|
let result;
|
|
11432
11976
|
if (task.signal) {
|
|
11433
11977
|
const abortPromise = new Promise((_, rej) => {
|
|
@@ -11493,6 +12037,25 @@ var PriorityTaskQueue = class {
|
|
|
11493
12037
|
};
|
|
11494
12038
|
|
|
11495
12039
|
// src/infrastructure/browser/browser-task-scheduler.ts
|
|
12040
|
+
var WORKER_DEATH_BACKSTOP_GRACE_MS = 5e3;
|
|
12041
|
+
async function executePoolTaskWithDeathBackstop(execute, taskTimeoutMs, label) {
|
|
12042
|
+
const backstopMs = taskTimeoutMs + WORKER_DEATH_BACKSTOP_GRACE_MS;
|
|
12043
|
+
let backstopId;
|
|
12044
|
+
try {
|
|
12045
|
+
return await Promise.race([
|
|
12046
|
+
execute(),
|
|
12047
|
+
new Promise((_, reject) => {
|
|
12048
|
+
backstopId = setTimeout(
|
|
12049
|
+
() => reject(new Error(`${label} task unanswered after ${backstopMs}ms \u2014 worker likely died mid-task`)),
|
|
12050
|
+
backstopMs
|
|
12051
|
+
);
|
|
12052
|
+
if (backstopId.unref) backstopId.unref();
|
|
12053
|
+
})
|
|
12054
|
+
]);
|
|
12055
|
+
} finally {
|
|
12056
|
+
if (backstopId) clearTimeout(backstopId);
|
|
12057
|
+
}
|
|
12058
|
+
}
|
|
11496
12059
|
var BrowserTaskScheduler = class {
|
|
11497
12060
|
// 5 minutes (was 30min — wasted RAM)
|
|
11498
12061
|
constructor(schedulerId, stateManager, container = getServiceContainer()) {
|
|
@@ -11620,15 +12183,18 @@ var BrowserTaskScheduler = class {
|
|
|
11620
12183
|
});
|
|
11621
12184
|
logger.debug(`[BrowserTaskScheduler] Executing search: "${query}" (Timeout: ${timeoutMs}ms)`);
|
|
11622
12185
|
let result;
|
|
12186
|
+
const settled = new AbortController();
|
|
12187
|
+
const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
|
|
11623
12188
|
try {
|
|
11624
12189
|
const queue = this.getPriorityQueue(config);
|
|
11625
12190
|
result = await Promise.race([
|
|
11626
12191
|
queue.enqueue("search", async () => {
|
|
11627
|
-
return await
|
|
11628
|
-
pool.execute({ type: "search", query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
|
|
11629
|
-
|
|
11630
|
-
|
|
11631
|
-
|
|
12192
|
+
return await executePoolTaskWithDeathBackstop(
|
|
12193
|
+
() => pool.execute({ type: "search", query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
|
|
12194
|
+
timeoutMs,
|
|
12195
|
+
"Search"
|
|
12196
|
+
);
|
|
12197
|
+
}, queueSignal),
|
|
11632
12198
|
timeoutPromise
|
|
11633
12199
|
]);
|
|
11634
12200
|
logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${Date.now() - startTime}ms`);
|
|
@@ -11648,6 +12214,7 @@ var BrowserTaskScheduler = class {
|
|
|
11648
12214
|
throw error;
|
|
11649
12215
|
} finally {
|
|
11650
12216
|
if (timeoutId) clearTimeout(timeoutId);
|
|
12217
|
+
settled.abort();
|
|
11651
12218
|
}
|
|
11652
12219
|
const duration = Date.now() - startTime;
|
|
11653
12220
|
metrics.observe("browser_search_duration_ms", duration, { status: "success" });
|
|
@@ -11681,15 +12248,18 @@ var BrowserTaskScheduler = class {
|
|
|
11681
12248
|
if (timeoutId.unref) timeoutId.unref();
|
|
11682
12249
|
});
|
|
11683
12250
|
let result;
|
|
12251
|
+
const settled = new AbortController();
|
|
12252
|
+
const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
|
|
11684
12253
|
try {
|
|
11685
12254
|
const queue = this.getPriorityQueue(config);
|
|
11686
12255
|
result = await Promise.race([
|
|
11687
12256
|
queue.enqueue("scrape", async () => {
|
|
11688
|
-
return await
|
|
11689
|
-
pool.execute({ type: "scrape", url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
|
|
11690
|
-
|
|
11691
|
-
|
|
11692
|
-
|
|
12257
|
+
return await executePoolTaskWithDeathBackstop(
|
|
12258
|
+
() => pool.execute({ type: "scrape", url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
|
|
12259
|
+
timeoutMs,
|
|
12260
|
+
"Scrape"
|
|
12261
|
+
);
|
|
12262
|
+
}, queueSignal),
|
|
11693
12263
|
timeoutPromise
|
|
11694
12264
|
]);
|
|
11695
12265
|
} catch (error) {
|
|
@@ -11706,6 +12276,7 @@ var BrowserTaskScheduler = class {
|
|
|
11706
12276
|
throw error;
|
|
11707
12277
|
} finally {
|
|
11708
12278
|
if (timeoutId) clearTimeout(timeoutId);
|
|
12279
|
+
settled.abort();
|
|
11709
12280
|
}
|
|
11710
12281
|
const duration = Date.now() - startTime;
|
|
11711
12282
|
metrics.observe("browser_scrape_duration_ms", duration, { status: "success" });
|
|
@@ -11735,17 +12306,18 @@ var BrowserTaskScheduler = class {
|
|
|
11735
12306
|
if (timeoutId.unref) timeoutId.unref();
|
|
11736
12307
|
});
|
|
11737
12308
|
let result;
|
|
12309
|
+
const settled = new AbortController();
|
|
12310
|
+
const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
|
|
11738
12311
|
try {
|
|
11739
12312
|
const queue = this.getPriorityQueue(config);
|
|
11740
12313
|
result = await Promise.race([
|
|
11741
12314
|
queue.enqueue("healthcheck", async () => {
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
|
|
11745
|
-
execPromise
|
|
11746
|
-
|
|
11747
|
-
|
|
11748
|
-
}, signal),
|
|
12315
|
+
return await executePoolTaskWithDeathBackstop(() => {
|
|
12316
|
+
const execPromise = pool.execute({ type: "healthcheck", queuedAt: startTime, taskTimeoutMs: timeoutMs });
|
|
12317
|
+
execPromise.catch((err) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
|
|
12318
|
+
return execPromise;
|
|
12319
|
+
}, timeoutMs, "Healthcheck");
|
|
12320
|
+
}, queueSignal),
|
|
11749
12321
|
timeoutPromise
|
|
11750
12322
|
]);
|
|
11751
12323
|
} catch (error) {
|
|
@@ -11762,6 +12334,7 @@ var BrowserTaskScheduler = class {
|
|
|
11762
12334
|
throw error;
|
|
11763
12335
|
} finally {
|
|
11764
12336
|
if (timeoutId) clearTimeout(timeoutId);
|
|
12337
|
+
settled.abort();
|
|
11765
12338
|
}
|
|
11766
12339
|
const duration = Date.now() - startTime;
|
|
11767
12340
|
metrics.observe("browser_healthcheck_duration_ms", duration, { status: "success" });
|
|
@@ -11802,12 +12375,16 @@ var BrowserTaskScheduler = class {
|
|
|
11802
12375
|
this.leadershipTimer = null;
|
|
11803
12376
|
}
|
|
11804
12377
|
if (!this.container.isDisposing) {
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
11810
|
-
|
|
12378
|
+
try {
|
|
12379
|
+
const schedulerService = await getService(ServiceNames.SCHEDULER, void 0, this.container);
|
|
12380
|
+
const currentScheduler = schedulerService.getSchedulerInstance();
|
|
12381
|
+
if (currentScheduler && "schedulerId" in currentScheduler && currentScheduler.schedulerId === this.schedulerId) {
|
|
12382
|
+
schedulerService.setSchedulerInstance(null);
|
|
12383
|
+
schedulerService.setSchedulerVersion(null);
|
|
12384
|
+
schedulerService.setSchedulerInitializationPromise(null);
|
|
12385
|
+
}
|
|
12386
|
+
} catch (err) {
|
|
12387
|
+
logger.debug("[Scheduler] Could not clear scheduler service reference during shutdown:", err);
|
|
11811
12388
|
}
|
|
11812
12389
|
}
|
|
11813
12390
|
let serverInfo = null;
|
|
@@ -12053,6 +12630,15 @@ async function getScheduler(config, container = getServiceContainer()) {
|
|
|
12053
12630
|
let wonElection = false;
|
|
12054
12631
|
let winnerPort = port;
|
|
12055
12632
|
let winnerAuthSecret;
|
|
12633
|
+
const myStartTime = await (async () => {
|
|
12634
|
+
try {
|
|
12635
|
+
const processLifecycle = await getService(ServiceNames.PROCESS_LIFECYCLE, void 0, container);
|
|
12636
|
+
return await processLifecycle.getCurrentProcessStartTime();
|
|
12637
|
+
} catch (err) {
|
|
12638
|
+
logger.debug("[Scheduler] Could not read own process start time (PID-reuse guard degrades to signal-0):", err);
|
|
12639
|
+
return null;
|
|
12640
|
+
}
|
|
12641
|
+
})();
|
|
12056
12642
|
try {
|
|
12057
12643
|
await stateManager.updateState(async (state) => {
|
|
12058
12644
|
if (state.browserServer) {
|
|
@@ -12065,7 +12651,13 @@ async function getScheduler(config, container = getServiceContainer()) {
|
|
|
12065
12651
|
}
|
|
12066
12652
|
}
|
|
12067
12653
|
const secret = getBrowserServerAuthSecret();
|
|
12068
|
-
state.browserServer = {
|
|
12654
|
+
state.browserServer = {
|
|
12655
|
+
port,
|
|
12656
|
+
pid: process.pid,
|
|
12657
|
+
schedulerId,
|
|
12658
|
+
authSecret: secret,
|
|
12659
|
+
...myStartTime !== null ? { startTime: myStartTime } : {}
|
|
12660
|
+
};
|
|
12069
12661
|
state.schedulerVersion = schedulerVersion;
|
|
12070
12662
|
wonElection = true;
|
|
12071
12663
|
return state;
|
|
@@ -12163,8 +12755,9 @@ init_service_interfaces();
|
|
|
12163
12755
|
// src/infrastructure/state/state-manager.ts
|
|
12164
12756
|
init_config();
|
|
12165
12757
|
init_logger();
|
|
12758
|
+
init_atomic_replace();
|
|
12166
12759
|
init_metrics();
|
|
12167
|
-
import * as
|
|
12760
|
+
import * as fs8 from "node:fs/promises";
|
|
12168
12761
|
import * as path8 from "node:path";
|
|
12169
12762
|
import * as crypto5 from "node:crypto";
|
|
12170
12763
|
import process2 from "node:process";
|
|
@@ -12215,7 +12808,16 @@ var SingletonStateSchema = Type3.Object({
|
|
|
12215
12808
|
// cross-embed and corrupt the vector space) — it falls back to a local
|
|
12216
12809
|
// in-process embedder instead. Optional for entries written by older
|
|
12217
12810
|
// processes, which are tolerated as matching.
|
|
12218
|
-
model: Type3.Optional(Type3.String())
|
|
12811
|
+
model: Type3.Optional(Type3.String()),
|
|
12812
|
+
// Shared secret a follower must present on every embed request. Mirrors
|
|
12813
|
+
// browserServer.authSecret: the server listens on loopback, but loopback is
|
|
12814
|
+
// not an authorization boundary — any local process could otherwise enqueue
|
|
12815
|
+
// embedding work, and because the serial queue steps the leader down when
|
|
12816
|
+
// poisoned, sustained abuse denies the knowledge store rather than merely
|
|
12817
|
+
// burning GPU. Optional so an entry written by an older leader still parses;
|
|
12818
|
+
// a client that finds none simply sends no header, and such a leader does not
|
|
12819
|
+
// check one either.
|
|
12820
|
+
authSecret: Type3.Optional(Type3.String())
|
|
12219
12821
|
}))
|
|
12220
12822
|
});
|
|
12221
12823
|
|
|
@@ -12432,7 +13034,7 @@ var StateManager = class {
|
|
|
12432
13034
|
];
|
|
12433
13035
|
for (const dir of dirs) {
|
|
12434
13036
|
try {
|
|
12435
|
-
await
|
|
13037
|
+
await fs8.mkdir(dir, { recursive: true, mode: 448 });
|
|
12436
13038
|
} catch (err) {
|
|
12437
13039
|
logger.error(`[StateManager] Failed to create directory ${dir}:`, err);
|
|
12438
13040
|
throw err;
|
|
@@ -12462,14 +13064,14 @@ var StateManager = class {
|
|
|
12462
13064
|
const stateDir = path8.dirname(this.stateFilePath);
|
|
12463
13065
|
const cutoff = Date.now() - STALE_TMP_MS;
|
|
12464
13066
|
try {
|
|
12465
|
-
const entries = await
|
|
13067
|
+
const entries = await fs8.readdir(stateDir);
|
|
12466
13068
|
for (const entry of entries) {
|
|
12467
13069
|
if (!entry.startsWith("research-state-") || !entry.endsWith(".tmp")) continue;
|
|
12468
13070
|
const filePath = path8.join(stateDir, entry);
|
|
12469
13071
|
try {
|
|
12470
|
-
const stats = await
|
|
13072
|
+
const stats = await fs8.stat(filePath);
|
|
12471
13073
|
if (!stats.isFile() || stats.mtimeMs >= cutoff) continue;
|
|
12472
|
-
await
|
|
13074
|
+
await fs8.unlink(filePath);
|
|
12473
13075
|
logger.debug(`[StateManager] Removed orphaned state temp file: ${entry}`);
|
|
12474
13076
|
} catch {
|
|
12475
13077
|
}
|
|
@@ -12500,7 +13102,7 @@ var StateManager = class {
|
|
|
12500
13102
|
return this.getDefaultState();
|
|
12501
13103
|
}
|
|
12502
13104
|
try {
|
|
12503
|
-
const content = await
|
|
13105
|
+
const content = await fs8.readFile(this.stateFilePath, "utf-8");
|
|
12504
13106
|
const state = JSON.parse(content);
|
|
12505
13107
|
return this.validator.migrateAndValidate(state);
|
|
12506
13108
|
} catch (error) {
|
|
@@ -12529,7 +13131,7 @@ var StateManager = class {
|
|
|
12529
13131
|
);
|
|
12530
13132
|
try {
|
|
12531
13133
|
await this.recoverFromCorruptionDirect();
|
|
12532
|
-
const recovered = await
|
|
13134
|
+
const recovered = await fs8.readFile(this.stateFilePath, "utf-8");
|
|
12533
13135
|
return this.validator.migrateAndValidate(JSON.parse(recovered));
|
|
12534
13136
|
} catch (recoveryError) {
|
|
12535
13137
|
logger.error(
|
|
@@ -12590,26 +13192,24 @@ var StateManager = class {
|
|
|
12590
13192
|
const tempFileName = `research-state-${crypto5.randomBytes(16).toString("hex")}.tmp`;
|
|
12591
13193
|
tempFilePath = path8.join(path8.dirname(this.stateFilePath), tempFileName);
|
|
12592
13194
|
const content = JSON.stringify(state, null, 2);
|
|
12593
|
-
const fh = await
|
|
13195
|
+
const fh = await fs8.open(tempFilePath, "w", 384);
|
|
12594
13196
|
try {
|
|
12595
13197
|
await fh.writeFile(content, { encoding: "utf-8" });
|
|
12596
13198
|
await fh.sync();
|
|
12597
13199
|
} finally {
|
|
12598
13200
|
await fh.close();
|
|
12599
13201
|
}
|
|
12600
|
-
|
|
12601
|
-
|
|
12602
|
-
|
|
12603
|
-
|
|
12604
|
-
|
|
12605
|
-
|
|
12606
|
-
|
|
12607
|
-
|
|
12608
|
-
|
|
12609
|
-
|
|
12610
|
-
}
|
|
12611
|
-
throw renameErr;
|
|
12612
|
-
}
|
|
13202
|
+
const outcome = await replaceFile(tempFilePath, this.stateFilePath);
|
|
13203
|
+
if (outcome === "copied") {
|
|
13204
|
+
logger.warn(
|
|
13205
|
+
"[StateManager] State written via non-atomic copy: the rename was blocked by another process holding the state file. A concurrent reader may observe a partial file."
|
|
13206
|
+
);
|
|
13207
|
+
metrics.increment("state_write_non_atomic_total", 1);
|
|
13208
|
+
const tmp = tempFilePath;
|
|
13209
|
+
tempFilePath = null;
|
|
13210
|
+
await fs8.unlink(tmp).catch((err) => {
|
|
13211
|
+
logger.warn("[StateManager] Failed to remove temp file after copy fallback (write succeeded):", err);
|
|
13212
|
+
});
|
|
12613
13213
|
}
|
|
12614
13214
|
tempFilePath = null;
|
|
12615
13215
|
await this.backupManager.cleanupOldBackups();
|
|
@@ -12617,7 +13217,7 @@ var StateManager = class {
|
|
|
12617
13217
|
throw new Error(`Failed to write state: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
12618
13218
|
} finally {
|
|
12619
13219
|
if (tempFilePath) {
|
|
12620
|
-
await
|
|
13220
|
+
await fs8.unlink(tempFilePath).catch((err) => {
|
|
12621
13221
|
logger.warn("[StateManager] Failed to clean up temp file:", err);
|
|
12622
13222
|
});
|
|
12623
13223
|
}
|
|
@@ -13047,8 +13647,23 @@ var MetricsService = class {
|
|
|
13047
13647
|
// src/infrastructure/process-lifecycle-service.ts
|
|
13048
13648
|
init_service_registry();
|
|
13049
13649
|
init_service_names();
|
|
13050
|
-
import * as
|
|
13650
|
+
import * as fs9 from "node:fs/promises";
|
|
13051
13651
|
import { execFile } from "node:child_process";
|
|
13652
|
+
function parseEtime(value) {
|
|
13653
|
+
const m = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/.exec(value.trim());
|
|
13654
|
+
if (!m) return null;
|
|
13655
|
+
const days = m[1] ? parseInt(m[1], 10) : 0;
|
|
13656
|
+
const hours = m[2] ? parseInt(m[2], 10) : 0;
|
|
13657
|
+
const minutes = parseInt(m[3], 10);
|
|
13658
|
+
const seconds = parseInt(m[4], 10);
|
|
13659
|
+
if ([days, hours, minutes, seconds].some((n) => isNaN(n))) return null;
|
|
13660
|
+
return days * 86400 + hours * 3600 + minutes * 60 + seconds;
|
|
13661
|
+
}
|
|
13662
|
+
function isDefinitelyGone(err) {
|
|
13663
|
+
const code = err?.code;
|
|
13664
|
+
if (code === "EPERM" || code === "EACCES") return false;
|
|
13665
|
+
return true;
|
|
13666
|
+
}
|
|
13052
13667
|
var ProcessLifecycleService = class {
|
|
13053
13668
|
name = ServiceNames.PROCESS_LIFECYCLE;
|
|
13054
13669
|
lifecycle = "uninitialized" /* UNINITIALIZED */;
|
|
@@ -13069,7 +13684,7 @@ var ProcessLifecycleService = class {
|
|
|
13069
13684
|
async getProcessStartTime(pid) {
|
|
13070
13685
|
if (process.platform === "linux") {
|
|
13071
13686
|
try {
|
|
13072
|
-
const stat5 = await
|
|
13687
|
+
const stat5 = await fs9.readFile(`/proc/${pid}/stat`, "utf8");
|
|
13073
13688
|
const lastParenIndex = stat5.lastIndexOf(")");
|
|
13074
13689
|
if (lastParenIndex === -1) return null;
|
|
13075
13690
|
const partsAfterName = stat5.substring(lastParenIndex + 2).trim().split(/\s+/);
|
|
@@ -13097,23 +13712,35 @@ var ProcessLifecycleService = class {
|
|
|
13097
13712
|
return null;
|
|
13098
13713
|
}
|
|
13099
13714
|
}
|
|
13100
|
-
|
|
13101
|
-
|
|
13102
|
-
|
|
13103
|
-
|
|
13104
|
-
|
|
13105
|
-
|
|
13106
|
-
|
|
13107
|
-
|
|
13108
|
-
|
|
13109
|
-
|
|
13110
|
-
|
|
13111
|
-
|
|
13112
|
-
|
|
13113
|
-
|
|
13114
|
-
|
|
13115
|
-
|
|
13715
|
+
const elapsedSec = await this.readElapsedSeconds(pid);
|
|
13716
|
+
if (elapsedSec === null) return null;
|
|
13717
|
+
return Math.floor(Date.now() / 1e3) - elapsedSec;
|
|
13718
|
+
}
|
|
13719
|
+
/** Elapsed seconds for `pid` via ps, or null. Tries `etimes`, then POSIX `etime`. */
|
|
13720
|
+
async readElapsedSeconds(pid) {
|
|
13721
|
+
const run = async (fmt) => {
|
|
13722
|
+
try {
|
|
13723
|
+
return await new Promise((resolve8, reject) => {
|
|
13724
|
+
const child = execFile(
|
|
13725
|
+
"ps",
|
|
13726
|
+
["-o", fmt, "-p", String(pid)],
|
|
13727
|
+
{ encoding: "utf8", timeout: 3e3 },
|
|
13728
|
+
(err, stdout) => err ? reject(err) : resolve8(stdout)
|
|
13729
|
+
);
|
|
13730
|
+
child.unref?.();
|
|
13731
|
+
});
|
|
13732
|
+
} catch {
|
|
13733
|
+
return null;
|
|
13734
|
+
}
|
|
13735
|
+
};
|
|
13736
|
+
const etimes = await run("etimes=");
|
|
13737
|
+
if (etimes && etimes.trim()) {
|
|
13738
|
+
const n = parseInt(etimes.trim(), 10);
|
|
13739
|
+
if (!isNaN(n)) return n;
|
|
13116
13740
|
}
|
|
13741
|
+
const etime = await run("etime=");
|
|
13742
|
+
if (etime && etime.trim()) return parseEtime(etime.trim());
|
|
13743
|
+
return null;
|
|
13117
13744
|
}
|
|
13118
13745
|
/**
|
|
13119
13746
|
* Get the start time for the current process
|
|
@@ -13134,17 +13761,25 @@ var ProcessLifecycleService = class {
|
|
|
13134
13761
|
async isProcessAlive(pid, expectedStartTime) {
|
|
13135
13762
|
try {
|
|
13136
13763
|
process.kill(pid, 0);
|
|
13764
|
+
} catch (err) {
|
|
13765
|
+
if (isDefinitelyGone(err)) return false;
|
|
13766
|
+
}
|
|
13767
|
+
try {
|
|
13137
13768
|
if (expectedStartTime !== void 0 && expectedStartTime !== null) {
|
|
13138
13769
|
const actualStartTime = await this.getProcessStartTime(pid);
|
|
13139
13770
|
if (actualStartTime === null) {
|
|
13140
|
-
|
|
13141
|
-
|
|
13771
|
+
try {
|
|
13772
|
+
process.kill(pid, 0);
|
|
13773
|
+
return true;
|
|
13774
|
+
} catch (reconfirmErr) {
|
|
13775
|
+
return !isDefinitelyGone(reconfirmErr);
|
|
13776
|
+
}
|
|
13142
13777
|
}
|
|
13143
13778
|
return Math.abs(actualStartTime - expectedStartTime) <= 1;
|
|
13144
13779
|
}
|
|
13145
13780
|
return true;
|
|
13146
13781
|
} catch {
|
|
13147
|
-
return
|
|
13782
|
+
return true;
|
|
13148
13783
|
}
|
|
13149
13784
|
}
|
|
13150
13785
|
/**
|
|
@@ -13154,8 +13789,8 @@ var ProcessLifecycleService = class {
|
|
|
13154
13789
|
try {
|
|
13155
13790
|
process.kill(pid, 0);
|
|
13156
13791
|
return true;
|
|
13157
|
-
} catch {
|
|
13158
|
-
return
|
|
13792
|
+
} catch (err) {
|
|
13793
|
+
return !isDefinitelyGone(err);
|
|
13159
13794
|
}
|
|
13160
13795
|
}
|
|
13161
13796
|
/**
|
|
@@ -13652,7 +14287,7 @@ var StateValidator = class {
|
|
|
13652
14287
|
init_logger();
|
|
13653
14288
|
init_metrics();
|
|
13654
14289
|
init_config();
|
|
13655
|
-
import { dirname as dirname7, join as
|
|
14290
|
+
import { dirname as dirname7, join as join12 } from "node:path";
|
|
13656
14291
|
import { existsSync as existsSync7 } from "node:fs";
|
|
13657
14292
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
13658
14293
|
import { FixedClusterPool, WorkerChoiceStrategies } from "poolifier";
|
|
@@ -13663,7 +14298,7 @@ import { spawn } from "node:child_process";
|
|
|
13663
14298
|
import { createRequire as createRequire3 } from "node:module";
|
|
13664
14299
|
import { createHash as createHash2 } from "node:crypto";
|
|
13665
14300
|
import { existsSync as existsSync6, readdirSync as readdirSync2, statSync as statSync3, fstatSync, openSync as openSync3, closeSync as closeSync3, rmSync } from "node:fs";
|
|
13666
|
-
import { join as
|
|
14301
|
+
import { join as join10, dirname as dirname6 } from "node:path";
|
|
13667
14302
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
13668
14303
|
var FETCH_TIMEOUT_MS = 15 * 60 * 1e3;
|
|
13669
14304
|
var STALE_LOCK_MS = 20 * 60 * 1e3;
|
|
@@ -13676,7 +14311,7 @@ function isBrowserBinaryPresent() {
|
|
|
13676
14311
|
if (!existsSync6(cacheDir)) return false;
|
|
13677
14312
|
return readdirSync2(cacheDir).some((entry) => {
|
|
13678
14313
|
try {
|
|
13679
|
-
return statSync3(
|
|
14314
|
+
return statSync3(join10(cacheDir, entry)).isDirectory();
|
|
13680
14315
|
} catch {
|
|
13681
14316
|
return false;
|
|
13682
14317
|
}
|
|
@@ -13689,7 +14324,7 @@ function resolveCamoufoxBin() {
|
|
|
13689
14324
|
try {
|
|
13690
14325
|
const require2 = createRequire3(import.meta.url);
|
|
13691
14326
|
const pkgJson = require2.resolve("camoufox-js/package.json");
|
|
13692
|
-
const bin =
|
|
14327
|
+
const bin = join10(dirname6(pkgJson), "..", ".bin", process.platform === "win32" ? "camoufox-js.cmd" : "camoufox-js");
|
|
13693
14328
|
if (existsSync6(bin)) return bin;
|
|
13694
14329
|
} catch {
|
|
13695
14330
|
}
|
|
@@ -13733,7 +14368,7 @@ async function provision() {
|
|
|
13733
14368
|
return;
|
|
13734
14369
|
}
|
|
13735
14370
|
const cacheKey = createHash2("sha256").update(getCamoufoxBinaryPath()).digest("hex").slice(0, 16);
|
|
13736
|
-
const lockPath =
|
|
14371
|
+
const lockPath = join10(tmpdir2(), `pi-research-camoufox-fetch-${cacheKey}.lock`);
|
|
13737
14372
|
let haveLock = false;
|
|
13738
14373
|
try {
|
|
13739
14374
|
closeSync3(openSync3(lockPath, "wx", 384));
|
|
@@ -13788,12 +14423,26 @@ function ensureBrowserInstalled() {
|
|
|
13788
14423
|
|
|
13789
14424
|
// src/infrastructure/browser/cleanup-utils.ts
|
|
13790
14425
|
init_logger();
|
|
13791
|
-
import * as
|
|
14426
|
+
import * as fs10 from "node:fs/promises";
|
|
13792
14427
|
import * as path9 from "node:path";
|
|
13793
14428
|
import * as os4 from "node:os";
|
|
13794
14429
|
var DEFAULT_STALE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
13795
14430
|
var LOCK_FILE_NAMES = ["lock", ".parentlock", "parent.lock"];
|
|
13796
14431
|
var LOCK_FRESHNESS_MS = 2 * 60 * 1e3;
|
|
14432
|
+
async function isWindowsParentLockHeld(lockPath) {
|
|
14433
|
+
let handle;
|
|
14434
|
+
try {
|
|
14435
|
+
handle = await fs10.open(lockPath, "r");
|
|
14436
|
+
} catch (e) {
|
|
14437
|
+
const code = e?.code;
|
|
14438
|
+
if (code === "ENOENT") return false;
|
|
14439
|
+
if (code === "EBUSY" || code === "EPERM" || code === "EACCES") return true;
|
|
14440
|
+
return null;
|
|
14441
|
+
}
|
|
14442
|
+
await handle.close().catch(() => {
|
|
14443
|
+
});
|
|
14444
|
+
return false;
|
|
14445
|
+
}
|
|
13797
14446
|
function parsePidFromLockTarget(target) {
|
|
13798
14447
|
const m = /\+(\d+)\s*$/.exec(target);
|
|
13799
14448
|
if (!m) return null;
|
|
@@ -13813,17 +14462,22 @@ async function isProfileLive(profileDir) {
|
|
|
13813
14462
|
const lockPath = path9.join(profileDir, name);
|
|
13814
14463
|
let lst;
|
|
13815
14464
|
try {
|
|
13816
|
-
lst = await
|
|
14465
|
+
lst = await fs10.lstat(lockPath);
|
|
13817
14466
|
} catch {
|
|
13818
14467
|
continue;
|
|
13819
14468
|
}
|
|
13820
14469
|
if (lst.isSymbolicLink()) {
|
|
13821
14470
|
try {
|
|
13822
|
-
const pid = parsePidFromLockTarget(await
|
|
14471
|
+
const pid = parsePidFromLockTarget(await fs10.readlink(lockPath));
|
|
13823
14472
|
if (pid !== null) return isPidAlive(pid);
|
|
13824
14473
|
} catch {
|
|
13825
14474
|
}
|
|
13826
14475
|
}
|
|
14476
|
+
if (process.platform === "win32" && name === "parent.lock") {
|
|
14477
|
+
const held = await isWindowsParentLockHeld(lockPath);
|
|
14478
|
+
if (held !== null) return held;
|
|
14479
|
+
return true;
|
|
14480
|
+
}
|
|
13827
14481
|
return Date.now() - lst.mtimeMs < LOCK_FRESHNESS_MS;
|
|
13828
14482
|
}
|
|
13829
14483
|
return false;
|
|
@@ -13835,19 +14489,19 @@ async function cleanupStaleProfiles(baseDir, staleMs = DEFAULT_STALE_MS) {
|
|
|
13835
14489
|
let removed = 0;
|
|
13836
14490
|
let errors = 0;
|
|
13837
14491
|
try {
|
|
13838
|
-
const entries = await
|
|
14492
|
+
const entries = await fs10.readdir(scanDir);
|
|
13839
14493
|
const now = Date.now();
|
|
13840
14494
|
for (const entry of entries) {
|
|
13841
14495
|
if (usePrefix && !entry.startsWith(prefix)) continue;
|
|
13842
14496
|
const fullPath = path9.join(scanDir, entry);
|
|
13843
14497
|
try {
|
|
13844
|
-
const stats = await
|
|
14498
|
+
const stats = await fs10.stat(fullPath);
|
|
13845
14499
|
if (!stats.isDirectory()) continue;
|
|
13846
14500
|
const isProfileLike = entry.startsWith(prefix) || entry.startsWith("playwright-artifacts-");
|
|
13847
14501
|
if (isProfileLike) {
|
|
13848
|
-
const inner = await
|
|
14502
|
+
const inner = await fs10.readdir(fullPath);
|
|
13849
14503
|
if (inner.length === 0) {
|
|
13850
|
-
await
|
|
14504
|
+
await fs10.rmdir(fullPath);
|
|
13851
14505
|
removed++;
|
|
13852
14506
|
continue;
|
|
13853
14507
|
}
|
|
@@ -13855,12 +14509,12 @@ async function cleanupStaleProfiles(baseDir, staleMs = DEFAULT_STALE_MS) {
|
|
|
13855
14509
|
logger.debug(`[Cleanup] Skipping profile ${entry} \u2014 owned by a live browser`);
|
|
13856
14510
|
continue;
|
|
13857
14511
|
}
|
|
13858
|
-
await
|
|
14512
|
+
await fs10.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
13859
14513
|
removed++;
|
|
13860
14514
|
continue;
|
|
13861
14515
|
}
|
|
13862
14516
|
if (now - stats.mtimeMs > staleMs && !await isProfileLive(fullPath)) {
|
|
13863
|
-
await
|
|
14517
|
+
await fs10.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
13864
14518
|
removed++;
|
|
13865
14519
|
}
|
|
13866
14520
|
} catch (_e) {
|
|
@@ -13880,7 +14534,7 @@ async function cleanupStaleProfiles(baseDir, staleMs = DEFAULT_STALE_MS) {
|
|
|
13880
14534
|
init_log_utils();
|
|
13881
14535
|
import process3 from "node:process";
|
|
13882
14536
|
import cluster from "node:cluster";
|
|
13883
|
-
import * as
|
|
14537
|
+
import * as fs11 from "node:fs/promises";
|
|
13884
14538
|
var workerId = "";
|
|
13885
14539
|
var BENIGN_IPC_CODES = /* @__PURE__ */ new Set(["EPIPE", "ERR_IPC_CHANNEL_CLOSED", "ECONNRESET"]);
|
|
13886
14540
|
function isBenignClusterIpcError(err) {
|
|
@@ -13941,7 +14595,7 @@ function logToDebugFile(level, ...args) {
|
|
|
13941
14595
|
return String(arg);
|
|
13942
14596
|
}).join(" "))
|
|
13943
14597
|
};
|
|
13944
|
-
|
|
14598
|
+
fs11.appendFile(logFile, `${JSON.stringify(entry)}
|
|
13945
14599
|
`).catch(() => {
|
|
13946
14600
|
});
|
|
13947
14601
|
} catch {
|
|
@@ -14016,6 +14670,7 @@ var WorkerPoolManager = class {
|
|
|
14016
14670
|
try {
|
|
14017
14671
|
if (this.pool && this.currentWorkerCount !== maxWorkers) {
|
|
14018
14672
|
logger.log(`[WorkerPoolManager] Worker count changed from ${this.currentWorkerCount} to ${maxWorkers}, recreating pool...`);
|
|
14673
|
+
this.poolEpoch++;
|
|
14019
14674
|
await this.pool.destroy();
|
|
14020
14675
|
this.pool = null;
|
|
14021
14676
|
}
|
|
@@ -14030,7 +14685,7 @@ var WorkerPoolManager = class {
|
|
|
14030
14685
|
void cleanupStaleProfiles(void 0, STALE_PROFILE_MS).catch((e) => logger.debug("[WorkerPoolManager] Profile sweep (legacy tmp) failed:", e));
|
|
14031
14686
|
void cleanupOrphanedCamoufoxProcesses().catch((e) => logger.debug("[WorkerPoolManager] Startup orphan sweep failed:", e));
|
|
14032
14687
|
const workerConcurrency = (config || getConfig()).WORKER_CONCURRENCY;
|
|
14033
|
-
const workerPath =
|
|
14688
|
+
const workerPath = join12(__dirname, "./thread-worker.mjs");
|
|
14034
14689
|
if (!existsSync7(workerPath)) {
|
|
14035
14690
|
throw new Error(
|
|
14036
14691
|
`Browser worker bundle is missing at ${workerPath}. It is produced by the build step; reinstall the extension (e.g. \`pi update --extensions\`) or run \`npm install\` in the extension directory so the prepare/build script regenerates it.`
|
|
@@ -14060,7 +14715,7 @@ var WorkerPoolManager = class {
|
|
|
14060
14715
|
}
|
|
14061
14716
|
},
|
|
14062
14717
|
exitHandler: (code) => {
|
|
14063
|
-
if (code !== 0 &&
|
|
14718
|
+
if (code !== 0 && this.poolEpoch === myEpoch) {
|
|
14064
14719
|
logger.error(`[WorkerPoolManager] Worker exited with code ${code}`);
|
|
14065
14720
|
this.consecutiveErrors++;
|
|
14066
14721
|
if (this.consecutiveErrors >= 3) {
|
|
@@ -14083,6 +14738,7 @@ var WorkerPoolManager = class {
|
|
|
14083
14738
|
logger.warn("[WorkerPoolManager] Pool initialized but a shutdown intervened. Destroying...");
|
|
14084
14739
|
const built = this.pool;
|
|
14085
14740
|
if (this.pool === built) this.pool = null;
|
|
14741
|
+
this.poolEpoch++;
|
|
14086
14742
|
await built.destroy().catch((err) => logger.debug("Swallowed pool destroy error:", err));
|
|
14087
14743
|
throw new Error("Worker pool is shutting down");
|
|
14088
14744
|
}
|
|
@@ -14143,6 +14799,7 @@ var WorkerPoolManager = class {
|
|
|
14143
14799
|
const t = setTimeout(async () => {
|
|
14144
14800
|
try {
|
|
14145
14801
|
if (this.isShuttingDown || this.generation !== myGen) return;
|
|
14802
|
+
this.poolEpoch++;
|
|
14146
14803
|
if (deadPool) await deadPool.destroy();
|
|
14147
14804
|
logger.info("[WorkerPoolManager] Auto-recovery: old pool destroyed.");
|
|
14148
14805
|
} catch (err) {
|
|
@@ -14183,6 +14840,7 @@ var WorkerPoolManager = class {
|
|
|
14183
14840
|
}
|
|
14184
14841
|
if (this.pool) {
|
|
14185
14842
|
try {
|
|
14843
|
+
this.poolEpoch++;
|
|
14186
14844
|
const destroyPromise = this.pool.destroy();
|
|
14187
14845
|
destroyPromise.catch((err) => logger.debug(`[WorkerPoolManager] Background pool destroy rejection: ${err.message}`));
|
|
14188
14846
|
await raceWithDeadline(destroyPromise, 5e3);
|
|
@@ -14360,7 +15018,7 @@ var ResearchRunSemaphore = class {
|
|
|
14360
15018
|
if (this._initialized) return;
|
|
14361
15019
|
this.lifecycle = "initializing" /* INITIALIZING */;
|
|
14362
15020
|
try {
|
|
14363
|
-
await import("node:fs/promises").then((
|
|
15021
|
+
await import("node:fs/promises").then((fs18) => fs18.mkdir(this.slotDir, { recursive: true, mode: 448 }));
|
|
14364
15022
|
} catch (err) {
|
|
14365
15023
|
logger.warn(`[ResearchRunSemaphore] Could not create slot dir ${this.slotDir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
14366
15024
|
}
|
|
@@ -14495,9 +15153,10 @@ function sleep(ms, signal) {
|
|
|
14495
15153
|
|
|
14496
15154
|
// src/infrastructure/state/state-backup-manager.ts
|
|
14497
15155
|
init_logger();
|
|
15156
|
+
init_atomic_replace();
|
|
14498
15157
|
init_service_registry();
|
|
14499
15158
|
init_service_names();
|
|
14500
|
-
import * as
|
|
15159
|
+
import * as fs12 from "node:fs/promises";
|
|
14501
15160
|
import * as crypto6 from "node:crypto";
|
|
14502
15161
|
import * as path11 from "node:path";
|
|
14503
15162
|
var QUARANTINE_KEEP = 5;
|
|
@@ -14521,7 +15180,7 @@ var StateBackupManager = class {
|
|
|
14521
15180
|
return;
|
|
14522
15181
|
}
|
|
14523
15182
|
this.lifecycle = "initializing" /* INITIALIZING */;
|
|
14524
|
-
await
|
|
15183
|
+
await fs12.mkdir(this.backupDirPath, { recursive: true, mode: 448 });
|
|
14525
15184
|
this._initialized = true;
|
|
14526
15185
|
this.lifecycle = "initialized" /* INITIALIZED */;
|
|
14527
15186
|
}
|
|
@@ -14535,15 +15194,15 @@ var StateBackupManager = class {
|
|
|
14535
15194
|
async createBackup() {
|
|
14536
15195
|
try {
|
|
14537
15196
|
try {
|
|
14538
|
-
await
|
|
15197
|
+
await fs12.access(this.stateFilePath);
|
|
14539
15198
|
} catch {
|
|
14540
15199
|
return;
|
|
14541
15200
|
}
|
|
14542
|
-
await
|
|
15201
|
+
await fs12.mkdir(this.backupDirPath, { recursive: true, mode: 448 });
|
|
14543
15202
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14544
15203
|
const backupFileName = `research-state-${timestamp}-${crypto6.randomBytes(4).toString("hex")}.json`;
|
|
14545
15204
|
const backupFilePath = path11.join(this.backupDirPath, backupFileName);
|
|
14546
|
-
await
|
|
15205
|
+
await fs12.copyFile(this.stateFilePath, backupFilePath);
|
|
14547
15206
|
} catch (error) {
|
|
14548
15207
|
throw new Error(`Failed to create backup: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
14549
15208
|
}
|
|
@@ -14553,14 +15212,14 @@ var StateBackupManager = class {
|
|
|
14553
15212
|
*/
|
|
14554
15213
|
async cleanupOldBackups() {
|
|
14555
15214
|
try {
|
|
14556
|
-
const entries = await
|
|
15215
|
+
const entries = await fs12.readdir(this.backupDirPath);
|
|
14557
15216
|
if (entries.length <= this.maxBackups) {
|
|
14558
15217
|
return;
|
|
14559
15218
|
}
|
|
14560
15219
|
const backupFiles = [];
|
|
14561
15220
|
for (const entry of entries) {
|
|
14562
15221
|
const filePath = path11.join(this.backupDirPath, entry);
|
|
14563
|
-
const stats = await
|
|
15222
|
+
const stats = await fs12.stat(filePath);
|
|
14564
15223
|
if (stats.isFile() && entry.startsWith("research-state-") && entry.endsWith(".json")) {
|
|
14565
15224
|
backupFiles.push({ name: entry, mtime: stats.mtime });
|
|
14566
15225
|
}
|
|
@@ -14569,7 +15228,7 @@ var StateBackupManager = class {
|
|
|
14569
15228
|
const backupsToRemove = backupFiles.slice(this.maxBackups);
|
|
14570
15229
|
for (const backupFile of backupsToRemove) {
|
|
14571
15230
|
const filePath = path11.join(this.backupDirPath, backupFile.name);
|
|
14572
|
-
await
|
|
15231
|
+
await fs12.unlink(filePath);
|
|
14573
15232
|
}
|
|
14574
15233
|
} catch (error) {
|
|
14575
15234
|
logger.error(`Failed to cleanup old backups: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -14582,10 +15241,10 @@ var StateBackupManager = class {
|
|
|
14582
15241
|
try {
|
|
14583
15242
|
let backups = [];
|
|
14584
15243
|
try {
|
|
14585
|
-
const entries = await
|
|
15244
|
+
const entries = await fs12.readdir(this.backupDirPath);
|
|
14586
15245
|
for (const entry of entries) {
|
|
14587
15246
|
const filePath = path11.join(this.backupDirPath, entry);
|
|
14588
|
-
const stats = await
|
|
15247
|
+
const stats = await fs12.stat(filePath);
|
|
14589
15248
|
if (stats.isFile() && entry.startsWith("research-state-") && entry.endsWith(".json")) {
|
|
14590
15249
|
backups.push({ name: entry, mtime: stats.mtime });
|
|
14591
15250
|
}
|
|
@@ -14597,7 +15256,7 @@ var StateBackupManager = class {
|
|
|
14597
15256
|
const backupPath = path11.join(this.backupDirPath, backup.name);
|
|
14598
15257
|
let restoreContent;
|
|
14599
15258
|
try {
|
|
14600
|
-
const content = await
|
|
15259
|
+
const content = await fs12.readFile(backupPath, "utf-8");
|
|
14601
15260
|
const validated = this.validator.migrateAndValidate(JSON.parse(content));
|
|
14602
15261
|
restoreContent = JSON.stringify(validated, null, 2);
|
|
14603
15262
|
} catch (err) {
|
|
@@ -14625,27 +15284,27 @@ var StateBackupManager = class {
|
|
|
14625
15284
|
*/
|
|
14626
15285
|
async quarantineFutureState(fileVersion) {
|
|
14627
15286
|
try {
|
|
14628
|
-
await
|
|
15287
|
+
await fs12.access(this.stateFilePath);
|
|
14629
15288
|
} catch {
|
|
14630
15289
|
return;
|
|
14631
15290
|
}
|
|
14632
|
-
await
|
|
15291
|
+
await fs12.mkdir(this.backupDirPath, { recursive: true, mode: 448 });
|
|
14633
15292
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14634
15293
|
const name = `research-state-future-v${fileVersion}-${timestamp}.quarantine`;
|
|
14635
15294
|
const dest = path11.join(this.backupDirPath, name);
|
|
14636
|
-
await
|
|
15295
|
+
await fs12.copyFile(this.stateFilePath, dest);
|
|
14637
15296
|
logger.warn(`[StateManager] Quarantined newer-build state (v${fileVersion}) copy at ${dest}`);
|
|
14638
15297
|
await this.pruneQuarantineFiles(QUARANTINE_KEEP);
|
|
14639
15298
|
}
|
|
14640
15299
|
/** Keep only the newest `keep` *.quarantine files; delete the rest. Best-effort. */
|
|
14641
15300
|
async pruneQuarantineFiles(keep) {
|
|
14642
15301
|
try {
|
|
14643
|
-
const entries = await
|
|
15302
|
+
const entries = await fs12.readdir(this.backupDirPath);
|
|
14644
15303
|
const quarantines = [];
|
|
14645
15304
|
for (const entry of entries) {
|
|
14646
15305
|
if (!entry.endsWith(".quarantine")) continue;
|
|
14647
15306
|
try {
|
|
14648
|
-
const stats = await
|
|
15307
|
+
const stats = await fs12.stat(path11.join(this.backupDirPath, entry));
|
|
14649
15308
|
if (stats.isFile()) quarantines.push({ name: entry, mtime: stats.mtimeMs });
|
|
14650
15309
|
} catch {
|
|
14651
15310
|
}
|
|
@@ -14654,7 +15313,7 @@ var StateBackupManager = class {
|
|
|
14654
15313
|
quarantines.sort((a, b) => b.mtime - a.mtime);
|
|
14655
15314
|
for (const stale of quarantines.slice(keep)) {
|
|
14656
15315
|
try {
|
|
14657
|
-
await
|
|
15316
|
+
await fs12.unlink(path11.join(this.backupDirPath, stale.name));
|
|
14658
15317
|
} catch {
|
|
14659
15318
|
}
|
|
14660
15319
|
}
|
|
@@ -14667,10 +15326,10 @@ var StateBackupManager = class {
|
|
|
14667
15326
|
*/
|
|
14668
15327
|
async atomicWriteStateFile(content) {
|
|
14669
15328
|
const stateDir = path11.dirname(this.stateFilePath);
|
|
14670
|
-
await
|
|
15329
|
+
await fs12.mkdir(stateDir, { recursive: true, mode: 448 });
|
|
14671
15330
|
const tempFile = `research-state-${crypto6.randomBytes(16).toString("hex")}.tmp`;
|
|
14672
15331
|
const tempPath = path11.join(stateDir, tempFile);
|
|
14673
|
-
const fh = await
|
|
15332
|
+
const fh = await fs12.open(tempPath, "w", 384);
|
|
14674
15333
|
try {
|
|
14675
15334
|
await fh.writeFile(content, { encoding: "utf-8" });
|
|
14676
15335
|
await fh.sync();
|
|
@@ -14678,17 +15337,14 @@ var StateBackupManager = class {
|
|
|
14678
15337
|
await fh.close();
|
|
14679
15338
|
}
|
|
14680
15339
|
try {
|
|
14681
|
-
await
|
|
14682
|
-
|
|
14683
|
-
if (process.platform === "win32") {
|
|
14684
|
-
await fs11.copyFile(tempPath, this.stateFilePath);
|
|
14685
|
-
await fs11.unlink(tempPath).catch(() => {
|
|
14686
|
-
});
|
|
14687
|
-
} else {
|
|
14688
|
-
await fs11.unlink(tempPath).catch(() => {
|
|
15340
|
+
if (await replaceFile(tempPath, this.stateFilePath) === "copied") {
|
|
15341
|
+
await fs12.unlink(tempPath).catch(() => {
|
|
14689
15342
|
});
|
|
14690
|
-
throw renameErr;
|
|
14691
15343
|
}
|
|
15344
|
+
} catch (renameErr) {
|
|
15345
|
+
await fs12.unlink(tempPath).catch(() => {
|
|
15346
|
+
});
|
|
15347
|
+
throw renameErr;
|
|
14692
15348
|
}
|
|
14693
15349
|
}
|
|
14694
15350
|
/**
|
|
@@ -14995,6 +15651,41 @@ async function recoverFromLeaderHandover(container) {
|
|
|
14995
15651
|
(err) => logger.debug("Wait for browser idle timed out or failed:", err)
|
|
14996
15652
|
);
|
|
14997
15653
|
}
|
|
15654
|
+
async function isRegisteredLeaderAliveUncached(container) {
|
|
15655
|
+
try {
|
|
15656
|
+
const stateManager = await getService(ServiceNames.STATE_MANAGER, void 0, container);
|
|
15657
|
+
const info = await stateManager.getBrowserServer();
|
|
15658
|
+
if (!info) return false;
|
|
15659
|
+
const schedulerService = await getService(ServiceNames.SCHEDULER, void 0, container);
|
|
15660
|
+
const cached = schedulerService.getSchedulerInstance();
|
|
15661
|
+
if (cached instanceof BrowserClient && cached.targetPort !== info.port) return false;
|
|
15662
|
+
return await stateManager.isPidAlive(info.pid, info.schedulerId);
|
|
15663
|
+
} catch {
|
|
15664
|
+
return false;
|
|
15665
|
+
}
|
|
15666
|
+
}
|
|
15667
|
+
var LEADER_LIVENESS_COALESCE_MS = 1e3;
|
|
15668
|
+
var leaderLivenessProbe = null;
|
|
15669
|
+
async function isRegisteredLeaderAlive(container) {
|
|
15670
|
+
const now = Date.now();
|
|
15671
|
+
if (leaderLivenessProbe && now - leaderLivenessProbe.startedAt < LEADER_LIVENESS_COALESCE_MS) {
|
|
15672
|
+
return leaderLivenessProbe.result;
|
|
15673
|
+
}
|
|
15674
|
+
const result = isRegisteredLeaderAliveUncached(container);
|
|
15675
|
+
leaderLivenessProbe = { startedAt: now, result };
|
|
15676
|
+
return result;
|
|
15677
|
+
}
|
|
15678
|
+
async function restartSchedulerWithHerdGuard(container) {
|
|
15679
|
+
const now = Date.now();
|
|
15680
|
+
if (now - lastRestartTime > RESTART_COOLDOWN_MS || !await isRegisteredLeaderAlive(container)) {
|
|
15681
|
+
lastRestartTime = now;
|
|
15682
|
+
logger.warn("[BrowserManager] Forcing scheduler restart and retrying...");
|
|
15683
|
+
await forceSchedulerRestart2(false, container);
|
|
15684
|
+
} else {
|
|
15685
|
+
logger.warn("[BrowserManager] Scheduler restart triggered recently and the registered leader is still alive \u2014 waiting for pool idle instead...");
|
|
15686
|
+
}
|
|
15687
|
+
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15688
|
+
}
|
|
14998
15689
|
async function getScheduler2(config, container = getServiceContainer()) {
|
|
14999
15690
|
try {
|
|
15000
15691
|
const factory = await getService(ServiceNames.SCHEDULER_FACTORY, void 0, container);
|
|
@@ -15057,16 +15748,7 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
|
|
|
15057
15748
|
logger.warn(`[BrowserManager] Pool is draining \u2014 waiting for pool idle before retry...`);
|
|
15058
15749
|
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15059
15750
|
} else {
|
|
15060
|
-
|
|
15061
|
-
if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
|
|
15062
|
-
lastRestartTime = now;
|
|
15063
|
-
logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
|
|
15064
|
-
await forceSchedulerRestart2(false, container);
|
|
15065
|
-
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15066
|
-
} else {
|
|
15067
|
-
logger.warn(`[BrowserManager] Scheduler restart recently triggered, waiting for pool idle...`);
|
|
15068
|
-
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15069
|
-
}
|
|
15751
|
+
await restartSchedulerWithHerdGuard(container);
|
|
15070
15752
|
}
|
|
15071
15753
|
const jitter = 100 + Math.floor(Math.random() * 400);
|
|
15072
15754
|
await new Promise((resolve8) => setTimeout(resolve8, jitter));
|
|
@@ -15092,15 +15774,7 @@ async function runBrowserHealthCheck(config, retries = 1, signal, container = ge
|
|
|
15092
15774
|
if (isPoolShutdownError(error)) {
|
|
15093
15775
|
await recoverFromLeaderHandover(container);
|
|
15094
15776
|
} else {
|
|
15095
|
-
|
|
15096
|
-
if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
|
|
15097
|
-
lastRestartTime = now;
|
|
15098
|
-
logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
|
|
15099
|
-
await forceSchedulerRestart2(false, container);
|
|
15100
|
-
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15101
|
-
} else {
|
|
15102
|
-
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15103
|
-
}
|
|
15777
|
+
await restartSchedulerWithHerdGuard(container);
|
|
15104
15778
|
}
|
|
15105
15779
|
const jitter = 100 + Math.floor(Math.random() * 400);
|
|
15106
15780
|
await new Promise((resolve8) => setTimeout(resolve8, jitter));
|
|
@@ -15144,15 +15818,7 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
|
|
|
15144
15818
|
if (isPoolShutdownError(error)) {
|
|
15145
15819
|
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15146
15820
|
} else {
|
|
15147
|
-
|
|
15148
|
-
if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
|
|
15149
|
-
lastRestartTime = now;
|
|
15150
|
-
logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
|
|
15151
|
-
await forceSchedulerRestart2(false, container);
|
|
15152
|
-
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15153
|
-
} else {
|
|
15154
|
-
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
15155
|
-
}
|
|
15821
|
+
await restartSchedulerWithHerdGuard(container);
|
|
15156
15822
|
}
|
|
15157
15823
|
const jitter = 100 + Math.floor(Math.random() * 400);
|
|
15158
15824
|
await new Promise((resolve8) => setTimeout(resolve8, jitter));
|
|
@@ -15168,7 +15834,7 @@ init_safe_unref();
|
|
|
15168
15834
|
init_url_utils();
|
|
15169
15835
|
init_metrics();
|
|
15170
15836
|
init_service_registry();
|
|
15171
|
-
async function performSearch(queries, config, signal, onProgress, container = getServiceContainer()) {
|
|
15837
|
+
async function performSearch(queries, config, signal, onProgress, container = getServiceContainer(), failures) {
|
|
15172
15838
|
const startTime = Date.now();
|
|
15173
15839
|
const resultMap = /* @__PURE__ */ new Map();
|
|
15174
15840
|
const seenUrls = /* @__PURE__ */ new Set();
|
|
@@ -15191,7 +15857,7 @@ async function performSearch(queries, config, signal, onProgress, container = ge
|
|
|
15191
15857
|
logger.log(`[Search] Orchestrating ${queries.length} queries across ${maxWorkers} worker processes...`);
|
|
15192
15858
|
if (onProgress) onProgress(0);
|
|
15193
15859
|
const QUERY_TIMEOUT_MS = (config?.SEARCH_TIMEOUT_MS ?? 45e3) + (config?.BROWSER_TASK_TIMEOUT_MS ?? 1e4);
|
|
15194
|
-
const filteredQueries = queries.filter((q) => q.trim());
|
|
15860
|
+
const filteredQueries = [...new Set(queries.filter((q) => q.trim()))];
|
|
15195
15861
|
let timeoutCount = 0;
|
|
15196
15862
|
let errorCount = 0;
|
|
15197
15863
|
let sampleWorkerError = "";
|
|
@@ -15236,11 +15902,24 @@ async function performSearch(queries, config, signal, onProgress, container = ge
|
|
|
15236
15902
|
metrics.increment("browser_search_queries_total", 1, { status });
|
|
15237
15903
|
if (isTimeout) {
|
|
15238
15904
|
logger.warn(`[Search] Query timed out after ${QUERY_TIMEOUT_MS}ms \u2014 likely starved waiting for a free browser worker under concurrent load, or a slow/blocked search provider: "${query}"`);
|
|
15905
|
+
failures?.set(query, {
|
|
15906
|
+
type: "timeout",
|
|
15907
|
+
message: `Search timed out after ${QUERY_TIMEOUT_MS}ms before returning anything. This says nothing about the query \u2014 the browser pool was saturated or the search provider was slow/blocked. Retrying the same query later is reasonable.`
|
|
15908
|
+
});
|
|
15239
15909
|
} else {
|
|
15240
15910
|
const msg2 = error instanceof Error ? error.message : String(error);
|
|
15241
15911
|
if (msg2 !== "Aborted") {
|
|
15242
15912
|
if (!sampleWorkerError) sampleWorkerError = msg2;
|
|
15243
15913
|
logger.error(`[Search] Worker failed for "${query}": ${msg2}`);
|
|
15914
|
+
failures?.set(query, {
|
|
15915
|
+
type: "service_unavailable",
|
|
15916
|
+
message: `The search backend failed for this query (${msg2}). This is an infrastructure failure, not a signal about the query itself \u2014 do not rewrite the query in response to it.`
|
|
15917
|
+
});
|
|
15918
|
+
} else if (!signal?.aborted) {
|
|
15919
|
+
failures?.set(query, {
|
|
15920
|
+
type: "service_unavailable",
|
|
15921
|
+
message: "The search task was cancelled on the worker side (pool drain or leader handover) before completing. This says nothing about the query \u2014 do not rewrite it; retrying the same query later is reasonable."
|
|
15922
|
+
});
|
|
15244
15923
|
}
|
|
15245
15924
|
}
|
|
15246
15925
|
resultMap.set(query, []);
|
|
@@ -15250,6 +15929,9 @@ async function performSearch(queries, config, signal, onProgress, container = ge
|
|
|
15250
15929
|
}
|
|
15251
15930
|
});
|
|
15252
15931
|
await Promise.all(searchTasks);
|
|
15932
|
+
if (signal?.aborted) {
|
|
15933
|
+
throw new Error("Aborted");
|
|
15934
|
+
}
|
|
15253
15935
|
const totalResults = Array.from(resultMap.values()).reduce((sum, r) => sum + r.length, 0);
|
|
15254
15936
|
if (totalResults === 0 && filteredQueries.length > 0) {
|
|
15255
15937
|
metrics.increment("browser_search_total_failures_total", 1);
|
|
@@ -15280,7 +15962,8 @@ async function search(queries, config, signal, onProgress, container = getServic
|
|
|
15280
15962
|
metrics.observe("search_query_count_per_request", queries.length);
|
|
15281
15963
|
const searchStart = Date.now();
|
|
15282
15964
|
try {
|
|
15283
|
-
const
|
|
15965
|
+
const failures = /* @__PURE__ */ new Map();
|
|
15966
|
+
const resultMap = await performSearch(queries, config, signal, onProgress, container, failures);
|
|
15284
15967
|
const searchDuration = Date.now() - searchStart;
|
|
15285
15968
|
metrics.observe("search_latency_ms", searchDuration);
|
|
15286
15969
|
let totalResults = 0;
|
|
@@ -15294,7 +15977,7 @@ async function search(queries, config, signal, onProgress, container = getServic
|
|
|
15294
15977
|
successfulQueries++;
|
|
15295
15978
|
metrics.observe("search_results_per_query", qResults.length);
|
|
15296
15979
|
} else {
|
|
15297
|
-
result.error = {
|
|
15980
|
+
result.error = failures.get(q) ?? {
|
|
15298
15981
|
type: "empty_results",
|
|
15299
15982
|
message: "Browser-based search returned no relevant results for this query (the query may be too narrow, or the site returned nothing usable)."
|
|
15300
15983
|
};
|
|
@@ -15309,10 +15992,13 @@ async function search(queries, config, signal, onProgress, container = getServic
|
|
|
15309
15992
|
return results;
|
|
15310
15993
|
} catch (error) {
|
|
15311
15994
|
const searchDuration = Date.now() - searchStart;
|
|
15995
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
15996
|
+
if (signal?.aborted || message === "Aborted") {
|
|
15997
|
+
throw error;
|
|
15998
|
+
}
|
|
15312
15999
|
metrics.observe("search_latency_ms", searchDuration, { status: "error" });
|
|
15313
16000
|
metrics.increment("search_queries_total", queries.length, { status: "error" });
|
|
15314
16001
|
metrics.increment("search_errors_total", 1, { error_type: "search_failed" });
|
|
15315
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
15316
16002
|
if (message.includes("Search completely failed")) {
|
|
15317
16003
|
metrics.increment("search_errors_total", 1, { error_type: "total_search_failure" });
|
|
15318
16004
|
throw error;
|
|
@@ -15687,7 +16373,9 @@ function createSearchTool(options) {
|
|
|
15687
16373
|
markdown += `## Query ${i + 1}: ${r.query}
|
|
15688
16374
|
`;
|
|
15689
16375
|
if (r.results.length === 0) {
|
|
15690
|
-
markdown +=
|
|
16376
|
+
markdown += r.error ? `*${r.error.message}*
|
|
16377
|
+
|
|
16378
|
+
` : `*No results found.*
|
|
15691
16379
|
|
|
15692
16380
|
`;
|
|
15693
16381
|
} else {
|
|
@@ -15831,7 +16519,7 @@ init_logger();
|
|
|
15831
16519
|
import * as dns from "node:dns/promises";
|
|
15832
16520
|
import * as dnsCb from "node:dns";
|
|
15833
16521
|
import * as net3 from "node:net";
|
|
15834
|
-
import
|
|
16522
|
+
import crypto9 from "node:crypto";
|
|
15835
16523
|
import { NodeHtmlMarkdown } from "node-html-markdown";
|
|
15836
16524
|
function isBenignScrapeFailure(error) {
|
|
15837
16525
|
const msg2 = error instanceof Error ? error.message : String(error);
|
|
@@ -15842,7 +16530,7 @@ function isBenignScrapeFailure(error) {
|
|
|
15842
16530
|
}
|
|
15843
16531
|
function getRandomUserAgent() {
|
|
15844
16532
|
try {
|
|
15845
|
-
const index =
|
|
16533
|
+
const index = crypto9.randomInt(0, USER_AGENTS.length);
|
|
15846
16534
|
const userAgent = USER_AGENTS[index];
|
|
15847
16535
|
return userAgent ?? USER_AGENTS[0];
|
|
15848
16536
|
} catch {
|
|
@@ -16084,6 +16772,7 @@ function validateContent(html, markdown, url) {
|
|
|
16084
16772
|
// src/web-research/web-scraper.ts
|
|
16085
16773
|
init_retry_utils();
|
|
16086
16774
|
init_safe_unref();
|
|
16775
|
+
init_http_body();
|
|
16087
16776
|
init_service_registry();
|
|
16088
16777
|
var FETCH_RETRY_DELAY_MS = 250;
|
|
16089
16778
|
var playwrightAvailable = false;
|
|
@@ -16180,48 +16869,6 @@ function decodeHtmlBody(bytes, contentTypeHeader) {
|
|
|
16180
16869
|
return new TextDecoder("utf-8").decode(bytes);
|
|
16181
16870
|
}
|
|
16182
16871
|
}
|
|
16183
|
-
var BodyTooLargeError = class extends Error {
|
|
16184
|
-
constructor(bytesRead, maxBytes) {
|
|
16185
|
-
super(`Response body exceeded ${maxBytes} byte cap`);
|
|
16186
|
-
this.bytesRead = bytesRead;
|
|
16187
|
-
this.maxBytes = maxBytes;
|
|
16188
|
-
this.name = "BodyTooLargeError";
|
|
16189
|
-
}
|
|
16190
|
-
bytesRead;
|
|
16191
|
-
maxBytes;
|
|
16192
|
-
};
|
|
16193
|
-
async function readBodyCapped(response, maxBytes) {
|
|
16194
|
-
const body = response.body;
|
|
16195
|
-
if (!body || typeof body.getReader !== "function") {
|
|
16196
|
-
return new Uint8Array(await response.arrayBuffer());
|
|
16197
|
-
}
|
|
16198
|
-
const reader = body.getReader();
|
|
16199
|
-
const chunks = [];
|
|
16200
|
-
let total = 0;
|
|
16201
|
-
try {
|
|
16202
|
-
for (; ; ) {
|
|
16203
|
-
const { done, value } = await reader.read();
|
|
16204
|
-
if (done) break;
|
|
16205
|
-
if (value) {
|
|
16206
|
-
total += value.byteLength;
|
|
16207
|
-
if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
|
|
16208
|
-
chunks.push(value);
|
|
16209
|
-
}
|
|
16210
|
-
}
|
|
16211
|
-
} finally {
|
|
16212
|
-
try {
|
|
16213
|
-
await reader.cancel();
|
|
16214
|
-
} catch {
|
|
16215
|
-
}
|
|
16216
|
-
}
|
|
16217
|
-
const out = new Uint8Array(total);
|
|
16218
|
-
let offset = 0;
|
|
16219
|
-
for (const c of chunks) {
|
|
16220
|
-
out.set(c, offset);
|
|
16221
|
-
offset += c.byteLength;
|
|
16222
|
-
}
|
|
16223
|
-
return out;
|
|
16224
|
-
}
|
|
16225
16872
|
function trackFetchLayerFailure(url, error) {
|
|
16226
16873
|
const isSsrf = error instanceof Error && error.message.includes("not allowed");
|
|
16227
16874
|
errorTracker.trackError(error instanceof Error ? error : String(error), {
|
|
@@ -16795,6 +17442,7 @@ init_retry_utils();
|
|
|
16795
17442
|
init_circuit_breaker();
|
|
16796
17443
|
init_safe_unref();
|
|
16797
17444
|
init_metrics();
|
|
17445
|
+
init_http_body();
|
|
16798
17446
|
|
|
16799
17447
|
// src/security/nvd-types.ts
|
|
16800
17448
|
function isWeaknessDescription(value) {
|
|
@@ -17083,18 +17731,20 @@ async function fetchPaginated(term, options, maxResults, severityVersion, admitE
|
|
|
17083
17731
|
const response = await fetchWithRetry(url, signal);
|
|
17084
17732
|
let data;
|
|
17085
17733
|
try {
|
|
17086
|
-
data = await response
|
|
17087
|
-
} catch {
|
|
17088
|
-
metrics.increment("nvd_search_errors_total", 1, {
|
|
17734
|
+
data = await readJsonCapped(response, MAX_BULK_JSON_BODY_BYTES);
|
|
17735
|
+
} catch (err) {
|
|
17736
|
+
metrics.increment("nvd_search_errors_total", 1, {
|
|
17737
|
+
error_type: err instanceof BodyTooLargeError ? "body_too_large" : "malformed_json"
|
|
17738
|
+
});
|
|
17089
17739
|
break;
|
|
17090
17740
|
}
|
|
17091
17741
|
const rawEntries = extractNVDEntries(data);
|
|
17092
17742
|
const admitted = rawEntries.filter(admitEntry).map((entry) => parseNVDEntry(entry, options));
|
|
17093
17743
|
if (rawEntries.length === 0) {
|
|
17094
|
-
metrics.increment("nvd_cache_misses_total", 1
|
|
17744
|
+
metrics.increment("nvd_cache_misses_total", 1);
|
|
17095
17745
|
break;
|
|
17096
17746
|
}
|
|
17097
|
-
metrics.increment("nvd_cache_hits_total", 1
|
|
17747
|
+
metrics.increment("nvd_cache_hits_total", 1);
|
|
17098
17748
|
allVulnerabilities.push(...admitted);
|
|
17099
17749
|
const responseData = data;
|
|
17100
17750
|
if (responseData?.["totalResults"] !== void 0 && typeof responseData["totalResults"] === "number" && startIndex + pageSize >= responseData["totalResults"]) {
|
|
@@ -17188,6 +17838,7 @@ async function searchNVD(terms, options) {
|
|
|
17188
17838
|
init_retry_utils();
|
|
17189
17839
|
init_circuit_breaker();
|
|
17190
17840
|
init_metrics();
|
|
17841
|
+
init_http_body();
|
|
17191
17842
|
var CISA_KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";
|
|
17192
17843
|
var cisaCircuitBreaker = new CircuitBreaker({
|
|
17193
17844
|
failureThreshold: 5,
|
|
@@ -17271,10 +17922,14 @@ async function searchCisaKev(terms = [], options) {
|
|
|
17271
17922
|
}, options?.signal);
|
|
17272
17923
|
let data;
|
|
17273
17924
|
try {
|
|
17274
|
-
data = await response
|
|
17275
|
-
} catch {
|
|
17925
|
+
data = await readJsonCapped(response, MAX_BULK_JSON_BODY_BYTES);
|
|
17926
|
+
} catch (err) {
|
|
17927
|
+
if (err instanceof BodyTooLargeError) {
|
|
17928
|
+
metrics.increment("cisa_kev_errors_total", 1, { error_type: "body_too_large" });
|
|
17929
|
+
throw err;
|
|
17930
|
+
}
|
|
17276
17931
|
metrics.increment("cisa_kev_errors_total", 1, { error_type: "malformed_json" });
|
|
17277
|
-
throw new Error("CISA KEV feed returned a malformed JSON response.");
|
|
17932
|
+
throw new Error("CISA KEV feed returned a malformed JSON response.", { cause: err });
|
|
17278
17933
|
}
|
|
17279
17934
|
const cisaData = extractCisaKevItems(data);
|
|
17280
17935
|
metrics.increment(cisaData.length > 0 ? "cisa_kev_cache_hits_total" : "cisa_kev_cache_misses_total", 1);
|
|
@@ -17397,6 +18052,7 @@ function normalizeEcosystem(value, target) {
|
|
|
17397
18052
|
init_retry_utils();
|
|
17398
18053
|
init_circuit_breaker();
|
|
17399
18054
|
init_metrics();
|
|
18055
|
+
init_http_body();
|
|
17400
18056
|
init_logger();
|
|
17401
18057
|
|
|
17402
18058
|
// src/security/github-advisory-types.ts
|
|
@@ -17618,10 +18274,14 @@ async function searchGitHubAdvisories(terms, options) {
|
|
|
17618
18274
|
}));
|
|
17619
18275
|
let data;
|
|
17620
18276
|
try {
|
|
17621
|
-
data = await response
|
|
17622
|
-
} catch {
|
|
18277
|
+
data = await readJsonCapped(response);
|
|
18278
|
+
} catch (err) {
|
|
18279
|
+
if (err instanceof BodyTooLargeError) {
|
|
18280
|
+
metrics.increment("github_search_errors_total", 1, { error_type: "body_too_large" });
|
|
18281
|
+
throw err;
|
|
18282
|
+
}
|
|
17623
18283
|
metrics.increment("github_search_errors_total", 1, { error_type: "malformed_json" });
|
|
17624
|
-
throw new Error("GitHub API returned a malformed JSON response (repo advisories).");
|
|
18284
|
+
throw new Error("GitHub API returned a malformed JSON response (repo advisories).", { cause: err });
|
|
17625
18285
|
}
|
|
17626
18286
|
let repoAdvisories = [];
|
|
17627
18287
|
if (isGitHubAdvisoryArray(data)) {
|
|
@@ -17687,10 +18347,14 @@ async function searchGitHubAdvisories(terms, options) {
|
|
|
17687
18347
|
}));
|
|
17688
18348
|
let data;
|
|
17689
18349
|
try {
|
|
17690
|
-
data = await response
|
|
17691
|
-
} catch {
|
|
18350
|
+
data = await readJsonCapped(response);
|
|
18351
|
+
} catch (err) {
|
|
18352
|
+
if (err instanceof BodyTooLargeError) {
|
|
18353
|
+
metrics.increment("github_search_errors_total", 1, { error_type: "body_too_large", endpoint: endpointType });
|
|
18354
|
+
throw err;
|
|
18355
|
+
}
|
|
17692
18356
|
metrics.increment("github_search_errors_total", 1, { error_type: "malformed_json", endpoint: endpointType });
|
|
17693
|
-
throw new Error(`GitHub API returned a malformed JSON response for term "${term}"
|
|
18357
|
+
throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`, { cause: err });
|
|
17694
18358
|
}
|
|
17695
18359
|
let items = [];
|
|
17696
18360
|
if (isGitHubAdvisoryArray(data)) {
|
|
@@ -17700,7 +18364,7 @@ async function searchGitHubAdvisories(terms, options) {
|
|
|
17700
18364
|
} else if (isGitHubAdvisoryListResponse(data) && isArray(data.items)) {
|
|
17701
18365
|
items = data.items.filter(isGitHubAdvisoryRaw);
|
|
17702
18366
|
}
|
|
17703
|
-
metrics.increment(items.length > 0 ? "github_cache_hits_total" : "github_cache_misses_total", 1, {
|
|
18367
|
+
metrics.increment(items.length > 0 ? "github_cache_hits_total" : "github_cache_misses_total", 1, { endpoint: endpointType });
|
|
17704
18368
|
return items.map(mapGitHubAdvisory);
|
|
17705
18369
|
};
|
|
17706
18370
|
const settled = await Promise.allSettled(terms.map(fetchTerm));
|
|
@@ -17771,6 +18435,7 @@ init_logger();
|
|
|
17771
18435
|
init_constants();
|
|
17772
18436
|
init_circuit_breaker();
|
|
17773
18437
|
init_metrics();
|
|
18438
|
+
init_http_body();
|
|
17774
18439
|
|
|
17775
18440
|
// src/security/cvss.ts
|
|
17776
18441
|
var AV = { N: 0.85, A: 0.62, L: 0.55, P: 0.2 };
|
|
@@ -18042,10 +18707,14 @@ async function searchOSV(terms, options) {
|
|
|
18042
18707
|
}
|
|
18043
18708
|
let data;
|
|
18044
18709
|
try {
|
|
18045
|
-
data = await response
|
|
18046
|
-
} catch {
|
|
18710
|
+
data = await readJsonCapped(response);
|
|
18711
|
+
} catch (err) {
|
|
18712
|
+
if (err instanceof BodyTooLargeError) {
|
|
18713
|
+
metrics.increment("osv_search_errors_total", 1, { error_type: "body_too_large" });
|
|
18714
|
+
throw err;
|
|
18715
|
+
}
|
|
18047
18716
|
metrics.increment("osv_search_errors_total", 1, { error_type: "malformed_json" });
|
|
18048
|
-
throw new Error(`OSV returned a malformed JSON response for "${term}"
|
|
18717
|
+
throw new Error(`OSV returned a malformed JSON response for "${term}".`, { cause: err });
|
|
18049
18718
|
}
|
|
18050
18719
|
let items;
|
|
18051
18720
|
if (isOsvVulnerability(data)) {
|
|
@@ -18057,7 +18726,7 @@ async function searchOSV(terms, options) {
|
|
|
18057
18726
|
return { vulns: [], skipped: false };
|
|
18058
18727
|
}
|
|
18059
18728
|
const endpoint = isIdLookup ? "vulns_by_id" : "query";
|
|
18060
|
-
metrics.increment(items.length > 0 ? "osv_cache_hits_total" : "osv_cache_misses_total", 1, {
|
|
18729
|
+
metrics.increment(items.length > 0 ? "osv_cache_hits_total" : "osv_cache_misses_total", 1, { endpoint });
|
|
18061
18730
|
const vulns = [];
|
|
18062
18731
|
for (const item of items) {
|
|
18063
18732
|
if (item.withdrawn) continue;
|
|
@@ -18650,6 +19319,7 @@ import { Value as Value8 } from "typebox/value";
|
|
|
18650
19319
|
init_logger();
|
|
18651
19320
|
init_circuit_breaker();
|
|
18652
19321
|
init_metrics();
|
|
19322
|
+
init_http_body();
|
|
18653
19323
|
var API_BASE = "https://api.stackexchange.com/2.3";
|
|
18654
19324
|
var StackExchangeClient = class {
|
|
18655
19325
|
_apiKey;
|
|
@@ -18714,8 +19384,9 @@ var StackExchangeClient = class {
|
|
|
18714
19384
|
});
|
|
18715
19385
|
let data;
|
|
18716
19386
|
try {
|
|
18717
|
-
data = await response
|
|
19387
|
+
data = await readJsonCapped(response);
|
|
18718
19388
|
} catch (parseErr) {
|
|
19389
|
+
if (parseErr instanceof BodyTooLargeError) throw parseErr;
|
|
18719
19390
|
if (!response.ok) throw new Error(`HTTP ${response.status} from Stack Exchange API`, { cause: parseErr });
|
|
18720
19391
|
throw parseErr;
|
|
18721
19392
|
}
|
|
@@ -19489,6 +20160,7 @@ init_retry_utils();
|
|
|
19489
20160
|
init_circuit_breaker();
|
|
19490
20161
|
init_logger();
|
|
19491
20162
|
init_metrics();
|
|
20163
|
+
init_http_body();
|
|
19492
20164
|
|
|
19493
20165
|
// src/youtube/potoken.ts
|
|
19494
20166
|
init_logger();
|
|
@@ -19721,7 +20393,7 @@ async function fetchOne(session, videoId, contentToken, lang, timeoutMs, signal,
|
|
|
19721
20393
|
});
|
|
19722
20394
|
throw new Error(`HTTP ${response.status} from timedtext endpoint`);
|
|
19723
20395
|
}
|
|
19724
|
-
const body = await response
|
|
20396
|
+
const body = await readTextCapped(response);
|
|
19725
20397
|
const text = parseJson3(body);
|
|
19726
20398
|
if (!text) {
|
|
19727
20399
|
throw new Error("Empty transcript body (bot-protection rejected the token).");
|
|
@@ -21110,7 +21782,7 @@ ${prompt}`);
|
|
|
21110
21782
|
modelRegistry: ctx.modelRegistry,
|
|
21111
21783
|
systemPrompt: prompt,
|
|
21112
21784
|
extensionCtx: ctx,
|
|
21113
|
-
excludeTools: this.options.excludeTools
|
|
21785
|
+
excludeTools: resolveExcludedTools(this.options.excludeTools),
|
|
21114
21786
|
config: this.config,
|
|
21115
21787
|
getGlobalState: () => ({
|
|
21116
21788
|
version: 1,
|
|
@@ -21439,6 +22111,9 @@ var DeepResearchOrchestrator = class {
|
|
|
21439
22111
|
excludeTools: this.options.excludeTools,
|
|
21440
22112
|
steeringMessages: steeringTexts
|
|
21441
22113
|
});
|
|
22114
|
+
} else if (this.currentRound >= maxRounds) {
|
|
22115
|
+
logger.log(`[DeepOrchestrator] Round cap (${maxRounds}) reached \u2014 skipping the evaluator and going straight to final synthesis ${this.elapsed()}`);
|
|
22116
|
+
break;
|
|
21442
22117
|
} else {
|
|
21443
22118
|
const synthesisService2 = await getService(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
|
|
21444
22119
|
observer?.onEvaluationStart?.(this.currentRound);
|
|
@@ -21564,7 +22239,7 @@ var DeepResearchOrchestrator = class {
|
|
|
21564
22239
|
options: {
|
|
21565
22240
|
...this.options,
|
|
21566
22241
|
config: this.config,
|
|
21567
|
-
excludeTools: this.options.excludeTools
|
|
22242
|
+
excludeTools: resolveExcludedTools(this.options.excludeTools),
|
|
21568
22243
|
observer: this.observer
|
|
21569
22244
|
},
|
|
21570
22245
|
currentRound: this.currentRound,
|
|
@@ -21741,7 +22416,7 @@ var ResearchOrchestrationService = class {
|
|
|
21741
22416
|
const { ctx, query, depth = 0, observer, onUpdate, sessionId, researchId, config, excludeTools } = options;
|
|
21742
22417
|
const researchConfig = config || getConfig(ctx.cwd);
|
|
21743
22418
|
const configDisabledTools = researchConfig.DISABLED_TOOLS ? researchConfig.DISABLED_TOOLS.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
21744
|
-
const mergedExcludeTools =
|
|
22419
|
+
const mergedExcludeTools = resolveExcludedTools(excludeTools, configDisabledTools);
|
|
21745
22420
|
const selectedModel = await this.resolveResearchModel(options);
|
|
21746
22421
|
logger.info(`[ResearchOrchestrationService] Using model: ${selectedModel.provider}/${selectedModel.id}`);
|
|
21747
22422
|
const researchStart = Date.now();
|
|
@@ -22306,6 +22981,23 @@ var ResearchSynthesisService = class _ResearchSynthesisService {
|
|
|
22306
22981
|
this.sessions.set(sessionId, reports);
|
|
22307
22982
|
return reports;
|
|
22308
22983
|
}
|
|
22984
|
+
/**
|
|
22985
|
+
* Read a session's reports WITHOUT creating an entry for an unknown id.
|
|
22986
|
+
*
|
|
22987
|
+
* The read-only accessors below must not go through `getSessionReports`: that
|
|
22988
|
+
* is create-on-read, so a query for an id we have never seen inserts an empty
|
|
22989
|
+
* map — and at MAX_SESSIONS that insert evicts the current head. The LRU touch
|
|
22990
|
+
* protects sessions that are read or written, but an in-flight run whose
|
|
22991
|
+
* researchers are all still executing (reports stored early, nothing touched
|
|
22992
|
+
* since) is exactly the stalest key, so a stray read for an unknown session
|
|
22993
|
+
* could evict a live run's reports and silently shrink its final synthesis.
|
|
22994
|
+
*
|
|
22995
|
+
* Returns a shared empty map for unknown ids — callers here only read.
|
|
22996
|
+
*/
|
|
22997
|
+
peekSessionReports(sessionId) {
|
|
22998
|
+
return this.sessions.get(sessionId) ?? _ResearchSynthesisService.EMPTY_REPORTS;
|
|
22999
|
+
}
|
|
23000
|
+
static EMPTY_REPORTS = /* @__PURE__ */ new Map();
|
|
22309
23001
|
/**
|
|
22310
23002
|
* Store a researcher report
|
|
22311
23003
|
* @param sessionId - Session identifier
|
|
@@ -22319,13 +23011,13 @@ var ResearchSynthesisService = class _ResearchSynthesisService {
|
|
|
22319
23011
|
* Get a report by ID
|
|
22320
23012
|
*/
|
|
22321
23013
|
getReport(sessionId, id) {
|
|
22322
|
-
return this.
|
|
23014
|
+
return this.peekSessionReports(sessionId).get(id);
|
|
22323
23015
|
}
|
|
22324
23016
|
/**
|
|
22325
23017
|
* Get all reports
|
|
22326
23018
|
*/
|
|
22327
23019
|
getAllReports(sessionId) {
|
|
22328
|
-
return new Map(this.
|
|
23020
|
+
return new Map(this.peekSessionReports(sessionId));
|
|
22329
23021
|
}
|
|
22330
23022
|
/**
|
|
22331
23023
|
* Get reports for a specific round
|
|
@@ -22333,7 +23025,7 @@ var ResearchSynthesisService = class _ResearchSynthesisService {
|
|
|
22333
23025
|
getReportsForRound(sessionId, round) {
|
|
22334
23026
|
const roundReports = /* @__PURE__ */ new Map();
|
|
22335
23027
|
const prefix = `${round}.`;
|
|
22336
|
-
for (const [key, report] of this.
|
|
23028
|
+
for (const [key, report] of this.peekSessionReports(sessionId).entries()) {
|
|
22337
23029
|
if (key.startsWith(prefix)) {
|
|
22338
23030
|
roundReports.set(key, report);
|
|
22339
23031
|
}
|
|
@@ -22344,13 +23036,13 @@ var ResearchSynthesisService = class _ResearchSynthesisService {
|
|
|
22344
23036
|
* Get total number of reports
|
|
22345
23037
|
*/
|
|
22346
23038
|
getReportCount(sessionId) {
|
|
22347
|
-
return this.
|
|
23039
|
+
return this.peekSessionReports(sessionId).size;
|
|
22348
23040
|
}
|
|
22349
23041
|
/**
|
|
22350
23042
|
* Check if there are any reports
|
|
22351
23043
|
*/
|
|
22352
23044
|
hasReports(sessionId) {
|
|
22353
|
-
return this.
|
|
23045
|
+
return this.peekSessionReports(sessionId).size > 0;
|
|
22354
23046
|
}
|
|
22355
23047
|
/**
|
|
22356
23048
|
* Clear reports for a session, or all reports if no sessionId provided
|
|
@@ -22369,7 +23061,7 @@ var ResearchSynthesisService = class _ResearchSynthesisService {
|
|
|
22369
23061
|
* @returns Fallback synthesis string
|
|
22370
23062
|
*/
|
|
22371
23063
|
buildFallbackSynthesis(sessionId, currentRound = 0) {
|
|
22372
|
-
const reports = this.
|
|
23064
|
+
const reports = this.peekSessionReports(sessionId);
|
|
22373
23065
|
const reportCount = reports.size;
|
|
22374
23066
|
const roundInfo = currentRound > 0 ? ` (up to Round ${currentRound})` : "";
|
|
22375
23067
|
let synthesis = `Research Findings${roundInfo}
|
|
@@ -22405,7 +23097,7 @@ Research performed using ${modelId}`;
|
|
|
22405
23097
|
* @returns Synthesis with guaranteed and verified CITED LINKS section
|
|
22406
23098
|
*/
|
|
22407
23099
|
ensureCitedLinks(sessionId, synthesis) {
|
|
22408
|
-
const reports = this.
|
|
23100
|
+
const reports = this.peekSessionReports(sessionId);
|
|
22409
23101
|
let globalCitations = reports.size > 0 ? normalizeCitations(reports).globalCitations : [];
|
|
22410
23102
|
if (reports.size > 0) {
|
|
22411
23103
|
const scraped = getScrapedLinks(sessionId);
|
|
@@ -22504,7 +23196,7 @@ ${guidanceSection}`;
|
|
|
22504
23196
|
extractAllCitations(sessionId) {
|
|
22505
23197
|
const seen = /* @__PURE__ */ new Set();
|
|
22506
23198
|
const allCitations = [];
|
|
22507
|
-
const reports = this.
|
|
23199
|
+
const reports = this.peekSessionReports(sessionId);
|
|
22508
23200
|
for (const report of reports.values()) {
|
|
22509
23201
|
const citations = parseCitations(report);
|
|
22510
23202
|
for (const cit of citations) {
|
|
@@ -23166,10 +23858,10 @@ init_url_utils();
|
|
|
23166
23858
|
// src/utils/research-export.ts
|
|
23167
23859
|
init_logger();
|
|
23168
23860
|
init_constants();
|
|
23169
|
-
import { promises as
|
|
23861
|
+
import { promises as fs15 } from "node:fs";
|
|
23170
23862
|
import { homedir as homedir4, tmpdir as tmpdir4, platform as platform3 } from "node:os";
|
|
23171
|
-
import { join as
|
|
23172
|
-
import { randomBytes as
|
|
23863
|
+
import { join as join19, resolve as resolve3, normalize, sep as sep2 } from "node:path";
|
|
23864
|
+
import { randomBytes as randomBytes6 } from "node:crypto";
|
|
23173
23865
|
var PREFERRED_SUBDIRS = [
|
|
23174
23866
|
"research",
|
|
23175
23867
|
"docs",
|
|
@@ -23232,16 +23924,16 @@ function isSafeToWriteDirectly(dir) {
|
|
|
23232
23924
|
async function findPreferredSubdir(cwd) {
|
|
23233
23925
|
let entries;
|
|
23234
23926
|
try {
|
|
23235
|
-
entries = await
|
|
23927
|
+
entries = await fs15.readdir(cwd);
|
|
23236
23928
|
} catch {
|
|
23237
23929
|
return null;
|
|
23238
23930
|
}
|
|
23239
23931
|
for (const target of PREFERRED_SUBDIRS) {
|
|
23240
23932
|
const match = entries.find((e) => e.toLowerCase() === target);
|
|
23241
23933
|
if (match === void 0) continue;
|
|
23242
|
-
const candidate =
|
|
23934
|
+
const candidate = join19(cwd, match);
|
|
23243
23935
|
try {
|
|
23244
|
-
const stat5 = await
|
|
23936
|
+
const stat5 = await fs15.stat(candidate);
|
|
23245
23937
|
if (stat5.isDirectory()) return candidate;
|
|
23246
23938
|
} catch {
|
|
23247
23939
|
}
|
|
@@ -23264,7 +23956,7 @@ function sanitizeQuery2(query) {
|
|
|
23264
23956
|
return query.toLowerCase().trim().slice(0, MAX_FILENAME_QUERY_LENGTH).replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
23265
23957
|
}
|
|
23266
23958
|
function generateHash() {
|
|
23267
|
-
return
|
|
23959
|
+
return randomBytes6(3).toString("hex");
|
|
23268
23960
|
}
|
|
23269
23961
|
async function exportResearchReport(query, result, _mode, cwd, explicitDir) {
|
|
23270
23962
|
const sanitizedQuery = sanitizeQuery2(query);
|
|
@@ -23273,7 +23965,7 @@ async function exportResearchReport(query, result, _mode, cwd, explicitDir) {
|
|
|
23273
23965
|
if (explicitDir && explicitDir.trim()) {
|
|
23274
23966
|
targetDir = resolve3(explicitDir.trim());
|
|
23275
23967
|
try {
|
|
23276
|
-
await
|
|
23968
|
+
await fs15.mkdir(targetDir, { recursive: true });
|
|
23277
23969
|
} catch (error) {
|
|
23278
23970
|
logger.error(`[export] Could not create export dir "${targetDir}":`, error);
|
|
23279
23971
|
return null;
|
|
@@ -23284,9 +23976,9 @@ async function exportResearchReport(query, result, _mode, cwd, explicitDir) {
|
|
|
23284
23976
|
for (let attempt = 0; attempt < MAX_EXPORT_RETRIES; attempt++) {
|
|
23285
23977
|
const hash = generateHash();
|
|
23286
23978
|
const filename = `${baseFilename}-${hash}.md`;
|
|
23287
|
-
const filepath =
|
|
23979
|
+
const filepath = join19(targetDir, filename);
|
|
23288
23980
|
try {
|
|
23289
|
-
await
|
|
23981
|
+
await fs15.writeFile(filepath, result, { flag: "wx" });
|
|
23290
23982
|
logger.log(`[export] Research report saved to: ${filepath}`);
|
|
23291
23983
|
return filepath;
|
|
23292
23984
|
} catch (error) {
|
|
@@ -23319,7 +24011,7 @@ import { getAgentDir as getAgentDir2 } from "@earendil-works/pi-coding-agent";
|
|
|
23319
24011
|
function readPkgVersion() {
|
|
23320
24012
|
try {
|
|
23321
24013
|
const pkgPath = fileURLToPath5(new URL("../package.json", import.meta.url));
|
|
23322
|
-
const pkg = JSON.parse(
|
|
24014
|
+
const pkg = JSON.parse(fs17.readFileSync(pkgPath, "utf-8"));
|
|
23323
24015
|
return typeof pkg.version === "string" && pkg.version ? pkg.version : "0.0.0";
|
|
23324
24016
|
} catch {
|
|
23325
24017
|
return "0.0.0";
|
|
@@ -23342,8 +24034,28 @@ var EXIT = {
|
|
|
23342
24034
|
* again shortly" apart from "this run crashed" and back off instead of
|
|
23343
24035
|
* re-running a broken command.
|
|
23344
24036
|
*/
|
|
23345
|
-
TEMPFAIL: 75
|
|
24037
|
+
TEMPFAIL: 75,
|
|
24038
|
+
/**
|
|
24039
|
+
* The run was CANCELLED via a programmatic abort (the caller's AbortSignal),
|
|
24040
|
+
* with no OS signal involved. 130 — 128+SIGINT — is the conventional
|
|
24041
|
+
* "interrupted" code and the natural default when there is no signal number to
|
|
24042
|
+
* derive one from.
|
|
24043
|
+
*
|
|
24044
|
+
* A cancellation caused by an actual SIGNAL does NOT use this constant: see
|
|
24045
|
+
* exitCodeForSignal below, which reports the true 128+N.
|
|
24046
|
+
*
|
|
24047
|
+
* Distinct from SOFTWARE for the same reason TEMPFAIL is: a deliberate stop is
|
|
24048
|
+
* not a fault. Reported as SOFTWARE (70), a cancel matched the skill contract's
|
|
24049
|
+
* "runtime error — suggest one retry for transient failures", so an agent
|
|
24050
|
+
* relaying the result would offer to re-run the very research the user had just
|
|
24051
|
+
* interrupted, and a --json consumer could not tell a cancel from a crash.
|
|
24052
|
+
*/
|
|
24053
|
+
CANCELLED: 130
|
|
23346
24054
|
};
|
|
24055
|
+
function exitCodeForSignal(sig) {
|
|
24056
|
+
const signum = os7.constants.signals[sig];
|
|
24057
|
+
return typeof signum === "number" ? 128 + signum : EXIT.CANCELLED;
|
|
24058
|
+
}
|
|
23347
24059
|
function toStderr(s) {
|
|
23348
24060
|
process.stderr.write(s);
|
|
23349
24061
|
}
|
|
@@ -23417,16 +24129,22 @@ function bridgeConfigEnv(explicitConfigPath) {
|
|
|
23417
24129
|
const cliOverlayPath = getInterfaceEnvFilePath("cli");
|
|
23418
24130
|
let loaded = false;
|
|
23419
24131
|
const merged = {};
|
|
24132
|
+
const fromExplicitBase = /* @__PURE__ */ new Set();
|
|
23420
24133
|
for (const filePath of [basePath, cliOverlayPath]) {
|
|
23421
24134
|
try {
|
|
23422
|
-
if (!
|
|
23423
|
-
|
|
24135
|
+
if (!fs17.existsSync(filePath)) continue;
|
|
24136
|
+
const parsed = parseDotEnv(fs17.readFileSync(filePath, "utf-8"));
|
|
24137
|
+
Object.assign(merged, parsed);
|
|
24138
|
+
for (const k of Object.keys(parsed)) {
|
|
24139
|
+
if (explicitConfigPath !== void 0 && filePath === basePath) fromExplicitBase.add(k);
|
|
24140
|
+
else fromExplicitBase.delete(k);
|
|
24141
|
+
}
|
|
23424
24142
|
loaded = true;
|
|
23425
24143
|
} catch {
|
|
23426
24144
|
}
|
|
23427
24145
|
}
|
|
23428
24146
|
for (const [k, v] of Object.entries(merged)) {
|
|
23429
|
-
if (isProjectScopedKey(k)) continue;
|
|
24147
|
+
if (isProjectScopedKey(k) && !fromExplicitBase.has(k)) continue;
|
|
23430
24148
|
if (process.env[k] === void 0 && (k.startsWith("PI_RESEARCH_") || k === "STACKEXCHANGE_API_KEY" || k === "GITHUB_TOKEN" || k === "NVD_API_KEY")) {
|
|
23431
24149
|
process.env[k] = v;
|
|
23432
24150
|
}
|
|
@@ -23448,7 +24166,7 @@ function resolvedConfigPaths() {
|
|
|
23448
24166
|
}
|
|
23449
24167
|
function fileExists(p) {
|
|
23450
24168
|
try {
|
|
23451
|
-
return
|
|
24169
|
+
return fs17.existsSync(p);
|
|
23452
24170
|
} catch {
|
|
23453
24171
|
return false;
|
|
23454
24172
|
}
|
|
@@ -23919,18 +24637,26 @@ async function reportError(err, what, json) {
|
|
|
23919
24637
|
const lower = msg2.toLowerCase();
|
|
23920
24638
|
const isTaggedResearchStop = err?.code === RESEARCH_STOPPED_ERROR_CODE;
|
|
23921
24639
|
const isCapacity = err instanceof Error && err.name === "ResearchRunCapacityError";
|
|
23922
|
-
const
|
|
23923
|
-
const
|
|
23924
|
-
const
|
|
24640
|
+
const isCancellation = cancellationRequested || err instanceof Error && /^(aborted|research (aborted|cancelled))$/i.test(msg2);
|
|
24641
|
+
const isConfigError = !isTaggedResearchStop && !isCapacity && !isCancellation && (lower.includes("no llm model available") || lower.includes("not found in pi") || lower.includes("no api key") || lower.includes("provider must be specified") || lower.includes("invalid model string") || lower.includes("authentication") || lower.includes("unauthorized") || lower.includes("api key"));
|
|
24642
|
+
const isRateLimit = !isCancellation && (lower.includes("429") || lower.includes("rate limit") || lower.includes("too many requests"));
|
|
24643
|
+
const exitCode = isCancellation ? cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED : isCapacity ? EXIT.TEMPFAIL : isConfigError && !isRateLimit ? EXIT.CONFIG : EXIT.SOFTWARE;
|
|
23925
24644
|
if (json) {
|
|
23926
24645
|
const payload = { ok: false, error: msg2, exitCode };
|
|
23927
|
-
if (isCapacity || isRateLimit) payload["retryable"] = true;
|
|
24646
|
+
if ((isCapacity || isRateLimit) && !isCancellation) payload["retryable"] = true;
|
|
24647
|
+
if (isCancellation) payload["cancelled"] = true;
|
|
23928
24648
|
if (process.env["PI_RESEARCH_DEBUG"] === "true" && err instanceof Error && err.stack) {
|
|
23929
24649
|
payload["stack"] = err.stack;
|
|
23930
24650
|
}
|
|
23931
24651
|
toStdout(pretty(payload));
|
|
23932
24652
|
return exitCode;
|
|
23933
24653
|
}
|
|
24654
|
+
if (isCancellation) {
|
|
24655
|
+
toStderr(`
|
|
24656
|
+
pi-research ${what} cancelled.
|
|
24657
|
+
`);
|
|
24658
|
+
return exitCode;
|
|
24659
|
+
}
|
|
23934
24660
|
if (isCapacity) {
|
|
23935
24661
|
toStderr(`
|
|
23936
24662
|
Error: pi-research ${what} did not start: ${msg2}
|
|
@@ -24037,12 +24763,14 @@ CONFIGURE
|
|
|
24037
24763
|
Run \`${BINARY_NAME} status\` to see exactly what is detected.
|
|
24038
24764
|
|
|
24039
24765
|
EXIT CODES
|
|
24040
|
-
0
|
|
24041
|
-
64
|
|
24042
|
-
75
|
|
24043
|
-
78
|
|
24044
|
-
|
|
24045
|
-
|
|
24766
|
+
0 success
|
|
24767
|
+
64 bad arguments
|
|
24768
|
+
75 at capacity \u2014 every concurrent run slot is busy; nothing is broken, retry later
|
|
24769
|
+
78 not configured \u2014 message prints the exact locations to fix
|
|
24770
|
+
(on \`knowledge\`, also returned when the store is deliberately disabled)
|
|
24771
|
+
130 cancelled \u2014 Ctrl-C; nothing failed, do not auto-retry
|
|
24772
|
+
(other stop signals report POSIX 128+N: 143 SIGTERM, 129 SIGHUP, 131 SIGQUIT)
|
|
24773
|
+
70 runtime error (network, provider, internal)
|
|
24046
24774
|
`;
|
|
24047
24775
|
}
|
|
24048
24776
|
function parseArgs(argv) {
|
|
@@ -24237,19 +24965,35 @@ var flushAndExit = (code) => {
|
|
|
24237
24965
|
});
|
|
24238
24966
|
};
|
|
24239
24967
|
var activeResearchAbortController = null;
|
|
24968
|
+
var cancellationRequested = false;
|
|
24969
|
+
var cancellationSignal = null;
|
|
24970
|
+
function _resetCancellationLatchForTests() {
|
|
24971
|
+
cancellationRequested = false;
|
|
24972
|
+
cancellationSignal = null;
|
|
24973
|
+
}
|
|
24974
|
+
function _markCancellationForTests(sig) {
|
|
24975
|
+
cancellationRequested = true;
|
|
24976
|
+
cancellationSignal = sig ?? null;
|
|
24977
|
+
}
|
|
24240
24978
|
async function main(argv) {
|
|
24241
24979
|
let _signalCleanupDone = false;
|
|
24242
24980
|
const onSignal = (sig) => {
|
|
24243
24981
|
if (_signalCleanupDone) return;
|
|
24244
24982
|
_signalCleanupDone = true;
|
|
24983
|
+
cancellationRequested = true;
|
|
24984
|
+
cancellationSignal = sig;
|
|
24245
24985
|
toStderr(`
|
|
24246
24986
|
[pi-research] ${sig} \u2014 cleaning up\u2026
|
|
24247
24987
|
`);
|
|
24248
24988
|
activeResearchAbortController?.abort();
|
|
24249
|
-
void safeShutdown().finally(() => flushAndExit(
|
|
24989
|
+
void safeShutdown().finally(() => flushAndExit(exitCodeForSignal(sig)));
|
|
24250
24990
|
};
|
|
24251
24991
|
process.once("SIGINT", () => onSignal("SIGINT"));
|
|
24252
24992
|
process.once("SIGTERM", () => onSignal("SIGTERM"));
|
|
24993
|
+
process.once("SIGHUP", () => onSignal("SIGHUP"));
|
|
24994
|
+
if (process.platform === "win32") {
|
|
24995
|
+
process.once("SIGBREAK", () => onSignal("SIGBREAK"));
|
|
24996
|
+
}
|
|
24253
24997
|
let parsed;
|
|
24254
24998
|
try {
|
|
24255
24999
|
parsed = parseArgs(argv);
|
|
@@ -24296,8 +25040,8 @@ ${buildHelp()}`);
|
|
|
24296
25040
|
var _isMain = (() => {
|
|
24297
25041
|
try {
|
|
24298
25042
|
if (!process.argv[1]) return false;
|
|
24299
|
-
const modulePath =
|
|
24300
|
-
const invokedPath =
|
|
25043
|
+
const modulePath = fs17.realpathSync(path18.resolve(fileURLToPath5(import.meta.url)));
|
|
25044
|
+
const invokedPath = fs17.realpathSync(path18.resolve(process.argv[1]));
|
|
24301
25045
|
return modulePath === invokedPath;
|
|
24302
25046
|
} catch {
|
|
24303
25047
|
return false;
|
|
@@ -24314,7 +25058,10 @@ Error: fatal: ${err instanceof Error ? err.message : String(err)}
|
|
|
24314
25058
|
export {
|
|
24315
25059
|
EXIT,
|
|
24316
25060
|
UsageError,
|
|
25061
|
+
_markCancellationForTests,
|
|
25062
|
+
_resetCancellationLatchForTests,
|
|
24317
25063
|
bridgeConfigEnv,
|
|
25064
|
+
exitCodeForSignal,
|
|
24318
25065
|
parseArgs,
|
|
24319
25066
|
reportError
|
|
24320
25067
|
};
|