@lincoln504/pi-research 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +166 -117
- package/dist/thread-worker.mjs +29 -2
- package/docs/CHANGELOG.md +49 -0
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cleanup/research-cleanup.ts +3 -35
- package/src/cli.ts +11 -3
- package/src/config.ts +28 -3
- package/src/core/interfaces/state-manager-interfaces.ts +0 -1
- package/src/core/service-registry.ts +104 -36
- package/src/infrastructure/browser/thread-worker-browser.ts +53 -4
- package/src/infrastructure/browser/thread-worker.mjs +29 -2
- package/src/infrastructure/browser/thread-worker.ts +8 -0
- package/src/infrastructure/knowledge-store-service.ts +78 -32
- package/src/infrastructure/state/state-browser-api.ts +0 -22
- package/src/infrastructure/state/state-browser-manager.ts +0 -21
- package/src/infrastructure/state/state-manager-service.ts +0 -7
- package/src/infrastructure/state/state-manager.ts +0 -11
- package/src/infrastructure/state/state-path-configuration.ts +7 -3
- package/src/knowledge/store.ts +33 -4
- package/src/orchestration/researcher-executor.ts +15 -5
- package/src/tools/research-tool-definition.ts +34 -8
- package/src/types/index.ts +0 -2
- package/src/types/research-panel-types.ts +0 -2
- package/src/utils/log-utils.ts +13 -1
- package/src/utils/url-utils.ts +15 -7
- package/src/web-research/scraper-utils.ts +9 -5
- package/src/web-research/web-scraper.ts +29 -4
package/dist/cli.mjs
CHANGED
|
@@ -215,7 +215,12 @@ function formatArg(arg) {
|
|
|
215
215
|
return String(arg);
|
|
216
216
|
}
|
|
217
217
|
function redactSecrets(message) {
|
|
218
|
-
let
|
|
218
|
+
let keep = MAX_LOG_MESSAGE_LENGTH;
|
|
219
|
+
if (message.length > keep) {
|
|
220
|
+
const lastKeptUnit = message.charCodeAt(keep - 1);
|
|
221
|
+
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
222
|
+
}
|
|
223
|
+
let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
|
|
219
224
|
out = out.replace(ANSI_PATTERN, "");
|
|
220
225
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
221
226
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -1102,6 +1107,7 @@ var init_service_registry = __esm({
|
|
|
1102
1107
|
factory,
|
|
1103
1108
|
instance: null,
|
|
1104
1109
|
initializationPromise: null,
|
|
1110
|
+
disposalPromise: null,
|
|
1105
1111
|
options: mergedOptions
|
|
1106
1112
|
});
|
|
1107
1113
|
this.dependencies.set(name, /* @__PURE__ */ new Set());
|
|
@@ -1149,6 +1155,10 @@ var init_service_registry = __esm({
|
|
|
1149
1155
|
if (!registration) {
|
|
1150
1156
|
throw new Error(`Service '${name}' is not registered`);
|
|
1151
1157
|
}
|
|
1158
|
+
while (registration.disposalPromise) {
|
|
1159
|
+
await registration.disposalPromise.catch(() => {
|
|
1160
|
+
});
|
|
1161
|
+
}
|
|
1152
1162
|
if (registration.instance) {
|
|
1153
1163
|
if (registration.initializationPromise && caller === void 0) {
|
|
1154
1164
|
await registration.initializationPromise.catch(() => {
|
|
@@ -1163,12 +1173,20 @@ var init_service_registry = __esm({
|
|
|
1163
1173
|
return registration.instance;
|
|
1164
1174
|
});
|
|
1165
1175
|
registration.initializationPromise = reinit;
|
|
1166
|
-
const
|
|
1167
|
-
|
|
1176
|
+
const isStillCurrent = () => registration.initializationPromise === reinit;
|
|
1177
|
+
reinit.then(
|
|
1178
|
+
() => {
|
|
1179
|
+
if (isStillCurrent()) registration.initializationPromise = null;
|
|
1180
|
+
},
|
|
1181
|
+
() => {
|
|
1182
|
+
if (!isStillCurrent()) return;
|
|
1168
1183
|
registration.initializationPromise = null;
|
|
1184
|
+
if (registration.instance) {
|
|
1185
|
+
registration.instance.lifecycle = "uninitialized" /* UNINITIALIZED */;
|
|
1186
|
+
registration.instance = null;
|
|
1187
|
+
}
|
|
1169
1188
|
}
|
|
1170
|
-
|
|
1171
|
-
reinit.then(clearIfCurrent, clearIfCurrent);
|
|
1189
|
+
);
|
|
1172
1190
|
return reinit;
|
|
1173
1191
|
}
|
|
1174
1192
|
return registration.instance;
|
|
@@ -1237,20 +1255,30 @@ var init_service_registry = __esm({
|
|
|
1237
1255
|
if (!registration) {
|
|
1238
1256
|
throw new Error(`Service '${name}' is not registered`);
|
|
1239
1257
|
}
|
|
1240
|
-
if (registration.
|
|
1241
|
-
|
|
1242
|
-
});
|
|
1243
|
-
}
|
|
1244
|
-
if (registration.instance && registration.instance.dispose) {
|
|
1245
|
-
await registration.instance.dispose().catch((err) => {
|
|
1246
|
-
logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
|
|
1247
|
-
});
|
|
1248
|
-
}
|
|
1249
|
-
registration.instance = null;
|
|
1250
|
-
registration.initializationPromise = null;
|
|
1251
|
-
if (registration.options.enableLogging) {
|
|
1252
|
-
logger.debug(`[ServiceContainer] Cleared service '${name}'`);
|
|
1258
|
+
if (registration.disposalPromise) {
|
|
1259
|
+
return registration.disposalPromise;
|
|
1253
1260
|
}
|
|
1261
|
+
registration.disposalPromise = (async () => {
|
|
1262
|
+
try {
|
|
1263
|
+
if (registration.initializationPromise) {
|
|
1264
|
+
await registration.initializationPromise.catch(() => {
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
if (registration.instance && registration.instance.dispose) {
|
|
1268
|
+
await registration.instance.dispose().catch((err) => {
|
|
1269
|
+
logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
|
|
1270
|
+
});
|
|
1271
|
+
}
|
|
1272
|
+
registration.instance = null;
|
|
1273
|
+
registration.initializationPromise = null;
|
|
1274
|
+
if (registration.options.enableLogging) {
|
|
1275
|
+
logger.debug(`[ServiceContainer] Cleared service '${name}'`);
|
|
1276
|
+
}
|
|
1277
|
+
} finally {
|
|
1278
|
+
registration.disposalPromise = null;
|
|
1279
|
+
}
|
|
1280
|
+
})();
|
|
1281
|
+
return registration.disposalPromise;
|
|
1254
1282
|
}
|
|
1255
1283
|
/**
|
|
1256
1284
|
* Replace a service instance with a new one
|
|
@@ -1260,20 +1288,30 @@ var init_service_registry = __esm({
|
|
|
1260
1288
|
if (!registration) {
|
|
1261
1289
|
throw new Error(`Service '${name}' is not registered`);
|
|
1262
1290
|
}
|
|
1263
|
-
if (registration.
|
|
1264
|
-
await registration.
|
|
1265
|
-
});
|
|
1266
|
-
}
|
|
1267
|
-
if (registration.instance && registration.instance.dispose) {
|
|
1268
|
-
await registration.instance.dispose().catch((err) => {
|
|
1269
|
-
logger.warn(`[ServiceContainer] Error disposing old service '${name}':`, err);
|
|
1270
|
-
});
|
|
1271
|
-
}
|
|
1272
|
-
registration.instance = newInstance;
|
|
1273
|
-
registration.initializationPromise = null;
|
|
1274
|
-
if (registration.options.enableLogging) {
|
|
1275
|
-
logger.debug(`[ServiceContainer] Replaced service '${name}'`);
|
|
1291
|
+
if (registration.disposalPromise) {
|
|
1292
|
+
await registration.disposalPromise;
|
|
1276
1293
|
}
|
|
1294
|
+
registration.disposalPromise = (async () => {
|
|
1295
|
+
try {
|
|
1296
|
+
if (registration.initializationPromise) {
|
|
1297
|
+
await registration.initializationPromise.catch(() => {
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
if (registration.instance && registration.instance.dispose) {
|
|
1301
|
+
await registration.instance.dispose().catch((err) => {
|
|
1302
|
+
logger.warn(`[ServiceContainer] Error disposing old service '${name}':`, err);
|
|
1303
|
+
});
|
|
1304
|
+
}
|
|
1305
|
+
registration.instance = newInstance;
|
|
1306
|
+
registration.initializationPromise = null;
|
|
1307
|
+
if (registration.options.enableLogging) {
|
|
1308
|
+
logger.debug(`[ServiceContainer] Replaced service '${name}'`);
|
|
1309
|
+
}
|
|
1310
|
+
} finally {
|
|
1311
|
+
registration.disposalPromise = null;
|
|
1312
|
+
}
|
|
1313
|
+
})();
|
|
1314
|
+
return registration.disposalPromise;
|
|
1277
1315
|
}
|
|
1278
1316
|
/**
|
|
1279
1317
|
* Dispose all services using a true Directed Acyclic Graph (DAG) teardown.
|
|
@@ -1853,13 +1891,12 @@ function stripTrailingLlmPunctuation(input) {
|
|
|
1853
1891
|
}
|
|
1854
1892
|
if (last === ")" || last === "]" || last === "}") {
|
|
1855
1893
|
const open5 = last === ")" ? "(" : last === "]" ? "[" : "{";
|
|
1856
|
-
let
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
if (ch ===
|
|
1860
|
-
else if (ch === last) closes++;
|
|
1894
|
+
let balance = 0;
|
|
1895
|
+
for (const ch of s.slice(0, -1)) {
|
|
1896
|
+
if (ch === open5) balance++;
|
|
1897
|
+
else if (ch === last) balance = Math.max(0, balance - 1);
|
|
1861
1898
|
}
|
|
1862
|
-
if (
|
|
1899
|
+
if (balance === 0) {
|
|
1863
1900
|
s = s.slice(0, -1);
|
|
1864
1901
|
continue;
|
|
1865
1902
|
}
|
|
@@ -2211,9 +2248,18 @@ import { Value } from "typebox/value";
|
|
|
2211
2248
|
function isProjectScopedKey(key) {
|
|
2212
2249
|
return LOCAL_SCOPE_KEYS.has(key);
|
|
2213
2250
|
}
|
|
2251
|
+
function getStateDir() {
|
|
2252
|
+
const override = process.env["PI_RESEARCH_STATE_DIR"];
|
|
2253
|
+
if (override) {
|
|
2254
|
+
if (path5.isAbsolute(override)) return override;
|
|
2255
|
+
logger.warn(
|
|
2256
|
+
`[config] PI_RESEARCH_STATE_DIR must be an absolute path \u2014 ignoring relative value "${override}" and using the default state directory.`
|
|
2257
|
+
);
|
|
2258
|
+
}
|
|
2259
|
+
return path5.join(getGlobalConfigDir(), "state");
|
|
2260
|
+
}
|
|
2214
2261
|
function getProjectSettingsRegistryPath() {
|
|
2215
|
-
|
|
2216
|
-
return path5.join(stateDir, "project-settings.json");
|
|
2262
|
+
return path5.join(getStateDir(), "project-settings.json");
|
|
2217
2263
|
}
|
|
2218
2264
|
function loadProjectSettingsRegistry() {
|
|
2219
2265
|
const registryPath = getProjectSettingsRegistryPath();
|
|
@@ -5718,7 +5764,18 @@ var init_store = __esm({
|
|
|
5718
5764
|
metrics.increment("knowledge_store_cache_hits_total", 1, { status: "miss" });
|
|
5719
5765
|
return null;
|
|
5720
5766
|
}
|
|
5721
|
-
|
|
5767
|
+
const chunkIndexOf = (row) => {
|
|
5768
|
+
try {
|
|
5769
|
+
const parsed = JSON.parse(row.metadata);
|
|
5770
|
+
return typeof parsed.chunkIndex === "number" ? parsed.chunkIndex : 0;
|
|
5771
|
+
} catch {
|
|
5772
|
+
return 0;
|
|
5773
|
+
}
|
|
5774
|
+
};
|
|
5775
|
+
results.sort((a, b) => {
|
|
5776
|
+
const tsDiff = Number(b.timestamp) - Number(a.timestamp);
|
|
5777
|
+
return tsDiff !== 0 ? tsDiff : chunkIndexOf(a) - chunkIndexOf(b);
|
|
5778
|
+
});
|
|
5722
5779
|
const r = results[0];
|
|
5723
5780
|
const rawAgeMs = Date.now() - Number(r.timestamp);
|
|
5724
5781
|
const hasAge = Number.isFinite(rawAgeMs);
|
|
@@ -5733,7 +5790,7 @@ var init_store = __esm({
|
|
|
5733
5790
|
try {
|
|
5734
5791
|
const metadata = JSON.parse(r.metadata);
|
|
5735
5792
|
const description = typeof metadata.description === "string" ? metadata.description : null;
|
|
5736
|
-
const textToReturn = r.content || r.text ||
|
|
5793
|
+
const textToReturn = r.content || description || r.text || "";
|
|
5737
5794
|
metrics.increment("knowledge_store_cache_hits_total", 1, { status: "hit" });
|
|
5738
5795
|
logger.log(`[store] Cache hit: synthesis-description for ${url} (${textToReturn.length} chars, ~${ageDays ?? "?"}d old)`);
|
|
5739
5796
|
return { text: textToReturn, description, metadata, ageDays };
|
|
@@ -5852,7 +5909,7 @@ var init_store = __esm({
|
|
|
5852
5909
|
const scopeFilter = this.getScopeFilter();
|
|
5853
5910
|
if (scopeFilter === "1 = 0") return;
|
|
5854
5911
|
const ttlDays = this.options.ttlDays ?? 30;
|
|
5855
|
-
const cutoffTimestamp = Date.now() - ttlDays * 24 * 60 * 60 * 1e3;
|
|
5912
|
+
const cutoffTimestamp = Math.trunc(Date.now() - ttlDays * 24 * 60 * 60 * 1e3);
|
|
5856
5913
|
await table.delete(`timestamp < ${BigInt(cutoffTimestamp)} AND (${scopeFilter})`);
|
|
5857
5914
|
logger.log(`[store] Ran eviction for records older than ${ttlDays} days [scope: ${scopeFilter}]`);
|
|
5858
5915
|
} catch (err) {
|
|
@@ -8449,7 +8506,17 @@ var init_knowledge_store_service = __esm({
|
|
|
8449
8506
|
_initializedMode = void 0;
|
|
8450
8507
|
// Initialization promise to prevent concurrent initialization
|
|
8451
8508
|
_initializationPromise = null;
|
|
8509
|
+
// Disposal promise: lets a concurrent initialize() wait out an in-flight
|
|
8510
|
+
// dispose() instead of racing it. See dispose()'s own comment for the
|
|
8511
|
+
// failure this closes — none of initialize()'s lifecycle guards below
|
|
8512
|
+
// recognize DISPOSING, so without this an initialize() arriving while
|
|
8513
|
+
// dispose() is mid-teardown would start rebuilding immediately, and the
|
|
8514
|
+
// two would race to close/null each other's components.
|
|
8515
|
+
_disposalPromise = null;
|
|
8452
8516
|
async initialize(ctx) {
|
|
8517
|
+
while (this._disposalPromise) {
|
|
8518
|
+
await this._disposalPromise;
|
|
8519
|
+
}
|
|
8453
8520
|
const newCwd = ctx?.cwd || this._cwd;
|
|
8454
8521
|
if ((this.lifecycle === "initialized" /* INITIALIZED */ || this.lifecycle === "disabled" /* DISABLED */) && this._cwd === newCwd) {
|
|
8455
8522
|
const liveMode = (ctx?.config || getConfig(this._cwd)).KNOWLEDGE_STORE_MODE;
|
|
@@ -8588,6 +8655,9 @@ var init_knowledge_store_service = __esm({
|
|
|
8588
8655
|
if (this.lifecycle === "disposed" /* DISPOSED */ || this.lifecycle === "uninitialized" /* UNINITIALIZED */) {
|
|
8589
8656
|
return;
|
|
8590
8657
|
}
|
|
8658
|
+
if (this._disposalPromise) {
|
|
8659
|
+
return this._disposalPromise;
|
|
8660
|
+
}
|
|
8591
8661
|
if (this._initializationPromise) {
|
|
8592
8662
|
try {
|
|
8593
8663
|
await this._initializationPromise;
|
|
@@ -8596,30 +8666,38 @@ var init_knowledge_store_service = __esm({
|
|
|
8596
8666
|
}
|
|
8597
8667
|
this.lifecycle = "disposing" /* DISPOSING */;
|
|
8598
8668
|
logger.debug("[KnowledgeStoreService] Disposing...");
|
|
8599
|
-
|
|
8600
|
-
|
|
8601
|
-
|
|
8602
|
-
|
|
8603
|
-
|
|
8604
|
-
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
8611
|
-
|
|
8669
|
+
const embedder = this._embedder;
|
|
8670
|
+
const store = this._store;
|
|
8671
|
+
const writerQueue = this._writerQueue;
|
|
8672
|
+
const initLock = this._initLock;
|
|
8673
|
+
this._disposalPromise = (async () => {
|
|
8674
|
+
try {
|
|
8675
|
+
if (writerQueue) {
|
|
8676
|
+
await writerQueue.dispose?.();
|
|
8677
|
+
}
|
|
8678
|
+
if (store) {
|
|
8679
|
+
await store.close();
|
|
8680
|
+
}
|
|
8681
|
+
if (embedder) {
|
|
8682
|
+
await embedder.dispose?.();
|
|
8683
|
+
await clearEmbeddingInstance();
|
|
8684
|
+
}
|
|
8685
|
+
if (initLock) {
|
|
8686
|
+
await initLock.dispose();
|
|
8687
|
+
}
|
|
8688
|
+
if (this._embedder === embedder) this._embedder = null;
|
|
8689
|
+
if (this._store === store) this._store = null;
|
|
8690
|
+
if (this._writerQueue === writerQueue) this._writerQueue = null;
|
|
8691
|
+
if (this._initLock === initLock) this._initLock = null;
|
|
8692
|
+
logger.debug("[KnowledgeStoreService] Disposed");
|
|
8693
|
+
} catch (err) {
|
|
8694
|
+
logger.error("[KnowledgeStoreService] Error during disposal:", err);
|
|
8695
|
+
} finally {
|
|
8696
|
+
this.lifecycle = "disposed" /* DISPOSED */;
|
|
8697
|
+
this._disposalPromise = null;
|
|
8612
8698
|
}
|
|
8613
|
-
|
|
8614
|
-
|
|
8615
|
-
this._writerQueue = null;
|
|
8616
|
-
this._initLock = null;
|
|
8617
|
-
logger.debug("[KnowledgeStoreService] Disposed");
|
|
8618
|
-
} catch (err) {
|
|
8619
|
-
logger.error("[KnowledgeStoreService] Error during disposal:", err);
|
|
8620
|
-
} finally {
|
|
8621
|
-
this.lifecycle = "disposed" /* DISPOSED */;
|
|
8622
|
-
}
|
|
8699
|
+
})();
|
|
8700
|
+
return this._disposalPromise;
|
|
8623
8701
|
}
|
|
8624
8702
|
/**
|
|
8625
8703
|
* Check if the knowledge store is ready
|
|
@@ -13106,20 +13184,6 @@ var StateBrowserApi = class {
|
|
|
13106
13184
|
const state = await readState();
|
|
13107
13185
|
return this.browserManager.getBrowserServer(state);
|
|
13108
13186
|
}
|
|
13109
|
-
/**
|
|
13110
|
-
* Set the current browser server information (atomic: only overwrites if no live server exists)
|
|
13111
|
-
* @param port The browser server port
|
|
13112
|
-
* @param pid The browser server process ID
|
|
13113
|
-
* @param schedulerId Optional scheduler ID
|
|
13114
|
-
* @param updateState Function to update state atomically
|
|
13115
|
-
* @param getStartTime Function to get process start time
|
|
13116
|
-
*/
|
|
13117
|
-
async setBrowserServer(port, pid, schedulerId, updateState, getStartTime, authSecret) {
|
|
13118
|
-
const startTime = await getStartTime(pid);
|
|
13119
|
-
await updateState((state) => {
|
|
13120
|
-
return this.browserManager.setBrowserServer(state, port, pid, schedulerId, startTime, authSecret);
|
|
13121
|
-
});
|
|
13122
|
-
}
|
|
13123
13187
|
/**
|
|
13124
13188
|
* Clear the browser server information
|
|
13125
13189
|
* @param updateState Function to update state atomically
|
|
@@ -13428,16 +13492,6 @@ var StateManager = class {
|
|
|
13428
13492
|
async getBrowserServer() {
|
|
13429
13493
|
return this.browserApi.getBrowserServer(this.readState.bind(this));
|
|
13430
13494
|
}
|
|
13431
|
-
async setBrowserServer(port, pid, schedulerId, authSecret) {
|
|
13432
|
-
await this.browserApi.setBrowserServer(
|
|
13433
|
-
port,
|
|
13434
|
-
pid,
|
|
13435
|
-
schedulerId,
|
|
13436
|
-
this.updateState.bind(this),
|
|
13437
|
-
this.processLifecycle.getProcessStartTime.bind(this.processLifecycle),
|
|
13438
|
-
authSecret
|
|
13439
|
-
);
|
|
13440
|
-
}
|
|
13441
13495
|
async clearBrowserServer(expected) {
|
|
13442
13496
|
await this.browserApi.clearBrowserServer(this.updateState.bind(this), expected);
|
|
13443
13497
|
}
|
|
@@ -13637,12 +13691,6 @@ var StateManagerService = class {
|
|
|
13637
13691
|
if (!this._stateManager) return null;
|
|
13638
13692
|
return this.getStateManager().getBrowserServer();
|
|
13639
13693
|
}
|
|
13640
|
-
/**
|
|
13641
|
-
* Set the current browser server information
|
|
13642
|
-
*/
|
|
13643
|
-
async setBrowserServer(port, pid, schedulerId, authSecret) {
|
|
13644
|
-
return this.getStateManager().setBrowserServer(port, pid, schedulerId, authSecret);
|
|
13645
|
-
}
|
|
13646
13694
|
/**
|
|
13647
13695
|
* Clear the browser server information
|
|
13648
13696
|
*/
|
|
@@ -14300,19 +14348,6 @@ var StateBrowserManager = class {
|
|
|
14300
14348
|
getBrowserServer(state) {
|
|
14301
14349
|
return state.browserServer ?? null;
|
|
14302
14350
|
}
|
|
14303
|
-
/**
|
|
14304
|
-
* Set the current browser server information
|
|
14305
|
-
* @param state The current state
|
|
14306
|
-
* @param port The browser server port
|
|
14307
|
-
* @param pid The browser server process ID
|
|
14308
|
-
* @param schedulerId Optional scheduler ID
|
|
14309
|
-
* @param startTime Optional process start time
|
|
14310
|
-
* @returns Updated state with browser server info
|
|
14311
|
-
*/
|
|
14312
|
-
setBrowserServer(state, port, pid, schedulerId, startTime, authSecret) {
|
|
14313
|
-
state.browserServer = { port, pid, schedulerId, startTime: startTime ?? void 0, authSecret };
|
|
14314
|
-
return state;
|
|
14315
|
-
}
|
|
14316
14351
|
/**
|
|
14317
14352
|
* Clear the browser server information
|
|
14318
14353
|
* @param state The current state
|
|
@@ -15115,7 +15150,7 @@ var StatePathConfiguration = class {
|
|
|
15115
15150
|
lockFilePath;
|
|
15116
15151
|
projectSettingsPath;
|
|
15117
15152
|
constructor(stateDir) {
|
|
15118
|
-
const resolvedStateDir = stateDir
|
|
15153
|
+
const resolvedStateDir = stateDir && path10.isAbsolute(stateDir) ? stateDir : getStateDir();
|
|
15119
15154
|
this.stateDir = resolvedStateDir;
|
|
15120
15155
|
this.stateFilePath = path10.join(resolvedStateDir, "research-state.json");
|
|
15121
15156
|
this.lockDirPath = path10.join(resolvedStateDir, ".locks");
|
|
@@ -16769,6 +16804,7 @@ function isBenignScrapeFailure(error) {
|
|
|
16769
16804
|
const msg2 = error instanceof Error ? error.message : String(error);
|
|
16770
16805
|
return msg2.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
|
|
16771
16806
|
msg2.startsWith("Fetch returned stub:") || // nav-only / near-empty page
|
|
16807
|
+
msg2.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
|
|
16772
16808
|
/^HTTP \d{3}\b/.test(msg2) || // remote 4xx/5xx status
|
|
16773
16809
|
isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
|
|
16774
16810
|
}
|
|
@@ -17088,7 +17124,7 @@ ${doc.toMarkdown(i)}
|
|
|
17088
17124
|
return markdown;
|
|
17089
17125
|
} catch (e) {
|
|
17090
17126
|
const msg2 = e instanceof Error ? e.message : String(e);
|
|
17091
|
-
logger.
|
|
17127
|
+
logger.debug(`[Scrapers] PDF extraction failed: ${msg2}`);
|
|
17092
17128
|
metrics.increment("scrape_pdf_errors_total", 1, { error_type: "extraction_failed" });
|
|
17093
17129
|
errorTracker.trackError(e instanceof Error ? e : String(e), {
|
|
17094
17130
|
component: "scrapers",
|
|
@@ -17410,7 +17446,14 @@ async function scrape(urls, maxConcurrency = 5, signal, config, sessionId, onUrl
|
|
|
17410
17446
|
const batchStart = Date.now();
|
|
17411
17447
|
const results = [];
|
|
17412
17448
|
for (let i = 0; i < urls.length; i += maxConcurrency) {
|
|
17413
|
-
if (signal?.aborted)
|
|
17449
|
+
if (signal?.aborted) {
|
|
17450
|
+
for (const url of urls.slice(i)) {
|
|
17451
|
+
const aborted = { url, success: false, error: "Aborted", markdown: "" };
|
|
17452
|
+
onUrlComplete?.(aborted);
|
|
17453
|
+
results.push(aborted);
|
|
17454
|
+
}
|
|
17455
|
+
break;
|
|
17456
|
+
}
|
|
17414
17457
|
const batch = urls.slice(i, i + maxConcurrency);
|
|
17415
17458
|
const batchRes = await Promise.all(
|
|
17416
17459
|
batch.map(async (url) => {
|
|
@@ -21911,11 +21954,14 @@ ${responseText}`);
|
|
|
21911
21954
|
observer?.onResearcherComplete?.(id, responseText);
|
|
21912
21955
|
return;
|
|
21913
21956
|
} catch (err) {
|
|
21914
|
-
const researcherDuration = Date.now() - researcherExecutionStartMs;
|
|
21915
|
-
metrics.observe("researcher_execution_latency_ms", researcherDuration, { mode: "deep", complexity: String(complexity), round: String(round), status: "error" });
|
|
21916
|
-
metrics.increment("researcher_errors_total", 1, { mode: "deep", complexity: String(complexity), round: String(round) });
|
|
21917
21957
|
lastError = err;
|
|
21918
21958
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
21959
|
+
const tornDown = Boolean(signal?.aborted || isAbortSentinel(errMsg) || errMsg.includes("during container disposal") || container?.isDisposing);
|
|
21960
|
+
const researcherDuration = Date.now() - researcherExecutionStartMs;
|
|
21961
|
+
metrics.observe("researcher_execution_latency_ms", researcherDuration, { mode: "deep", complexity: String(complexity), round: String(round), status: tornDown ? "cancelled" : "error" });
|
|
21962
|
+
if (!tornDown) {
|
|
21963
|
+
metrics.increment("researcher_errors_total", 1, { mode: "deep", complexity: String(complexity), round: String(round) });
|
|
21964
|
+
}
|
|
21919
21965
|
try {
|
|
21920
21966
|
const partialResponse = ensureAssistantResponse(session, id);
|
|
21921
21967
|
const ungrounded = scrapeEnabled && successfulScrapeCount === 0 && nonScrapeGroundingHits === 0 && historicalUrls.length === 0;
|
|
@@ -21929,7 +21975,7 @@ ${responseText}`);
|
|
|
21929
21975
|
} catch (salvageErr) {
|
|
21930
21976
|
logger.debug(`[ResearcherExecutor] Researcher ${id} salvage attempt failed:`, salvageErr);
|
|
21931
21977
|
}
|
|
21932
|
-
if (
|
|
21978
|
+
if (tornDown) {
|
|
21933
21979
|
logger.debug(`[ResearcherExecutor] Researcher ${id} aborted or container disposing, skipping retries.`);
|
|
21934
21980
|
break;
|
|
21935
21981
|
}
|
|
@@ -24678,7 +24724,7 @@ function resolvedConfigPaths() {
|
|
|
24678
24724
|
piIfaceEnv: getInterfaceEnvFilePath("pi"),
|
|
24679
24725
|
piAuth: path18.join(agentDir, "auth.json"),
|
|
24680
24726
|
piModels: path18.join(agentDir, "models.json"),
|
|
24681
|
-
piState:
|
|
24727
|
+
piState: getStateDir()
|
|
24682
24728
|
};
|
|
24683
24729
|
}
|
|
24684
24730
|
function fileExists(p) {
|
|
@@ -24919,6 +24965,8 @@ Configure in:
|
|
|
24919
24965
|
}
|
|
24920
24966
|
let result;
|
|
24921
24967
|
let exit = EXIT.OK;
|
|
24968
|
+
const abortController = new AbortController();
|
|
24969
|
+
activeResearchAbortController = abortController;
|
|
24922
24970
|
try {
|
|
24923
24971
|
await initResearchSDK({
|
|
24924
24972
|
model: det.model,
|
|
@@ -24929,10 +24977,11 @@ Configure in:
|
|
|
24929
24977
|
});
|
|
24930
24978
|
toStderr(`[pi-research] searching knowledge store (${queries.length} query/${queries.length === 1 ? "" : "s"})\u2026
|
|
24931
24979
|
`);
|
|
24932
|
-
result = await searchKnowledge(queries);
|
|
24980
|
+
result = await searchKnowledge(queries, abortController.signal);
|
|
24933
24981
|
} catch (err) {
|
|
24934
24982
|
return reportError(err, "knowledge search", json);
|
|
24935
24983
|
} finally {
|
|
24984
|
+
if (activeResearchAbortController === abortController) activeResearchAbortController = null;
|
|
24936
24985
|
await safeShutdown();
|
|
24937
24986
|
}
|
|
24938
24987
|
if (json) {
|
package/dist/thread-worker.mjs
CHANGED
|
@@ -45,7 +45,12 @@ function formatArg(arg) {
|
|
|
45
45
|
return String(arg);
|
|
46
46
|
}
|
|
47
47
|
function redactSecrets(message) {
|
|
48
|
-
let
|
|
48
|
+
let keep = MAX_LOG_MESSAGE_LENGTH;
|
|
49
|
+
if (message.length > keep) {
|
|
50
|
+
const lastKeptUnit = message.charCodeAt(keep - 1);
|
|
51
|
+
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
52
|
+
}
|
|
53
|
+
let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
|
|
49
54
|
out = out.replace(ANSI_PATTERN, "");
|
|
50
55
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
51
56
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -1483,6 +1488,7 @@ function isBenignScrapeFailure(error) {
|
|
|
1483
1488
|
const msg = error instanceof Error ? error.message : String(error);
|
|
1484
1489
|
return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
|
|
1485
1490
|
msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
|
|
1491
|
+
msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
|
|
1486
1492
|
/^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
|
|
1487
1493
|
isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
|
|
1488
1494
|
}
|
|
@@ -2500,9 +2506,21 @@ var browser = null;
|
|
|
2500
2506
|
var context = null;
|
|
2501
2507
|
var initPromise = null;
|
|
2502
2508
|
var workerId3 = "";
|
|
2509
|
+
var activeTaskCount = 0;
|
|
2510
|
+
var resetPending = false;
|
|
2503
2511
|
function setWorkerId3(id) {
|
|
2504
2512
|
workerId3 = id;
|
|
2505
2513
|
}
|
|
2514
|
+
function taskStarted() {
|
|
2515
|
+
activeTaskCount++;
|
|
2516
|
+
}
|
|
2517
|
+
function taskFinished() {
|
|
2518
|
+
activeTaskCount = Math.max(0, activeTaskCount - 1);
|
|
2519
|
+
if (activeTaskCount === 0 && resetPending) {
|
|
2520
|
+
resetPending = false;
|
|
2521
|
+
doResetBrowser();
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2506
2524
|
var isBrowserConnected = () => {
|
|
2507
2525
|
try {
|
|
2508
2526
|
return browser && typeof browser.isConnected === "function" && browser.isConnected();
|
|
@@ -2628,12 +2646,19 @@ async function initBrowser() {
|
|
|
2628
2646
|
function getContext() {
|
|
2629
2647
|
return context;
|
|
2630
2648
|
}
|
|
2631
|
-
function
|
|
2649
|
+
function doResetBrowser() {
|
|
2632
2650
|
if (context) Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during reset: ${err.message}`));
|
|
2633
2651
|
if (browser) Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during reset: ${err.message}`));
|
|
2634
2652
|
context = null;
|
|
2635
2653
|
browser = null;
|
|
2636
2654
|
}
|
|
2655
|
+
function resetBrowser() {
|
|
2656
|
+
if (activeTaskCount > 1) {
|
|
2657
|
+
resetPending = true;
|
|
2658
|
+
return;
|
|
2659
|
+
}
|
|
2660
|
+
doResetBrowser();
|
|
2661
|
+
}
|
|
2637
2662
|
async function cleanupBrowser2() {
|
|
2638
2663
|
const timeoutMs = 2e3;
|
|
2639
2664
|
const cleanup = async () => {
|
|
@@ -2695,6 +2720,7 @@ async function runTask(data) {
|
|
|
2695
2720
|
if (taskTimeoutMs && taskTimeoutMs > 0) {
|
|
2696
2721
|
taskTimer = setTimeout(() => abortController.abort(new Error(`Task timed out after ${taskTimeoutMs}ms`)), taskTimeoutMs);
|
|
2697
2722
|
}
|
|
2723
|
+
taskStarted();
|
|
2698
2724
|
try {
|
|
2699
2725
|
await initBrowser();
|
|
2700
2726
|
const initMs = Date.now() - startTime;
|
|
@@ -2731,6 +2757,7 @@ async function runTask(data) {
|
|
|
2731
2757
|
};
|
|
2732
2758
|
} finally {
|
|
2733
2759
|
if (taskTimer !== void 0) clearTimeout(taskTimer);
|
|
2760
|
+
taskFinished();
|
|
2734
2761
|
}
|
|
2735
2762
|
}
|
|
2736
2763
|
var worker = new ClusterWorker(runTask, {
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,55 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.3.2] - 2026-08-07
|
|
9
|
+
|
|
10
|
+
A full-tree re-audit of 1.3.1: 10 independent reviews, each assigned a
|
|
11
|
+
disjoint slice of `src/`, calibrated toward the concurrency/IPC/resource-
|
|
12
|
+
lifecycle defect classes prior rounds kept finding. Every fix below is
|
|
13
|
+
paired with a regression test confirmed (via a local revert-and-rerun) to
|
|
14
|
+
fail against the pre-fix code.
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- **`knowledge` subcommand never wired cancellation.** `cmdKnowledge` called `searchKnowledge()` with no `AbortSignal`, so Ctrl-C during a knowledge search only raced `safeShutdown()`'s service teardown against the still-running, unsignaled search — the same class of gap 1.3.0/1.3.1 closed for `cmdResearch`/the pi-extension tool, one call site earlier. Now wired the same way `cmdResearch` already is.
|
|
19
|
+
- **A relative `PI_RESEARCH_STATE_DIR` silently fragmented cross-process coordination.** Three independent, unguarded reads of the env var could each resolve a relative override against whatever `process.cwd()` a given process happened to have — splitting browser/embedding-server leader election and the run-cap semaphore across several non-communicating state directories. `scripts/cleanup.cjs` already treated a relative override as untrusted (falling back to the default); the runtime now agrees, via a single `getStateDir()` all three call sites share.
|
|
20
|
+
- **`KnowledgeStoreService` had no mutual exclusion between `initialize()` and `dispose()`.** A concurrent re-init (an ordinary multi-workspace run, or a `/research-config` model/device change) could build fresh embedder/store/writer-queue handles that an in-flight `dispose()` — reading them dynamically, not captured — then closed out from under it, or the reverse: a disposal silently resurrected moments later with a stale instance.
|
|
21
|
+
- **`rebuildDocument()` could return one chunk's fragment instead of the complete cached document.** Multiple chunks from the same ingest share an identical timestamp; sorting by timestamp alone is a no-op on that tie, so whichever chunk the store's internal scan happened to return first won — only chunk 0 carries the full original content. Ties now break on chunk index, and the fallback chain prefers the (always-complete) synthesized description over a lone chunk's own fragment.
|
|
22
|
+
- **Log truncation could split a UTF-16 surrogate pair.** The 10k-character cap on every log line sliced at a raw offset with no surrogate check, silently corrupting to a replacement character (console) or an invalid escape (the JSON log file) whenever the cut landed inside an emoji or other non-BMP character in scraped content.
|
|
23
|
+
- **A citation URL's genuinely balanced trailing `)` could be stripped.** The LLM-noise-bracket stripper compared flat open/close counts across the whole string rather than tracking a running balance, so an unrelated stray bracket earlier in the same URL could tip the count and truncate a legitimately balanced trailing parenthesis.
|
|
24
|
+
- **The scrape tool could silently drop URLs cut off by cancellation.** A cancel mid-batch stopped dispatching further URLs but never accounted for the ones that never started — the tool's own "Successful: X, Failed: Y" summary undercounted with no indication a run was interrupted. Every URL now gets an entry either way.
|
|
25
|
+
- **A single crashed browser tab could take down a healthy sibling task's scrape/search.** Cluster workers run more than one task at a time per process by default; a page-scoped browser error from one task used to reset the *whole shared browser*, failing any concurrently-running sibling task with a spurious, un-retried error. The reset now defers until no sibling task is still active.
|
|
26
|
+
- **The service registry could hand a caller an instance mid-disposal, or leave a failed re-init registered indefinitely.** A `get()` call arriving while `clear()`/`replace()` was still tearing an instance down could either receive a reference to the instance being disposed or (with a scope-change context) re-initialize it while its `dispose()` was still running; separately, a failed context-scoped re-initialization used to leave the same now-inconsistent instance registered rather than clearing it for a clean rebuild, unlike an ordinary first-init failure.
|
|
27
|
+
|
|
28
|
+
Also removed as dead code (confirmed zero production call sites): the
|
|
29
|
+
wave-animation timer plumbing in `CleanupContext`/`ResearchPanelState`
|
|
30
|
+
(`updateWaveTimer`, `stopWaveAnimation`), superseded by pulse-driven
|
|
31
|
+
animation; and `StateManager.setBrowserServer` and its full delegation
|
|
32
|
+
chain, superseded by the browser scheduler's own inline leader-registration
|
|
33
|
+
logic.
|
|
34
|
+
|
|
35
|
+
Patch, not minor: nothing here changes a documented CLI flag, tool output
|
|
36
|
+
schema, or config key — every fix is either an internal robustness/
|
|
37
|
+
correctness correction invisible to well-formed usage, or (the
|
|
38
|
+
`cmdKnowledge` cancellation wiring) closes a race whose observable exit
|
|
39
|
+
code was in practice already forced correct by the signal handler's own
|
|
40
|
+
independent shutdown path.
|
|
41
|
+
|
|
42
|
+
## [1.3.1] - 2026-08-06
|
|
43
|
+
|
|
44
|
+
A dedicated post-release investigation pass over 1.3.0 — three independent
|
|
45
|
+
agents (fresh-eyes code review, packaging/release-artifact audit, isolated
|
|
46
|
+
runtime CLI smoke test) plus a full from-scratch local gate run — to confirm
|
|
47
|
+
the shipped artifact and codebase were correct before considering the
|
|
48
|
+
release closed. The packaging audit and runtime smoke test found nothing;
|
|
49
|
+
the code review found one real, currently-live defect, fixed below.
|
|
50
|
+
|
|
51
|
+
### Fixed
|
|
52
|
+
|
|
53
|
+
- **The pi-extension research tool could report a cancelled-but-partial run as an ordinary success.** `DeepResearchOrchestrator.run()` does not throw on cancellation once at least one researcher has produced a report — it resolves normally with a fallback synthesis (the same documented contract 1.3.0's `cmdResearch` fix was built around). `research-tool-definition.ts`'s abort check only lived in its `catch` block, so this resolves-normally path fell straight through to unconditional success handling: metrics recorded `status: 'success'`, and the text handed back to the calling agent carried no indication it was a partial report. This matters more here than at the CLI: `AgentToolResult`'s `details` field is logs/UI-only by its own type contract, so `content[].text` is the *only* channel the calling agent has to learn a report is partial rather than complete — a silent miscategorization here could lead an agent to treat cut-short research as a finished answer. Same bug class as the 1.3.0 CLI fix, at a call site that round's review missed. Fixed by checking the abort signal immediately after `runResearch()` resolves: the metrics status now reflects the real outcome, and a cancellation note is prepended to the report text itself.
|
|
54
|
+
|
|
55
|
+
Not a minor bump: unlike 1.3.0's `cmdResearch` fix, this does not touch the CLI's exit-code contract — it is scoped entirely to the pi-extension tool call path.
|
|
56
|
+
|
|
8
57
|
## [1.3.0] - 2026-08-06
|
|
9
58
|
|
|
10
59
|
Bundles three unreleased fix rounds that accumulated on top of 1.2.0 (each
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: pi-research
|
|
3
3
|
description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
|
|
4
4
|
license: MIT
|
|
5
|
-
metadata: { "author": "Lincoln Deen", "version": "1.3.
|
|
5
|
+
metadata: { "author": "Lincoln Deen", "version": "1.3.2", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
|
|
6
6
|
allowed-tools: Bash(node:*)
|
|
7
7
|
---
|
|
8
8
|
|