@lincoln504/pi-research 1.3.2 → 1.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +1 -1
  2. package/dist/cli.mjs +178 -73
  3. package/dist/thread-worker.mjs +56 -37
  4. package/docs/CHANGELOG.md +62 -0
  5. package/package.json +4 -4
  6. package/skills/pi-research/SKILL.md +1 -1
  7. package/src/core/llm/agentic-repair.ts +5 -1
  8. package/src/core/llm/model-registry-factory.ts +17 -7
  9. package/src/core/pi-version.ts +15 -2
  10. package/src/core/service-registry.ts +37 -3
  11. package/src/healthcheck/index.ts +9 -7
  12. package/src/infrastructure/browser/task-execution-service.ts +21 -5
  13. package/src/infrastructure/browser/thread-worker-browser.ts +20 -1
  14. package/src/infrastructure/browser/thread-worker-messaging.ts +30 -22
  15. package/src/infrastructure/browser/thread-worker.mjs +56 -37
  16. package/src/infrastructure/knowledge-store-service.ts +27 -2
  17. package/src/knowledge/chunker.ts +14 -1
  18. package/src/orchestration/quick-research-orchestrator.ts +13 -2
  19. package/src/orchestration/research-orchestration-service.ts +16 -3
  20. package/src/sdk.ts +11 -2
  21. package/src/security/github-advisories.ts +13 -4
  22. package/src/skill-install/skill-installer.ts +40 -5
  23. package/src/stackexchange/index.ts +15 -0
  24. package/src/stackexchange/rest-client.ts +13 -24
  25. package/src/tools/research-knowledge-search.ts +6 -2
  26. package/src/utils/input-validation.ts +0 -10
  27. package/src/utils/json-utils.ts +0 -78
  28. package/src/utils/log-utils.ts +32 -33
  29. package/src/utils/metrics.ts +0 -8
  30. package/src/utils/shared-links.ts +0 -14
  31. package/src/utils/stdio-capture.ts +44 -7
  32. package/src/utils/url-utils.ts +7 -1
  33. package/src/web-research/scraper-utils.ts +8 -3
  34. package/src/web-research/web-scraper.ts +33 -1
package/README.md CHANGED
@@ -61,7 +61,7 @@ ask in natural language and the tool picks the right one:
61
61
  ### Requirements / limitations
62
62
 
63
63
  - Node.js >= 22.19.0
64
- - An LLM with a 100k+ context window (bring your own key)
64
+ - An LLM with a 100k+ context window (bring your own key or use a local model)
65
65
  - Internet access
66
66
  - A residential IP address — search, scraping, and YouTube transcripts all rely on a residential connection. A datacenter/VPS/cloud IP gets bot-blocked by the providers these features depend on.
67
67
  - The pi runtime the engine builds on — `@earendil-works/pi-ai`, `@earendil-works/pi-coding-agent`, and `@earendil-works/pi-tui`. The pi extension uses the host's copies; the standalone CLI / agent skill install them as dependencies.
package/dist/cli.mjs CHANGED
@@ -215,12 +215,7 @@ function formatArg(arg) {
215
215
  return String(arg);
216
216
  }
217
217
  function redactSecrets(message) {
218
- let keep = MAX_LOG_MESSAGE_LENGTH;
219
- if (message.length > keep) {
220
- const lastKeptUnit = message.charCodeAt(keep - 1);
221
- if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
222
- }
223
- let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
218
+ let out = message;
224
219
  out = out.replace(ANSI_PATTERN, "");
225
220
  out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
226
221
  out = out.replace(JWT_PATTERN, "[REDACTED]");
@@ -231,6 +226,14 @@ function redactSecrets(message) {
231
226
  out = out.replace(KNOWN_TOKEN_PATTERN, "[REDACTED]");
232
227
  out = out.replace(PROVIDER_TOKEN_PATTERN, "[REDACTED]");
233
228
  out = out.replace(LONG_HEX_SECRET_PATTERN, "[REDACTED]");
229
+ let keep = MAX_LOG_MESSAGE_LENGTH;
230
+ if (out.length > keep) {
231
+ const lastKeptUnit = out.charCodeAt(keep - 1);
232
+ if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
233
+ }
234
+ if (out.length > MAX_LOG_MESSAGE_LENGTH) {
235
+ out = `${out.slice(0, keep)}\u2026[truncated ${out.length - keep} chars]`;
236
+ }
234
237
  return out;
235
238
  }
236
239
  function stripTerminalEscapes(message) {
@@ -245,7 +248,7 @@ var init_log_utils = __esm({
245
248
  ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
246
249
  URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
247
250
  COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
248
- SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
251
+ SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
249
252
  KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
250
253
  JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
251
254
  BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
@@ -497,7 +500,10 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
497
500
  unkeyedCaptureDepth++;
498
501
  isAnyLoggerCapturingOutput = true;
499
502
  }
500
- const decoder = new TextDecoder2();
503
+ const stderrDecoder = new TextDecoder2();
504
+ const stdoutDecoder = new TextDecoder2();
505
+ const fd1SyncDecoder = new TextDecoder2();
506
+ const fd2SyncDecoder = new TextDecoder2();
501
507
  const originalStderrWrite = process.stderr.write;
502
508
  const originalStdoutWrite = process.stdout.write;
503
509
  const originalFsWriteSync = fs3.writeSync;
@@ -543,7 +549,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
543
549
  console.debug = createConsolePatch("debug", logFile, hasSufficientDiskSpace);
544
550
  process.stderr.write = (chunk, encodingOrCb, callback) => {
545
551
  const cb = typeof encodingOrCb === "function" ? encodingOrCb : callback;
546
- const message = typeof chunk === "string" ? chunk : decoder.decode(chunk);
552
+ const message = typeof chunk === "string" ? chunk : stderrDecoder.decode(chunk, { stream: true });
553
+ if (message.length === 0) {
554
+ if (typeof cb === "function") cb();
555
+ return true;
556
+ }
547
557
  if (!hasSufficientDiskSpace()) {
548
558
  if (typeof cb === "function") cb();
549
559
  return true;
@@ -564,7 +574,7 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
564
574
  return true;
565
575
  };
566
576
  process.stdout.write = (chunk, encodingOrCb, callback) => {
567
- const message = typeof chunk === "string" ? chunk : decoder.decode(chunk);
577
+ const message = typeof chunk === "string" ? chunk : stdoutDecoder.decode(chunk, { stream: true });
568
578
  const isNative = isNativeLog(message);
569
579
  const complexTui = isComplexTui(message);
570
580
  const boxDrawing = isBoxDrawing(message);
@@ -618,7 +628,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
618
628
  if (!descriptor || (descriptor.writable || descriptor.set)) {
619
629
  fs3.writeSync = (fd, chunk, ...args) => {
620
630
  if (fd === 1 || fd === 2) {
621
- const message = typeof chunk === "string" ? chunk : decoder.decode(chunk);
631
+ const fdDecoder = fd === 1 ? fd1SyncDecoder : fd2SyncDecoder;
632
+ const message = typeof chunk === "string" ? chunk : fdDecoder.decode(chunk, { stream: true });
633
+ if (fd === 2 && message.length === 0) {
634
+ return typeof chunk === "string" ? Buffer.from(chunk).length : chunk.length;
635
+ }
622
636
  const isNative = isNativeLog(message);
623
637
  const boxDrawing = isBoxDrawing(message);
624
638
  const complexTui = isComplexTui(message);
@@ -1349,6 +1363,24 @@ var init_service_registry = __esm({
1349
1363
  ]);
1350
1364
  if (settleTimer) clearTimeout(settleTimer);
1351
1365
  }
1366
+ const inFlightDisposals = Array.from(this.services.values()).map((r) => r.disposalPromise).filter((p) => p != null);
1367
+ if (inFlightDisposals.length > 0) {
1368
+ if (this.defaultOptions.enableLogging) {
1369
+ logger.debug(`[ServiceContainer] Waiting for ${inFlightDisposals.length} in-flight disposal(s) before teardown`);
1370
+ }
1371
+ let disposalSettleTimer;
1372
+ await Promise.race([
1373
+ Promise.allSettled(inFlightDisposals),
1374
+ new Promise((resolve9) => {
1375
+ disposalSettleTimer = setTimeout(() => {
1376
+ logger.warn("[ServiceContainer] Timed out waiting for in-flight service disposal; proceeding with teardown");
1377
+ resolve9();
1378
+ }, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
1379
+ disposalSettleTimer.unref?.();
1380
+ })
1381
+ ]);
1382
+ if (disposalSettleTimer) clearTimeout(disposalSettleTimer);
1383
+ }
1352
1384
  const activeServices = Array.from(this.services.keys()).filter(
1353
1385
  (name) => this.services.get(name)?.instance !== null
1354
1386
  );
@@ -1932,7 +1964,7 @@ function normalizeUrl(url) {
1932
1964
  }
1933
1965
  return result;
1934
1966
  } catch (_err) {
1935
- logger.debug(`[url-utils] normalizing unparseable URL (${url.slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
1967
+ logger.debug(`[url-utils] normalizing unparseable URL (${redactSecrets(url).slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
1936
1968
  let cleaned = url.trim().replace(/^[*_~`"']{1,20}/, "").replace(/[*_~`"',.)}\]]{1,20}$/, "");
1937
1969
  const hashIdx = cleaned.indexOf("#");
1938
1970
  if (hashIdx >= 0) cleaned = cleaned.substring(0, hashIdx);
@@ -1996,6 +2028,7 @@ var init_url_utils = __esm({
1996
2028
  "src/utils/url-utils.ts"() {
1997
2029
  "use strict";
1998
2030
  init_logger();
2031
+ init_log_utils();
1999
2032
  MAX_INITIAL_LINKS = 20;
2000
2033
  MAX_INITIAL_LINK_CHARS = 2048;
2001
2034
  }
@@ -3565,7 +3598,7 @@ async function buildModelRegistry(apiKey, provider) {
3565
3598
  allowModelNetwork: false
3566
3599
  });
3567
3600
  if (apiKey && provider) {
3568
- await runtime.setRuntimeApiKey(provider, apiKey, { allowNetwork: false });
3601
+ await runtime.setRuntimeApiKey(provider, apiKey);
3569
3602
  }
3570
3603
  const registry = new ModelRegistry(runtime);
3571
3604
  registryRuntimes.set(registry, runtime);
@@ -6434,7 +6467,7 @@ var init_chunker = __esm({
6434
6467
  if (code >= 56320 && code <= 57343) {
6435
6468
  if (start - 1 > iterationStart) {
6436
6469
  start -= 1;
6437
- } else if (start + 1 < end) {
6470
+ } else if (start + 1 <= end) {
6438
6471
  start += 1;
6439
6472
  }
6440
6473
  }
@@ -8663,6 +8696,9 @@ var init_knowledge_store_service = __esm({
8663
8696
  await this._initializationPromise;
8664
8697
  } catch {
8665
8698
  }
8699
+ if (this._disposalPromise) {
8700
+ return this._disposalPromise;
8701
+ }
8666
8702
  }
8667
8703
  this.lifecycle = "disposing" /* DISPOSING */;
8668
8704
  logger.debug("[KnowledgeStoreService] Disposing...");
@@ -8670,7 +8706,8 @@ var init_knowledge_store_service = __esm({
8670
8706
  const store = this._store;
8671
8707
  const writerQueue = this._writerQueue;
8672
8708
  const initLock = this._initLock;
8673
- this._disposalPromise = (async () => {
8709
+ let disposePromise = null;
8710
+ disposePromise = (async () => {
8674
8711
  try {
8675
8712
  if (writerQueue) {
8676
8713
  await writerQueue.dispose?.();
@@ -8694,9 +8731,12 @@ var init_knowledge_store_service = __esm({
8694
8731
  logger.error("[KnowledgeStoreService] Error during disposal:", err);
8695
8732
  } finally {
8696
8733
  this.lifecycle = "disposed" /* DISPOSED */;
8697
- this._disposalPromise = null;
8734
+ if (this._disposalPromise === disposePromise) {
8735
+ this._disposalPromise = null;
8736
+ }
8698
8737
  }
8699
8738
  })();
8739
+ this._disposalPromise = disposePromise;
8700
8740
  return this._disposalPromise;
8701
8741
  }
8702
8742
  /**
@@ -9740,16 +9780,31 @@ function installSkill(toolIds, opts = {}) {
9740
9780
  continue;
9741
9781
  }
9742
9782
  fs16.unlinkSync(absSkillPath);
9743
- fs16.symlinkSync(source, absSkillPath, process.platform === "win32" ? "junction" : "dir");
9744
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: "symlink", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
9745
- results.push({ tool: id, path: absSkillPath, type: "symlink", status: "installed", message: "re-pointed stale symlink to current source" });
9783
+ try {
9784
+ fs16.symlinkSync(source, absSkillPath, process.platform === "win32" ? "junction" : "dir");
9785
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: "symlink", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
9786
+ results.push({ tool: id, path: absSkillPath, type: "symlink", status: "installed", message: "re-pointed stale symlink to current source" });
9787
+ } catch (relinkErr) {
9788
+ try {
9789
+ copyDir(source, absSkillPath);
9790
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: "copy", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
9791
+ results.push({ tool: id, path: absSkillPath, type: "copy", status: "installed", message: `symlink re-point unavailable (${relinkErr?.code ?? relinkErr?.message}); copied instead` });
9792
+ } catch (copyErr) {
9793
+ results.push({ tool: id, path: absSkillPath, type: "symlink", status: "error", message: `symlink re-point failed (${relinkErr?.code ?? relinkErr?.message}) and the copy fallback also failed: ${copyErr?.message ?? String(copyErr)}` });
9794
+ }
9795
+ }
9746
9796
  continue;
9747
9797
  }
9748
9798
  }
9749
9799
  } catch {
9750
9800
  }
9751
9801
  results.push({ tool: id, path: absSkillPath, type, status: "already-installed" });
9752
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? "symlink" : "copy", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
9802
+ {
9803
+ const isSymlinkNow = isSymlinkPresent(absSkillPath);
9804
+ const existingEntry = manifest.entries.find((e) => e.path === absSkillPath);
9805
+ const versionField = isSymlinkNow ? skillVersion ? { version: skillVersion } : {} : existingEntry?.version ? { version: existingEntry.version } : {};
9806
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkNow ? "symlink" : "copy", source, createdAt: now, ...versionField });
9807
+ }
9753
9808
  continue;
9754
9809
  }
9755
9810
  if (fs16.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
@@ -15867,6 +15922,7 @@ init_logger();
15867
15922
 
15868
15923
  // src/infrastructure/browser/task-execution-service.ts
15869
15924
  init_logger();
15925
+ init_log_utils();
15870
15926
  init_error_tracker();
15871
15927
 
15872
15928
  // src/infrastructure/browser/browser-lifecycle.ts
@@ -15914,6 +15970,10 @@ var lastRestartTime = 0;
15914
15970
  var RESTART_COOLDOWN_MS = 1e4;
15915
15971
  var LEADER_HANDOVER_RETRIES = 3;
15916
15972
  var HANDOVER_BACKOFF_MS = 1200;
15973
+ function briefErrorMessage(error) {
15974
+ const msg2 = error instanceof Error ? error.message : String(error);
15975
+ return redactSecrets(msg2).substring(0, 100);
15976
+ }
15917
15977
  async function recoverFromLeaderHandover(container) {
15918
15978
  logger.warn("[BrowserManager] Leader handover detected (pool draining) \u2014 dropping stale scheduler handle and re-resolving...");
15919
15979
  await forceSchedulerRestart2(false, container).catch(
@@ -16003,7 +16063,7 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
16003
16063
  taskType: type,
16004
16064
  errorType: "leader_handover"
16005
16065
  });
16006
- logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
16066
+ logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
16007
16067
  await recoverFromLeaderHandover(container);
16008
16068
  await new Promise((resolve9) => setTimeout(resolve9, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
16009
16069
  return runBrowserTask(taskOrUrl, type, config, signal, retries, container, handoverRetries - 1);
@@ -16015,7 +16075,7 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
16015
16075
  taskType: type,
16016
16076
  errorType: "transient_socket_error"
16017
16077
  });
16018
- logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
16078
+ logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${briefErrorMessage(error)}...`);
16019
16079
  if (isPoolShutdownError(error)) {
16020
16080
  logger.warn(`[BrowserManager] Pool is draining \u2014 waiting for pool idle before retry...`);
16021
16081
  await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
@@ -16042,7 +16102,7 @@ async function runBrowserHealthCheck(config, retries = 1, signal, container = ge
16042
16102
  operation: "healthcheck",
16043
16103
  errorType: "transient_socket_error"
16044
16104
  });
16045
- logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
16105
+ logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${briefErrorMessage(error)}...`);
16046
16106
  if (isPoolShutdownError(error)) {
16047
16107
  await recoverFromLeaderHandover(container);
16048
16108
  } else {
@@ -16074,7 +16134,7 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
16074
16134
  query,
16075
16135
  errorType: "leader_handover"
16076
16136
  });
16077
- logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
16137
+ logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
16078
16138
  await recoverFromLeaderHandover(container);
16079
16139
  await new Promise((resolve9) => setTimeout(resolve9, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
16080
16140
  return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
@@ -16086,7 +16146,7 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
16086
16146
  query,
16087
16147
  errorType: "transient_socket_error"
16088
16148
  });
16089
- logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
16149
+ logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${briefErrorMessage(error)}...`);
16090
16150
  if (isPoolShutdownError(error)) {
16091
16151
  await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
16092
16152
  } else {
@@ -16513,13 +16573,11 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
16513
16573
  registry.register("StateManager", async () => {
16514
16574
  try {
16515
16575
  const stateManager = await getService(ServiceNames.STATE_MANAGER, { container }, container);
16516
- const stats = await stateManager.getMetrics();
16517
16576
  const gpuOwner = await stateManager.getGpuOwner();
16518
16577
  return {
16519
16578
  healthy: true,
16520
16579
  diagnostic: {
16521
16580
  status: "operational",
16522
- sessions: stats.activeSessions,
16523
16581
  gpuLocked: !!gpuOwner,
16524
16582
  gpuOwner: gpuOwner?.sessionId || "none"
16525
16583
  }
@@ -16805,6 +16863,7 @@ function isBenignScrapeFailure(error) {
16805
16863
  return msg2.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
16806
16864
  msg2.startsWith("Fetch returned stub:") || // nav-only / near-empty page
16807
16865
  msg2.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
16866
+ msg2.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
16808
16867
  /^HTTP \d{3}\b/.test(msg2) || // remote 4xx/5xx status
16809
16868
  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
16810
16869
  }
@@ -17097,8 +17156,22 @@ async function extractPdfToMarkdown(bytes) {
17097
17156
  throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
17098
17157
  }
17099
17158
  const pdfExtractionStart = Date.now();
17159
+ let WasmPdfDocument;
17160
+ try {
17161
+ ({ WasmPdfDocument } = await import("pdf-oxide-wasm"));
17162
+ } catch (e) {
17163
+ const msg2 = e instanceof Error ? e.message : String(e);
17164
+ logger.error(`[Scrapers] pdf-oxide-wasm failed to load \u2014 PDF extraction is unavailable (not just for this URL): ${msg2}`);
17165
+ metrics.increment("scrape_pdf_errors_total", 1, { error_type: "native_module_unavailable" });
17166
+ errorTracker.trackError(e instanceof Error ? e : String(e), {
17167
+ component: "scrapers",
17168
+ operation: "pdf-extract",
17169
+ contentType: "pdf",
17170
+ errorType: "native_module_unavailable"
17171
+ });
17172
+ throw new Error(`PDF extraction unavailable (pdf-oxide-wasm failed to load: ${msg2})`, { cause: e });
17173
+ }
17100
17174
  try {
17101
- const { WasmPdfDocument } = await import("pdf-oxide-wasm");
17102
17175
  const doc = new WasmPdfDocument(bytes);
17103
17176
  const pageCount = doc.pageCount();
17104
17177
  let markdown = `# PDF Document
@@ -18548,9 +18621,9 @@ async function searchGitHubAdvisories(terms, options) {
18548
18621
  throw new Error(`Invalid repo format: "${options.repo}". Expected "owner/name".`);
18549
18622
  }
18550
18623
  const [owner, name] = repoParts;
18551
- const repoNameRe = /^[A-Za-z0-9._-]+$/;
18624
+ const repoNameRe = /^(?!\.+$)[A-Za-z0-9._-]+$/;
18552
18625
  if (!repoNameRe.test(owner) || !repoNameRe.test(name)) {
18553
- throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-'.`);
18626
+ throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-', and cannot be all dots.`);
18554
18627
  }
18555
18628
  const url = `${GITHUB_API_BASE}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/security-advisories?per_page=${perPage}`;
18556
18629
  const response = await githubCircuitBreaker.execute(() => retryWithBackoff(async () => {
@@ -19674,6 +19747,7 @@ init_logger();
19674
19747
  init_circuit_breaker();
19675
19748
  init_metrics();
19676
19749
  init_http_body();
19750
+ init_retry_utils();
19677
19751
  var API_BASE = "https://api.stackexchange.com/2.3";
19678
19752
  var QUOTA_PROBE_COOLDOWN_MS = 10 * 60 * 1e3;
19679
19753
  var StackExchangeClient = class {
@@ -19721,21 +19795,11 @@ var StackExchangeClient = class {
19721
19795
  if (!url.searchParams.has("site") && !options.endpoint.startsWith("/sites")) {
19722
19796
  url.searchParams.set("site", "stackoverflow.com");
19723
19797
  }
19724
- const controller = new AbortController();
19725
- const timeoutId = setTimeout(() => {
19726
- controller.abort();
19727
- }, this._timeout);
19728
- const abortHandler = () => {
19729
- clearTimeout(timeoutId);
19730
- controller.abort();
19731
- };
19732
- if (signal) {
19733
- signal.addEventListener("abort", abortHandler, { once: true });
19734
- }
19798
+ const requestSignal = createTimeoutSignal(this._timeout, signal);
19735
19799
  try {
19736
19800
  const response = await fetch(url.toString(), {
19737
19801
  method: options.method,
19738
- signal: controller.signal,
19802
+ signal: requestSignal,
19739
19803
  headers: {
19740
19804
  "Accept": "application/json"
19741
19805
  }
@@ -19790,11 +19854,6 @@ var StackExchangeClient = class {
19790
19854
  throw new Error(`Request timeout after ${this._timeout}ms`, { cause: error });
19791
19855
  }
19792
19856
  throw error;
19793
- } finally {
19794
- clearTimeout(timeoutId);
19795
- if (signal) {
19796
- signal.removeEventListener("abort", abortHandler);
19797
- }
19798
19857
  }
19799
19858
  });
19800
19859
  }
@@ -20035,6 +20094,33 @@ function formatCompactQuestions(questions) {
20035
20094
  }
20036
20095
 
20037
20096
  // src/stackexchange/output/compact.ts
20097
+ function formatQuestionsCompact(questions) {
20098
+ if (questions.length === 0) {
20099
+ return "No questions found.";
20100
+ }
20101
+ const lines = [];
20102
+ let index = 1;
20103
+ for (const q of questions) {
20104
+ const accepted = q.accepted_answer_id ? "[accepted]" : " ";
20105
+ lines.push(`${index}. ${accepted} [${q.title}](${q.link}) (${q.score} pts, ${q.answer_count} ans, ${q.view_count} views)`);
20106
+ index++;
20107
+ }
20108
+ return lines.join("\n");
20109
+ }
20110
+ function formatAnswersCompact(answers) {
20111
+ if (answers.length === 0) {
20112
+ return "No answers found.";
20113
+ }
20114
+ const lines = [];
20115
+ let index = 1;
20116
+ for (const a of answers) {
20117
+ const accepted = a.is_accepted ? "[accepted]" : " ";
20118
+ const displayName = a.owner?.display_name ?? "Unknown";
20119
+ lines.push(`${index}. ${accepted} by ${displayName} (${a.score} pts)`);
20120
+ index++;
20121
+ }
20122
+ return lines.join("\n");
20123
+ }
20038
20124
  function formatUsersCompact(users) {
20039
20125
  if (users.length === 0) {
20040
20126
  return "No users found.";
@@ -20318,6 +20404,16 @@ function formatCompact(result) {
20318
20404
  return formatCompactQuestions(result);
20319
20405
  }
20320
20406
  }
20407
+ } else if (typeof result === "object" && result !== null) {
20408
+ if ("question" in result && "answers" in result) {
20409
+ const r = result;
20410
+ let output = formatQuestionsCompact([r.question]);
20411
+ if (r.answers.length > 0) {
20412
+ output += `
20413
+ ${formatAnswersCompact(r.answers)}`;
20414
+ }
20415
+ return output;
20416
+ }
20321
20417
  }
20322
20418
  return JSON.stringify(result);
20323
20419
  }
@@ -22362,32 +22458,36 @@ ${result}`);
22362
22458
  if (this.config.KNOWLEDGE_STORE_MODE !== "none") {
22363
22459
  const ksService = await getService(ServiceNames.KNOWLEDGE_STORE, ctx, container);
22364
22460
  if (ksService.isReady()) {
22365
- const writer = await getService(ServiceNames.WRITER_QUEUE, ctx, container);
22366
- const citations = parseCitations(result);
22367
- if (citations.length === 0) {
22368
- logger.warn("[QuickOrchestrator] Researcher produced no parseable CITED LINKS \u2014 no descriptions stored for this session");
22369
- }
22370
- let enqueued = 0;
22371
- for (const cit of citations) {
22372
- if (cit.url) {
22373
- const fullContent = getCachedScrapedContent(this.options.researchId, cit.url);
22374
- const markdown = cit.description || `(source: ${cit.url})`;
22375
- writer.enqueue({
22376
- url: normalizeUrl(cit.url),
22377
- markdown,
22378
- content: fullContent,
22379
- metadata: {
22380
- ingestionType: "synthesis-description",
22381
- source: "researcher",
22382
- synthesizedAt: (/* @__PURE__ */ new Date()).toISOString(),
22383
- description: cit.description || "",
22384
- fullContentSnippet: fullContent?.substring(0, 5e3)
22385
- }
22386
- });
22387
- enqueued++;
22461
+ const writer = await ksService.getWriterQueue();
22462
+ if (!writer) {
22463
+ logger.debug("[QuickOrchestrator] Writer queue unavailable, skipping link descriptions");
22464
+ } else {
22465
+ const citations = parseCitations(result);
22466
+ if (citations.length === 0) {
22467
+ logger.warn("[QuickOrchestrator] Researcher produced no parseable CITED LINKS \u2014 no descriptions stored for this session");
22388
22468
  }
22469
+ let enqueued = 0;
22470
+ for (const cit of citations) {
22471
+ if (cit.url) {
22472
+ const fullContent = getCachedScrapedContent(this.options.researchId, cit.url);
22473
+ const markdown = cit.description || `(source: ${cit.url})`;
22474
+ writer.enqueue({
22475
+ url: normalizeUrl(cit.url),
22476
+ markdown,
22477
+ content: fullContent,
22478
+ metadata: {
22479
+ ingestionType: "synthesis-description",
22480
+ source: "researcher",
22481
+ synthesizedAt: (/* @__PURE__ */ new Date()).toISOString(),
22482
+ description: cit.description || "",
22483
+ fullContentSnippet: fullContent?.substring(0, 5e3)
22484
+ }
22485
+ });
22486
+ enqueued++;
22487
+ }
22488
+ }
22489
+ if (enqueued > 0) await writer.drain();
22389
22490
  }
22390
- if (enqueued > 0) await writer.drain();
22391
22491
  }
22392
22492
  }
22393
22493
  } catch (err) {
@@ -23180,7 +23280,11 @@ var ResearchOrchestrationService = class {
23180
23280
  return;
23181
23281
  }
23182
23282
  const synthesisService = await getService(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
23183
- const writer = await getService(ServiceNames.WRITER_QUEUE, ctx, container);
23283
+ const writer = await ksService.getWriterQueue();
23284
+ if (!writer) {
23285
+ logger.debug("[ResearchOrchestrationService] Writer queue unavailable, skipping link descriptions");
23286
+ return;
23287
+ }
23184
23288
  const roundPrefix = `${round}.`;
23185
23289
  let enqueued = 0;
23186
23290
  let researcherCount = 0;
@@ -23833,6 +23937,7 @@ async function initializeOrchestrationServices(ctx, container = getServiceContai
23833
23937
  init_service_registry();
23834
23938
  init_service_interfaces();
23835
23939
  init_logger();
23940
+ init_log_utils();
23836
23941
  init_config();
23837
23942
  init_metrics();
23838
23943
 
@@ -24175,7 +24280,7 @@ function getLastResearcherOutcome() {
24175
24280
  }
24176
24281
  function logRunErrorSummary(report, depthLabel, status) {
24177
24282
  if (!report || report.totalErrors === 0) return;
24178
- const top = report.patterns.slice(0, 3).map((p) => `"${String(p.message ?? p.signature ?? "error").slice(0, 48)}" \xD7${p.count}`).join(", ");
24283
+ const top = report.patterns.slice(0, 3).map((p) => `"${redactSecrets(String(p.message ?? p.signature ?? "error")).slice(0, 48)}" \xD7${p.count}`).join(", ");
24179
24284
  logger.warn(`[SDK] run ${status} with ${report.totalErrors} tracked error(s) across ${report.uniquePatterns} pattern(s) [${depthLabel}]${top ? ` \u2014 top: ${top}` : ""}`);
24180
24285
  }
24181
24286
  function wrapObserverForOutcomeCapture(observer, onPlanned) {