@lincoln504/pi-research 1.5.2 → 1.6.0

package/src/knowledge/store.ts

@@ -23,6 +23,7 @@ import { MigrationStrategy, MigrationResult } from './migration.ts';
 import { metrics } from '../utils/metrics.ts';
 import { createStoreTable, CURRENT_SCHEMA_VERSION } from './store-schema.ts';
 import { addDocumentsToStore, assertValidEmbeddingVectors, searchStore, findDocumentsByUrl, findRelevantUrls } from './store-operations.ts';
+import type { AddDocumentsOutcome } from './store-operations.ts';
 import { isEmbedderUnreachable } from './embedder-utils.ts';
 import { ServiceLifecycle } from '../core/service-registry.ts';
 import { ServiceNames } from '../core/service-interfaces.ts';
@@ -713,13 +714,14 @@ export class KnowledgeStore implements IKnowledgeStore {
       // Now check isClosing after registering
       if (this.isClosing) {
         logger.warn(`[store] Skipping addDocuments — store is closing`);
+        metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
         return;
       }
 
       let table = await this.getFreshTable();
       const workspace = this.getWorkspace();
 
-      await this.withEmbedderReconnect(async (embedder) => {
+      const outcome = await this.withEmbedderReconnect<AddDocumentsOutcome>(async (embedder) => {
         let retryCount = 0;
         const MAX_RETRIES = 5;
         const BASE_DELAY = 100;
@@ -727,7 +729,7 @@ export class KnowledgeStore implements IKnowledgeStore {
         while (retryCount <= MAX_RETRIES) {
           try {
             const isGlobal = this.options.knowledgeMode === 'global';
-          await addDocumentsToStore(
+            return await addDocumentsToStore(
               table, 
               docs, 
               embedder, 
@@ -735,7 +737,6 @@ export class KnowledgeStore implements IKnowledgeStore {
               workspace,
               isGlobal
             );
-            return;
           } catch (err) {
             // Detect commit conflicts typical of cross-process write contention
             if (isLanceCommitConflict(err)) {
@@ -758,7 +759,24 @@ export class KnowledgeStore implements IKnowledgeStore {
             throw err;
           }
         }
+        // Unreachable: the loop only exits via return or throw. Present so the
+        // outcome type stays non-optional for the accounting below.
+        throw new Error('[store] addDocuments retry loop exited without result');
       });
+
+      // Exactly one outcome per logical addDocuments call. Attempts that failed and
+      // were then recovered by the reconnect/conflict retries are not counted here
+      // and are not logged at ERROR — only a write that is actually abandoned is.
+      // An empty batch is a no-op and is not counted at all, as before.
+      if (outcome !== 'empty') {
+        metrics.increment('knowledge_store_add_documents_total', 1, {
+          status: outcome === 'ignored_closing' ? 'ignored_closing' : 'success',
+        });
+      }
+    } catch (err) {
+      metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
+      logger.error('[store] Failed to add documents:', err);
+      throw err;
     } finally {
       this.activeWrites.delete(writePromise);
       writeResolve!();

package/src/skill-install/skill-installer.ts

@@ -49,8 +49,11 @@ export interface HarnessDef {
 export const HARNESSES: readonly HarnessDef[] = [
   { id: 'claude', label: 'Claude', baseDir: '.claude', skillsDir: path.join('.claude', 'skills'), confidence: 'confirmed' },
   { id: 'openclaw', label: 'OpenClaw', baseDir: '.openclaw', skillsDir: path.join('.openclaw', 'skills'), confidence: 'confirmed', note: 'OpenClaw reads ~/.openclaw/skills as a managed skill root and accepts symlinked skill folders (docs.openclaw.ai/tools/skills).' },
-  { id: 'pi', label: 'pi', baseDir: '.pi', skillsDir: path.join('.pi', 'skills'), confidence: 'confirmed' },
-  { id: 'cursor', label: 'Cursor', baseDir: '.cursor', skillsDir: path.join('.cursor', 'skills'), confidence: 'partial', note: 'Cursor project-level skills are documented; the global ~/.cursor/skills path is community-reported.' },
+  // pi's own skill root is under its agent dir (~/.pi/agent/skills), NOT ~/.pi/skills.
+  // Listed for DETECTION only — pi is never an install target (see SKILL_AGENT_TARGETS):
+  // the host already exposes research as a native tool, so a skill there would just
+  // duplicate it through a slower subprocess.
+  { id: 'pi', label: 'pi', baseDir: '.pi', skillsDir: path.join('.pi', 'agent', 'skills'), confidence: 'confirmed' },
   { id: 'codex', label: 'OpenAI Codex CLI', baseDir: '.codex', skillsDir: path.join('.codex', 'skills'), confidence: 'unverified', note: 'Codex skills support is emerging; path not confirmed by official docs.' },
   { id: 'agents', label: 'Cross-tool (~/.agents/skills)', baseDir: '.agents', skillsDir: path.join('.agents', 'skills'), confidence: 'unverified', note: 'Proposed Universal Agents convention; not yet a ratified standard.' },
 ] as const;
@@ -58,6 +61,31 @@ export const HARNESSES: readonly HarnessDef[] = [
 const SKILL_NAME = 'pi-research';
 const PACKAGE_NAME = '@lincoln504/pi-research';
 
+/**
+ * Directory inside this package that holds the shipped skill.
+ *
+ * Deliberately NOT `skills/`: pi treats a package-root `skills/` directory as a
+ * resource root and convention-scans it whenever its manifest omits a `skills`
+ * key, so the old name made pi load this skill — a slower duplicate of the
+ * extension's own native research tool — as soon as anything rewrote the
+ * package's settings entry into object form (which `pi config` does on any
+ * toggle). The name must stay outside pi's reserved resource types
+ * (extensions, skills, prompts, themes).
+ */
+const SKILL_SOURCE_DIR = 'agent-skill';
+/** Pre-1.5.3 location. Still recognised so existing installs stay manageable. */
+const LEGACY_SKILL_SOURCE_DIR = 'skills';
+
+/**
+ * Shape an owned symlink's destination must have: `…/pi-research/<dir>/pi-research`
+ * where `<dir>` is the current or the legacy source directory. Kept in sync with
+ * the duplicate in scripts/cleanup.cjs, which runs on preuninstall without this
+ * module available.
+ */
+const OWNED_SYMLINK_DEST = new RegExp(
+  `[/\\\\]${SKILL_NAME}[/\\\\](?:${SKILL_SOURCE_DIR}|${LEGACY_SKILL_SOURCE_DIR})[/\\\\]${SKILL_NAME}$`,
+);
+
 /**
  * This package's version, used to stamp manifest entries so a stale COPY can be
  * detected and refreshed. Read from the SKILL.md we are about to install rather
@@ -77,18 +105,17 @@ function readSkillVersion(sourceDir: string): string | null {
 }
 
 /**
- * The coding agents the in-app installer (the /research-config TUI) targets:
- * Claude, Codex, and OpenClaw. Each loads skills from a home-directory
- * `~/.<agent>/skills/` path — OpenClaw reads `~/.openclaw/skills` as a managed
- * skill root that accepts symlinked skill folders — so a single global symlink
- * works. Every target is gated on the agent's config dir already existing (see
- * skillInstallCandidates), so a tool that isn't set up is never offered and we
+ * The coding agents the in-app installer (the /research-config TUI) targets. Each
+ * loads skills from a home-directory `~/.<agent>/skills/` path, so a single global
+ * symlink works. Every target is gated on the agent's config dir already existing
+ * (see skillInstallCandidates), so a tool that isn't set up is never offered and we
  * never create its home dir.
  *
- * Cursor is deliberately excluded: it has no personal/global skills directory —
- * Cursor only loads skills from a project-level `.cursor/skills/`, so a global
- * `~/.cursor/skills` symlink would never be read. `pi` (the host itself) and the
- * speculative `~/.agents` convention are likewise excluded from the one-click flow.
+ * `pi` (the host itself) and the speculative `~/.agents` convention are excluded:
+ * pi already exposes research as a native tool, and a skill in either root would
+ * only add a slower duplicate. Agents that expose no personal/global skills
+ * directory — only project-level ones — cannot be served by a global symlink and
+ * are likewise not targeted.
  */
 export const SKILL_AGENT_TARGETS = ['claude', 'codex', 'openclaw'] as const;
 
@@ -166,7 +193,11 @@ function homeDir(opts?: { home?: string }): string {
  * Locate the bundled skill source directory (the one containing SKILL.md).
  * Robust across both the bundled CLI (dist/cli.mjs) and unbundled test runs:
  * honors PI_RESEARCH_SKILL_DIR, then walks up from this module and from cwd
- * looking for `skills/pi-research/SKILL.md`.
+ * looking for `agent-skill/pi-research/SKILL.md`.
+ *
+ * The legacy `skills/` location is still probed, after the current one, so a
+ * mixed tree (an older package still on disk beside a newer one) resolves rather
+ * than throwing. It is deliberately second: the current layout always wins.
  */
 export function resolveSkillSourceDir(explicit?: string): string {
   const candidates: string[] = [];
@@ -181,7 +212,8 @@ export function resolveSkillSourceDir(explicit?: string): string {
     if (!start) continue;
     let dir = start;
     for (let i = 0; i < 8; i++) {
-      candidates.push(path.join(dir, 'skills', SKILL_NAME));
+      candidates.push(path.join(dir, SKILL_SOURCE_DIR, SKILL_NAME));
+      candidates.push(path.join(dir, LEGACY_SKILL_SOURCE_DIR, SKILL_NAME));
       const parent = path.dirname(dir);
       if (parent === dir) break;
       dir = parent;
@@ -193,7 +225,7 @@ export function resolveSkillSourceDir(explicit?: string): string {
       if (fs.existsSync(path.join(c, 'SKILL.md'))) return path.resolve(c);
     } catch { /* next */ }
   }
-  throw new Error('Could not locate the bundled research skill (skills/pi-research/SKILL.md). Set PI_RESEARCH_SKILL_DIR to its path.');
+  throw new Error('Could not locate the bundled research skill (agent-skill/pi-research/SKILL.md). Set PI_RESEARCH_SKILL_DIR to its path.');
 }
 
 export function getManifestPath(opts?: { home?: string }): string {
@@ -251,7 +283,7 @@ function upsertEntry(manifest: Manifest, entry: ManifestEntry): void {
 // Ownership checks (never delete what isn't ours)
 // ---------------------------------------------------------------------------
 
-/** A symlink we own points at the skill source (our package's skills/pi-research). */
+/** A symlink we own points at the skill source (our package's agent-skill/pi-research). */
 function isOwnedSymlink(targetPath: string, expectedSource?: string): boolean {
   try {
     const st = fs.lstatSync(targetPath);
@@ -259,11 +291,17 @@ function isOwnedSymlink(targetPath: string, expectedSource?: string): boolean {
     const dest = path.resolve(path.dirname(targetPath), fs.readlinkSync(targetPath));
     if (expectedSource && path.resolve(expectedSource) === dest) return true;
     // Fallback: own it only if it points at OUR package layout, i.e. the dest
-    // ends in `pi-research/skills/pi-research`. Requiring `pi-research` to be the
-    // immediate parent segment of `skills/pi-research` avoids the substring false
-    // positive where a foreign path like `/home/pi-researcher/.../skills/pi-research`
-    // (note: pi-researcher) would otherwise be misclassified as owned and deleted.
-    return /[/\\]pi-research[/\\]skills[/\\]pi-research$/.test(dest);
+    // ends in `pi-research/<source-dir>/pi-research`. Requiring `pi-research` to be
+    // the immediate parent segment avoids the substring false positive where a
+    // foreign path like `/home/pi-researcher/.../skills/pi-research` (note:
+    // pi-researcher) would otherwise be misclassified as owned and deleted.
+    //
+    // BOTH the current and legacy source-dir names are accepted. Three of this
+    // function's four call sites pass no expectedSource and rely on this branch
+    // alone, so dropping the legacy name would strand every link created before
+    // 1.5.3: reconcile would stop re-pointing it, uninstall would refuse to
+    // remove it, and status would report it as a foreign skill.
+    return OWNED_SYMLINK_DEST.test(dest);
   } catch { return false; }
 }
 

package/src/web-research/scraper-utils.ts

@@ -19,45 +19,13 @@ import {
   INTERNAL_NETWORK_PATTERNS,
   type NativeHtmlToMarkdownModule,
 } from './scraper-types.ts';
-import { isCloudflareBlockError, isTaskTimeoutError, isPoolShutdownError } from '../infrastructure/browser/browser-error-utils.ts';
 
 /**
- * True when a scrape failure is an EXPECTED per-URL outcome rather than an engine
- * fault. Scraping the open web routinely hits bot-protection interstitials, thin
- * "stub" pages, HTTP 4xx/5xx, slow/unresponsive remotes, and (on quit) pool-drain
- * races. None of these is a pi-research bug: the URL is already accounted for as
- * "not scraped" (scrape_results_total{outcome=total_failure}) and surfaced to the
- * user that way, and no operator action follows. Callers use this to log such
- * failures at debug instead of error — keeping the ERROR log (and, via
- * logger.error's error-tracker re-track, the diagnostic error count) focused on
- * genuine faults. The messages matched here are produced by validateContent
- * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
- * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
- * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
- * exceeds the deliberate 100MB policy cap, is routine on the open web and
- * just as non-actionable as a stub page or a 404; the 25MB HTML policy caps
- * (`HTML response too large` from the fetch layer, `Browser HTML too large`
- * from the browser layer) are the same deliberate-cap class; the remaining
- * cases delegate to the shared browser-layer predicates. Deliberately does NOT
- * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
- * failure) — that is an infrastructure fault affecting every URL, not a
- * routine per-PDF outcome, and must still surface at ERROR.
+ * Re-exported from the browser error utilities, where it lives beside the three
+ * predicates it delegates to. Kept exported here because this is where every scrape
+ * caller already imports its error helpers from.
  */
-export function isBenignScrapeFailure(error: unknown): boolean {
-  const msg = error instanceof Error ? error.message : String(error);
-  return (
-    msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-    msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
-    msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
-    msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
-    msg.startsWith('HTML response too large') ||          // fetch-layer 25MB policy cap — same class as the PDF cap
-    msg.startsWith('Browser HTML too large') ||           // browser-layer 25MB policy cap
-    /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
-    isCloudflareBlockError(error) ||
-    isTaskTimeoutError(error) ||
-    isPoolShutdownError(error)
-  );
-}
+export { isBenignScrapeFailure } from '../infrastructure/browser/browser-error-utils.ts';
 
 /**
  * Get a random user agent

package/src/youtube/potoken.ts

@@ -6,8 +6,8 @@
  * HTTP 200 with an EMPTY body (a silent failure). This module mints tokens with
  * `bgutils-js`, running the BotGuard VM under a `jsdom` DOM.
  *
- * Hard-won implementation details (validated live against youtubei.js@17 /
- * bgutils-js@3 / jsdom@29):
+ * Hard-won implementation details (re-validated live against youtubei.js@18 /
+ * bgutils-js@4 / jsdom@29):
  *
  *  1. Cross-realm typed arrays. The BotGuard VM must run in the SAME JS realm as
  *     bgutils' `base64ToU8` (the Node realm). So the interpreter is eval'd via
@@ -171,9 +171,17 @@ async function doMint(
   const started = Date.now();
 
   // Lazy-load heavy deps — never imported at module load (keeps startup native-free).
-  const bgMod = (await import('bgutils-js')) as unknown as BgModule;
-  const BG = bgMod.BG ?? bgMod.default;
-  const { buildURL, getHeaders } = bgMod;
+  // bgutils-js 4 dropped the barrel entry (package.json "exports" has no "."), so each
+  // piece comes from its own subpath: the challenge fetcher and VM client from
+  // /botguard, the minter from /webpo, the attestation URL/header helpers from /utils.
+  const [bgGuard, bgWebPo, bgUtils] = (await Promise.all([
+    import('bgutils-js/botguard'),
+    import('bgutils-js/webpo'),
+    import('bgutils-js/utils'),
+  ])) as unknown as [BgGuardModule, BgWebPoModule, BgUtilsModule];
+  const { getChallenge, BotGuardClient } = bgGuard;
+  const { WebPoMinter } = bgWebPo;
+  const { buildURL, getHeaders } = bgUtils;
   const { JSDOM } = await import('jsdom');
 
   const dom = new JSDOM('<!DOCTYPE html><html lang="en"><body></body></html>', {
@@ -211,18 +219,18 @@ async function doMint(
 
   bridge();
   try {
-    const bgConfig: BgConfig = {
-      fetch: (input: unknown, init?: unknown) =>
+    // bgutils-js 4 replaced BG.Challenge.create(config) with a free getChallenge()
+    // whose config is narrower: no globalObj and no identifier (the content binding is
+    // supplied per-mint, which is what this module already did), and `fetch` renamed
+    // to `fetchFunction`.
+    const challenge = await getChallenge({
+      requestKey,
+      fetchFunction: ((input: unknown, init?: unknown) =>
         fetchImpl(input as Parameters<typeof fetch>[0], {
           ...(init as RequestInit | undefined),
           signal: (init as RequestInit | undefined)?.signal ?? opts.signal,
-        }),
-      globalObj: globalThis,
-      identifier: identifiers[0]!,
-      requestKey,
-    };
-
-    const challenge = await BG.Challenge.create(bgConfig);
+        })) as unknown as typeof fetch,
+    });
     if (!challenge) throw new Error('BotGuard challenge unavailable');
 
     const interpreter = challenge.interpreterJavascript?.privateDoNotAccessOrElseSafeScriptWrappedValue;
@@ -233,10 +241,16 @@ async function doMint(
     // PoToken generation; trust rests on that TLS connection.
     new Function(interpreter)();
 
-    const botguard = await BG.BotGuardClient.create({
+    if (!challenge.program || !challenge.globalName) {
+      throw new Error('BotGuard challenge missing program/globalName');
+    }
+    // `globalObj` was renamed `globalObject` in bgutils-js 4. Passing the old name
+    // leaves the VM without a global to attach to and the minter factory silently
+    // returns a non-function, so this rename is load-bearing, not cosmetic.
+    const botguard = await BotGuardClient.create({
       program: challenge.program,
       globalName: challenge.globalName,
-      globalObj: globalThis,
+      globalObject: globalThis,
     });
 
     const webPoSignalOutput: unknown[] = [];
@@ -260,7 +274,7 @@ async function doMint(
       throw new Error('Integrity token rejected by attestation server (likely a non-residential IP)');
     }
 
-    const minter = await BG.WebPoMinter.create({ integrityToken }, webPoSignalOutput);
+    const minter = await WebPoMinter.create({ integrityToken }, webPoSignalOutput);
 
     const tokens = new Map<string, string>();
     for (const id of identifiers) {
@@ -294,26 +308,27 @@ async function doMint(
 // Minimal structural types for the loosely-typed bgutils-js surface we touch.
 // ---------------------------------------------------------------------------
 
-interface BgConfig {
-  fetch: (input: unknown, init?: unknown) => Promise<Response>;
-  globalObj: unknown;
-  identifier: string;
-  requestKey: string;
+interface BgChallenge {
+  interpreterJavascript?: { privateDoNotAccessOrElseSafeScriptWrappedValue?: string };
+  program?: string;
+  globalName?: string;
 }
 
-interface BgNamespace {
-  Challenge: {
-    create(config: BgConfig): Promise<{
-      interpreterJavascript?: { privateDoNotAccessOrElseSafeScriptWrappedValue?: string };
-      program: string;
-      globalName: string;
-    } | undefined>;
-  };
+interface BgGuardModule {
+  getChallenge(config: {
+    requestKey: string;
+    fetchFunction: typeof fetch;
+    interpreterHash?: string;
+    useYouTubeAPI?: boolean;
+  }): Promise<BgChallenge | undefined>;
   BotGuardClient: {
-    create(opts: { program: string; globalName: string; globalObj: unknown }): Promise<{
+    create(opts: { program: string; globalName: string; globalObject: unknown }): Promise<{
       snapshot(args: { webPoSignalOutput: unknown[] }): Promise<string>;
     }>;
   };
+}
+
+interface BgWebPoModule {
   WebPoMinter: {
     create(
       integrity: { integrityToken: string },
@@ -322,9 +337,7 @@ interface BgNamespace {
   };
 }
 
-interface BgModule {
-  BG?: BgNamespace;
-  default: BgNamespace;
+interface BgUtilsModule {
   buildURL(endpoint: string, useYouTubeAPI?: boolean): string;
   getHeaders(): Record<string, string>;
 }