@lincoln504/pi-research 1.0.4 → 1.0.6

package/README.md

@@ -63,7 +63,7 @@ pi install npm:@lincoln504/pi-research
 
 See [Agent skill](docs/AGENT-SKILL.md) or [SDK](docs/SDK.md) for standalone (no pi extension) installation.
 
-The pi extension works out of the box — it runs on your pi session's model and pi's own configuration. Standalone use (agent skill / CLI / SDK) requires a configured model: set `PI_RESEARCH_MODEL` (e.g. `openai/gpt-4o`) as an environment variable or in `~/.pi/research/config.env`, plus `PI_RESEARCH_API_KEY` if you don't use pi. See [Configuration](docs/CONFIGURATION.md).
+The pi extension works out of the box — it runs on your pi session's model and pi's own configuration. Standalone use (agent skill / CLI) requires a configured model: set `PI_RESEARCH_MODEL` (e.g. `openai/gpt-4o`) as an environment variable or in `~/.pi/research/config.env` (the CLI also accepts a per-run `--model`), plus `PI_RESEARCH_API_KEY` if you don't use pi; the SDK takes a `model` option. See [Configuration](docs/CONFIGURATION.md).
 
 The first install pulls the stealth browser engine, which takes a few minutes.
 

package/dist/cli.mjs

@@ -4712,9 +4712,6 @@ var init_store = __esm({
                   if (strategy !== "drop" && strategy !== "backup") {
                     logger.warn("[store] Falling back to backup strategy after migration failure");
                     await this.handleModelChange(storedModel || "unknown", this.options.modelName, "backup");
-                  } else if (strategy === "backup") {
-                    logger.warn("[store] Falling back to drop strategy after backup failure");
-                    await this.handleModelChange(storedModel || "unknown", this.options.modelName, "drop");
                   } else {
                     throw new Error(errorMsg, { cause: err });
                   }
@@ -4763,7 +4760,12 @@ var init_store = __esm({
         if (!this.table || !this.db) {
           throw new Error("Table not connected");
         }
-        const count = await this.table.countRows();
+        let count = -1;
+        try {
+          count = await this.table.countRows();
+        } catch (err) {
+          logger.warn(`[store] countRows failed before backup (continuing): ${err instanceof Error ? err.message : String(err)}`);
+        }
         this.table = null;
         const oldDir = path13.join(this.options.dbDir, `${this.tableName}.lance`);
         const backupDir = path13.join(this.options.dbDir, `${backupTableName}.lance`);
@@ -4778,8 +4780,8 @@ var init_store = __esm({
         }
         this.table = await this.createTable();
         await this.saveManifest();
-        logger.info(`[store] Migration complete: ${count} documents backed up, fresh table created with model ${newModel}`);
-        return { strategy: "backup", success: true, documentsProcessed: count };
+        logger.info(`[store] Migration complete: ${count >= 0 ? count : "unknown count of"} documents backed up, fresh table created with model ${newModel}`);
+        return { strategy: "backup", success: true, documentsProcessed: Math.max(0, count) };
       }
       async migrationDrop(_oldModel, newModel) {
         logger.warn(`[store] Dropping table and recreating with model ${newModel} (data will be lost)`);
@@ -9368,6 +9370,7 @@ init_constants();
 var sessionLinks = /* @__PURE__ */ new Map();
 var sessionScrapedContent = /* @__PURE__ */ new Map();
 var researcherScrapes = /* @__PURE__ */ new Map();
+var transcribedUrls = /* @__PURE__ */ new Map();
 var sessionTimestamps = /* @__PURE__ */ new Map();
 var SESSION_MAX_AGE_MS = 2 * 60 * 60 * 1e3;
 var MAX_CACHED_CONTENT_PER_SESSION = 500;
@@ -9383,6 +9386,7 @@ function startCleanupTimer() {
         sessionScrapedContent.delete(researchId);
         sessionTimestamps.delete(researchId);
         researcherScrapes.delete(researchId);
+        transcribedUrls.delete(researchId);
         cleaned++;
       }
     }
@@ -9420,6 +9424,17 @@ function registerScrapedLinks(researchId, links) {
 function getScrapedLinks(researchId) {
   return Array.from(sessionLinks.get(researchId) || []);
 }
+function registerTranscribedLinks(researchId, links) {
+  if (!transcribedUrls.has(researchId)) {
+    transcribedUrls.set(researchId, /* @__PURE__ */ new Set());
+  }
+  sessionTimestamps.set(researchId, Date.now());
+  const pool = transcribedUrls.get(researchId);
+  links.forEach((l) => pool.add(normalizeUrl(l)));
+}
+function isTranscribedLink(researchId, url) {
+  return transcribedUrls.get(researchId)?.has(normalizeUrl(url)) ?? false;
+}
 function deduplicateUrls(urls, researchId) {
   const pool = sessionLinks.get(researchId) || /* @__PURE__ */ new Set();
   const kept = [];
@@ -9439,6 +9454,7 @@ function cleanupSharedLinks(researchId) {
   sessionScrapedContent.delete(researchId);
   sessionTimestamps.delete(researchId);
   researcherScrapes.delete(researchId);
+  transcribedUrls.delete(researchId);
   logger.debug(`[Shared Links] Cleaned up session: ${researchId}`);
 }
 function clearAllSharedLinks() {
@@ -9450,6 +9466,7 @@ function clearAllSharedLinks() {
   sessionScrapedContent.clear();
   sessionTimestamps.clear();
   researcherScrapes.clear();
+  transcribedUrls.clear();
   logger.debug("[Shared Links] All global state cleared");
 }
 function registerResearcherScrapes(researchId, researcherId, urls) {
@@ -15709,6 +15726,48 @@ function decodeHtmlBody(bytes, contentTypeHeader) {
     return new TextDecoder("utf-8").decode(bytes);
   }
 }
+var BodyTooLargeError = class extends Error {
+  constructor(bytesRead, maxBytes) {
+    super(`Response body exceeded ${maxBytes} byte cap`);
+    this.bytesRead = bytesRead;
+    this.maxBytes = maxBytes;
+    this.name = "BodyTooLargeError";
+  }
+  bytesRead;
+  maxBytes;
+};
+async function readBodyCapped(response, maxBytes) {
+  const body = response.body;
+  if (!body || typeof body.getReader !== "function") {
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  const reader = body.getReader();
+  const chunks = [];
+  let total = 0;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value) {
+        total += value.byteLength;
+        if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
+        chunks.push(value);
+      }
+    }
+  } finally {
+    try {
+      await reader.cancel();
+    } catch {
+    }
+  }
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const c of chunks) {
+    out.set(c, offset);
+    offset += c.byteLength;
+  }
+  return out;
+}
 async function scrapeWithFetch(url, signal) {
   await validateUrlForSSRF(url);
   const dispatcher = await getSsrfSafeDispatcher();
@@ -15744,6 +15803,10 @@ async function scrapeWithFetch(url, signal) {
         if (!location) break;
         const resolved = new URL(location, currentUrl).href;
         await validateUrlForSSRF(resolved);
+        try {
+          await response.body?.cancel();
+        } catch {
+        }
         currentUrl = resolved;
         continue;
       }
@@ -15751,6 +15814,10 @@ async function scrapeWithFetch(url, signal) {
     }
     if (!response.ok) {
       metrics.increment("scrape_errors_total", 1, { error_type: "http_error", status_code: String(response.status) });
+      try {
+        await response.body?.cancel();
+      } catch {
+      }
       throw new Error(`HTTP ${response.status}`);
     }
     const contentType = response.headers.get("content-type") ?? "";
@@ -15763,24 +15830,62 @@ async function scrapeWithFetch(url, signal) {
           const sizeMB = Math.round(size / 1024 / 1024);
           logger.warn(`[Scrapers] PDF too large (Content-Length: ${sizeMB}MB, max 100MB), skipping`);
           metrics.increment("scrape_pdf_errors_total", 1, { error_type: "size_exceeded" });
+          try {
+            await response.body?.cancel();
+          } catch {
+          }
           throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
         }
       } else if (size > MAX_HTML_SIZE) {
         const sizeMB = Math.round(size / 1024 / 1024);
         logger.warn(`[Scrapers] HTML response too large (Content-Length: ${sizeMB}MB, max 25MB), skipping`);
         metrics.increment("scrape_errors_total", 1, { error_type: "size_exceeded", content_type: contentType.split(";")[0] || "unknown" });
+        try {
+          await response.body?.cancel();
+        } catch {
+        }
         throw new Error(`HTML response too large (${sizeMB}MB, max 25MB)`);
       }
     }
     if (contentType.includes("application/pdf") || url.toLowerCase().endsWith(".pdf")) {
-      const buffer = await response.arrayBuffer();
-      const markdown2 = await extractPdfToMarkdown(new Uint8Array(buffer));
+      let pdfBytes;
+      try {
+        pdfBytes = await readBodyCapped(response, MAX_PDF_SIZE);
+      } catch (e) {
+        if (e instanceof BodyTooLargeError) {
+          const sizeMB = Math.round(e.bytesRead / 1024 / 1024);
+          logger.warn(`[Scrapers] PDF too large (streamed ${sizeMB}MB, max 100MB), skipping`);
+          metrics.increment("scrape_pdf_errors_total", 1, { error_type: "size_exceeded" });
+          throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`, { cause: e });
+        }
+        throw e;
+      }
+      const markdown2 = await extractPdfToMarkdown(pdfBytes);
       validateContent("", markdown2, url);
       metrics.increment("scrape_operations_total", 1, { layer: "fetch", content_type: "pdf", status: "success" });
       metrics.observe("scrape_latency_ms", fetchDuration, { layer: "fetch", content_type: "pdf", status: "success" });
       return { source: "fetch", layer: "fetch", markdown: markdown2 };
     }
-    const bodyBytes = new Uint8Array(await response.arrayBuffer());
+    let bodyBytes;
+    try {
+      bodyBytes = await readBodyCapped(response, MAX_HTML_SIZE);
+    } catch (e) {
+      if (e instanceof BodyTooLargeError) {
+        const sizeMB = Math.round(e.bytesRead / 1024 / 1024);
+        logger.warn(`[Scrapers] HTML response too large (streamed ${sizeMB}MB, max 25MB), skipping`);
+        metrics.increment("scrape_errors_total", 1, { error_type: "size_exceeded", content_type: "html" });
+        errorTracker.trackError(new Error(`HTML response too large (${sizeMB}MB, max 25MB)`), {
+          component: "scrapers",
+          operation: "fetch",
+          url,
+          domain: extractDomain(url),
+          layer: "fetch",
+          contentType: "html"
+        });
+        throw new Error(`HTML response too large (${sizeMB}MB, max 25MB)`, { cause: e });
+      }
+      throw e;
+    }
     if (bodyBytes.length > MAX_HTML_SIZE) {
       const sizeMB = Math.round(bodyBytes.length / 1024 / 1024);
       logger.warn(`[Scrapers] HTML response too large (actual: ${sizeMB}MB, max 25MB), skipping`);
@@ -16478,6 +16583,8 @@ function handleFetchError(error) {
 }
 function handleResponseStatus(response) {
   if (!response.ok) {
+    void response.body?.cancel()?.catch(() => {
+    });
     if (response.status === 429) {
       throw new Error("NVD API rate limit exceeded (HTTP 429). Retrying with backoff...");
     }
@@ -16672,6 +16779,8 @@ async function fetchWithRetryImpl(url, options, signal) {
       async () => {
         const response = await fetch(url, options);
         if (!response.ok) {
+          void response.body?.cancel()?.catch(() => {
+          });
           const error = new Error(`HTTP ${response.status}: ${response.statusText}`);
           error.status = response.status;
           metrics.increment("cisa_kev_errors_total", 1, { endpoint, status_code: response.status.toString() });
@@ -17028,6 +17137,8 @@ async function searchGitHubAdvisories(terms, options) {
           signal: createTimeoutSignal(1e4, options?.signal)
         });
         if (!resp.ok) {
+          void resp.body?.cancel()?.catch(() => {
+          });
           if (resp.status === 404) {
             throw new Error(`Repository "${owner}/${name}" not found or no access to security advisories.`);
           }
@@ -17095,6 +17206,8 @@ async function searchGitHubAdvisories(terms, options) {
             signal: createTimeoutSignal(1e4, options?.signal)
           });
           if (!resp.ok) {
+            void resp.body?.cancel()?.catch(() => {
+            });
             if (resp.status === 404) {
               throw new Error("Advisory not found (HTTP 404)");
             }
@@ -17415,6 +17528,8 @@ async function fetchWithRetry2(url, options, signal) {
       async () => {
         const response = await fetch(url, options);
         if (!response.ok) {
+          void response.body?.cancel()?.catch(() => {
+          });
           const error = new Error(`HTTP ${response.status}: ${response.statusText}`);
           error.status = response.status;
           metrics.increment("osv_errors_total", 1, { endpoint, status_code: response.status.toString() });
@@ -19153,6 +19268,8 @@ async function fetchOne(session, videoId, contentToken, lang, timeoutMs, signal,
             headers: { "User-Agent": "Mozilla/5.0", "Accept-Language": lang }
           });
           if (!response.ok) {
+            void response.body?.cancel()?.catch(() => {
+            });
             throw new Error(`HTTP ${response.status} from timedtext endpoint`);
           }
           const body = await response.text();
@@ -19329,7 +19446,7 @@ function formatResults(results, rejected) {
     if (meta.length) md += `**${meta.join(" \xB7 ")}**
 `;
     if (r.title) {
-      md += `**Cite as:** '${r.title}'${r.author ? ` by ${r.author}` : ""}
+      md += `**Cite as:** ${r.url} \u2014 '${r.title}'${r.author ? ` by ${r.author}` : ""}
 `;
     }
     md += `
@@ -19388,7 +19505,7 @@ function createYoutubeTranscriptTool(options) {
       "You may call this tool only ONCE, so choose the videos carefully before calling.",
       "Pass full watch URLs, youtu.be links, or shorts/embed/live links. Non-YouTube links are ignored.",
       "Videos without captions, or that are private/age-restricted, are reported as unavailable \u2014 the rest still succeed.",
-      `When you cite a YouTube source in CITED LINKS, use the video's ACTUAL title and channel (both shown in this tool's output) as the description \u2014 e.g. "'<exact title>' by <channel>". Do NOT paraphrase or invent a title; the watch URL alone is not human-identifiable.`
+      `When you cite a YouTube source in CITED LINKS, the entry's link MUST be the video's watch URL (shown in this tool's output) \u2014 a title-only entry loses the source from the final report. Put the video's ACTUAL title and channel in the Description \u2014 e.g. "'<exact title>' by <channel>". Do NOT paraphrase or invent a title.`
     ],
     parameters: YoutubeParamsSchema,
     executionMode: "parallel",
@@ -19439,6 +19556,7 @@ function createYoutubeTranscriptTool(options) {
         });
         if (researchId && cachedUrls.length > 0) {
           registerScrapedLinks(researchId, cachedUrls);
+          registerTranscribedLinks(researchId, cachedUrls);
           options.updateGlobalLinks?.(cachedUrls);
           if (options.researcherId) {
             registerResearcherScrapes(researchId, options.researcherId, cachedUrls);
@@ -20309,6 +20427,45 @@ init_constants();
 
 // src/orchestration/headless-observer.ts
 init_logger();
+function makeSafeObserver(observer) {
+  return new Proxy({}, {
+    get(_target, prop) {
+      let value;
+      try {
+        value = Reflect.get(observer, prop, observer);
+      } catch (err) {
+        logger.warn(`[observer] Ignored error reading observer.${String(prop)}:`, err);
+        return void 0;
+      }
+      if (typeof value !== "function") return value;
+      return (...args) => {
+        try {
+          const out = value.apply(observer, args);
+          if (out && typeof out.then === "function") {
+            return out.catch((err) => {
+              logger.warn(`[observer] Ignored rejection from observer.${String(prop)}:`, err);
+            });
+          }
+          return out;
+        } catch (err) {
+          logger.warn(`[observer] Ignored error from observer.${String(prop)}:`, err);
+          return void 0;
+        }
+      };
+    },
+    has(_target, prop) {
+      try {
+        return Reflect.has(observer, prop);
+      } catch {
+        return false;
+      }
+    },
+    // Keep `instanceof` transparent (legal on an extensible target).
+    getPrototypeOf() {
+      return Reflect.getPrototypeOf(observer);
+    }
+  });
+}
 var HeadlessObserver = class {
   constructor(options = {}) {
     this.options = options;
@@ -20398,9 +20555,9 @@ var QuickResearchOrchestrator = class {
     this.options = options;
     this.config = options.config || getConfig(options.ctx.cwd);
     if (options.observer && typeof options.observer.onProgress === "function" && !(options.observer instanceof HeadlessObserver)) {
-      this.observer = new HeadlessObserver(options.observer);
+      this.observer = makeSafeObserver(new HeadlessObserver(options.observer));
     } else {
-      this.observer = options.observer;
+      this.observer = options.observer ? makeSafeObserver(options.observer) : void 0;
     }
   }
   options;
@@ -20714,9 +20871,9 @@ var DeepResearchOrchestrator = class {
     this.options = options;
     this.config = options.config || getConfig(options.ctx.cwd);
     if (options.observer && typeof options.observer.onProgress === "function" && !(options.observer instanceof HeadlessObserver)) {
-      this.observer = new HeadlessObserver(options.observer);
+      this.observer = makeSafeObserver(new HeadlessObserver(options.observer));
     } else {
-      this.observer = options.observer;
+      this.observer = options.observer ? makeSafeObserver(options.observer) : void 0;
     }
     if (options.orchestrationService) {
       this.orchestrationService = options.orchestrationService;
@@ -20989,18 +21146,22 @@ var DeepResearchOrchestrator = class {
         observer?.onEvaluationDecision?.("synthesize", finalReport, maxRounds);
       }
       let result = finalReport.content || "";
+      const synthesisCheckService = await getService(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
+      const hasReports = synthesisCheckService.hasReports(researchId);
+      const failedIds = getFailedResearchers(this.options.sessionId, researchId);
+      if (!hasReports && failedIds.length > 0) {
+        const reasons = getResearcherFailureReasons(this.options.sessionId, researchId);
+        const causes = failedIds.map((id) => `researcher ${id}: ${reasons[id] ?? "no usable report"}`).join("; ");
+        throw new Error(
+          `Research produced no report \u2014 no source material was collected. Causes: ${causes}`
+        );
+      }
       if (!result.trim()) {
-        const synthesisService2 = await getService(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
-        if (synthesisService2.hasReports(researchId)) {
-          logger.warn(`[DeepOrchestrator] LLM returned empty synthesis for ${researchId}, building fallback from ${synthesisService2.getReportCount(researchId)} reports`);
-          result = synthesisService2.buildFallbackSynthesis(researchId, this.currentRound);
+        if (hasReports) {
+          logger.warn(`[DeepOrchestrator] LLM returned empty synthesis for ${researchId}, building fallback from ${synthesisCheckService.getReportCount(researchId)} reports`);
+          result = synthesisCheckService.buildFallbackSynthesis(researchId, this.currentRound);
         } else {
-          const failedIds = getFailedResearchers(this.options.sessionId, researchId);
-          const reasons = getResearcherFailureReasons(this.options.sessionId, researchId);
-          const causes = failedIds.map((id) => `researcher ${id}: ${reasons[id] ?? "no usable report"}`).join("; ");
-          throw new Error(
-            "Research produced no report \u2014 no source material was collected" + (causes ? `. Causes: ${causes}` : ".")
-          );
+          throw new Error("Research produced no report \u2014 no source material was collected.");
         }
       }
       try {
@@ -21731,9 +21892,14 @@ Research performed using ${modelId}`;
     if (globalCitations.length === 0) {
       const scraped = getScrapedLinks(sessionId);
       if (scraped.length > 0) {
-        globalCitations = scraped.map((url, i) => ({ id: i + 1, url, description: "", source: "Scrape" }));
+        globalCitations = scraped.map((url, i) => ({
+          id: i + 1,
+          url,
+          description: "",
+          source: isTranscribedLink(sessionId, url) ? "YouTube Transcript" : "Scrape"
+        }));
         logger.warn(
-          `[ResearchSynthesisService] No citations parsed from reports; rebuilt ${scraped.length} source(s) from scrape provenance.`
+          `[ResearchSynthesisService] No citations parsed from reports; rebuilt ${scraped.length} source(s) from scrape/transcript provenance.`
         );
       }
     }
@@ -22267,6 +22433,7 @@ async function _doShutdown() {
   _lastResearchId = null;
   _lastRunSummary = null;
   _lastErrorReport = null;
+  resetConfig();
   _removeSignalHandlers();
   if (errors.length > 0) {
     throw new AggregateError(errors, `${errors.length} error(s) during SDK shutdown`);
@@ -22641,7 +22808,7 @@ function piKeysAvailable(piAuthPresent, provider) {
     return piAuthPresent;
   }
 }
-function detectCredentials() {
+function detectCredentials(explicitModel) {
   const paths = resolvedConfigPaths();
   const piAuthPresent = fileExists(paths.piAuth);
   const piModelsPresent = fileExists(paths.piModels);
@@ -22649,8 +22816,8 @@ function detectCredentials() {
   const providerEnv = process.env["PI_RESEARCH_PROVIDER"];
   const modelEnv = process.env["PI_RESEARCH_MODEL"];
   const modelConfig = getConfig(process.cwd(), "cli").RESEARCH_MODEL;
-  const model = modelEnv ?? modelConfig;
-  const modelFrom = modelEnv ? "PI_RESEARCH_MODEL" : modelConfig ? "config.env" : "unset";
+  const model = explicitModel ?? modelEnv ?? modelConfig;
+  const modelFrom = explicitModel ? "--model" : modelEnv ? "PI_RESEARCH_MODEL" : modelConfig ? "config.env" : "unset";
   const provider = providerEnv ?? (model && model.includes("/") ? model.slice(0, model.indexOf("/")) : void 0);
   const piKeysUsable = piKeysAvailable(piAuthPresent, provider);
   if (apiKey) {
@@ -22675,7 +22842,7 @@ function detectCredentials() {
         modelFrom,
         piAuthPresent,
         piModelsPresent,
-        problem: 'PI_RESEARCH_API_KEY is set but no model is configured. Set PI_RESEARCH_MODEL to a "provider/model-id" (e.g. openai/gpt-4o).'
+        problem: 'PI_RESEARCH_API_KEY is set but no model is configured. Set PI_RESEARCH_MODEL to a "provider/model-id" (e.g. openai/gpt-4o), or pass --model <provider/id> for a single run.'
       };
     }
     return {
@@ -22698,7 +22865,7 @@ function detectCredentials() {
         modelFrom,
         piAuthPresent,
         piModelsPresent,
-        problem: `pi credentials were found (in pi's configuration), but no research model is configured. The standalone CLI / agent skill run only on an explicitly configured model \u2014 they do not follow the model selected inside the pi extension. Set PI_RESEARCH_MODEL to a "provider/model-id" (in the environment, or in config.env / cli.env below).`
+        problem: `pi credentials were found (in pi's configuration), but no research model is configured. The standalone CLI / agent skill run only on an explicitly configured model \u2014 they do not follow the model selected inside the pi extension. Set PI_RESEARCH_MODEL to a "provider/model-id" (in the environment, or in config.env / cli.env below), or pass --model <provider/id> for a single run.`
       };
     }
     return {
@@ -22738,13 +22905,13 @@ function configBlock(det, extraNote) {
   lines.push(`  \u2022 api key configured: ${det.apiKeyConfigured}`);
   lines.push(`  \u2022 provider:           ${det.provider ?? "(unset \u2014 inferred from a provider/model-id model)"}`);
   lines.push(`  \u2022 model:              ${det.model ? `${det.model}  [from: ${det.modelFrom}]` : "(not set \u2014 set PI_RESEARCH_MODEL, required)"}`);
-  lines.push(`  \u2022 pi auth.json:       ${det.piAuthPresent ? "present" : "absent"}`);
-  lines.push(`  \u2022 pi models.json:     ${det.piModelsPresent ? "present" : "absent"}`);
+  lines.push(`  \u2022 pi config (auth.json):   ${det.piAuthPresent ? "present" : "absent"}`);
+  lines.push(`  \u2022 pi config (models.json): ${det.piModelsPresent ? "present" : "absent"}`);
   if (extraNote) lines.push(`  \u2022 note:               ${extraNote}`);
   return lines.join("\n");
 }
 async function cmdResearch(args) {
-  const det = detectCredentials();
+  const det = detectCredentials(args.model);
   if (det.problem) {
     if (args.json) toStdout(pretty({ ok: false, error: det.problem, exitCode: EXIT.CONFIG }));
     else toStderr(`
@@ -22766,7 +22933,7 @@ Error: ${vmsg}
     return EXIT.USAGE;
   }
   const depth = args.depth ?? getConfig(process.cwd(), "cli").DEFAULT_RESEARCH_DEPTH;
-  const runModel = args.model ?? det.model;
+  const runModel = det.model;
   toStderr(`[pi-research] starting research (depth ${depth})${runModel ? ` with ${runModel}` : ""}\u2026
 `);
   const observer = makeProgressObserver();
@@ -23141,7 +23308,7 @@ USAGE
 COMMANDS
   research  "<query>"            Run multi-agent web research (default depth: 1).
     --depth <0-3>                0 = quick, 1 = normal (default), 2 = deep, 3 = ultra.
-    --model <provider/id>        Override the model for this run.
+    --model <provider/id>        Model for this run (outranks PI_RESEARCH_MODEL; satisfies the required-model check).
     --exclude-tools <a,b>        Disable internal tools (e.g. security_search,stackexchange).
     --initial-links <url ...>    Seed URLs to investigate first; requires a query (-- ends options).
     --config <path>              Use this config file instead of the base config.env.
@@ -23178,9 +23345,9 @@ CONFIGURE
     pi config (keys):    ${p.piAuth}
     pi config (models):  ${p.piModels}
 
-  A model is REQUIRED: set PI_RESEARCH_MODEL to a "provider/model-id". This CLI and
-  the agent skill run only on that configured model \u2014 they do not follow the model
-  selected inside the pi extension.
+  A model is REQUIRED: set PI_RESEARCH_MODEL to a "provider/model-id" (or pass
+  --model per run). This CLI and the agent skill run only on that configured
+  model \u2014 they do not follow the model selected inside the pi extension.
 
   The pi extension has its own optional overlay file:
     pi extension: ${p.piIfaceEnv}

package/dist/prompts/researcher.md

@@ -58,7 +58,7 @@ Write your report immediately after scraping is complete, or as soon as a tool r
 ## Guidelines
 
 - **Available Tools**:
-  - `scrape`: Fetch and read web pages (primary tool). Focus your energy here.
+  - `scrape`: Fetch and read web pages (primary tool). Focus your energy here. Do NOT scrape YouTube video links (watch / youtu.be / shorts) — a watch page returns YouTube's app shell, not the video's content; read videos with `youtube_transcript`. Scraping a YouTube channel or playlist page to DISCOVER video links is fine.
   - `youtube_transcript`: Read the captions/transcript of YouTube videos from your source list — it is the `scrape` for video. ONE call only, batching up to a few of the most relevant YouTube links. Cite these as `Source: YouTube Transcript`.
   - `stackexchange`: Use ONLY for Stack Overflow or Stack Exchange URLs.
   - `security_search`: Query NVD, CVE, OSV, CISA databases.

package/docs/ARCHITECTURE.md

@@ -25,8 +25,8 @@ pi CLI
    in-session, the calling agent picks the depth (1–3) from the user's wording and the
    task's complexity, guided by the tool's usage prompt (`src/prompts/research-tool-usage.md`).
    CLI and SDK callers pass the depth explicitly.
-2. Depth 0 takes the quick path; depth 1–3 takes the deep path (below). The TUI and agent
-   skill are restricted to levels 1–3.
+2. Depth 0 takes the quick path; depth 1–3 takes the deep path (below). The pi extension's
+   tool and the TUI are restricted to levels 1–3; the CLI, SDK, and agent skill can pass 0.
 3. On the deep path the coordinator plans the research tracks and runs one initial
    search burst, then hands each researcher a set of result URLs to start from.
 4. Researchers scrape and read those pages through the scrape tooling and return cited
@@ -154,7 +154,8 @@ WriterQueue (async, non-blocking)
 Key files: `src/knowledge/store.ts` (LanceDB operations), `embedder.ts` (model load +
 batched inference), `writer-queue.ts` (async writes + content-hash dedup), `chunker.ts`
 (chunking), `webgpu-viability.ts` (out-of-process GPU probe + cached verdict),
-`migration.ts` (model-change migration: drop / backup / re-embed).
+`migration.ts` (migration strategy types — the drop / backup / re-embed logic
+itself lives in `store.ts`).
 
 The store needs native ONNX-runtime and LanceDB bindings. On platforms with no prebuilt
 binary — notably Intel macOS (`darwin-x64`) — it is absent: the health check reports it

package/docs/CONFIGURATION.md

@@ -75,7 +75,7 @@ Research
 | `PI_RESEARCH_RETRY_DELAY_MS` | `2000` | 100–10000 | Base delay between retries. |
 | `PI_RESEARCH_WORKER_THREADS` | `4` | 1–10 | Browser worker processes. Higher = more throughput, more CPU/RAM. |
 | `PI_RESEARCH_WORKER_CONCURRENCY` | `2` | 1–10 | Tasks per worker process. |
-| `PI_RESEARCH_MODEL` | _(pi: session model; CLI/skill: required)_ | — | The model research runs on. **Required for the standalone CLI / agent skill** — they use only this configured model (never the model selected inside the pi extension) and refuse to start without one. On the SDK it selects the session model when no `model` option is given. In the pi extension it overrides researcher sub-agents and knowledge synthesis, while the coordinator and evaluator keep using the session model. Accepts `provider/id` or a bare model id. |
+| `PI_RESEARCH_MODEL` | _(pi: session model; CLI/skill: required)_ | — | The model research runs on. **Required for the standalone CLI / agent skill** — they use only this configured model (never the model selected inside the pi extension) and refuse to start without one (the CLI's per-run `--model` flag also satisfies this). On the SDK it selects the session model when no `model` option is given. In the pi extension it overrides researcher sub-agents and knowledge synthesis, while the coordinator and evaluator keep using the session model. Accepts `provider/id` or a bare model id. |
 | `PI_RESEARCH_DISABLED_TOOLS` | _(none)_ | — | Comma-separated research tools to disable for a run (`search`, `scrape`, `security_search`, `stackexchange`, `youtube_transcript`, `grep`). Removed from every researcher's toolset and named in the coordinator/evaluator prompt. |
 | `PI_RESEARCH_REPORT_EXPORT_ENABLED` (TUI) | `false` | — | Front-ends write a Markdown report to disk and surface its path. |
 | `PI_RESEARCH_REPORT_EXPORT_DIR` | _(smart cwd)_ | — | Pin exported reports to a fixed directory, bypassing the cwd-relative resolution. Useful for the agent skill, which runs from the host agent's arbitrary directory. |

package/docs/KNOWLEDGE-STORE.md

@@ -174,9 +174,11 @@ built with, the store is migrated according to `PI_RESEARCH_MIGRATION_STRATEGY`:
 | `drop` | The old table is discarded and a fresh one created. Fast; no backup. |
 | `re-embed` | Every stored document is re-embedded with the new model into a new table, preserving history. Slowest. |
 
-If the chosen strategy fails, pi-research falls back to `backup`, then to `drop`,
-rather than leaving the store in a broken state. Changing the model from the
-`/research-config` menu always clears the current store and starts fresh.
+If `re-embed` fails, pi-research falls back to `backup`. A failed `backup` (or
+`drop`) aborts the migration instead: the store stays on the old model and the
+next open retries — data is never dropped unless `drop` was chosen explicitly.
+Changing the model from the `/research-config` menu always clears the current
+store and starts fresh.
 
 ### Managing the store
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "A set of web research tools (search, scrape, and others) and research agent system that uses them safely and efficiently.",
   "type": "module",
   "private": false,

package/skills/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.0.4", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.0.6", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---