@openontology/opencode-palantir 0.1.4-next.9 → 0.1.5-next.4

package/README.md

@@ -36,7 +36,7 @@ Restart OpenCode.
 After enabling the plugin, OpenCode will automatically register:
 
 - Tools: `get_doc_page`, `list_all_docs`
-- Commands: `/refresh-docs`, `/setup-palantir-mcp`, `/rescan-palantir-mcp-tools`
+- Commands: `/refresh-docs`, `/refresh-docs-rescrape`, `/setup-palantir-mcp`, `/rescan-palantir-mcp-tools`
 - Agents: `foundry-librarian`, `foundry`
 
 ### Versions: how to get the latest
@@ -131,17 +131,22 @@ it’s `export`ed in the environment where OpenCode is launched.
 
 ## Docs tools (Palantir public docs)
 
-This package does **not** ship with docs bundled. The docs DB is a local file:
+The docs DB is a local file:
 
 - `data/docs.parquet` (in your repo root)
 
-### Fetch docs
+### First-run behavior
 
-In OpenCode, run:
+On startup and tool usage, the plugin automatically ensures `data/docs.parquet` exists by using a
+prebuilt snapshot (download/copy). In most repos, docs tools should work without any manual setup.
 
-- `/refresh-docs`
+### Refresh commands
 
-This downloads the docs and writes `data/docs.parquet`.
+- `/refresh-docs` (recommended)
+  - Force refresh from a prebuilt snapshot (no live rescrape)
+- `/refresh-docs-rescrape` (unsafe/experimental fallback)
+  - Live-rescrapes palantir.com docs and rebuilds `data/docs.parquet`
+  - Use only when snapshot download/copy is blocked
 
 ### Tools
 
@@ -150,8 +155,6 @@ This downloads the docs and writes `data/docs.parquet`.
 - `list_all_docs`
   - List docs with pagination and optional query/scope filtering
 
-If `data/docs.parquet` is missing, both tools will tell you to run `/refresh-docs`.
-
 ## Foundry MCP helpers
 
 This plugin registers Foundry commands and agents automatically at startup (config-driven).

package/dist/index.js

@@ -1,6 +1,6 @@
 // @bun
 // src/index.ts
-import path4 from "path";
+import path5 from "path";
 import { tool } from "@opencode-ai/plugin/tool";
 
 // node_modules/hyparquet/src/constants.js
@@ -6088,6 +6088,9 @@ var BASE_DELAY_MS = 1000;
 var BACKOFF_FACTOR = 2;
 var JITTER_RANGE = 0.25;
 var BATCH_SIZE = 100;
+function formatError(error) {
+  return error instanceof Error ? error.toString() : String(error);
+}
 function decompressPagefind(data) {
   const decompressed = gunzipSync(Buffer.from(data));
   if (decompressed.length < PAGEFIND_HEADER_SIZE) {
@@ -6202,8 +6205,11 @@ async function withConcurrencyLimit(tasks, limit) {
     next();
   });
 }
-async function fetchAllDocs(dbPath) {
+async function fetchAllDocs(dbPath, options = {}) {
   const entry = await fetchEntryPoint();
+  const onProgress = options.onProgress;
+  const concurrency = typeof options.concurrency === "number" && options.concurrency > 0 ? options.concurrency : DEFAULT_CONCURRENCY;
+  const progressEvery = typeof options.progressEvery === "number" && options.progressEvery > 0 ? Math.floor(options.progressEvery) : BATCH_SIZE;
   const langKey = Object.keys(entry.languages)[0];
   if (!langKey) {
     throw new Error("No languages found in Pagefind entry");
@@ -6211,17 +6217,25 @@ async function fetchAllDocs(dbPath) {
   const langHash = entry.languages[langKey].hash;
   const pageHashes = await fetchAndParseMeta(langHash);
   const totalPages = pageHashes.length;
+  onProgress?.({ type: "discovered", totalPages });
   const fetchedRecords = [];
   const failedUrls = [];
-  let done = 0;
-  const tasks = pageHashes.map((hash) => () => fetchFragment(hash).then((record) => {
-    done++;
-    if (done % BATCH_SIZE === 0 || done === totalPages) {
-      console.log(`Fetched ${done}/${totalPages} pages...`);
+  let processedPages = 0;
+  const tasks = pageHashes.map((hash) => () => fetchFragment(hash).catch((error) => {
+    const url = `${PAGEFIND_BASE}/fragment/${hash}.pf_fragment`;
+    onProgress?.({
+      type: "page-failed",
+      url,
+      error: formatError(error)
+    });
+    throw error;
+  }).finally(() => {
+    processedPages += 1;
+    if (processedPages % progressEvery === 0 || processedPages === totalPages) {
+      onProgress?.({ type: "progress", processedPages, totalPages });
     }
-    return record;
   }));
-  const results = await withConcurrencyLimit(tasks, DEFAULT_CONCURRENCY);
+  const results = await withConcurrencyLimit(tasks, concurrency);
   for (let i = 0;i < results.length; i++) {
     const result = results[i];
     if (result.status === "fulfilled") {
@@ -6229,10 +6243,15 @@ async function fetchAllDocs(dbPath) {
     } else {
       const url = `${PAGEFIND_BASE}/fragment/${pageHashes[i]}.pf_fragment`;
       failedUrls.push(url);
-      console.log(`[ERROR] Failed to fetch ${url}: ${result.reason.message}`);
     }
   }
   await writeParquet(fetchedRecords, dbPath);
+  onProgress?.({
+    type: "completed",
+    totalPages,
+    fetchedPages: fetchedRecords.length,
+    failedPages: failedUrls.length
+  });
   return {
     totalPages,
     fetchedPages: fetchedRecords.length,
@@ -6241,8 +6260,189 @@ async function fetchAllDocs(dbPath) {
   };
 }
 
+// src/docs/snapshot.ts
+import fs from "fs/promises";
+import path from "path";
+var DEFAULT_DOCS_SNAPSHOT_URLS = [
+  "https://raw.githubusercontent.com/anand-testcompare/opencode-palantir/main/data/docs.parquet"
+];
+var MIN_SNAPSHOT_BYTES = 64;
+var inFlightByPath = new Map;
+function formatError2(err) {
+  return err instanceof Error ? err.toString() : String(err);
+}
+function emit(onEvent, event) {
+  if (!onEvent)
+    return;
+  onEvent(event);
+}
+function normalizeSnapshotUrls(customUrls) {
+  const envSingleRaw = process.env.OPENCODE_PALANTIR_DOCS_SNAPSHOT_URL;
+  const envManyRaw = process.env.OPENCODE_PALANTIR_DOCS_SNAPSHOT_URLS;
+  const envSingle = typeof envSingleRaw === "string" && envSingleRaw.trim().length > 0 ? [envSingleRaw.trim()] : [];
+  const envMany = typeof envManyRaw === "string" && envManyRaw.trim().length > 0 ? envManyRaw.split(",").map((x) => x.trim()).filter((x) => x.length > 0) : [];
+  const resolved = customUrls ?? [...envMany, ...envSingle, ...DEFAULT_DOCS_SNAPSHOT_URLS];
+  return Array.from(new Set(resolved));
+}
+async function ensureDirectoryExists(dbPath) {
+  await fs.mkdir(path.dirname(dbPath), { recursive: true });
+}
+async function statIfExists(filePath) {
+  try {
+    return await fs.stat(filePath);
+  } catch (err) {
+    if (err.code === "ENOENT")
+      return null;
+    throw err;
+  }
+}
+function assertValidSnapshotSize(bytes, source) {
+  if (bytes < MIN_SNAPSHOT_BYTES) {
+    throw new Error(`Snapshot from ${source} is unexpectedly small (${bytes} bytes). Expected at least ${MIN_SNAPSHOT_BYTES} bytes.`);
+  }
+}
+function tempPathFor(dbPath) {
+  const base = path.basename(dbPath);
+  return path.join(path.dirname(dbPath), `.${base}.tmp.${process.pid}.${Date.now()}`);
+}
+async function writeBufferAtomic(dbPath, bytes) {
+  const tmp = tempPathFor(dbPath);
+  await fs.writeFile(tmp, bytes);
+  await fs.rename(tmp, dbPath);
+}
+async function copyFileAtomic(sourcePath, dbPath) {
+  const tmp = tempPathFor(dbPath);
+  await fs.copyFile(sourcePath, tmp);
+  await fs.rename(tmp, dbPath);
+}
+function bundledSnapshotCandidates(dbPath, pluginDirectory) {
+  const candidates = [];
+  if (pluginDirectory && pluginDirectory.trim().length > 0) {
+    candidates.push(path.resolve(pluginDirectory, "data", "docs.parquet"));
+  } else {
+    candidates.push(path.resolve(import.meta.dir, "..", "..", "data", "docs.parquet"));
+  }
+  const target = path.resolve(dbPath);
+  const deduped = Array.from(new Set(candidates.map((x) => path.resolve(x))));
+  return deduped.filter((candidate) => candidate !== target);
+}
+async function tryDownloadSnapshot(dbPath, urls, onEvent) {
+  const errors = [];
+  for (const url of urls) {
+    emit(onEvent, { type: "download-start", url });
+    try {
+      const response = await fetch(url);
+      if (!response.ok) {
+        const reason = `HTTP ${response.status} ${response.statusText}`.trim();
+        emit(onEvent, { type: "download-failed", url, error: reason });
+        errors.push(`${url}: ${reason}`);
+        continue;
+      }
+      const bytes = new Uint8Array(await response.arrayBuffer());
+      assertValidSnapshotSize(bytes.byteLength, url);
+      await writeBufferAtomic(dbPath, bytes);
+      emit(onEvent, { type: "download-success", url, bytes: bytes.byteLength });
+      return {
+        dbPath,
+        changed: true,
+        source: "download",
+        bytes: bytes.byteLength,
+        downloadUrl: url
+      };
+    } catch (err) {
+      const reason = formatError2(err);
+      emit(onEvent, { type: "download-failed", url, error: reason });
+      errors.push(`${url}: ${reason}`);
+    }
+  }
+  if (errors.length === 0)
+    return null;
+  throw new Error([
+    "Unable to download prebuilt docs snapshot from configured source URLs.",
+    ...errors.map((line) => `- ${line}`)
+  ].join(`
+`));
+}
+async function tryCopyBundledSnapshot(dbPath, pluginDirectory, onEvent) {
+  const candidates = bundledSnapshotCandidates(dbPath, pluginDirectory);
+  for (const sourcePath of candidates) {
+    const stat = await statIfExists(sourcePath);
+    if (!stat || !stat.isFile())
+      continue;
+    emit(onEvent, { type: "copy-start", sourcePath });
+    assertValidSnapshotSize(stat.size, sourcePath);
+    await copyFileAtomic(sourcePath, dbPath);
+    emit(onEvent, { type: "copy-success", sourcePath, bytes: stat.size });
+    return {
+      dbPath,
+      changed: true,
+      source: "bundled-copy",
+      bytes: stat.size
+    };
+  }
+  return null;
+}
+async function ensureDocsParquetInternal(options) {
+  const dbPath = path.resolve(options.dbPath);
+  const force = options.force === true;
+  const onEvent = options.onEvent;
+  emit(onEvent, { type: "start", force });
+  await ensureDirectoryExists(dbPath);
+  if (!force) {
+    const existing = await statIfExists(dbPath);
+    if (existing && existing.isFile()) {
+      assertValidSnapshotSize(existing.size, dbPath);
+      const result = {
+        dbPath,
+        changed: false,
+        source: "existing",
+        bytes: existing.size
+      };
+      emit(onEvent, { type: "skip-existing", bytes: existing.size });
+      emit(onEvent, { type: "done", result });
+      return result;
+    }
+  }
+  const snapshotUrls = normalizeSnapshotUrls(options.snapshotUrls);
+  let downloadError = null;
+  try {
+    const downloaded = await tryDownloadSnapshot(dbPath, snapshotUrls, onEvent);
+    if (downloaded) {
+      emit(onEvent, { type: "done", result: downloaded });
+      return downloaded;
+    }
+  } catch (err) {
+    downloadError = err instanceof Error ? err : new Error(String(err));
+  }
+  const copied = await tryCopyBundledSnapshot(dbPath, options.pluginDirectory, onEvent);
+  if (copied) {
+    emit(onEvent, { type: "done", result: copied });
+    return copied;
+  }
+  const fallbackHint = "No bundled snapshot was found. You can run /refresh-docs-rescrape as a fallback.";
+  if (downloadError) {
+    throw new Error(`${downloadError.message}
+${fallbackHint}`);
+  }
+  throw new Error(`No docs snapshot sources were available. ${fallbackHint} ` + `Checked URLs=${snapshotUrls.length}, bundled candidates=${bundledSnapshotCandidates(dbPath, options.pluginDirectory).length}.`);
+}
+async function ensureDocsParquet(options) {
+  const dbPath = path.resolve(options.dbPath);
+  const existing = inFlightByPath.get(dbPath);
+  if (existing)
+    return existing;
+  let promise;
+  promise = ensureDocsParquetInternal({ ...options, dbPath }).finally(() => {
+    if (inFlightByPath.get(dbPath) === promise) {
+      inFlightByPath.delete(dbPath);
+    }
+  });
+  inFlightByPath.set(dbPath, promise);
+  return promise;
+}
+
 // src/palantir-mcp/commands.ts
-import path3 from "path";
+import path4 from "path";
 
 // src/palantir-mcp/allowlist.ts
 function isMutatingTool(toolName) {
@@ -6304,7 +6504,7 @@ function computeAllowedTools(profile, toolNames) {
 }
 
 // src/palantir-mcp/mcp-client.ts
-function formatError(err) {
+function formatError3(err) {
   return err instanceof Error ? err.toString() : String(err);
 }
 function withTimeout(p, ms, label) {
@@ -6445,7 +6645,7 @@ ${errText}`));
     return Array.from(new Set(names)).sort((a, b) => a.localeCompare(b));
   } catch (err) {
     const stderrText = stderrChunks.join("");
-    throw new Error(`[ERROR] Failed to list palantir-mcp tools: ${formatError(err)}
+    throw new Error(`[ERROR] Failed to list palantir-mcp tools: ${formatError3(err)}
 ${stderrText}`);
   } finally {
     try {
@@ -6484,8 +6684,8 @@ function normalizeFoundryBaseUrl(raw) {
 }
 
 // src/palantir-mcp/opencode-config.ts
-import fs from "fs/promises";
-import path from "path";
+import fs2 from "fs/promises";
+import path2 from "path";
 
 // node_modules/jsonc-parser/lib/esm/impl/scanner.js
 function createScanner(text, ignoreTrivia = false) {
@@ -7297,28 +7497,28 @@ var OPENCODE_JSON_FILENAME = "opencode.json";
 function isRecord(value) {
   return !!value && typeof value === "object" && !Array.isArray(value);
 }
-function formatError2(err) {
+function formatError4(err) {
   return err instanceof Error ? err.toString() : String(err);
 }
 async function pathExists(p) {
   try {
-    await fs.access(p);
+    await fs2.access(p);
     return true;
   } catch {
     return false;
   }
 }
 async function readOpencodeJsonc(worktree) {
-  const configPath = path.join(worktree, OPENCODE_JSONC_FILENAME);
+  const configPath = path2.join(worktree, OPENCODE_JSONC_FILENAME);
   if (!await pathExists(configPath))
     return { ok: false, missing: true };
   let text;
   try {
-    text = await fs.readFile(configPath, "utf8");
+    text = await fs2.readFile(configPath, "utf8");
   } catch (err) {
     return {
       ok: false,
-      error: `[ERROR] Failed reading ${OPENCODE_JSONC_FILENAME}: ${formatError2(err)}`
+      error: `[ERROR] Failed reading ${OPENCODE_JSONC_FILENAME}: ${formatError4(err)}`
     };
   }
   const errors = [];
@@ -7333,16 +7533,16 @@ async function readOpencodeJsonc(worktree) {
   return { ok: true, path: configPath, text, data };
 }
 async function readLegacyOpencodeJson(worktree) {
-  const legacyPath = path.join(worktree, OPENCODE_JSON_FILENAME);
+  const legacyPath = path2.join(worktree, OPENCODE_JSON_FILENAME);
   if (!await pathExists(legacyPath))
     return { ok: false, missing: true };
   let text;
   try {
-    text = await fs.readFile(legacyPath, "utf8");
+    text = await fs2.readFile(legacyPath, "utf8");
   } catch (err) {
     return {
       ok: false,
-      error: `[ERROR] Failed reading ${OPENCODE_JSON_FILENAME}: ${formatError2(err)}`
+      error: `[ERROR] Failed reading ${OPENCODE_JSON_FILENAME}: ${formatError4(err)}`
     };
   }
   let data;
@@ -7351,7 +7551,7 @@ async function readLegacyOpencodeJson(worktree) {
   } catch (err) {
     return {
       ok: false,
-      error: `[ERROR] Failed parsing ${OPENCODE_JSON_FILENAME}: ${formatError2(err)}`
+      error: `[ERROR] Failed parsing ${OPENCODE_JSON_FILENAME}: ${formatError4(err)}`
     };
   }
   return { ok: true, path: legacyPath, text, data };
@@ -7380,24 +7580,24 @@ function mergeLegacyIntoJsonc(legacyData, jsoncData) {
   return deepMergePreferTarget(base, legacy);
 }
 async function writeFileAtomic(filePath, text) {
-  const dir = path.dirname(filePath);
-  const base = path.basename(filePath);
-  const tmp = path.join(dir, `.${base}.tmp.${process.pid}.${Date.now()}`);
-  await fs.writeFile(tmp, text, "utf8");
-  await fs.rename(tmp, filePath);
+  const dir = path2.dirname(filePath);
+  const base = path2.basename(filePath);
+  const tmp = path2.join(dir, `.${base}.tmp.${process.pid}.${Date.now()}`);
+  await fs2.writeFile(tmp, text, "utf8");
+  await fs2.rename(tmp, filePath);
 }
 async function renameLegacyToBak(worktree) {
-  const legacyPath = path.join(worktree, OPENCODE_JSON_FILENAME);
+  const legacyPath = path2.join(worktree, OPENCODE_JSON_FILENAME);
   if (!await pathExists(legacyPath))
     return null;
-  const baseBak = path.join(worktree, `${OPENCODE_JSON_FILENAME}.bak`);
+  const baseBak = path2.join(worktree, `${OPENCODE_JSON_FILENAME}.bak`);
   let bakPath = baseBak;
   let i = 1;
   while (await pathExists(bakPath)) {
     bakPath = `${baseBak}.${i}`;
     i += 1;
   }
-  await fs.rename(legacyPath, bakPath);
+  await fs2.rename(legacyPath, bakPath);
   return bakPath;
 }
 function toolKey(toolName) {
@@ -7637,11 +7837,11 @@ function stringifyJsonc(data) {
 }
 
 // src/palantir-mcp/repo-scan.ts
-import fs2 from "fs/promises";
-import path2 from "path";
+import fs3 from "fs/promises";
+import path3 from "path";
 async function pathExists2(p) {
   try {
-    await fs2.access(p);
+    await fs3.access(p);
     return true;
   } catch {
     return false;
@@ -7649,7 +7849,7 @@ async function pathExists2(p) {
 }
 async function readTextFileBounded(p, maxBytes) {
   try {
-    const file = await fs2.open(p, "r");
+    const file = await fs3.open(p, "r");
     try {
       const buf = Buffer.alloc(maxBytes);
       const { bytesRead } = await file.read(buf, 0, maxBytes, 0);
@@ -7732,14 +7932,14 @@ async function collectSampleFiles(root, limit) {
     visitedDirs += 1;
     let entries;
     try {
-      entries = await fs2.readdir(dir, { withFileTypes: true });
+      entries = await fs3.readdir(dir, { withFileTypes: true });
     } catch {
       continue;
     }
     for (const ent of entries) {
       if (results.length >= limit)
         break;
-      const full = path2.join(dir, ent.name);
+      const full = path3.join(dir, ent.name);
       if (ent.isDirectory()) {
         if (ignoreDirs.has(ent.name))
           continue;
@@ -7748,7 +7948,7 @@ async function collectSampleFiles(root, limit) {
       }
       if (!ent.isFile())
         continue;
-      const ext = path2.extname(ent.name);
+      const ext = path3.extname(ent.name);
       if (!allowedExts.has(ext))
         continue;
       results.push(full);
@@ -7771,13 +7971,13 @@ async function scanRepoForProfile(root) {
     { p: "lerna.json", profile: "all", score: 3, reason: "Found lerna.json" }
   ];
   for (const c of candidates) {
-    if (await pathExists2(path2.join(root, c.p))) {
+    if (await pathExists2(path3.join(root, c.p))) {
       addScore(scores, reasons, c.profile, c.score, c.reason);
     }
   }
-  const packageJsonPath = path2.join(root, "package.json");
-  const pyprojectPath = path2.join(root, "pyproject.toml");
-  const requirementsPath = path2.join(root, "requirements.txt");
+  const packageJsonPath = path3.join(root, "package.json");
+  const pyprojectPath = path3.join(root, "pyproject.toml");
+  const requirementsPath = path3.join(root, "requirements.txt");
   const hasPackageJson = await pathExists2(packageJsonPath);
   const hasPyproject = await pathExists2(pyprojectPath);
   if (hasPackageJson && hasPyproject) {
@@ -7817,22 +8017,22 @@ async function scanRepoForProfile(root) {
       }
     }
   }
-  if (await pathExists2(path2.join(root, "pipelines"))) {
+  if (await pathExists2(path3.join(root, "pipelines"))) {
     addScore(scores, reasons, "pipelines_transforms", 3, "Found pipelines/ directory");
   }
-  if (await pathExists2(path2.join(root, "transforms"))) {
+  if (await pathExists2(path3.join(root, "transforms"))) {
     addScore(scores, reasons, "pipelines_transforms", 3, "Found transforms/ directory");
   }
-  if (await pathExists2(path2.join(root, "internal", "pipeline"))) {
+  if (await pathExists2(path3.join(root, "internal", "pipeline"))) {
     addScore(scores, reasons, "pipelines_transforms", 3, "Found internal/pipeline/ directory");
   }
-  if (await pathExists2(path2.join(root, "internal", "transforms"))) {
+  if (await pathExists2(path3.join(root, "internal", "transforms"))) {
     addScore(scores, reasons, "pipelines_transforms", 3, "Found internal/transforms/ directory");
   }
-  if (await pathExists2(path2.join(root, "functions"))) {
+  if (await pathExists2(path3.join(root, "functions"))) {
     addScore(scores, reasons, "osdk_functions_ts", 2, "Found functions/ directory");
   }
-  if (await pathExists2(path2.join(root, "src", "functions"))) {
+  if (await pathExists2(path3.join(root, "src", "functions"))) {
     addScore(scores, reasons, "osdk_functions_ts", 2, "Found src/functions/ directory");
   }
   const sampleFiles = await collectSampleFiles(root, 50);
@@ -7863,7 +8063,7 @@ async function scanRepoForProfile(root) {
 }
 
 // src/palantir-mcp/commands.ts
-function formatError3(err) {
+function formatError5(err) {
   return err instanceof Error ? err.toString() : String(err);
 }
 function isRecord2(value) {
@@ -7912,7 +8112,7 @@ async function resolveProfile(worktree) {
   } catch (err) {
     return {
       profile: "unknown",
-      reasons: [`Repo scan failed; falling back to unknown: ${formatError3(err)}`]
+      reasons: [`Repo scan failed; falling back to unknown: ${formatError5(err)}`]
     };
   }
 }
@@ -7976,7 +8176,7 @@ async function autoBootstrapPalantirMcpIfConfigured(worktree) {
     const changed = needsMigration || stableJsonStringify(merged) !== stableJsonStringify(patch.data);
     if (!changed)
       return;
-    const outPath = path3.join(worktree, OPENCODE_JSONC_FILENAME);
+    const outPath = path4.join(worktree, OPENCODE_JSONC_FILENAME);
     const text = stringifyJsonc(patch.data);
     await writeFileAtomic(outPath, text);
     if (readLegacy.ok) {
@@ -8037,7 +8237,7 @@ async function setupPalantirMcp(worktree, rawArgs) {
   try {
     toolNames = await listPalantirMcpTools(discoveryUrl);
   } catch (err) {
-    return `[ERROR] ${formatError3(err)}`;
+    return `[ERROR] ${formatError5(err)}`;
   }
   if (toolNames.length === 0)
     return "[ERROR] palantir-mcp tool discovery returned no tools.";
@@ -8048,12 +8248,12 @@ async function setupPalantirMcp(worktree, rawArgs) {
     profile,
     allowlist
   });
-  const outPath = path3.join(worktree, OPENCODE_JSONC_FILENAME);
+  const outPath = path4.join(worktree, OPENCODE_JSONC_FILENAME);
   const text = stringifyJsonc(patch.data);
   try {
     await writeFileAtomic(outPath, text);
   } catch (err) {
-    return `[ERROR] Failed writing ${OPENCODE_JSONC_FILENAME}: ${formatError3(err)}`;
+    return `[ERROR] Failed writing ${OPENCODE_JSONC_FILENAME}: ${formatError5(err)}`;
   }
   let bakInfo = "";
   if (readLegacy.ok) {
@@ -8064,7 +8264,7 @@ async function setupPalantirMcp(worktree, rawArgs) {
 Migrated legacy ${readLegacy.path} -> ${bakPath}`;
     } catch (err) {
       bakInfo = `
-[ERROR] Wrote ${OPENCODE_JSONC_FILENAME}, but failed to rename legacy ${readLegacy.path}: ${formatError3(err)}`;
+[ERROR] Wrote ${OPENCODE_JSONC_FILENAME}, but failed to rename legacy ${readLegacy.path}: ${formatError5(err)}`;
     }
   }
   const warnings = [...normalized.warnings, ...patch.warnings];
@@ -8118,18 +8318,18 @@ async function rescanPalantirMcpTools(worktree) {
   try {
     toolNames = await listPalantirMcpTools(normalized.url);
   } catch (err) {
-    return `[ERROR] ${formatError3(err)}`;
+    return `[ERROR] ${formatError5(err)}`;
   }
   if (toolNames.length === 0)
     return "[ERROR] palantir-mcp tool discovery returned no tools.";
   const allowlist = computeAllowedTools(profile, toolNames);
   const patch = patchConfigForRescan(baseData, { toolNames, profile, allowlist });
-  const outPath = path3.join(worktree, OPENCODE_JSONC_FILENAME);
+  const outPath = path4.join(worktree, OPENCODE_JSONC_FILENAME);
   const text = stringifyJsonc(patch.data);
   try {
     await writeFileAtomic(outPath, text);
   } catch (err) {
-    return `[ERROR] Failed writing ${OPENCODE_JSONC_FILENAME}: ${formatError3(err)}`;
+    return `[ERROR] Failed writing ${OPENCODE_JSONC_FILENAME}: ${formatError5(err)}`;
   }
   const warnings = [...normalized.warnings, ...patch.warnings];
   return [
@@ -8142,18 +8342,41 @@ async function rescanPalantirMcpTools(worktree) {
 }
 
 // src/index.ts
-var NO_DB_MESSAGE = "Documentation database not found. Run /refresh-docs to download Palantir Foundry documentation.";
 var plugin = async (input) => {
-  const dbPath = path4.join(input.worktree, "data", "docs.parquet");
+  const dbPath = path5.join(input.worktree, "data", "docs.parquet");
   let dbInstance = null;
-  let autoBootstrapStarted = false;
+  let dbInitPromise = null;
+  let autoBootstrapMcpStarted = false;
+  let autoBootstrapDocsStarted = false;
+  function formatError6(err) {
+    return err instanceof Error ? err.toString() : String(err);
+  }
+  function formatBytes(bytes) {
+    if (!Number.isFinite(bytes) || bytes < 0)
+      return "0 B";
+    const units = ["B", "KB", "MB", "GB"];
+    let value = bytes;
+    let index = 0;
+    while (value >= 1024 && index < units.length - 1) {
+      value /= 1024;
+      index += 1;
+    }
+    const decimals = value >= 10 || index === 0 ? 0 : 1;
+    return `${value.toFixed(decimals)} ${units[index]}`;
+  }
   function ensureCommandDefinitions(cfg) {
     if (!cfg.command)
       cfg.command = {};
     if (!cfg.command["refresh-docs"]) {
       cfg.command["refresh-docs"] = {
-        template: "Refresh Palantir documentation database.",
-        description: "Download Palantir docs and write data/docs.parquet (local)."
+        template: "Refresh Palantir docs snapshot (recommended).",
+        description: "Force refresh data/docs.parquet from a prebuilt snapshot (download/copy; no rescrape)."
+      };
+    }
+    if (!cfg.command["refresh-docs-rescrape"]) {
+      cfg.command["refresh-docs-rescrape"] = {
+        template: "Refresh docs by live rescrape (unsafe/experimental).",
+        description: "Explicit fallback: rescrape palantir.com docs and rebuild data/docs.parquet. Slower and less reliable than /refresh-docs."
       };
     }
     if (!cfg.command["setup-palantir-mcp"]) {
@@ -8220,8 +8443,8 @@ var plugin = async (input) => {
     ensureAgentDefaults2(foundry, "foundry");
     cfg.agent.foundry = foundry;
   }
-  function maybeStartAutoBootstrap() {
-    if (autoBootstrapStarted)
+  function maybeStartAutoBootstrapMcp() {
+    if (autoBootstrapMcpStarted)
       return;
     const token = process.env.FOUNDRY_TOKEN;
     const url = process.env.FOUNDRY_URL;
@@ -8229,21 +8452,106 @@ var plugin = async (input) => {
       return;
     if (!url || url.trim().length === 0)
       return;
-    autoBootstrapStarted = true;
+    autoBootstrapMcpStarted = true;
     autoBootstrapPalantirMcpIfConfigured(input.worktree);
   }
-  async function getDb() {
-    if (!dbInstance) {
-      dbInstance = await createDatabase(dbPath);
-    }
-    return dbInstance;
+  function maybeStartAutoBootstrapDocs() {
+    if (autoBootstrapDocsStarted)
+      return;
+    autoBootstrapDocsStarted = true;
+    ensureDocsAvailable().catch(() => {});
   }
-  async function dbExists() {
-    return Bun.file(dbPath).exists();
+  function resetDb() {
+    if (dbInstance)
+      closeDatabase(dbInstance);
+    dbInstance = null;
+    dbInitPromise = null;
+  }
+  async function getDb() {
+    if (dbInstance)
+      return dbInstance;
+    if (dbInitPromise)
+      return dbInitPromise;
+    dbInitPromise = createDatabase(dbPath).then((created) => {
+      dbInstance = created;
+      return created;
+    }).finally(() => {
+      dbInitPromise = null;
+    });
+    return dbInitPromise;
   }
   function pushText(output, text) {
     output.parts.push({ type: "text", text });
   }
+  function formatSnapshotFailure(err) {
+    return [
+      "[ERROR] Unable to obtain Palantir docs snapshot.",
+      "",
+      `Reason: ${formatError6(err)}`,
+      "",
+      "Next steps:",
+      "- Retry /refresh-docs (recommended prebuilt snapshot path).",
+      "- If snapshot download is blocked, run /refresh-docs-rescrape (unsafe/experimental)."
+    ].join(`
+`);
+  }
+  function formatSnapshotRefreshEvent(event) {
+    if (event.type === "skip-existing")
+      return `snapshot_status=already_present bytes=${event.bytes}`;
+    if (event.type === "download-start")
+      return `download_attempt url=${event.url}`;
+    if (event.type === "download-failed")
+      return `download_failed url=${event.url} error=${event.error}`;
+    if (event.type === "download-success")
+      return `download_succeeded url=${event.url} bytes=${event.bytes}`;
+    if (event.type === "copy-start")
+      return `copy_attempt source=${event.sourcePath}`;
+    if (event.type === "copy-success")
+      return `copy_succeeded source=${event.sourcePath} bytes=${event.bytes}`;
+    return null;
+  }
+  async function ensureDocsAvailable(options = {}) {
+    return ensureDocsParquet({
+      dbPath,
+      force: options.force === true,
+      pluginDirectory: input.directory,
+      onEvent: options.onEvent
+    });
+  }
+  async function ensureDocsReadyForTool() {
+    try {
+      await ensureDocsAvailable();
+      return null;
+    } catch (err) {
+      return formatSnapshotFailure(err);
+    }
+  }
+  function formatRescrapeFailure(err) {
+    return [
+      "[ERROR] /refresh-docs-rescrape failed.",
+      "",
+      `Reason: ${formatError6(err)}`,
+      "",
+      "Try /refresh-docs for the recommended prebuilt snapshot flow."
+    ].join(`
+`);
+  }
+  function formatRescrapeProgressEvent(event, progressLines, failureSamples) {
+    if (event.type === "discovered") {
+      progressLines.push(`discovered_pages=${event.totalPages}`);
+      return;
+    }
+    if (event.type === "progress") {
+      progressLines.push(`processed_pages=${event.processedPages}/${event.totalPages}`);
+      return;
+    }
+    if (event.type === "page-failed") {
+      if (failureSamples.length < 5) {
+        failureSamples.push(`url=${event.url} error=${event.error}`);
+      }
+      return;
+    }
+  }
   function toPathname(inputUrl) {
     const trimmed = inputUrl.trim();
     if (trimmed.length === 0)
@@ -8297,8 +8605,8 @@ var plugin = async (input) => {
   function isInScope(pageUrl, scope) {
     if (scope === "all")
       return true;
-    const path5 = toPathname(pageUrl);
-    return path5.startsWith(`/${scope}/`) || path5.startsWith(`/docs/${scope}/`);
+    const path6 = toPathname(pageUrl);
+    return path6.startsWith(`/${scope}/`) || path6.startsWith(`/docs/${scope}/`);
   }
   function tokenizeQuery(query) {
     const tokens = query.toLowerCase().trim().split(/[\s/._-]+/g).map((t) => t.trim()).filter((t) => t.length > 0);
@@ -8308,13 +8616,13 @@ var plugin = async (input) => {
     const q = query.toLowerCase().trim();
     if (q.length === 0)
       return 0;
-    const path5 = toPathname(page.url).toLowerCase();
+    const path6 = toPathname(page.url).toLowerCase();
     const title = page.title.toLowerCase();
-    if (path5 === q)
+    if (path6 === q)
       return 2000;
-    if (path5 === toPathname(q).toLowerCase())
+    if (path6 === toPathname(q).toLowerCase())
       return 2000;
-    if (path5.includes(q))
+    if (path6.includes(q))
       return 1200;
     if (title.includes(q))
       return 1000;
@@ -8325,10 +8633,10 @@ var plugin = async (input) => {
     for (const t of tokens) {
       if (title.includes(t))
         score += 40;
-      if (path5.includes(t))
+      if (path6.includes(t))
         score += 30;
     }
-    if (path5.startsWith(q))
+    if (path6.startsWith(q))
       score += 100;
     if (title.startsWith(q))
       score += 100;
@@ -8338,7 +8646,8 @@ var plugin = async (input) => {
     config: async (cfg) => {
       ensureCommandDefinitions(cfg);
       ensureAgentDefinitions(cfg);
-      maybeStartAutoBootstrap();
+      maybeStartAutoBootstrapMcp();
+      maybeStartAutoBootstrapDocs();
     },
     tool: {
       get_doc_page: tool({
@@ -8349,8 +8658,9 @@ var plugin = async (input) => {
           scope: tool.schema.enum(["foundry", "apollo", "gotham", "all"]).optional().describe("Scope to search within when using query or fuzzy matching (default: foundry).")
         },
         async execute(args) {
-          if (!await dbExists())
-            return NO_DB_MESSAGE;
+          const docsError = await ensureDocsReadyForTool();
+          if (docsError)
+            return docsError;
           const scope = parseScope(args.scope);
           if (!scope) {
             return [
@@ -8422,8 +8732,9 @@ ${bestPage.content}`;
           query: tool.schema.string().optional().describe("Optional query to filter/rank results by title/URL (case-insensitive).")
         },
         async execute(args) {
-          if (!await dbExists())
-            return NO_DB_MESSAGE;
+          const docsError = await ensureDocsReadyForTool();
+          if (docsError)
+            return docsError;
           const scope = parseScope(args.scope);
           if (!scope) {
             return [
@@ -8502,12 +8813,65 @@ ${bestPage.content}`;
     },
     "command.execute.before": async (hookInput, output) => {
       if (hookInput.command === "refresh-docs") {
-        const result = await fetchAllDocs(dbPath);
-        if (dbInstance) {
-          closeDatabase(dbInstance);
-          dbInstance = null;
+        const progressLines = [];
+        try {
+          const result = await ensureDocsAvailable({
+            force: true,
+            onEvent: (event) => {
+              const line = formatSnapshotRefreshEvent(event);
+              if (line)
+                progressLines.push(line);
+            }
+          });
+          resetDb();
+          const db = await getDb();
+          const indexedPages = getAllPages(db).length;
+          pushText(output, [
+            "refresh-docs complete (recommended snapshot path).",
+            "",
+            ...progressLines.map((line) => `- ${line}`),
+            ...progressLines.length > 0 ? [""] : [],
+            `snapshot_source=${result.source}`,
+            result.downloadUrl ? `snapshot_url=${result.downloadUrl}` : null,
+            `snapshot_bytes=${result.bytes} (${formatBytes(result.bytes)})`,
+            `indexed_pages=${indexedPages}`
+          ].filter((line) => !!line).join(`
+`));
+        } catch (err) {
+          pushText(output, formatSnapshotFailure(err));
+        }
+        return;
+      }
+      if (hookInput.command === "refresh-docs-rescrape") {
+        const progressLines = [];
+        const failureSamples = [];
+        try {
+          const result = await fetchAllDocs(dbPath, {
+            progressEvery: 250,
+            onProgress: (event) => {
+              formatRescrapeProgressEvent(event, progressLines, failureSamples);
+            }
+          });
+          resetDb();
+          const db = await getDb();
+          const indexedPages = getAllPages(db).length;
+          pushText(output, [
+            "refresh-docs-rescrape complete (unsafe/experimental).",
+            "",
+            "Warning: this command live-scrapes palantir.com and is slower/less reliable than /refresh-docs.",
+            "",
+            ...progressLines.map((line) => `- ${line}`),
+            ...progressLines.length > 0 ? [""] : [],
+            `total_pages=${result.totalPages}`,
+            `fetched_pages=${result.fetchedPages}`,
+            `failed_pages=${result.failedUrls.length}`,
+            `indexed_pages=${indexedPages}`,
+            ...failureSamples.length > 0 ? ["", "failure_samples:", ...failureSamples.map((line) => `- ${line}`)] : []
+          ].join(`
+`));
+        } catch (err) {
+          pushText(output, formatRescrapeFailure(err));
         }
-        pushText(output, `Refreshed documentation: ${result.fetchedPages}/${result.totalPages} pages fetched. ${result.failedUrls.length} failures.`);
         return;
       }
       if (hookInput.command === "setup-palantir-mcp") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openontology/opencode-palantir",
-  "version": "0.1.4-next.9",
+  "version": "0.1.5-next.4",
   "description": "collection of tools, agents, hooks to supercharge development in foundry",
   "license": "MIT",
   "author": {