@autorag/librarian 2.13.9 → 2.13.11

package/dist/index.js

@@ -3668,20 +3668,30 @@ var init_bundle = __esm(() => {
 });
 
 // src/filesystem/path-exclusions.ts
-import { posix as posix2, win32 as win322 } from "node:path";
+import { matchesGlob, posix as posix2, win32 as win322 } from "node:path";
+function isGlobExclusion(entry) {
+  return /[*?[]/u.test(entry);
+}
+function matchesGlobExclusion(path, glob) {
+  const slashed = path.replaceAll("\\", "/");
+  if (matchesGlob(slashed, glob))
+    return true;
+  return glob.startsWith("**/") && matchesGlob(slashed, glob.slice(3));
+}
 function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
   const paths = platform === "win32" ? win322 : posix2;
   const normalize = (path) => {
     const absolute = paths.resolve(path);
     return platform === "win32" ? absolute.toLowerCase() : absolute;
   };
-  const excluded = excludedPaths.map((path) => {
+  const globs = excludedPaths.filter(isGlobExclusion).map((glob) => platform === "win32" ? glob.toLowerCase() : glob);
+  const excluded = excludedPaths.filter((path) => !isGlobExclusion(path)).map((path) => {
     const absolute = normalize(path);
     return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
   });
   return (path) => {
     const absolute = normalize(path);
-    return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
+    return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix)) || globs.some((glob) => matchesGlobExclusion(absolute, glob));
   };
 }
 var init_path_exclusions = () => {};
@@ -10333,7 +10343,7 @@ var init_cloud = __esm(() => {
 // src/mirror/sync.ts
 import { createHash as createHash11 } from "node:crypto";
 import { opendir, readFile, stat as stat3 } from "node:fs/promises";
-import { basename as basename9, dirname as dirname12, extname as extname6, join as join27, resolve as resolve11, sep as sep6 } from "node:path";
+import { basename as basename9, dirname as dirname12, extname as extname6, join as join27, resolve as resolve11 } from "node:path";
 import { setTimeout as sleep3 } from "node:timers/promises";
 function errorCode(error) {
   if (!(error instanceof Error))
@@ -10880,13 +10890,14 @@ function normalizeExtension2(extension) {
   return lower.startsWith(".") ? lower : `.${lower}`;
 }
 function isPathExcluded(sourcePath, excluded) {
-  if (excluded === undefined)
+  if (excluded === undefined || excluded.size === 0)
     return false;
-  for (const excludedPath of excluded) {
-    if (sourcePath === excludedPath || sourcePath.startsWith(`${excludedPath}${sep6}`))
-      return true;
+  let matcher = matcherBySet.get(excluded);
+  if (matcher === undefined) {
+    matcher = createPathExclusionMatcher([...excluded]);
+    matcherBySet.set(excluded, matcher);
   }
-  return false;
+  return matcher(sourcePath);
 }
 function walkLimits(options) {
   return {
@@ -11053,10 +11064,11 @@ async function collectFiles(state, directory, depth) {
     warnDirectoryUnreadable(state, directory, code);
   }
 }
-var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
+var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, matcherBySet, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
 var init_sync = __esm(() => {
   init_cli2();
   init_cloud_placeholder();
+  init_path_exclusions();
   init_source_paths();
   init_binary();
   init_stage();
@@ -11115,6 +11127,7 @@ var init_sync = __esm(() => {
     }
   };
   STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
+  matcherBySet = new WeakMap;
   SEARCH_ROOT_UNAVAILABLE_MESSAGES = {
     gone: "A search root no longer exists; its previously indexed documents were left untouched.",
     "not-a-directory": "A search root is no longer a directory; its previously indexed documents were left untouched.",
@@ -16189,7 +16202,7 @@ var init_automation_profiles = __esm(() => {
 
 // src/automation/config.ts
 import { existsSync as existsSync19, readFileSync as readFileSync14, realpathSync as realpathSync7 } from "node:fs";
-import { basename as basename10, dirname as dirname15, isAbsolute as isAbsolute8, join as join30, matchesGlob, relative as relative5, resolve as resolve12 } from "node:path";
+import { basename as basename10, dirname as dirname15, isAbsolute as isAbsolute8, join as join30, matchesGlob as matchesGlob2, relative as relative5, resolve as resolve12 } from "node:path";
 function automationsConfigPath(env = process.env) {
   return join30(resolveAutoRAGHome(env), "automations.json");
 }
@@ -16365,9 +16378,9 @@ function deleteAutomation(configPath, id) {
 function globMatches(relPosix, pattern) {
   if (pattern === "**")
     return true;
-  if (matchesGlob(relPosix, pattern))
+  if (matchesGlob2(relPosix, pattern))
     return true;
-  if (pattern.startsWith("**/") && matchesGlob(relPosix, pattern.slice(3)))
+  if (pattern.startsWith("**/") && matchesGlob2(relPosix, pattern.slice(3)))
     return true;
   return false;
 }
@@ -17164,7 +17177,7 @@ var init_permission_judgment = __esm(() => {
 
 // src/agent/permission-gate.ts
 import { statSync as statSync7 } from "node:fs";
-import { isAbsolute as isAbsolute9, relative as relative6, sep as sep7 } from "node:path";
+import { isAbsolute as isAbsolute9, relative as relative6, sep as sep6 } from "node:path";
 import { matchesKey } from "@earendil-works/pi-tui";
 function pathExists(path) {
   try {
@@ -17227,7 +17240,7 @@ class PermissionController {
       if (path === root)
         return true;
       const rel = relative6(root, path);
-      if (rel !== "" && rel !== ".." && !rel.startsWith(`..${sep7}`) && !isAbsolute9(rel))
+      if (rel !== "" && rel !== ".." && !rel.startsWith(`..${sep6}`) && !isAbsolute9(rel))
         return true;
     }
     return false;
@@ -18679,7 +18692,7 @@ var init_connector = () => {};
 // src/datasource/sidecar/config.ts
 import { readFileSync as readFileSync17 } from "node:fs";
 import { homedir as homedir5 } from "node:os";
-import { dirname as dirname17, isAbsolute as isAbsolute10, join as join34, resolve as resolve14, sep as sep8 } from "node:path";
+import { dirname as dirname17, isAbsolute as isAbsolute10, join as join34, resolve as resolve14, sep as sep7 } from "node:path";
 import { parse as parseToml } from "smol-toml";
 function resolveSidecarOptions(input) {
   const { connector, workspaceRoot } = input;
@@ -18719,7 +18732,7 @@ function archivePathFromCrawlerConfig(input) {
 function expandHome(path) {
   if (path === "~")
     return homedir5();
-  const prefixed = path.startsWith("~/") || sep8 === "\\" && path.startsWith("~\\");
+  const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
   return prefixed ? join34(homedir5(), path.slice(2)) : path;
 }
 var init_config3 = __esm(() => {
@@ -25973,7 +25986,7 @@ import {
   writeFileSync as writeFileSync13
 } from "node:fs";
 import { opendir as opendir2, stat as stat4 } from "node:fs/promises";
-import { basename as basename11, dirname as dirname23, extname as extname8, join as join40, relative as relative7, sep as sep9 } from "node:path";
+import { basename as basename11, dirname as dirname23, extname as extname8, join as join40, relative as relative7, sep as sep8 } from "node:path";
 function errnoCode2(error) {
   const code = error instanceof Error ? error.code : undefined;
   return typeof code === "string" ? code : "UNKNOWN";
@@ -26043,7 +26056,7 @@ async function listVaultNotes(vaultPath) {
   return { notes, unreadableFolders, collisions };
 }
 function toRelative(vaultPath, path) {
-  return relative7(vaultPath, path).split(sep9).join("/").normalize("NFC");
+  return relative7(vaultPath, path).split(sep8).join("/").normalize("NFC");
 }
 function loadStagingState2(statePath) {
   try {
@@ -26127,7 +26140,7 @@ function removeUnexpected(filesRoot, directory, wanted) {
   return removed;
 }
 function toStagedRelative(filesRoot, path) {
-  return relative7(filesRoot, path).split(sep9).join("/").normalize("NFC");
+  return relative7(filesRoot, path).split(sep8).join("/").normalize("NFC");
 }
 var NOTE_EXTENSIONS, SKIPPED_DIRECTORIES, MAX_WALK_DEPTH = 64, VaultUnavailableError;
 var init_staging = __esm(() => {
@@ -28048,7 +28061,7 @@ function resolveExcludePaths(value, origin) {
   if (!Array.isArray(value) || value.some((path) => typeof path !== "string" || path.length === 0)) {
     throw new ConfigError("Config field 'excludePaths' must be a non-empty string array");
   }
-  return resolveSearchPaths(value, origin);
+  return value.map((entry) => isGlobExclusion(entry) ? entry : resolvePersistedPath(entry, origin));
 }
 function resolvePersistedPath(path, origin) {
   return isAbsolute11(path) ? path : resolve15(origin, path);
@@ -29353,6 +29366,7 @@ var init_config5 = __esm(() => {
   init_factory();
   init_embedding();
   init_file_lock();
+  init_path_exclusions();
   init_language();
   init_atomic_write();
   init_rerank();
@@ -38606,7 +38620,7 @@ var init_web_search_tool = __esm(() => {
 // src/agent/agent.ts
 import { randomUUID as randomUUID7 } from "node:crypto";
 import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
-import { basename as basename15, dirname as dirname27, join as join55, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
+import { basename as basename15, dirname as dirname27, join as join55, relative as relative8, resolve as resolve18, sep as sep9 } from "node:path";
 import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
 import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
 function describeDecisionModel(status) {
@@ -41279,7 +41293,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
         return { ok: false, reason: "cloud-root-skipped", stdout: "", stderr: "", code: null };
       }
       const canonicalSourcePath = canonicalPath(sourcePath);
-      const exclude = scope.nestedCloudRoots.map((root) => relative8(canonicalSourcePath, root.path).split(sep10).join("/")).filter((pattern) => pattern !== "" && !pattern.startsWith(".."));
+      const exclude = scope.nestedCloudRoots.map((root) => relative8(canonicalSourcePath, root.path).split(sep9).join("/")).filter((pattern) => pattern !== "" && !pattern.startsWith(".."));
       return client.prepare(sourcePath, { ...options, ...exclude.length === 0 ? {} : { exclude } });
     }));
   }
@@ -41793,6 +41807,8 @@ function toMinSyncReasonDiagnostic(reason) {
   };
 }
 function pinExcludedPath(path) {
+  if (isGlobExclusion(path))
+    return path;
   const resolvedPath = resolve18(path);
   try {
     return join55(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
@@ -41841,6 +41857,7 @@ var init_agent = __esm(() => {
   init_embedding();
   init_everything();
   init_cloud_placeholder();
+  init_path_exclusions();
   init_fsearch();
   init_diagnostics();
   init_jikji();
@@ -42058,6 +42075,12 @@ class AutoRAGLite {
   getMemorySchema() {
     return structuredClone(this.memory.getSchema());
   }
+  convertFiles(request, signal) {
+    return this.agent.convertFiles(request, signal);
+  }
+  listFileConversions(from, signal) {
+    return this.agent.listFileConversions(from, signal);
+  }
 }
 function createAutoRAGLite(options = {}) {
   const input = { flags: options.flags ?? {} };
@@ -42927,7 +42950,7 @@ import { z as z4 } from "zod";
 // src/filesystem/name-search.ts
 import { realpathSync as realpathSync9 } from "node:fs";
 import { lstat as lstat2, readdir as readdir5 } from "node:fs/promises";
-import { basename as basename16, dirname as dirname28, isAbsolute as isAbsolute14, join as join56, relative as relative9, resolve as resolve19, sep as sep11 } from "node:path";
+import { basename as basename16, dirname as dirname28, isAbsolute as isAbsolute14, join as join56, relative as relative9, resolve as resolve19, sep as sep10 } from "node:path";
 var SKIP_DIR_NAMES2 = {
   ".git": true,
   ".autorag": true,
@@ -42955,7 +42978,7 @@ function fsErrorCode(error) {
   return typeof code === "string" ? code : undefined;
 }
 function isContained(childReal, parentReal) {
-  const prefix = parentReal.endsWith(sep11) ? parentReal : `${parentReal}${sep11}`;
+  const prefix = parentReal.endsWith(sep10) ? parentReal : `${parentReal}${sep10}`;
   return childReal === parentReal || childReal.startsWith(prefix);
 }
 function pinExcludedPath2(path) {
@@ -42968,14 +42991,14 @@ function pinExcludedPath2(path) {
 }
 function isExcluded2(absPath, excluded) {
   for (const excludedPath of excluded) {
-    const prefix = excludedPath.endsWith(sep11) ? excludedPath : `${excludedPath}${sep11}`;
+    const prefix = excludedPath.endsWith(sep10) ? excludedPath : `${excludedPath}${sep10}`;
     if (absPath === excludedPath || absPath.startsWith(prefix))
       return true;
   }
   return false;
 }
 function matchesEntry(context, absPath, name) {
-  const haystackRaw = context.matchPath ? relative9(context.rootReal, absPath).split(sep11).join("/") : name;
+  const haystackRaw = context.matchPath ? relative9(context.rootReal, absPath).split(sep10).join("/") : name;
   const haystack = context.matchCase ? haystackRaw : haystackRaw.toLowerCase();
   const needle = context.matchCase ? context.query : context.query.toLowerCase();
   return haystack.includes(needle);
@@ -43116,7 +43139,7 @@ async function filterFileNameSearchMatches(roots, candidates, excludePaths = [])
       continue;
     if (isExcluded2(real, excluded))
       continue;
-    const segments = real.split(sep11);
+    const segments = real.split(sep10);
     if (segments.some((segment) => SKIP_DIR_NAMES2[segment] === true))
       continue;
     matches.push({ path: real, type: stat.isDirectory() ? "folder" : "file" });
@@ -43240,6 +43263,16 @@ var evidenceInput = z4.strictObject({
   sessionId: z4.string().trim().min(1),
   resultNumber: z4.number().int().positive().optional()
 });
+var convertInput = z4.strictObject({
+  inputs: z4.array(z4.string().trim().min(1)).min(1).max(500),
+  to: z4.string().trim().min(1),
+  outputDir: z4.string().trim().min(1).optional(),
+  recursive: z4.boolean().optional(),
+  quality: z4.number().int().min(1).max(100).optional(),
+  keepDates: z4.boolean().optional()
+});
+var convertFormatsInput = z4.strictObject({ from: z4.string().trim().min(1).optional() });
+var MAX_CONVERT_FILES = 200;
 function exactGroupsFromScan(scan) {
   const groups = new Map;
   for (const file of scan.files) {
@@ -43261,6 +43294,57 @@ function jsonResult(value, isError = false) {
 function toolError(code, message, extra = {}) {
   return jsonResult({ ok: false, errorCode: code, message, ...extra }, true);
 }
+var DIAGNOSTIC_EXAMPLES_PER_CODE = 3;
+function capDiagnostics(diagnostics) {
+  const byCode = new Map;
+  for (const entry of diagnostics) {
+    const list = byCode.get(entry.code);
+    if (list === undefined)
+      byCode.set(entry.code, [entry]);
+    else
+      list.push(entry);
+  }
+  const out = [];
+  for (const [code, entries] of byCode) {
+    out.push(...entries.slice(0, DIAGNOSTIC_EXAMPLES_PER_CODE));
+    const rest = entries.slice(DIAGNOSTIC_EXAMPLES_PER_CODE);
+    if (rest.length === 0)
+      continue;
+    const reasons = new Map;
+    for (const entry of rest) {
+      const key = entry.reason ?? "unspecified";
+      reasons.set(key, (reasons.get(key) ?? 0) + 1);
+    }
+    const split = [...reasons].map(([reason, count]) => `${reason}: ${count}`).join(", ");
+    const severity = entries.some((entry) => entry.severity === "error") ? "error" : entries.some((entry) => entry.severity === "warning") ? "warning" : "info";
+    out.push({
+      code,
+      severity,
+      message: `${rest.length} more ${code} entries omitted (${split}); the first ${DIAGNOSTIC_EXAMPLES_PER_CODE} are shown above.`
+    });
+  }
+  return out;
+}
+function convertResult(details) {
+  const { result, links } = details;
+  const files = result.files.slice(0, MAX_CONVERT_FILES).map((file) => ({
+    source: file.source,
+    state: file.state,
+    outputs: file.outputs,
+    ...file.error !== undefined ? { error: file.error } : {},
+    ...file.notice !== undefined ? { notice: file.notice } : {}
+  }));
+  const body = {
+    target: result.target,
+    converted: result.converted,
+    skipped: result.skipped,
+    failed: result.failed,
+    files,
+    ...result.files.length > files.length ? { omittedFiles: result.files.length - files.length } : {},
+    links
+  };
+  return result.converted === 0 && result.failed > 0 ? toolError("convert-failed", "No file was converted.", body) : jsonResult({ ok: true, ...body });
+}
 function isToolEnabled(name, options) {
   return options.tools === undefined || options.tools.includes(name);
 }
@@ -43386,7 +43470,10 @@ function createAutoRAGMcpServer(lite, options = {}) {
       inputSchema: emptyInput,
       outputSchema: objectOutput,
       annotations: { readOnlyHint: true, openWorldHint: false }
-    }, async () => jsonResult(await lite.getRefreshStatus()));
+    }, async () => {
+      const status = await lite.getRefreshStatus();
+      return jsonResult({ ...status, diagnostics: capDiagnostics(status.diagnostics) });
+    });
   }
   if (isToolEnabled("autorag.search", options)) {
     server.registerTool("autorag.search", {
@@ -43425,7 +43512,7 @@ function createAutoRAGMcpServer(lite, options = {}) {
             content: result.content
           })),
           unsearched: retrieved.unsearched,
-          diagnostics: [...status.diagnostics, ...retrieved.diagnostics]
+          diagnostics: capDiagnostics([...status.diagnostics, ...retrieved.diagnostics])
         });
       } catch (error) {
         return toolError(error instanceof RetrievalSelectionError ? "invalid-selection" : "search-failed", error instanceof Error ? error.message : String(error), { retryable: !(error instanceof RetrievalSelectionError) });
@@ -43480,7 +43567,7 @@ function createAutoRAGMcpServer(lite, options = {}) {
             content: result.content
           })),
           unsearched: retrieved.unsearched,
-          diagnostics: [...status.diagnostics, ...retrieved.diagnostics]
+          diagnostics: capDiagnostics([...status.diagnostics, ...retrieved.diagnostics])
         });
       } catch (error) {
         return toolError(error instanceof RetrievalSelectionError ? "invalid-selection" : "search-failed", error instanceof Error ? error.message : String(error), { datasourceId: entry.datasourceId, retryable: !(error instanceof RetrievalSelectionError) });
@@ -43555,6 +43642,46 @@ function createAutoRAGMcpServer(lite, options = {}) {
       return datasource === undefined ? toolError("datasource-not-found", `No configured datasource named ${datasourceId}.`, { datasourceId }) : jsonResult({ ok: true, datasource });
     });
   }
+  const convertEnabled = lite.config.filee?.enabled !== false;
+  if (convertEnabled && !readOnly && isToolEnabled("autorag.convert", options)) {
+    server.registerTool("autorag.convert", {
+      title: "Convert Files",
+      description: "Convert local files to another format offline with Filee: documents, spreadsheets, slides, HWP/HWPX, images, e-books, archives, audio, video, data. Writes new files next to each source (or into outputDir) and never replaces an existing file: a colliding name gets a number added. Each output is compared with its source by Dupey, so the result says whether the copy would count as a duplicate. Use autorag.convert.formats when unsure what a source can become.",
+      inputSchema: convertInput,
+      outputSchema: objectOutput,
+      annotations: { readOnlyHint: false, idempotentHint: false, destructiveHint: false, openWorldHint: false }
+    }, async ({ inputs, to, outputDir, recursive, quality, keepDates }, ctx) => {
+      try {
+        const details = await lite.convertFiles({
+          inputs,
+          to,
+          ...outputDir !== undefined ? { outputDir } : {},
+          ...recursive !== undefined ? { recursive } : {},
+          ...quality !== undefined ? { quality } : {},
+          ...keepDates !== undefined ? { keepDates } : {}
+        }, ctx.mcpReq.signal);
+        return convertResult(details);
+      } catch (error) {
+        return toolError("convert-failed", error instanceof Error ? error.message : String(error));
+      }
+    });
+  }
+  if (convertEnabled && isToolEnabled("autorag.convert.formats", options)) {
+    server.registerTool("autorag.convert.formats", {
+      title: "List Convertible Formats",
+      description: "List the file formats Filee knows, or what one source format (heic, docx, hwp, mkv) can be converted to, including targets that need an optional engine. Read-only.",
+      inputSchema: convertFormatsInput,
+      outputSchema: objectOutput,
+      annotations: { readOnlyHint: true, openWorldHint: false }
+    }, async ({ from }, ctx) => {
+      try {
+        const formats = await lite.listFileConversions(from, ctx.mcpReq.signal);
+        return jsonResult({ ok: true, ...from !== undefined ? { from } : {}, formats });
+      } catch (error) {
+        return toolError("convert-failed", error instanceof Error ? error.message : String(error));
+      }
+    });
+  }
   if (!readOnly && isToolEnabled("autorag.report", options)) {
     server.registerTool("autorag.report", {
       title: "Persist AutoRAG Curated Report",