@autorag/librarian 2.13.8 → 2.13.10

package/dist/profiles.js

@@ -14789,6 +14789,35 @@ var init_embedding = __esm(() => {
   created = new Set;
 });
 
+// src/filesystem/path-exclusions.ts
+import { matchesGlob, posix as posix3, win32 as win322 } from "node:path";
+function isGlobExclusion(entry) {
+  return /[*?[]/u.test(entry);
+}
+function matchesGlobExclusion(path, glob) {
+  const slashed = path.replaceAll("\\", "/");
+  if (matchesGlob(slashed, glob))
+    return true;
+  return glob.startsWith("**/") && matchesGlob(slashed, glob.slice(3));
+}
+function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
+  const paths = platform === "win32" ? win322 : posix3;
+  const normalize = (path) => {
+    const absolute = paths.resolve(path);
+    return platform === "win32" ? absolute.toLowerCase() : absolute;
+  };
+  const globs = excludedPaths.filter(isGlobExclusion).map((glob) => platform === "win32" ? glob.toLowerCase() : glob);
+  const excluded = excludedPaths.filter((path) => !isGlobExclusion(path)).map((path) => {
+    const absolute = normalize(path);
+    return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
+  });
+  return (path) => {
+    const absolute = normalize(path);
+    return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix)) || globs.some((glob) => matchesGlobExclusion(absolute, glob));
+  };
+}
+var init_path_exclusions = () => {};
+
 // src/language.ts
 function supportedList() {
   return AUTORAG_LANGUAGE_TAGS.join(", ");
@@ -15377,7 +15406,7 @@ function resolveExcludePaths(value, origin) {
   if (!Array.isArray(value) || value.some((path) => typeof path !== "string" || path.length === 0)) {
     throw new ConfigError("Config field 'excludePaths' must be a non-empty string array");
   }
-  return resolveSearchPaths(value, origin);
+  return value.map((entry) => isGlobExclusion(entry) ? entry : resolvePersistedPath(entry, origin));
 }
 function resolvePersistedPath(path, origin) {
   return isAbsolute5(path) ? path : resolve6(origin, path);
@@ -16682,6 +16711,7 @@ var init_config4 = __esm(() => {
   init_factory();
   init_embedding();
   init_file_lock();
+  init_path_exclusions();
   init_language();
   init_atomic_write();
   init_rerank();
@@ -16978,7 +17008,7 @@ var init_automation_profiles = __esm(() => {
 
 // src/automation/config.ts
 import { existsSync as existsSync14, readFileSync as readFileSync16, realpathSync as realpathSync2 } from "node:fs";
-import { basename as basename6, dirname as dirname12, isAbsolute as isAbsolute6, join as join24, matchesGlob, relative as relative4, resolve as resolve7 } from "node:path";
+import { basename as basename6, dirname as dirname12, isAbsolute as isAbsolute6, join as join24, matchesGlob as matchesGlob2, relative as relative4, resolve as resolve7 } from "node:path";
 function automationsConfigPath(env = process.env) {
   return join24(resolveAutoRAGHome(env), "automations.json");
 }
@@ -17154,9 +17184,9 @@ function deleteAutomation(configPath, id) {
 function globMatches(relPosix, pattern) {
   if (pattern === "**")
     return true;
-  if (matchesGlob(relPosix, pattern))
+  if (matchesGlob2(relPosix, pattern))
     return true;
-  if (pattern.startsWith("**/") && matchesGlob(relPosix, pattern.slice(3)))
+  if (pattern.startsWith("**/") && matchesGlob2(relPosix, pattern.slice(3)))
     return true;
   return false;
 }
@@ -18612,25 +18642,6 @@ var init_bundle = __esm(() => {
   init_atomic_write();
 });
 
-// src/filesystem/path-exclusions.ts
-import { posix as posix3, win32 as win322 } from "node:path";
-function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
-  const paths = platform === "win32" ? win322 : posix3;
-  const normalize = (path) => {
-    const absolute = paths.resolve(path);
-    return platform === "win32" ? absolute.toLowerCase() : absolute;
-  };
-  const excluded = excludedPaths.map((path) => {
-    const absolute = normalize(path);
-    return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
-  });
-  return (path) => {
-    const absolute = normalize(path);
-    return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
-  };
-}
-var init_path_exclusions = () => {};
-
 // src/filesystem/shared-exclusive-queue.ts
 class SharedExclusiveQueue {
   sharedSinceExclusive = [];
@@ -22145,6 +22156,34 @@ var init_memory = __esm(() => {
   });
 });
 
+// src/minsync/channel-union.ts
+function unionChannelResults(channels, options = {}) {
+  const byChunk = new Map;
+  const depth = Math.max(0, ...channels.map((channel) => channel.results.length));
+  for (let rank = 0;rank < depth; rank++) {
+    for (const channel of channels) {
+      const result = channel.results[rank];
+      if (result === undefined)
+        continue;
+      const key = `${result.source}\x00${result.id}`;
+      const existing = byChunk.get(key);
+      if (existing === undefined)
+        byChunk.set(key, { result, ranks: { [channel.channel]: rank + 1 } });
+      else
+        existing.ranks[channel.channel] ??= rank + 1;
+    }
+  }
+  const union = [...byChunk.values()].map(({ result, ranks }) => ({
+    ...result,
+    metadata: {
+      ...result.metadata,
+      minsyncChannels: Object.keys(ranks).sort(),
+      minsyncChannelRanks: ranks
+    }
+  }));
+  return options.limit === undefined ? union : union.slice(0, options.limit);
+}
+
 // src/retrieval/chunk-location.ts
 function sectionLabel(line) {
   const text = line.trim();
@@ -24183,7 +24222,7 @@ var init_cloud = __esm(() => {
 // src/mirror/sync.ts
 import { createHash as createHash17 } from "node:crypto";
 import { opendir as opendir2, readFile as readFile2, stat as stat4 } from "node:fs/promises";
-import { basename as basename13, dirname as dirname21, extname as extname9, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
+import { basename as basename13, dirname as dirname21, extname as extname9, join as join42, resolve as resolve15 } from "node:path";
 import { setTimeout as sleep4 } from "node:timers/promises";
 function errorCode(error) {
   if (!(error instanceof Error))
@@ -24730,13 +24769,14 @@ function normalizeExtension2(extension) {
   return lower.startsWith(".") ? lower : `.${lower}`;
 }
 function isPathExcluded(sourcePath, excluded) {
-  if (excluded === undefined)
+  if (excluded === undefined || excluded.size === 0)
     return false;
-  for (const excludedPath of excluded) {
-    if (sourcePath === excludedPath || sourcePath.startsWith(`${excludedPath}${sep9}`))
-      return true;
+  let matcher = matcherBySet.get(excluded);
+  if (matcher === undefined) {
+    matcher = createPathExclusionMatcher([...excluded]);
+    matcherBySet.set(excluded, matcher);
   }
-  return false;
+  return matcher(sourcePath);
 }
 function walkLimits(options) {
   return {
@@ -24903,10 +24943,11 @@ async function collectFiles(state, directory, depth) {
     warnDirectoryUnreadable(state, directory, code);
   }
 }
-var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
+var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, matcherBySet, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
 var init_sync = __esm(() => {
   init_cli2();
   init_cloud_placeholder();
+  init_path_exclusions();
   init_source_paths();
   init_binary();
   init_stage();
@@ -24965,6 +25006,7 @@ var init_sync = __esm(() => {
     }
   };
   STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
+  matcherBySet = new WeakMap;
   SEARCH_ROOT_UNAVAILABLE_MESSAGES = {
     gone: "A search root no longer exists; its previously indexed documents were left untouched.",
     "not-a-directory": "A search root is no longer a directory; its previously indexed documents were left untouched.",
@@ -38520,7 +38562,7 @@ var init_web_search_tool = __esm(() => {
 // src/agent/agent.ts
 import { randomUUID as randomUUID7 } from "node:crypto";
 import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
-import { basename as basename15, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
+import { basename as basename15, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep9 } from "node:path";
 import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
 import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
 function describeDecisionModel(status) {
@@ -38663,6 +38705,7 @@ class AutoRAGAgent {
   reranker;
   rerankTopN;
   minSyncMethod;
+  minSyncSparseMethod;
   minSyncEmbedderConfig;
   minSyncBinaryPath;
   jikjiClient;
@@ -38785,6 +38828,7 @@ class AutoRAGAgent {
       ...this.limits.recordCut === undefined ? {} : { recordCut: this.limits.recordCut }
     };
     this.minSyncMethod = new MinSyncVectorMethod(minSyncDefaults);
+    this.minSyncSparseMethod = isSparseOnlyEmbedder(options.minSync?.embedder) ? undefined : new MinSyncVectorMethod({ ...minSyncDefaults, mode: "sparse" });
     this.methodRegistry.register(new MinSyncHybridMethod(minSyncDefaults));
     const registeredDatasourceIds = new Set;
     for (const skill of this.datasourceSkills) {
@@ -40259,13 +40303,7 @@ Warning: the edited config did not validate, so the previous config was restored
 ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
 `)}`;
     const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
-      this.searchProgress.track("minsync", "MinSync semantic search", this.minSyncMethod.retrieve(searchQuery, retrieveOptions), {
-        count: (found) => found.length
-      }).catch((error) => {
-        if (error instanceof MinSyncRequiredError)
-          throw error;
-        return [];
-      }),
+      this.prefetchMinSyncCandidates(searchQuery, retrieveOptions),
       Promise.all(datasources.map((datasourceId) => this.searchProgress.track(`datasource:${datasourceId}`, `Datasource ${datasourceId}`, this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
         topK: this.limits.prefetch.datasourceTopK,
         scope: options.scope,
@@ -40380,6 +40418,30 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
 
 `);
   }
+  async prefetchMinSyncCandidates(searchQuery, options) {
+    const sparse = this.reranker === undefined ? undefined : this.minSyncSparseMethod;
+    const run = (id, label, method) => this.searchProgress.track(id, label, method.retrieve(searchQuery, options), { count: (found) => found.length }).catch((error) => {
+      if (error instanceof MinSyncRequiredError)
+        throw error;
+      if (sparse !== undefined) {
+        this.routingDiagnostics.push({
+          code: "retrieval-method-failed",
+          severity: "warning",
+          message: `MinSync ${id === "minsync" ? "dense" : "SPLADE"} search failed and its candidates were skipped; the other channel's were used: ${describeRetrievalError(error)}`,
+          source: id
+        });
+      }
+      return [];
+    });
+    const dense = run("minsync", "MinSync semantic search", this.minSyncMethod);
+    if (sparse === undefined)
+      return dense;
+    const [vector, splade] = await Promise.all([dense, run("minsync-sparse", "MinSync SPLADE search", sparse)]);
+    return unionChannelResults([
+      { channel: "vector", results: vector },
+      { channel: "sparse", results: splade }
+    ]);
+  }
   async prefetchWebSearch(searchQuery, signal) {
     const web = this.webSearchOptions;
     if (web === undefined)
@@ -41173,7 +41235,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
         return { ok: false, reason: "cloud-root-skipped", stdout: "", stderr: "", code: null };
       }
       const canonicalSourcePath = canonicalPath(sourcePath);
-      const exclude = scope.nestedCloudRoots.map((root) => relative8(canonicalSourcePath, root.path).split(sep10).join("/")).filter((pattern) => pattern !== "" && !pattern.startsWith(".."));
+      const exclude = scope.nestedCloudRoots.map((root) => relative8(canonicalSourcePath, root.path).split(sep9).join("/")).filter((pattern) => pattern !== "" && !pattern.startsWith(".."));
       return client.prepare(sourcePath, { ...options, ...exclude.length === 0 ? {} : { exclude } });
     }));
   }
@@ -41687,6 +41749,8 @@ function toMinSyncReasonDiagnostic(reason) {
   };
 }
 function pinExcludedPath(path) {
+  if (isGlobExclusion(path))
+    return path;
   const resolvedPath = resolve18(path);
   try {
     return join54(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
@@ -41735,6 +41799,7 @@ var init_agent = __esm(() => {
   init_embedding();
   init_everything();
   init_cloud_placeholder();
+  init_path_exclusions();
   init_fsearch();
   init_diagnostics();
   init_jikji();
@@ -41757,6 +41822,7 @@ var init_agent = __esm(() => {
   init_merger();
   init_rerank();
   init_scope();
+  init_skip();
   init_search();
   init_model_auth();
   init_answer_citations();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.13.8",
+  "version": "2.13.10",
   "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {