@autorag/librarian 2.13.8 → 2.13.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +10 -0
- package/dist/cli/commands/lite-find.d.ts +26 -0
- package/dist/cli/index.js +757 -42
- package/dist/core.d.ts +5 -0
- package/dist/core.js +102 -30
- package/dist/filesystem/path-exclusions.d.ts +12 -1
- package/dist/find/groups.d.ts +35 -0
- package/dist/find/index.d.ts +10 -0
- package/dist/find/relevance.d.ts +29 -0
- package/dist/find/run.d.ts +24 -0
- package/dist/find/text.d.ts +37 -0
- package/dist/find/types.d.ts +86 -0
- package/dist/index.js +177 -35
- package/dist/mcp/index.js +224 -38
- package/dist/mcp/reloading-lite.d.ts +25 -0
- package/dist/minsync/channel-union.d.ts +23 -0
- package/dist/mirror/sync.d.ts +1 -0
- package/dist/profiles.js +105 -39
- package/package.json +1 -1
package/dist/mcp/index.js
CHANGED
|
@@ -3669,20 +3669,30 @@ var init_bundle = __esm(() => {
|
|
|
3669
3669
|
});
|
|
3670
3670
|
|
|
3671
3671
|
// src/filesystem/path-exclusions.ts
|
|
3672
|
-
import { posix as posix2, win32 as win322 } from "node:path";
|
|
3672
|
+
import { matchesGlob, posix as posix2, win32 as win322 } from "node:path";
|
|
3673
|
+
function isGlobExclusion(entry) {
|
|
3674
|
+
return /[*?[]/u.test(entry);
|
|
3675
|
+
}
|
|
3676
|
+
function matchesGlobExclusion(path, glob) {
|
|
3677
|
+
const slashed = path.replaceAll("\\", "/");
|
|
3678
|
+
if (matchesGlob(slashed, glob))
|
|
3679
|
+
return true;
|
|
3680
|
+
return glob.startsWith("**/") && matchesGlob(slashed, glob.slice(3));
|
|
3681
|
+
}
|
|
3673
3682
|
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
3674
3683
|
const paths = platform === "win32" ? win322 : posix2;
|
|
3675
3684
|
const normalize = (path) => {
|
|
3676
3685
|
const absolute = paths.resolve(path);
|
|
3677
3686
|
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
3678
3687
|
};
|
|
3679
|
-
const
|
|
3688
|
+
const globs = excludedPaths.filter(isGlobExclusion).map((glob) => platform === "win32" ? glob.toLowerCase() : glob);
|
|
3689
|
+
const excluded = excludedPaths.filter((path) => !isGlobExclusion(path)).map((path) => {
|
|
3680
3690
|
const absolute = normalize(path);
|
|
3681
3691
|
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
3682
3692
|
});
|
|
3683
3693
|
return (path) => {
|
|
3684
3694
|
const absolute = normalize(path);
|
|
3685
|
-
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
3695
|
+
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix)) || globs.some((glob) => matchesGlobExclusion(absolute, glob));
|
|
3686
3696
|
};
|
|
3687
3697
|
}
|
|
3688
3698
|
var init_path_exclusions = () => {};
|
|
@@ -7619,6 +7629,34 @@ var init_memory = __esm(() => {
|
|
|
7619
7629
|
});
|
|
7620
7630
|
});
|
|
7621
7631
|
|
|
7632
|
+
// src/minsync/channel-union.ts
|
|
7633
|
+
function unionChannelResults(channels, options = {}) {
|
|
7634
|
+
const byChunk = new Map;
|
|
7635
|
+
const depth = Math.max(0, ...channels.map((channel) => channel.results.length));
|
|
7636
|
+
for (let rank = 0;rank < depth; rank++) {
|
|
7637
|
+
for (const channel of channels) {
|
|
7638
|
+
const result = channel.results[rank];
|
|
7639
|
+
if (result === undefined)
|
|
7640
|
+
continue;
|
|
7641
|
+
const key = `${result.source}\x00${result.id}`;
|
|
7642
|
+
const existing = byChunk.get(key);
|
|
7643
|
+
if (existing === undefined)
|
|
7644
|
+
byChunk.set(key, { result, ranks: { [channel.channel]: rank + 1 } });
|
|
7645
|
+
else
|
|
7646
|
+
existing.ranks[channel.channel] ??= rank + 1;
|
|
7647
|
+
}
|
|
7648
|
+
}
|
|
7649
|
+
const union = [...byChunk.values()].map(({ result, ranks }) => ({
|
|
7650
|
+
...result,
|
|
7651
|
+
metadata: {
|
|
7652
|
+
...result.metadata,
|
|
7653
|
+
minsyncChannels: Object.keys(ranks).sort(),
|
|
7654
|
+
minsyncChannelRanks: ranks
|
|
7655
|
+
}
|
|
7656
|
+
}));
|
|
7657
|
+
return options.limit === undefined ? union : union.slice(0, options.limit);
|
|
7658
|
+
}
|
|
7659
|
+
|
|
7622
7660
|
// src/minsync/local-embedder.ts
|
|
7623
7661
|
import { spawn as spawn11 } from "node:child_process";
|
|
7624
7662
|
import { request } from "node:http";
|
|
@@ -10306,7 +10344,7 @@ var init_cloud = __esm(() => {
|
|
|
10306
10344
|
// src/mirror/sync.ts
|
|
10307
10345
|
import { createHash as createHash11 } from "node:crypto";
|
|
10308
10346
|
import { opendir, readFile, stat as stat3 } from "node:fs/promises";
|
|
10309
|
-
import { basename as basename9, dirname as dirname12, extname as extname6, join as join27, resolve as resolve11
|
|
10347
|
+
import { basename as basename9, dirname as dirname12, extname as extname6, join as join27, resolve as resolve11 } from "node:path";
|
|
10310
10348
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
10311
10349
|
function errorCode(error) {
|
|
10312
10350
|
if (!(error instanceof Error))
|
|
@@ -10853,13 +10891,14 @@ function normalizeExtension2(extension) {
|
|
|
10853
10891
|
return lower.startsWith(".") ? lower : `.${lower}`;
|
|
10854
10892
|
}
|
|
10855
10893
|
function isPathExcluded(sourcePath, excluded) {
|
|
10856
|
-
if (excluded === undefined)
|
|
10894
|
+
if (excluded === undefined || excluded.size === 0)
|
|
10857
10895
|
return false;
|
|
10858
|
-
|
|
10859
|
-
|
|
10860
|
-
|
|
10896
|
+
let matcher = matcherBySet.get(excluded);
|
|
10897
|
+
if (matcher === undefined) {
|
|
10898
|
+
matcher = createPathExclusionMatcher([...excluded]);
|
|
10899
|
+
matcherBySet.set(excluded, matcher);
|
|
10861
10900
|
}
|
|
10862
|
-
return
|
|
10901
|
+
return matcher(sourcePath);
|
|
10863
10902
|
}
|
|
10864
10903
|
function walkLimits(options) {
|
|
10865
10904
|
return {
|
|
@@ -11026,10 +11065,11 @@ async function collectFiles(state, directory, depth) {
|
|
|
11026
11065
|
warnDirectoryUnreadable(state, directory, code);
|
|
11027
11066
|
}
|
|
11028
11067
|
}
|
|
11029
|
-
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
11068
|
+
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, matcherBySet, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
11030
11069
|
var init_sync = __esm(() => {
|
|
11031
11070
|
init_cli2();
|
|
11032
11071
|
init_cloud_placeholder();
|
|
11072
|
+
init_path_exclusions();
|
|
11033
11073
|
init_source_paths();
|
|
11034
11074
|
init_binary();
|
|
11035
11075
|
init_stage();
|
|
@@ -11088,6 +11128,7 @@ var init_sync = __esm(() => {
|
|
|
11088
11128
|
}
|
|
11089
11129
|
};
|
|
11090
11130
|
STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
|
|
11131
|
+
matcherBySet = new WeakMap;
|
|
11091
11132
|
SEARCH_ROOT_UNAVAILABLE_MESSAGES = {
|
|
11092
11133
|
gone: "A search root no longer exists; its previously indexed documents were left untouched.",
|
|
11093
11134
|
"not-a-directory": "A search root is no longer a directory; its previously indexed documents were left untouched.",
|
|
@@ -16162,7 +16203,7 @@ var init_automation_profiles = __esm(() => {
|
|
|
16162
16203
|
|
|
16163
16204
|
// src/automation/config.ts
|
|
16164
16205
|
import { existsSync as existsSync19, readFileSync as readFileSync14, realpathSync as realpathSync7 } from "node:fs";
|
|
16165
|
-
import { basename as basename10, dirname as dirname15, isAbsolute as isAbsolute8, join as join30, matchesGlob, relative as relative5, resolve as resolve12 } from "node:path";
|
|
16206
|
+
import { basename as basename10, dirname as dirname15, isAbsolute as isAbsolute8, join as join30, matchesGlob as matchesGlob2, relative as relative5, resolve as resolve12 } from "node:path";
|
|
16166
16207
|
function automationsConfigPath(env = process.env) {
|
|
16167
16208
|
return join30(resolveAutoRAGHome(env), "automations.json");
|
|
16168
16209
|
}
|
|
@@ -16338,9 +16379,9 @@ function deleteAutomation(configPath, id) {
|
|
|
16338
16379
|
function globMatches(relPosix, pattern) {
|
|
16339
16380
|
if (pattern === "**")
|
|
16340
16381
|
return true;
|
|
16341
|
-
if (
|
|
16382
|
+
if (matchesGlob2(relPosix, pattern))
|
|
16342
16383
|
return true;
|
|
16343
|
-
if (pattern.startsWith("**/") &&
|
|
16384
|
+
if (pattern.startsWith("**/") && matchesGlob2(relPosix, pattern.slice(3)))
|
|
16344
16385
|
return true;
|
|
16345
16386
|
return false;
|
|
16346
16387
|
}
|
|
@@ -17137,7 +17178,7 @@ var init_permission_judgment = __esm(() => {
|
|
|
17137
17178
|
|
|
17138
17179
|
// src/agent/permission-gate.ts
|
|
17139
17180
|
import { statSync as statSync7 } from "node:fs";
|
|
17140
|
-
import { isAbsolute as isAbsolute9, relative as relative6, sep as
|
|
17181
|
+
import { isAbsolute as isAbsolute9, relative as relative6, sep as sep6 } from "node:path";
|
|
17141
17182
|
import { matchesKey } from "@earendil-works/pi-tui";
|
|
17142
17183
|
function pathExists(path) {
|
|
17143
17184
|
try {
|
|
@@ -17200,7 +17241,7 @@ class PermissionController {
|
|
|
17200
17241
|
if (path === root)
|
|
17201
17242
|
return true;
|
|
17202
17243
|
const rel = relative6(root, path);
|
|
17203
|
-
if (rel !== "" && rel !== ".." && !rel.startsWith(`..${
|
|
17244
|
+
if (rel !== "" && rel !== ".." && !rel.startsWith(`..${sep6}`) && !isAbsolute9(rel))
|
|
17204
17245
|
return true;
|
|
17205
17246
|
}
|
|
17206
17247
|
return false;
|
|
@@ -18652,7 +18693,7 @@ var init_connector = () => {};
|
|
|
18652
18693
|
// src/datasource/sidecar/config.ts
|
|
18653
18694
|
import { readFileSync as readFileSync17 } from "node:fs";
|
|
18654
18695
|
import { homedir as homedir5 } from "node:os";
|
|
18655
|
-
import { dirname as dirname17, isAbsolute as isAbsolute10, join as join34, resolve as resolve14, sep as
|
|
18696
|
+
import { dirname as dirname17, isAbsolute as isAbsolute10, join as join34, resolve as resolve14, sep as sep7 } from "node:path";
|
|
18656
18697
|
import { parse as parseToml } from "smol-toml";
|
|
18657
18698
|
function resolveSidecarOptions(input) {
|
|
18658
18699
|
const { connector, workspaceRoot } = input;
|
|
@@ -18692,7 +18733,7 @@ function archivePathFromCrawlerConfig(input) {
|
|
|
18692
18733
|
function expandHome(path) {
|
|
18693
18734
|
if (path === "~")
|
|
18694
18735
|
return homedir5();
|
|
18695
|
-
const prefixed = path.startsWith("~/") ||
|
|
18736
|
+
const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
|
|
18696
18737
|
return prefixed ? join34(homedir5(), path.slice(2)) : path;
|
|
18697
18738
|
}
|
|
18698
18739
|
var init_config3 = __esm(() => {
|
|
@@ -25946,7 +25987,7 @@ import {
|
|
|
25946
25987
|
writeFileSync as writeFileSync13
|
|
25947
25988
|
} from "node:fs";
|
|
25948
25989
|
import { opendir as opendir2, stat as stat4 } from "node:fs/promises";
|
|
25949
|
-
import { basename as basename11, dirname as dirname23, extname as extname8, join as join40, relative as relative7, sep as
|
|
25990
|
+
import { basename as basename11, dirname as dirname23, extname as extname8, join as join40, relative as relative7, sep as sep8 } from "node:path";
|
|
25950
25991
|
function errnoCode2(error) {
|
|
25951
25992
|
const code = error instanceof Error ? error.code : undefined;
|
|
25952
25993
|
return typeof code === "string" ? code : "UNKNOWN";
|
|
@@ -26016,7 +26057,7 @@ async function listVaultNotes(vaultPath) {
|
|
|
26016
26057
|
return { notes, unreadableFolders, collisions };
|
|
26017
26058
|
}
|
|
26018
26059
|
function toRelative(vaultPath, path) {
|
|
26019
|
-
return relative7(vaultPath, path).split(
|
|
26060
|
+
return relative7(vaultPath, path).split(sep8).join("/").normalize("NFC");
|
|
26020
26061
|
}
|
|
26021
26062
|
function loadStagingState2(statePath) {
|
|
26022
26063
|
try {
|
|
@@ -26100,7 +26141,7 @@ function removeUnexpected(filesRoot, directory, wanted) {
|
|
|
26100
26141
|
return removed;
|
|
26101
26142
|
}
|
|
26102
26143
|
function toStagedRelative(filesRoot, path) {
|
|
26103
|
-
return relative7(filesRoot, path).split(
|
|
26144
|
+
return relative7(filesRoot, path).split(sep8).join("/").normalize("NFC");
|
|
26104
26145
|
}
|
|
26105
26146
|
var NOTE_EXTENSIONS, SKIPPED_DIRECTORIES, MAX_WALK_DEPTH = 64, VaultUnavailableError;
|
|
26106
26147
|
var init_staging = __esm(() => {
|
|
@@ -28021,7 +28062,7 @@ function resolveExcludePaths(value, origin) {
|
|
|
28021
28062
|
if (!Array.isArray(value) || value.some((path) => typeof path !== "string" || path.length === 0)) {
|
|
28022
28063
|
throw new ConfigError("Config field 'excludePaths' must be a non-empty string array");
|
|
28023
28064
|
}
|
|
28024
|
-
return
|
|
28065
|
+
return value.map((entry) => isGlobExclusion(entry) ? entry : resolvePersistedPath(entry, origin));
|
|
28025
28066
|
}
|
|
28026
28067
|
function resolvePersistedPath(path, origin) {
|
|
28027
28068
|
return isAbsolute11(path) ? path : resolve15(origin, path);
|
|
@@ -29326,6 +29367,7 @@ var init_config5 = __esm(() => {
|
|
|
29326
29367
|
init_factory();
|
|
29327
29368
|
init_embedding();
|
|
29328
29369
|
init_file_lock();
|
|
29370
|
+
init_path_exclusions();
|
|
29329
29371
|
init_language();
|
|
29330
29372
|
init_atomic_write();
|
|
29331
29373
|
init_rerank();
|
|
@@ -38579,7 +38621,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
38579
38621
|
// src/agent/agent.ts
|
|
38580
38622
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
38581
38623
|
import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
|
|
38582
|
-
import { basename as basename15, dirname as dirname27, join as join55, relative as relative8, resolve as resolve18, sep as
|
|
38624
|
+
import { basename as basename15, dirname as dirname27, join as join55, relative as relative8, resolve as resolve18, sep as sep9 } from "node:path";
|
|
38583
38625
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
38584
38626
|
import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
|
|
38585
38627
|
function describeDecisionModel(status) {
|
|
@@ -38722,6 +38764,7 @@ class AutoRAGAgent {
|
|
|
38722
38764
|
reranker;
|
|
38723
38765
|
rerankTopN;
|
|
38724
38766
|
minSyncMethod;
|
|
38767
|
+
minSyncSparseMethod;
|
|
38725
38768
|
minSyncEmbedderConfig;
|
|
38726
38769
|
minSyncBinaryPath;
|
|
38727
38770
|
jikjiClient;
|
|
@@ -38844,6 +38887,7 @@ class AutoRAGAgent {
|
|
|
38844
38887
|
...this.limits.recordCut === undefined ? {} : { recordCut: this.limits.recordCut }
|
|
38845
38888
|
};
|
|
38846
38889
|
this.minSyncMethod = new MinSyncVectorMethod(minSyncDefaults);
|
|
38890
|
+
this.minSyncSparseMethod = isSparseOnlyEmbedder(options.minSync?.embedder) ? undefined : new MinSyncVectorMethod({ ...minSyncDefaults, mode: "sparse" });
|
|
38847
38891
|
this.methodRegistry.register(new MinSyncHybridMethod(minSyncDefaults));
|
|
38848
38892
|
const registeredDatasourceIds = new Set;
|
|
38849
38893
|
for (const skill of this.datasourceSkills) {
|
|
@@ -40318,13 +40362,7 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
40318
40362
|
${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
40319
40363
|
`)}`;
|
|
40320
40364
|
const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
|
|
40321
|
-
this.
|
|
40322
|
-
count: (found) => found.length
|
|
40323
|
-
}).catch((error) => {
|
|
40324
|
-
if (error instanceof MinSyncRequiredError)
|
|
40325
|
-
throw error;
|
|
40326
|
-
return [];
|
|
40327
|
-
}),
|
|
40365
|
+
this.prefetchMinSyncCandidates(searchQuery, retrieveOptions),
|
|
40328
40366
|
Promise.all(datasources.map((datasourceId) => this.searchProgress.track(`datasource:${datasourceId}`, `Datasource ${datasourceId}`, this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
|
|
40329
40367
|
topK: this.limits.prefetch.datasourceTopK,
|
|
40330
40368
|
scope: options.scope,
|
|
@@ -40439,6 +40477,30 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
|
|
|
40439
40477
|
|
|
40440
40478
|
`);
|
|
40441
40479
|
}
|
|
40480
|
+
async prefetchMinSyncCandidates(searchQuery, options) {
|
|
40481
|
+
const sparse = this.reranker === undefined ? undefined : this.minSyncSparseMethod;
|
|
40482
|
+
const run = (id, label, method) => this.searchProgress.track(id, label, method.retrieve(searchQuery, options), { count: (found) => found.length }).catch((error) => {
|
|
40483
|
+
if (error instanceof MinSyncRequiredError)
|
|
40484
|
+
throw error;
|
|
40485
|
+
if (sparse !== undefined) {
|
|
40486
|
+
this.routingDiagnostics.push({
|
|
40487
|
+
code: "retrieval-method-failed",
|
|
40488
|
+
severity: "warning",
|
|
40489
|
+
message: `MinSync ${id === "minsync" ? "dense" : "SPLADE"} search failed and its candidates were skipped; the other channel's were used: ${describeRetrievalError(error)}`,
|
|
40490
|
+
source: id
|
|
40491
|
+
});
|
|
40492
|
+
}
|
|
40493
|
+
return [];
|
|
40494
|
+
});
|
|
40495
|
+
const dense = run("minsync", "MinSync semantic search", this.minSyncMethod);
|
|
40496
|
+
if (sparse === undefined)
|
|
40497
|
+
return dense;
|
|
40498
|
+
const [vector, splade] = await Promise.all([dense, run("minsync-sparse", "MinSync SPLADE search", sparse)]);
|
|
40499
|
+
return unionChannelResults([
|
|
40500
|
+
{ channel: "vector", results: vector },
|
|
40501
|
+
{ channel: "sparse", results: splade }
|
|
40502
|
+
]);
|
|
40503
|
+
}
|
|
40442
40504
|
async prefetchWebSearch(searchQuery, signal) {
|
|
40443
40505
|
const web = this.webSearchOptions;
|
|
40444
40506
|
if (web === undefined)
|
|
@@ -41232,7 +41294,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
41232
41294
|
return { ok: false, reason: "cloud-root-skipped", stdout: "", stderr: "", code: null };
|
|
41233
41295
|
}
|
|
41234
41296
|
const canonicalSourcePath = canonicalPath(sourcePath);
|
|
41235
|
-
const exclude = scope.nestedCloudRoots.map((root) => relative8(canonicalSourcePath, root.path).split(
|
|
41297
|
+
const exclude = scope.nestedCloudRoots.map((root) => relative8(canonicalSourcePath, root.path).split(sep9).join("/")).filter((pattern) => pattern !== "" && !pattern.startsWith(".."));
|
|
41236
41298
|
return client.prepare(sourcePath, { ...options, ...exclude.length === 0 ? {} : { exclude } });
|
|
41237
41299
|
}));
|
|
41238
41300
|
}
|
|
@@ -41746,6 +41808,8 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
41746
41808
|
};
|
|
41747
41809
|
}
|
|
41748
41810
|
function pinExcludedPath(path) {
|
|
41811
|
+
if (isGlobExclusion(path))
|
|
41812
|
+
return path;
|
|
41749
41813
|
const resolvedPath = resolve18(path);
|
|
41750
41814
|
try {
|
|
41751
41815
|
return join55(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
|
|
@@ -41794,6 +41858,7 @@ var init_agent = __esm(() => {
|
|
|
41794
41858
|
init_embedding();
|
|
41795
41859
|
init_everything();
|
|
41796
41860
|
init_cloud_placeholder();
|
|
41861
|
+
init_path_exclusions();
|
|
41797
41862
|
init_fsearch();
|
|
41798
41863
|
init_diagnostics();
|
|
41799
41864
|
init_jikji();
|
|
@@ -41816,6 +41881,7 @@ var init_agent = __esm(() => {
|
|
|
41816
41881
|
init_merger();
|
|
41817
41882
|
init_rerank();
|
|
41818
41883
|
init_scope();
|
|
41884
|
+
init_skip();
|
|
41819
41885
|
init_search();
|
|
41820
41886
|
init_model_auth();
|
|
41821
41887
|
init_answer_citations();
|
|
@@ -41953,6 +42019,12 @@ class AutoRAGLite {
|
|
|
41953
42019
|
getMemorySchema() {
|
|
41954
42020
|
return structuredClone(this.memory.getSchema());
|
|
41955
42021
|
}
|
|
42022
|
+
convertFiles(request, signal) {
|
|
42023
|
+
return this.agent.convertFiles(request, signal);
|
|
42024
|
+
}
|
|
42025
|
+
listFileConversions(from, signal) {
|
|
42026
|
+
return this.agent.listFileConversions(from, signal);
|
|
42027
|
+
}
|
|
41956
42028
|
}
|
|
41957
42029
|
function createAutoRAGLite(options = {}) {
|
|
41958
42030
|
const input = { flags: options.flags ?? {} };
|
|
@@ -42644,6 +42716,48 @@ import { realpathSync as realpathSync10 } from "node:fs";
|
|
|
42644
42716
|
import { fileURLToPath as fileURLToPath6 } from "node:url";
|
|
42645
42717
|
import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
|
|
42646
42718
|
|
|
42719
|
+
// src/mcp/reloading-lite.ts
|
|
42720
|
+
init_config5();
|
|
42721
|
+
import { statSync as statSync12 } from "node:fs";
|
|
42722
|
+
function createReloadingLite(options) {
|
|
42723
|
+
let attempted = options.signature();
|
|
42724
|
+
let current = options.create();
|
|
42725
|
+
const sync = () => {
|
|
42726
|
+
const next = options.signature();
|
|
42727
|
+
if (next === attempted)
|
|
42728
|
+
return current;
|
|
42729
|
+
attempted = next;
|
|
42730
|
+
try {
|
|
42731
|
+
current = options.create();
|
|
42732
|
+
} catch (error) {
|
|
42733
|
+
options.onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
42734
|
+
}
|
|
42735
|
+
return current;
|
|
42736
|
+
};
|
|
42737
|
+
return new Proxy({}, {
|
|
42738
|
+
get(_target, property) {
|
|
42739
|
+
const runtime = sync();
|
|
42740
|
+
const value = Reflect.get(runtime, property, runtime);
|
|
42741
|
+
return typeof value === "function" ? value.bind(runtime) : value;
|
|
42742
|
+
},
|
|
42743
|
+
has(_target, property) {
|
|
42744
|
+
return property in sync();
|
|
42745
|
+
}
|
|
42746
|
+
});
|
|
42747
|
+
}
|
|
42748
|
+
function configSignature(env = process.env) {
|
|
42749
|
+
const { configPath, legacyPath } = resolveConfigPath({ flags: {}, env });
|
|
42750
|
+
return [configPath, legacyPath].map((path) => path === undefined ? "-" : fingerprint(path)).join("|");
|
|
42751
|
+
}
|
|
42752
|
+
function fingerprint(path) {
|
|
42753
|
+
try {
|
|
42754
|
+
const stats = statSync12(path);
|
|
42755
|
+
return `${stats.ino}:${stats.size}:${stats.mtimeMs}`;
|
|
42756
|
+
} catch {
|
|
42757
|
+
return "absent";
|
|
42758
|
+
}
|
|
42759
|
+
}
|
|
42760
|
+
|
|
42647
42761
|
// src/mcp/server.ts
|
|
42648
42762
|
init_evidence();
|
|
42649
42763
|
init_report();
|
|
@@ -42657,7 +42771,7 @@ import { z as z4 } from "zod";
|
|
|
42657
42771
|
// src/filesystem/name-search.ts
|
|
42658
42772
|
import { realpathSync as realpathSync9 } from "node:fs";
|
|
42659
42773
|
import { lstat as lstat2, readdir as readdir5 } from "node:fs/promises";
|
|
42660
|
-
import { basename as basename16, dirname as dirname28, isAbsolute as isAbsolute14, join as join56, relative as relative9, resolve as resolve19, sep as
|
|
42774
|
+
import { basename as basename16, dirname as dirname28, isAbsolute as isAbsolute14, join as join56, relative as relative9, resolve as resolve19, sep as sep10 } from "node:path";
|
|
42661
42775
|
var SKIP_DIR_NAMES2 = {
|
|
42662
42776
|
".git": true,
|
|
42663
42777
|
".autorag": true,
|
|
@@ -42685,7 +42799,7 @@ function fsErrorCode(error) {
|
|
|
42685
42799
|
return typeof code === "string" ? code : undefined;
|
|
42686
42800
|
}
|
|
42687
42801
|
function isContained(childReal, parentReal) {
|
|
42688
|
-
const prefix = parentReal.endsWith(
|
|
42802
|
+
const prefix = parentReal.endsWith(sep10) ? parentReal : `${parentReal}${sep10}`;
|
|
42689
42803
|
return childReal === parentReal || childReal.startsWith(prefix);
|
|
42690
42804
|
}
|
|
42691
42805
|
function pinExcludedPath2(path) {
|
|
@@ -42698,14 +42812,14 @@ function pinExcludedPath2(path) {
|
|
|
42698
42812
|
}
|
|
42699
42813
|
function isExcluded2(absPath, excluded) {
|
|
42700
42814
|
for (const excludedPath of excluded) {
|
|
42701
|
-
const prefix = excludedPath.endsWith(
|
|
42815
|
+
const prefix = excludedPath.endsWith(sep10) ? excludedPath : `${excludedPath}${sep10}`;
|
|
42702
42816
|
if (absPath === excludedPath || absPath.startsWith(prefix))
|
|
42703
42817
|
return true;
|
|
42704
42818
|
}
|
|
42705
42819
|
return false;
|
|
42706
42820
|
}
|
|
42707
42821
|
function matchesEntry(context, absPath, name) {
|
|
42708
|
-
const haystackRaw = context.matchPath ? relative9(context.rootReal, absPath).split(
|
|
42822
|
+
const haystackRaw = context.matchPath ? relative9(context.rootReal, absPath).split(sep10).join("/") : name;
|
|
42709
42823
|
const haystack = context.matchCase ? haystackRaw : haystackRaw.toLowerCase();
|
|
42710
42824
|
const needle = context.matchCase ? context.query : context.query.toLowerCase();
|
|
42711
42825
|
return haystack.includes(needle);
|
|
@@ -42846,7 +42960,7 @@ async function filterFileNameSearchMatches(roots, candidates, excludePaths = [])
|
|
|
42846
42960
|
continue;
|
|
42847
42961
|
if (isExcluded2(real, excluded))
|
|
42848
42962
|
continue;
|
|
42849
|
-
const segments = real.split(
|
|
42963
|
+
const segments = real.split(sep10);
|
|
42850
42964
|
if (segments.some((segment) => SKIP_DIR_NAMES2[segment] === true))
|
|
42851
42965
|
continue;
|
|
42852
42966
|
matches.push({ path: real, type: stat.isDirectory() ? "folder" : "file" });
|
|
@@ -42970,6 +43084,16 @@ var evidenceInput = z4.strictObject({
|
|
|
42970
43084
|
sessionId: z4.string().trim().min(1),
|
|
42971
43085
|
resultNumber: z4.number().int().positive().optional()
|
|
42972
43086
|
});
|
|
43087
|
+
var convertInput = z4.strictObject({
|
|
43088
|
+
inputs: z4.array(z4.string().trim().min(1)).min(1).max(500),
|
|
43089
|
+
to: z4.string().trim().min(1),
|
|
43090
|
+
outputDir: z4.string().trim().min(1).optional(),
|
|
43091
|
+
recursive: z4.boolean().optional(),
|
|
43092
|
+
quality: z4.number().int().min(1).max(100).optional(),
|
|
43093
|
+
keepDates: z4.boolean().optional()
|
|
43094
|
+
});
|
|
43095
|
+
var convertFormatsInput = z4.strictObject({ from: z4.string().trim().min(1).optional() });
|
|
43096
|
+
var MAX_CONVERT_FILES = 200;
|
|
42973
43097
|
function exactGroupsFromScan(scan) {
|
|
42974
43098
|
const groups = new Map;
|
|
42975
43099
|
for (const file of scan.files) {
|
|
@@ -42991,6 +43115,26 @@ function jsonResult(value, isError = false) {
|
|
|
42991
43115
|
function toolError(code, message, extra = {}) {
|
|
42992
43116
|
return jsonResult({ ok: false, errorCode: code, message, ...extra }, true);
|
|
42993
43117
|
}
|
|
43118
|
+
function convertResult(details) {
|
|
43119
|
+
const { result, links } = details;
|
|
43120
|
+
const files = result.files.slice(0, MAX_CONVERT_FILES).map((file) => ({
|
|
43121
|
+
source: file.source,
|
|
43122
|
+
state: file.state,
|
|
43123
|
+
outputs: file.outputs,
|
|
43124
|
+
...file.error !== undefined ? { error: file.error } : {},
|
|
43125
|
+
...file.notice !== undefined ? { notice: file.notice } : {}
|
|
43126
|
+
}));
|
|
43127
|
+
const body = {
|
|
43128
|
+
target: result.target,
|
|
43129
|
+
converted: result.converted,
|
|
43130
|
+
skipped: result.skipped,
|
|
43131
|
+
failed: result.failed,
|
|
43132
|
+
files,
|
|
43133
|
+
...result.files.length > files.length ? { omittedFiles: result.files.length - files.length } : {},
|
|
43134
|
+
links
|
|
43135
|
+
};
|
|
43136
|
+
return result.converted === 0 && result.failed > 0 ? toolError("convert-failed", "No file was converted.", body) : jsonResult({ ok: true, ...body });
|
|
43137
|
+
}
|
|
42994
43138
|
function isToolEnabled(name, options) {
|
|
42995
43139
|
return options.tools === undefined || options.tools.includes(name);
|
|
42996
43140
|
}
|
|
@@ -43285,6 +43429,46 @@ function createAutoRAGMcpServer(lite, options = {}) {
|
|
|
43285
43429
|
return datasource === undefined ? toolError("datasource-not-found", `No configured datasource named ${datasourceId}.`, { datasourceId }) : jsonResult({ ok: true, datasource });
|
|
43286
43430
|
});
|
|
43287
43431
|
}
|
|
43432
|
+
const convertEnabled = lite.config.filee?.enabled !== false;
|
|
43433
|
+
if (convertEnabled && !readOnly && isToolEnabled("autorag.convert", options)) {
|
|
43434
|
+
server.registerTool("autorag.convert", {
|
|
43435
|
+
title: "Convert Files",
|
|
43436
|
+
description: "Convert local files to another format offline with Filee: documents, spreadsheets, slides, HWP/HWPX, images, e-books, archives, audio, video, data. Writes new files next to each source (or into outputDir) and never replaces an existing file: a colliding name gets a number added. Each output is compared with its source by Dupey, so the result says whether the copy would count as a duplicate. Use autorag.convert.formats when unsure what a source can become.",
|
|
43437
|
+
inputSchema: convertInput,
|
|
43438
|
+
outputSchema: objectOutput,
|
|
43439
|
+
annotations: { readOnlyHint: false, idempotentHint: false, destructiveHint: false, openWorldHint: false }
|
|
43440
|
+
}, async ({ inputs, to, outputDir, recursive, quality, keepDates }, ctx) => {
|
|
43441
|
+
try {
|
|
43442
|
+
const details = await lite.convertFiles({
|
|
43443
|
+
inputs,
|
|
43444
|
+
to,
|
|
43445
|
+
...outputDir !== undefined ? { outputDir } : {},
|
|
43446
|
+
...recursive !== undefined ? { recursive } : {},
|
|
43447
|
+
...quality !== undefined ? { quality } : {},
|
|
43448
|
+
...keepDates !== undefined ? { keepDates } : {}
|
|
43449
|
+
}, ctx.mcpReq.signal);
|
|
43450
|
+
return convertResult(details);
|
|
43451
|
+
} catch (error) {
|
|
43452
|
+
return toolError("convert-failed", error instanceof Error ? error.message : String(error));
|
|
43453
|
+
}
|
|
43454
|
+
});
|
|
43455
|
+
}
|
|
43456
|
+
if (convertEnabled && isToolEnabled("autorag.convert.formats", options)) {
|
|
43457
|
+
server.registerTool("autorag.convert.formats", {
|
|
43458
|
+
title: "List Convertible Formats",
|
|
43459
|
+
description: "List the file formats Filee knows, or what one source format (heic, docx, hwp, mkv) can be converted to, including targets that need an optional engine. Read-only.",
|
|
43460
|
+
inputSchema: convertFormatsInput,
|
|
43461
|
+
outputSchema: objectOutput,
|
|
43462
|
+
annotations: { readOnlyHint: true, openWorldHint: false }
|
|
43463
|
+
}, async ({ from }, ctx) => {
|
|
43464
|
+
try {
|
|
43465
|
+
const formats = await lite.listFileConversions(from, ctx.mcpReq.signal);
|
|
43466
|
+
return jsonResult({ ok: true, ...from !== undefined ? { from } : {}, formats });
|
|
43467
|
+
} catch (error) {
|
|
43468
|
+
return toolError("convert-failed", error instanceof Error ? error.message : String(error));
|
|
43469
|
+
}
|
|
43470
|
+
});
|
|
43471
|
+
}
|
|
43288
43472
|
if (!readOnly && isToolEnabled("autorag.report", options)) {
|
|
43289
43473
|
server.registerTool("autorag.report", {
|
|
43290
43474
|
title: "Persist AutoRAG Curated Report",
|
|
@@ -43372,9 +43556,11 @@ function isInvokedDirectly() {
|
|
|
43372
43556
|
async function main() {
|
|
43373
43557
|
const readOnly = parseBoolean(process.env.AUTORAG_MCP_READ_ONLY);
|
|
43374
43558
|
const tools = parseTools(process.env.AUTORAG_MCP_TOOLS);
|
|
43375
|
-
const lite =
|
|
43376
|
-
env: process.env,
|
|
43377
|
-
|
|
43559
|
+
const lite = createReloadingLite({
|
|
43560
|
+
create: () => createAutoRAGLite({ env: process.env, readOnly }),
|
|
43561
|
+
signature: () => configSignature(process.env),
|
|
43562
|
+
onError: (error) => process.stderr.write(`autorag-mcp: keeping the previous config, the new one failed: ${error.message}
|
|
43563
|
+
`)
|
|
43378
43564
|
});
|
|
43379
43565
|
const server = createAutoRAGMcpServer(lite, { readOnly, tools });
|
|
43380
43566
|
await server.connect(new StdioServerTransport);
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { AutoRAGLite } from "../core.js";
|
|
2
|
+
export interface ReloadingLiteOptions {
|
|
3
|
+
/** Build a runtime from the config as it is now. May throw (missing or unparsable config). */
|
|
4
|
+
readonly create: () => AutoRAGLite;
|
|
5
|
+
/** A cheap fingerprint of whatever `create` reads; a different value means the runtime is out of date. */
|
|
6
|
+
readonly signature: () => string;
|
|
7
|
+
/** A config that could not be loaded. Called once per distinct broken config, never on every call. */
|
|
8
|
+
readonly onError?: (error: Error) => void;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* A runtime that follows its config file. An MCP server is started once and then lives for as long as its host
|
|
12
|
+
* keeps it (Hermes keeps one per gateway, across chat sessions), so a config written after it started, which is
|
|
13
|
+
* exactly what first-time setup does, would otherwise never be seen: every search kept answering `index-not-ready`.
|
|
14
|
+
*
|
|
15
|
+
* Each access compares the config's signature (two `stat` calls) and rebuilds the runtime when it changed. The first
|
|
16
|
+
* build happens eagerly, so a bad config still fails at start. A later config that cannot be loaded (half-written,
|
|
17
|
+
* mistyped) keeps the last working runtime and is reported once, then retried when the file changes again.
|
|
18
|
+
*/
|
|
19
|
+
export declare function createReloadingLite(options: ReloadingLiteOptions): AutoRAGLite;
|
|
20
|
+
/**
|
|
21
|
+
* Fingerprint of the config file `createAutoRAGLite` would read: its inode, size and modification time, or `absent`.
|
|
22
|
+
* An explicit `AUTORAG_CONFIG`, else `<AUTORAG_HOME>/config.json` (and the legacy `autorag.config.json` beside the
|
|
23
|
+
* working directory) are covered, in the same order the resolver uses.
|
|
24
|
+
*/
|
|
25
|
+
export declare function configSignature(env?: NodeJS.ProcessEnv): string;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { RetrievalResult } from "../retrieval/types.js";
|
|
2
|
+
/** Ranked results of one MinSync query mode (`vector` or `sparse`), best first. */
|
|
3
|
+
export interface MinSyncChannelResults {
|
|
4
|
+
readonly channel: string;
|
|
5
|
+
readonly results: readonly RetrievalResult[];
|
|
6
|
+
}
|
|
7
|
+
export interface UnionChannelOptions {
|
|
8
|
+
/** Keep at most this many chunks after interleaving. Unset keeps the whole union. */
|
|
9
|
+
readonly limit?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Union of independently ranked MinSync channels, built for a reranker that decides the final order.
|
|
13
|
+
*
|
|
14
|
+
* Channels are interleaved by rank (rank 1 of each channel, then rank 2, ...) so a limit never cuts one
|
|
15
|
+
* channel's head to keep the other's tail, and a chunk found by several channels appears once, at its
|
|
16
|
+
* best interleaved position. Scores are left as the first channel reported them: dense and SPLADE scores
|
|
17
|
+
* are not comparable and nothing here ranks across them. Each chunk records which channels found it and
|
|
18
|
+
* at what 1-based rank (`minsyncChannels`, `minsyncChannelRanks`).
|
|
19
|
+
*
|
|
20
|
+
* Identity is `source` + `id`, the same key the cross-method merger uses, so two chunks of one document
|
|
21
|
+
* stay distinct while the same chunk reached by two channels collapses.
|
|
22
|
+
*/
|
|
23
|
+
export declare function unionChannelResults(channels: readonly MinSyncChannelResults[], options?: UnionChannelOptions): RetrievalResult[];
|
package/dist/mirror/sync.d.ts
CHANGED
|
@@ -187,4 +187,5 @@ export declare class MirrorWorkspaceError extends Error {
|
|
|
187
187
|
export declare function syncParsedMirrors(options: ParsedMirrorSyncOptions): Promise<ParsedMirrorSyncResult>;
|
|
188
188
|
/** Ask MinSync's read-only plan which current source revisions still need work. */
|
|
189
189
|
export declare function detectMirrorStaleness(options: ParsedMirrorSyncOptions): Promise<ParsedMirrorDiagnostic[]>;
|
|
190
|
+
/** A path is excluded when it equals an excluded path, sits under one, or matches an excluded glob. */
|
|
190
191
|
export declare function isPathExcluded(sourcePath: string, excluded: ReadonlySet<string> | undefined): boolean;
|