@autorag/librarian 2.10.3 → 2.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -3
- package/dist/agent/agent.d.ts +1 -1
- package/dist/agent/answer-guidelines.d.ts +17 -0
- package/dist/cli/commands/types.d.ts +2 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +897 -589
- package/dist/core.js +1613 -1324
- package/dist/datasource/sidecar/descriptors.d.ts +1 -0
- package/dist/datasource/skills/lazykatok/skill.d.ts +13 -0
- package/dist/everything/client.d.ts +5 -0
- package/dist/filesystem/path-exclusions.d.ts +2 -0
- package/dist/fsearch/client.d.ts +3 -2
- package/dist/fsearch/walk.d.ts +2 -0
- package/dist/index.js +1614 -1325
- package/dist/mcp/index.js +1614 -1325
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -2657,7 +2657,7 @@ function chatCrawler(id) {
|
|
|
2657
2657
|
hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
|
|
2658
2658
|
};
|
|
2659
2659
|
}
|
|
2660
|
-
var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, CRAWLER_DESCRIPTORS;
|
|
2660
|
+
var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, kakao, CRAWLER_DESCRIPTORS;
|
|
2661
2661
|
var init_descriptors = __esm(() => {
|
|
2662
2662
|
slack = {
|
|
2663
2663
|
id: "slack",
|
|
@@ -2688,10 +2688,32 @@ var init_descriptors = __esm(() => {
|
|
|
2688
2688
|
str(metadata.channelName) ?? str(metadata.channelId) ?? "unknown"
|
|
2689
2689
|
]
|
|
2690
2690
|
};
|
|
2691
|
+
kakao = {
|
|
2692
|
+
id: "kakao",
|
|
2693
|
+
keyExpression: "m.chunk_id",
|
|
2694
|
+
chunkId: (key) => key,
|
|
2695
|
+
fromClause: "chunks m",
|
|
2696
|
+
liveWhere: "length(trim(coalesce(m.text, ''))) > 0 and not (json_valid(m.text) and (json_extract(m.text, '$.feedType') is not null or json_extract(m.text, '$.csIP') is not null))",
|
|
2697
|
+
textExpression: "coalesce(m.text, '')",
|
|
2698
|
+
titleExpression: "m.chat_name",
|
|
2699
|
+
epochSecondsExpression: "unixepoch(m.started_at)",
|
|
2700
|
+
fts: { table: "chunks_fts", joinOn: "chunks_fts.rowid = m.rowid" },
|
|
2701
|
+
metadata: [
|
|
2702
|
+
{ sql: "m.chat_id", key: "chatId" },
|
|
2703
|
+
{ sql: "m.chat_name", key: "chatName" },
|
|
2704
|
+
{ sql: "m.sender_nickname", key: "senderNickname" },
|
|
2705
|
+
{ sql: "m.started_at", key: "startedAt" },
|
|
2706
|
+
{ sql: "m.ended_at", key: "endedAt" }
|
|
2707
|
+
],
|
|
2708
|
+
channelIdExpression: "m.chat_id",
|
|
2709
|
+
channelNameExpression: "m.chat_name",
|
|
2710
|
+
hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
|
|
2711
|
+
};
|
|
2691
2712
|
CRAWLER_DESCRIPTORS = {
|
|
2692
2713
|
slack,
|
|
2693
2714
|
telegram: chatCrawler("telegram"),
|
|
2694
|
-
whatsapp: chatCrawler("whatsapp")
|
|
2715
|
+
whatsapp: chatCrawler("whatsapp"),
|
|
2716
|
+
kakao
|
|
2695
2717
|
};
|
|
2696
2718
|
});
|
|
2697
2719
|
|
|
@@ -8054,6 +8076,504 @@ var init_skip = __esm(() => {
|
|
|
8054
8076
|
MINSYNC_METHOD_NAMES = new Set(["minsync", "hybrid", "bm25"]);
|
|
8055
8077
|
});
|
|
8056
8078
|
|
|
8079
|
+
// src/datasource/sidecar/fusion.ts
|
|
8080
|
+
function encodeVector(vector) {
|
|
8081
|
+
const floats = Float32Array.from(vector);
|
|
8082
|
+
return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
|
|
8083
|
+
}
|
|
8084
|
+
function decodeVector(blob) {
|
|
8085
|
+
if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
|
|
8086
|
+
return;
|
|
8087
|
+
return new Float32Array(new Uint8Array(blob).buffer);
|
|
8088
|
+
}
|
|
8089
|
+
function cosine2(a, b) {
|
|
8090
|
+
if (a.length === 0 || a.length !== b.length)
|
|
8091
|
+
return 0;
|
|
8092
|
+
let dot = 0;
|
|
8093
|
+
let normA = 0;
|
|
8094
|
+
let normB = 0;
|
|
8095
|
+
for (let index = 0;index < a.length; index += 1) {
|
|
8096
|
+
const x = a[index] ?? 0;
|
|
8097
|
+
const y = b[index] ?? 0;
|
|
8098
|
+
dot += x * y;
|
|
8099
|
+
normA += x * x;
|
|
8100
|
+
normB += y * y;
|
|
8101
|
+
}
|
|
8102
|
+
if (normA === 0 || normB === 0)
|
|
8103
|
+
return 0;
|
|
8104
|
+
return dot / Math.sqrt(normA * normB);
|
|
8105
|
+
}
|
|
8106
|
+
function topKByCosine(query, rows, k) {
|
|
8107
|
+
if (k <= 0)
|
|
8108
|
+
return [];
|
|
8109
|
+
const scored = [];
|
|
8110
|
+
for (const row of rows) {
|
|
8111
|
+
const score = cosine2(query, row.vector);
|
|
8112
|
+
if (score > 0)
|
|
8113
|
+
scored.push({ key: row.key, score });
|
|
8114
|
+
}
|
|
8115
|
+
scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
|
|
8116
|
+
return scored.slice(0, k);
|
|
8117
|
+
}
|
|
8118
|
+
function reciprocalRankFusion(lists) {
|
|
8119
|
+
const scores = new Map;
|
|
8120
|
+
for (const list of lists) {
|
|
8121
|
+
const seen = new Set;
|
|
8122
|
+
for (const [index, key] of list.entries()) {
|
|
8123
|
+
if (seen.has(key))
|
|
8124
|
+
continue;
|
|
8125
|
+
seen.add(key);
|
|
8126
|
+
scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K + index + 1));
|
|
8127
|
+
}
|
|
8128
|
+
}
|
|
8129
|
+
return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
|
|
8130
|
+
}
|
|
8131
|
+
var RRF_K = 60;
|
|
8132
|
+
|
|
8133
|
+
// src/datasource/sidecar/index.ts
|
|
8134
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
8135
|
+
import { mkdirSync as mkdirSync7, statSync } from "node:fs";
|
|
8136
|
+
import { dirname as dirname11 } from "node:path";
|
|
8137
|
+
import { DatabaseSync } from "node:sqlite";
|
|
8138
|
+
import { pathToFileURL } from "node:url";
|
|
8139
|
+
|
|
8140
|
+
class CrawlerSidecar {
|
|
8141
|
+
descriptor;
|
|
8142
|
+
options;
|
|
8143
|
+
select;
|
|
8144
|
+
where;
|
|
8145
|
+
opened;
|
|
8146
|
+
attached;
|
|
8147
|
+
lastSemantic;
|
|
8148
|
+
constructor(options) {
|
|
8149
|
+
this.options = options;
|
|
8150
|
+
const { descriptor } = options;
|
|
8151
|
+
this.descriptor = descriptor;
|
|
8152
|
+
const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
|
|
8153
|
+
this.select = [
|
|
8154
|
+
`${descriptor.keyExpression} as k`,
|
|
8155
|
+
`${descriptor.textExpression} as content`,
|
|
8156
|
+
`${descriptor.titleExpression ?? "null"} as title`,
|
|
8157
|
+
`${descriptor.epochSecondsExpression ?? "null"} as epoch`,
|
|
8158
|
+
...metadataColumns
|
|
8159
|
+
].join(", ");
|
|
8160
|
+
const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
|
|
8161
|
+
const allowlist = [
|
|
8162
|
+
...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
|
|
8163
|
+
...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
|
|
8164
|
+
];
|
|
8165
|
+
this.where = [
|
|
8166
|
+
descriptor.liveWhere,
|
|
8167
|
+
...scope ? [`${descriptor.scopeExpression} = ?`] : [],
|
|
8168
|
+
...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
|
|
8169
|
+
].join(" and ");
|
|
8170
|
+
}
|
|
8171
|
+
get db() {
|
|
8172
|
+
if (this.opened !== undefined)
|
|
8173
|
+
return this.opened;
|
|
8174
|
+
mkdirSync7(dirname11(this.options.sidecarDbPath), { recursive: true });
|
|
8175
|
+
const db = new DatabaseSync(this.options.sidecarDbPath);
|
|
8176
|
+
db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
8177
|
+
db.exec("pragma journal_mode = wal");
|
|
8178
|
+
db.exec(`
|
|
8179
|
+
create table if not exists sidecar_meta (k text primary key, v text not null);
|
|
8180
|
+
create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
|
|
8181
|
+
`);
|
|
8182
|
+
this.opened = db;
|
|
8183
|
+
return db;
|
|
8184
|
+
}
|
|
8185
|
+
attachCrawler() {
|
|
8186
|
+
let file;
|
|
8187
|
+
try {
|
|
8188
|
+
const { dev, ino } = statSync(this.options.crawlerDbPath);
|
|
8189
|
+
file = { dev, ino };
|
|
8190
|
+
} catch {
|
|
8191
|
+
this.detachCrawler();
|
|
8192
|
+
return false;
|
|
8193
|
+
}
|
|
8194
|
+
if (this.attached !== undefined) {
|
|
8195
|
+
if (this.attached.dev === file.dev && this.attached.ino === file.ino)
|
|
8196
|
+
return true;
|
|
8197
|
+
this.detachCrawler();
|
|
8198
|
+
}
|
|
8199
|
+
const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
|
|
8200
|
+
try {
|
|
8201
|
+
this.db.exec(`attach database '${crawlerUri}' as crawler`);
|
|
8202
|
+
} catch {
|
|
8203
|
+
return false;
|
|
8204
|
+
}
|
|
8205
|
+
const { descriptor } = this.options;
|
|
8206
|
+
try {
|
|
8207
|
+
this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
|
|
8208
|
+
this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
|
|
8209
|
+
} catch {
|
|
8210
|
+
this.db.exec("detach database crawler");
|
|
8211
|
+
return false;
|
|
8212
|
+
}
|
|
8213
|
+
this.attached = file;
|
|
8214
|
+
this.lastSemantic = undefined;
|
|
8215
|
+
return true;
|
|
8216
|
+
}
|
|
8217
|
+
detachCrawler() {
|
|
8218
|
+
if (this.attached === undefined)
|
|
8219
|
+
return;
|
|
8220
|
+
this.attached = undefined;
|
|
8221
|
+
this.lastSemantic = undefined;
|
|
8222
|
+
try {
|
|
8223
|
+
this.opened?.exec("detach database crawler");
|
|
8224
|
+
} catch {}
|
|
8225
|
+
}
|
|
8226
|
+
get semanticEnabled() {
|
|
8227
|
+
return this.options.embedder !== undefined;
|
|
8228
|
+
}
|
|
8229
|
+
close() {
|
|
8230
|
+
this.opened?.close();
|
|
8231
|
+
this.opened = undefined;
|
|
8232
|
+
this.attached = undefined;
|
|
8233
|
+
this.lastSemantic = undefined;
|
|
8234
|
+
}
|
|
8235
|
+
whereParams() {
|
|
8236
|
+
const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
|
|
8237
|
+
return [
|
|
8238
|
+
...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
|
|
8239
|
+
...channelIds,
|
|
8240
|
+
...channelNames
|
|
8241
|
+
];
|
|
8242
|
+
}
|
|
8243
|
+
searchLexical(query, topK) {
|
|
8244
|
+
const match = toMatchQuery(query);
|
|
8245
|
+
if (match === undefined || topK <= 0 || !this.attachCrawler())
|
|
8246
|
+
return [];
|
|
8247
|
+
const { descriptor } = this.options;
|
|
8248
|
+
const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
|
|
8249
|
+
from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
|
|
8250
|
+
where ${descriptor.fts.table} match ? and ${this.where}
|
|
8251
|
+
order by rank limit ?`).all(match, ...this.whereParams(), topK);
|
|
8252
|
+
return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
|
|
8253
|
+
}
|
|
8254
|
+
async searchSemantic(query, topK) {
|
|
8255
|
+
const trimmed = query.trim();
|
|
8256
|
+
if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
|
|
8257
|
+
return [];
|
|
8258
|
+
const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
|
|
8259
|
+
if (ranked.length === 0)
|
|
8260
|
+
return [];
|
|
8261
|
+
const live = this.hydrate(ranked.map((entry) => entry.key));
|
|
8262
|
+
const hits = [];
|
|
8263
|
+
for (const entry of ranked) {
|
|
8264
|
+
const row = live.get(entry.key);
|
|
8265
|
+
if (row !== undefined)
|
|
8266
|
+
hits.push(this.toHit(row, entry.score));
|
|
8267
|
+
if (hits.length >= topK)
|
|
8268
|
+
break;
|
|
8269
|
+
}
|
|
8270
|
+
return hits;
|
|
8271
|
+
}
|
|
8272
|
+
async rankSemantic(query, pool) {
|
|
8273
|
+
const now = Date.now();
|
|
8274
|
+
let cached = this.lastSemantic;
|
|
8275
|
+
if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
|
|
8276
|
+
const embedder = this.options.embedder;
|
|
8277
|
+
if (embedder === undefined)
|
|
8278
|
+
return [];
|
|
8279
|
+
cached = {
|
|
8280
|
+
query,
|
|
8281
|
+
at: now,
|
|
8282
|
+
vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
|
|
8283
|
+
return;
|
|
8284
|
+
}),
|
|
8285
|
+
ranked: undefined
|
|
8286
|
+
};
|
|
8287
|
+
this.lastSemantic = cached;
|
|
8288
|
+
}
|
|
8289
|
+
const vector = await cached.vector;
|
|
8290
|
+
if (vector === undefined) {
|
|
8291
|
+
if (this.lastSemantic === cached)
|
|
8292
|
+
this.lastSemantic = undefined;
|
|
8293
|
+
return [];
|
|
8294
|
+
}
|
|
8295
|
+
if (cached.ranked === undefined || cached.ranked.pool < pool) {
|
|
8296
|
+
const wide = pool * POOL_FACTOR;
|
|
8297
|
+
cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
|
|
8298
|
+
}
|
|
8299
|
+
return cached.ranked.keys.slice(0, pool);
|
|
8300
|
+
}
|
|
8301
|
+
async searchHybrid(query, topK) {
|
|
8302
|
+
if (topK <= 0)
|
|
8303
|
+
return [];
|
|
8304
|
+
const pool = topK * POOL_FACTOR;
|
|
8305
|
+
const lexical = this.searchLexical(query, pool);
|
|
8306
|
+
const semantic = await this.searchSemantic(query, pool);
|
|
8307
|
+
if (semantic.length === 0)
|
|
8308
|
+
return lexical.slice(0, topK);
|
|
8309
|
+
const byKey = new Map;
|
|
8310
|
+
for (const hit of [...semantic, ...lexical])
|
|
8311
|
+
byKey.set(hit.key, hit);
|
|
8312
|
+
const fused = [];
|
|
8313
|
+
for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
|
|
8314
|
+
const hit = byKey.get(entry.key);
|
|
8315
|
+
if (hit !== undefined)
|
|
8316
|
+
fused.push({ ...hit, score: entry.score });
|
|
8317
|
+
if (fused.length >= topK)
|
|
8318
|
+
break;
|
|
8319
|
+
}
|
|
8320
|
+
return fused;
|
|
8321
|
+
}
|
|
8322
|
+
async sync() {
|
|
8323
|
+
const { embedder } = this.options;
|
|
8324
|
+
if (embedder === undefined)
|
|
8325
|
+
return { ok: false, message: "no embedder configured" };
|
|
8326
|
+
if (!this.attachCrawler())
|
|
8327
|
+
return { ok: false, message: "crawler archive not available yet" };
|
|
8328
|
+
let identity;
|
|
8329
|
+
try {
|
|
8330
|
+
identity = await embedder.identity();
|
|
8331
|
+
} catch (error) {
|
|
8332
|
+
return { ok: false, message: errorMessage(error) };
|
|
8333
|
+
}
|
|
8334
|
+
this.adoptIdentity(identity);
|
|
8335
|
+
const pruned = this.prune();
|
|
8336
|
+
const pending = this.pendingKeys();
|
|
8337
|
+
const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
|
|
8338
|
+
const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
|
|
8339
|
+
const target = Math.min(pending.length, budget);
|
|
8340
|
+
const embed = async (rows) => {
|
|
8341
|
+
const vectors = await embedder.embed(rows.map((row) => row.input));
|
|
8342
|
+
if (vectors.length !== rows.length) {
|
|
8343
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
|
|
8344
|
+
}
|
|
8345
|
+
const wrong = vectors.find((vector) => vector.length !== identity.dimension);
|
|
8346
|
+
if (wrong !== undefined) {
|
|
8347
|
+
throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
|
|
8348
|
+
}
|
|
8349
|
+
return vectors;
|
|
8350
|
+
};
|
|
8351
|
+
let embedded = 0;
|
|
8352
|
+
let skipped = 0;
|
|
8353
|
+
for (let offset = 0;offset < target; offset += batchSize) {
|
|
8354
|
+
const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
|
|
8355
|
+
if (rows.length === 0)
|
|
8356
|
+
continue;
|
|
8357
|
+
try {
|
|
8358
|
+
this.store(rows, await embed(rows));
|
|
8359
|
+
embedded += rows.length;
|
|
8360
|
+
} catch (batchError) {
|
|
8361
|
+
const rejected = [];
|
|
8362
|
+
let lastError = batchError;
|
|
8363
|
+
for (const row of rows) {
|
|
8364
|
+
try {
|
|
8365
|
+
this.store([row], await embed([row]));
|
|
8366
|
+
embedded += 1;
|
|
8367
|
+
} catch (error) {
|
|
8368
|
+
rejected.push(row);
|
|
8369
|
+
lastError = error;
|
|
8370
|
+
}
|
|
8371
|
+
}
|
|
8372
|
+
if (rejected.length === 0)
|
|
8373
|
+
continue;
|
|
8374
|
+
if (embedded === 0)
|
|
8375
|
+
return { ok: false, message: errorMessage(lastError) };
|
|
8376
|
+
this.store(rejected, undefined);
|
|
8377
|
+
skipped += rejected.length;
|
|
8378
|
+
}
|
|
8379
|
+
}
|
|
8380
|
+
return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
|
|
8381
|
+
}
|
|
8382
|
+
adoptIdentity(identity) {
|
|
8383
|
+
const serialized = JSON.stringify(identity);
|
|
8384
|
+
const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
|
|
8385
|
+
if (current?.v === serialized)
|
|
8386
|
+
return;
|
|
8387
|
+
this.transaction(() => {
|
|
8388
|
+
this.db.exec("delete from sidecar_vec");
|
|
8389
|
+
this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
|
|
8390
|
+
});
|
|
8391
|
+
this.lastSemantic = undefined;
|
|
8392
|
+
}
|
|
8393
|
+
prune() {
|
|
8394
|
+
const { descriptor } = this.options;
|
|
8395
|
+
const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
|
|
8396
|
+
select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
|
|
8397
|
+
)`).run(...this.whereParams());
|
|
8398
|
+
if (Number(result.changes) > 0)
|
|
8399
|
+
this.lastSemantic = undefined;
|
|
8400
|
+
return Number(result.changes);
|
|
8401
|
+
}
|
|
8402
|
+
pendingKeys() {
|
|
8403
|
+
const { descriptor } = this.options;
|
|
8404
|
+
const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
|
|
8405
|
+
from ${descriptor.fromClause}
|
|
8406
|
+
left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
|
|
8407
|
+
where ${this.where}`).iterate(...this.whereParams());
|
|
8408
|
+
const pending = [];
|
|
8409
|
+
for (const row of rows) {
|
|
8410
|
+
if (row.stored_hash !== this.toRow(row).hash)
|
|
8411
|
+
pending.push(String(row.k));
|
|
8412
|
+
}
|
|
8413
|
+
return pending;
|
|
8414
|
+
}
|
|
8415
|
+
hydrate(keys) {
|
|
8416
|
+
const { descriptor } = this.options;
|
|
8417
|
+
const out = new Map;
|
|
8418
|
+
if (keys.length === 0)
|
|
8419
|
+
return out;
|
|
8420
|
+
const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
|
|
8421
|
+
where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
|
|
8422
|
+
const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
|
|
8423
|
+
for (const key of keys) {
|
|
8424
|
+
const row = byKey.get(key);
|
|
8425
|
+
if (row !== undefined)
|
|
8426
|
+
out.set(key, row);
|
|
8427
|
+
}
|
|
8428
|
+
return out;
|
|
8429
|
+
}
|
|
8430
|
+
store(rows, vectors) {
|
|
8431
|
+
const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
|
|
8432
|
+
this.transaction(() => {
|
|
8433
|
+
for (const [index, row] of rows.entries()) {
|
|
8434
|
+
upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
|
|
8435
|
+
}
|
|
8436
|
+
});
|
|
8437
|
+
this.lastSemantic = undefined;
|
|
8438
|
+
}
|
|
8439
|
+
*storedVectors() {
|
|
8440
|
+
const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
|
|
8441
|
+
for (const row of rows) {
|
|
8442
|
+
const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
|
|
8443
|
+
if (vector !== undefined)
|
|
8444
|
+
yield { key: String(row.message_key), vector };
|
|
8445
|
+
}
|
|
8446
|
+
}
|
|
8447
|
+
transaction(work) {
|
|
8448
|
+
this.db.exec("begin immediate");
|
|
8449
|
+
try {
|
|
8450
|
+
work();
|
|
8451
|
+
this.db.exec("commit");
|
|
8452
|
+
} catch (error) {
|
|
8453
|
+
this.db.exec("rollback");
|
|
8454
|
+
throw error;
|
|
8455
|
+
}
|
|
8456
|
+
}
|
|
8457
|
+
toRow(row) {
|
|
8458
|
+
const metadata = {};
|
|
8459
|
+
for (const [index, column] of this.options.descriptor.metadata.entries()) {
|
|
8460
|
+
const value = row[`m${index}`];
|
|
8461
|
+
if (typeof value === "string" && value.length > 0 || typeof value === "number")
|
|
8462
|
+
metadata[column.key] = String(value);
|
|
8463
|
+
}
|
|
8464
|
+
const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
|
|
8465
|
+
const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
|
|
8466
|
+
const content = String(row.content ?? "");
|
|
8467
|
+
const input = `${title ?? ""}
|
|
8468
|
+
${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
|
|
8469
|
+
return {
|
|
8470
|
+
key: String(row.k),
|
|
8471
|
+
content,
|
|
8472
|
+
title,
|
|
8473
|
+
epoch,
|
|
8474
|
+
metadata,
|
|
8475
|
+
input,
|
|
8476
|
+
hash: createHash6("sha1").update(input).digest("hex")
|
|
8477
|
+
};
|
|
8478
|
+
}
|
|
8479
|
+
toHit(row, score) {
|
|
8480
|
+
return {
|
|
8481
|
+
key: row.key,
|
|
8482
|
+
content: row.content,
|
|
8483
|
+
score,
|
|
8484
|
+
...row.title !== undefined ? { title: row.title } : {},
|
|
8485
|
+
...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
|
|
8486
|
+
metadata: row.metadata
|
|
8487
|
+
};
|
|
8488
|
+
}
|
|
8489
|
+
}
|
|
8490
|
+
function toMatchQuery(text) {
|
|
8491
|
+
const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
|
|
8492
|
+
return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
|
|
8493
|
+
}
|
|
8494
|
+
function errorMessage(error) {
|
|
8495
|
+
return error instanceof Error ? error.message : String(error);
|
|
8496
|
+
}
|
|
8497
|
+
var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
|
|
8498
|
+
var init_sidecar = () => {};
|
|
8499
|
+
|
|
8500
|
+
// src/datasource/sidecar/methods.ts
|
|
8501
|
+
function sidecarMethodName(datasourceId, mode) {
|
|
8502
|
+
return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
|
|
8503
|
+
}
|
|
8504
|
+
|
|
8505
|
+
class SidecarRetrievalMethod {
|
|
8506
|
+
options;
|
|
8507
|
+
constructor(options) {
|
|
8508
|
+
this.options = options;
|
|
8509
|
+
}
|
|
8510
|
+
describe() {
|
|
8511
|
+
const { datasourceId, skillType, mode, tags } = this.options;
|
|
8512
|
+
const { type, text, capabilities } = DESCRIPTIONS[mode];
|
|
8513
|
+
return {
|
|
8514
|
+
name: sidecarMethodName(datasourceId, mode),
|
|
8515
|
+
type,
|
|
8516
|
+
description: `${text} over ${skillType} via a read-only sidecar index`,
|
|
8517
|
+
status: "active",
|
|
8518
|
+
capabilities: [...capabilities, "scoped", "path-opaque-sources"],
|
|
8519
|
+
datasourceId,
|
|
8520
|
+
tags
|
|
8521
|
+
};
|
|
8522
|
+
}
|
|
8523
|
+
async retrieve(query, options) {
|
|
8524
|
+
const trimmed = query.trim();
|
|
8525
|
+
if (trimmed.length === 0)
|
|
8526
|
+
return [];
|
|
8527
|
+
const topK = options.topK ?? DEFAULT_TOP_K5;
|
|
8528
|
+
const { sidecar, mode } = this.options;
|
|
8529
|
+
const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
|
|
8530
|
+
const results = [];
|
|
8531
|
+
for (const hit of hits) {
|
|
8532
|
+
const result = this.toResult(hit, options);
|
|
8533
|
+
if (result !== undefined)
|
|
8534
|
+
results.push(result);
|
|
8535
|
+
}
|
|
8536
|
+
return results;
|
|
8537
|
+
}
|
|
8538
|
+
toResult(hit, options) {
|
|
8539
|
+
const { descriptor, datasourceId, instanceId, mode } = this.options;
|
|
8540
|
+
const chunkId = descriptor.chunkId(hit.key, hit.metadata);
|
|
8541
|
+
const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
|
|
8542
|
+
if (!matchesDatasourceScope(source, options.scope))
|
|
8543
|
+
return;
|
|
8544
|
+
const method = sidecarMethodName(datasourceId, mode);
|
|
8545
|
+
return {
|
|
8546
|
+
id: `${datasourceId}:${instanceId}:${chunkId}`,
|
|
8547
|
+
content: hit.content,
|
|
8548
|
+
source,
|
|
8549
|
+
score: hit.score,
|
|
8550
|
+
metadata: {
|
|
8551
|
+
...hit.metadata,
|
|
8552
|
+
method,
|
|
8553
|
+
datasourceId,
|
|
8554
|
+
instanceId,
|
|
8555
|
+
backend: "sidecar",
|
|
8556
|
+
...hit.title !== undefined ? { title: hit.title } : {},
|
|
8557
|
+
hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
|
|
8558
|
+
...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
|
|
8559
|
+
}
|
|
8560
|
+
};
|
|
8561
|
+
}
|
|
8562
|
+
}
|
|
8563
|
+
var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
|
|
8564
|
+
var init_methods4 = __esm(() => {
|
|
8565
|
+
init_scope2();
|
|
8566
|
+
DESCRIPTIONS = {
|
|
8567
|
+
bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
|
|
8568
|
+
semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
|
|
8569
|
+
hybrid: {
|
|
8570
|
+
type: "hybrid",
|
|
8571
|
+
text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
|
|
8572
|
+
capabilities: ["lexical", "semantic", "hybrid"]
|
|
8573
|
+
}
|
|
8574
|
+
};
|
|
8575
|
+
});
|
|
8576
|
+
|
|
8057
8577
|
// src/datasource/skills/lazykatok/skill.ts
|
|
8058
8578
|
class LazykatokSkill {
|
|
8059
8579
|
client;
|
|
@@ -8061,6 +8581,7 @@ class LazykatokSkill {
|
|
|
8061
8581
|
instances;
|
|
8062
8582
|
pollingIntervalMs;
|
|
8063
8583
|
tags;
|
|
8584
|
+
sidecar;
|
|
8064
8585
|
lastIndexedAt;
|
|
8065
8586
|
constructor(options) {
|
|
8066
8587
|
this.client = options.client;
|
|
@@ -8069,6 +8590,11 @@ class LazykatokSkill {
|
|
|
8069
8590
|
this.pollingIntervalMs = options.pollingIntervalMs ?? DEFAULT_POLLING_INTERVAL_MS3;
|
|
8070
8591
|
this.tags = options.tags ?? DEFAULT_KAKAO_TAGS2;
|
|
8071
8592
|
this.lastIndexedAt = options.lastIndexedAt;
|
|
8593
|
+
this.sidecar = options.sidecar === undefined ? undefined : new CrawlerSidecar({
|
|
8594
|
+
...options.sidecar,
|
|
8595
|
+
channelIds: options.channelIds ?? options.sidecar.channelIds ?? [],
|
|
8596
|
+
channelNames: options.channelNames ?? options.sidecar.channelNames ?? []
|
|
8597
|
+
});
|
|
8072
8598
|
}
|
|
8073
8599
|
describe() {
|
|
8074
8600
|
return {
|
|
@@ -8100,6 +8626,17 @@ class LazykatokSkill {
|
|
|
8100
8626
|
const sync = await this.client.sync(options?.signal);
|
|
8101
8627
|
if (!sync.ok)
|
|
8102
8628
|
return this.fail("datasource-index-failed", sync);
|
|
8629
|
+
if (this.sidecar !== undefined) {
|
|
8630
|
+
this.lastIndexedAt = Date.now();
|
|
8631
|
+
return {
|
|
8632
|
+
ok: true,
|
|
8633
|
+
instanceId: this.instanceId,
|
|
8634
|
+
skill: KAKAO_DATASOURCE_ID2,
|
|
8635
|
+
chunkCount: syncChunkCount(sync),
|
|
8636
|
+
indexedAt: this.lastIndexedAt,
|
|
8637
|
+
diagnostics: await this.syncSidecar()
|
|
8638
|
+
};
|
|
8639
|
+
}
|
|
8103
8640
|
const indexResult = await this.client.index(options?.signal);
|
|
8104
8641
|
if (!indexResult.ok)
|
|
8105
8642
|
return this.fail("datasource-index-failed", indexResult);
|
|
@@ -8123,12 +8660,75 @@ class LazykatokSkill {
|
|
|
8123
8660
|
});
|
|
8124
8661
|
}
|
|
8125
8662
|
}
|
|
8663
|
+
close() {
|
|
8664
|
+
this.sidecar?.close();
|
|
8665
|
+
}
|
|
8126
8666
|
retrievalMethods() {
|
|
8667
|
+
const cliBm25 = new LazykatokBm25Method({ client: this.client, instanceId: this.instanceId, tags: this.tags });
|
|
8668
|
+
const { sidecar } = this;
|
|
8669
|
+
if (sidecar === undefined) {
|
|
8670
|
+
return [
|
|
8671
|
+
cliBm25,
|
|
8672
|
+
new LazykatokSemanticMethod({ client: this.client, instanceId: this.instanceId, tags: this.tags })
|
|
8673
|
+
];
|
|
8674
|
+
}
|
|
8675
|
+
const shared = {
|
|
8676
|
+
sidecar,
|
|
8677
|
+
descriptor: sidecar.descriptor,
|
|
8678
|
+
datasourceId: KAKAO_DATASOURCE_ID2,
|
|
8679
|
+
skillType: KAKAO_SKILL_TYPE,
|
|
8680
|
+
instanceId: this.instanceId,
|
|
8681
|
+
tags: this.tags
|
|
8682
|
+
};
|
|
8683
|
+
const bm25 = new SidecarRetrievalMethod({ ...shared, mode: "bm25" });
|
|
8684
|
+
if (!sidecar.semanticEnabled)
|
|
8685
|
+
return [cliBm25, bm25];
|
|
8127
8686
|
return [
|
|
8128
|
-
|
|
8129
|
-
|
|
8687
|
+
cliBm25,
|
|
8688
|
+
bm25,
|
|
8689
|
+
new SidecarRetrievalMethod({ ...shared, mode: "semantic" }),
|
|
8690
|
+
new SidecarRetrievalMethod({ ...shared, mode: "hybrid" })
|
|
8130
8691
|
];
|
|
8131
8692
|
}
|
|
8693
|
+
async syncSidecar() {
|
|
8694
|
+
if (this.sidecar === undefined || !this.sidecar.semanticEnabled)
|
|
8695
|
+
return [];
|
|
8696
|
+
const base = { instanceId: this.instanceId, source: KAKAO_DATASOURCE_ID2 };
|
|
8697
|
+
let result;
|
|
8698
|
+
try {
|
|
8699
|
+
result = await this.sidecar.sync();
|
|
8700
|
+
} catch (error) {
|
|
8701
|
+
result = { ok: false, message: error instanceof Error ? error.message : String(error) };
|
|
8702
|
+
}
|
|
8703
|
+
if (!result.ok) {
|
|
8704
|
+
return [
|
|
8705
|
+
{
|
|
8706
|
+
code: "datasource-index-failed",
|
|
8707
|
+
severity: "warning",
|
|
8708
|
+
message: `semantic-unavailable: ${result.message}`.slice(0, 300),
|
|
8709
|
+
...base
|
|
8710
|
+
}
|
|
8711
|
+
];
|
|
8712
|
+
}
|
|
8713
|
+
const diagnostics = [];
|
|
8714
|
+
if (result.skipped > 0) {
|
|
8715
|
+
diagnostics.push({
|
|
8716
|
+
code: "datasource-index-partial",
|
|
8717
|
+
severity: "warning",
|
|
8718
|
+
message: `semantic index skipped ${result.skipped} message(s) the embedder rejected; they stay searchable by BM25 and are retried when edited`,
|
|
8719
|
+
...base
|
|
8720
|
+
});
|
|
8721
|
+
}
|
|
8722
|
+
if (result.remaining > 0) {
|
|
8723
|
+
diagnostics.push({
|
|
8724
|
+
code: "datasource-index-partial",
|
|
8725
|
+
severity: "info",
|
|
8726
|
+
message: `semantic index catching up: ${result.embedded} embedded this refresh, ${result.remaining} remaining; hybrid search uses BM25 for the rest until a later refresh`,
|
|
8727
|
+
...base
|
|
8728
|
+
});
|
|
8729
|
+
}
|
|
8730
|
+
return diagnostics;
|
|
8731
|
+
}
|
|
8132
8732
|
describeSources() {
|
|
8133
8733
|
return this.instances.map((instanceId) => ({
|
|
8134
8734
|
source: datasourceSourcePath(KAKAO_DATASOURCE_ID2, instanceId),
|
|
@@ -8202,10 +8802,22 @@ class LazykatokSkill {
|
|
|
8202
8802
|
};
|
|
8203
8803
|
}
|
|
8204
8804
|
}
|
|
8805
|
+
function syncChunkCount(sync) {
|
|
8806
|
+
if (!("data" in sync) || typeof sync.data !== "object" || sync.data === null)
|
|
8807
|
+
return 0;
|
|
8808
|
+
const meta = "metadata" in sync.data && typeof sync.data.metadata === "object" && sync.data.metadata !== null ? sync.data.metadata : undefined;
|
|
8809
|
+
if (typeof meta?.chunks === "number")
|
|
8810
|
+
return meta.chunks;
|
|
8811
|
+
if ("messageCount" in sync.data && typeof sync.data.messageCount === "number")
|
|
8812
|
+
return sync.data.messageCount;
|
|
8813
|
+
return 0;
|
|
8814
|
+
}
|
|
8205
8815
|
var KAKAO_DATASOURCE_ID2 = "kakao", KAKAO_SKILL_TYPE = "kakaotalk", DEFAULT_INSTANCE_ID4 = "default", DEFAULT_POLLING_INTERVAL_MS3, DEFAULT_KAKAO_TAGS2;
|
|
8206
8816
|
var init_skill7 = __esm(() => {
|
|
8207
8817
|
init_skip();
|
|
8208
8818
|
init_scope2();
|
|
8819
|
+
init_sidecar();
|
|
8820
|
+
init_methods4();
|
|
8209
8821
|
init_methods3();
|
|
8210
8822
|
DEFAULT_POLLING_INTERVAL_MS3 = 15 * 60 * 1000;
|
|
8211
8823
|
DEFAULT_KAKAO_TAGS2 = ["kakaotalk", "personal", "pii"];
|
|
@@ -8221,7 +8833,7 @@ var init_lazykatok = __esm(() => {
|
|
|
8221
8833
|
|
|
8222
8834
|
// src/datasource/skills/mail-export/connector.ts
|
|
8223
8835
|
import { readdir as readdir2, readFile as readFile4, stat as stat3 } from "node:fs/promises";
|
|
8224
|
-
import { basename as basename5, dirname as
|
|
8836
|
+
import { basename as basename5, dirname as dirname12, extname as extname3, join as join14 } from "node:path";
|
|
8225
8837
|
import { simpleParser as simpleParser2 } from "mailparser";
|
|
8226
8838
|
|
|
8227
8839
|
class MailExportConnector {
|
|
@@ -8282,7 +8894,7 @@ class MailExportConnector {
|
|
|
8282
8894
|
continue;
|
|
8283
8895
|
}
|
|
8284
8896
|
const stem = basename5(file, extname3(file));
|
|
8285
|
-
const mailboxDir = basename5(
|
|
8897
|
+
const mailboxDir = basename5(dirname12(file));
|
|
8286
8898
|
const hierarchy = ["mailboxes", mailboxDir, stem];
|
|
8287
8899
|
const extension = extname3(file).toLowerCase();
|
|
8288
8900
|
const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
|
|
@@ -8392,14 +9004,14 @@ var init_mail_export = __esm(() => {
|
|
|
8392
9004
|
});
|
|
8393
9005
|
|
|
8394
9006
|
// src/datasource/skills/mailcrawl/config.ts
|
|
8395
|
-
import { chmodSync, mkdirSync as
|
|
9007
|
+
import { chmodSync, mkdirSync as mkdirSync8 } from "node:fs";
|
|
8396
9008
|
function validateMailcrawlInstanceId(instanceId) {
|
|
8397
9009
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(instanceId)) {
|
|
8398
9010
|
throw new Error("mailcrawl instanceId must be a safe single path segment");
|
|
8399
9011
|
}
|
|
8400
9012
|
}
|
|
8401
9013
|
function ensurePrivateMailcrawlDataDir(path) {
|
|
8402
|
-
|
|
9014
|
+
mkdirSync8(path, { recursive: true, mode: 448 });
|
|
8403
9015
|
if (process.platform !== "win32")
|
|
8404
9016
|
chmodSync(path, 448);
|
|
8405
9017
|
}
|
|
@@ -8827,7 +9439,7 @@ class MailcrawlMethod {
|
|
|
8827
9439
|
}
|
|
8828
9440
|
}
|
|
8829
9441
|
var modes, MAILCRAWL_MODES;
|
|
8830
|
-
var
|
|
9442
|
+
var init_methods5 = __esm(() => {
|
|
8831
9443
|
init_errors2();
|
|
8832
9444
|
init_scope2();
|
|
8833
9445
|
modes = ["bm25", "semantic", "hybrid"];
|
|
@@ -8977,515 +9589,17 @@ var init_skill9 = __esm(() => {
|
|
|
8977
9589
|
init_scope2();
|
|
8978
9590
|
init_client5();
|
|
8979
9591
|
init_config3();
|
|
8980
|
-
|
|
9592
|
+
init_methods5();
|
|
8981
9593
|
TAGS = ["mailcrawl", "email", "pii"];
|
|
8982
9594
|
});
|
|
8983
9595
|
|
|
8984
9596
|
// src/datasource/skills/mailcrawl/index.ts
|
|
8985
9597
|
var init_mailcrawl = __esm(() => {
|
|
8986
9598
|
init_client5();
|
|
8987
|
-
|
|
9599
|
+
init_methods5();
|
|
8988
9600
|
init_skill9();
|
|
8989
9601
|
});
|
|
8990
9602
|
|
|
8991
|
-
// src/datasource/sidecar/fusion.ts
|
|
8992
|
-
function encodeVector(vector) {
|
|
8993
|
-
const floats = Float32Array.from(vector);
|
|
8994
|
-
return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
|
|
8995
|
-
}
|
|
8996
|
-
function decodeVector(blob) {
|
|
8997
|
-
if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
|
|
8998
|
-
return;
|
|
8999
|
-
return new Float32Array(new Uint8Array(blob).buffer);
|
|
9000
|
-
}
|
|
9001
|
-
function cosine2(a, b) {
|
|
9002
|
-
if (a.length === 0 || a.length !== b.length)
|
|
9003
|
-
return 0;
|
|
9004
|
-
let dot = 0;
|
|
9005
|
-
let normA = 0;
|
|
9006
|
-
let normB = 0;
|
|
9007
|
-
for (let index = 0;index < a.length; index += 1) {
|
|
9008
|
-
const x = a[index] ?? 0;
|
|
9009
|
-
const y = b[index] ?? 0;
|
|
9010
|
-
dot += x * y;
|
|
9011
|
-
normA += x * x;
|
|
9012
|
-
normB += y * y;
|
|
9013
|
-
}
|
|
9014
|
-
if (normA === 0 || normB === 0)
|
|
9015
|
-
return 0;
|
|
9016
|
-
return dot / Math.sqrt(normA * normB);
|
|
9017
|
-
}
|
|
9018
|
-
function topKByCosine(query, rows, k) {
|
|
9019
|
-
if (k <= 0)
|
|
9020
|
-
return [];
|
|
9021
|
-
const scored = [];
|
|
9022
|
-
for (const row of rows) {
|
|
9023
|
-
const score = cosine2(query, row.vector);
|
|
9024
|
-
if (score > 0)
|
|
9025
|
-
scored.push({ key: row.key, score });
|
|
9026
|
-
}
|
|
9027
|
-
scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
|
|
9028
|
-
return scored.slice(0, k);
|
|
9029
|
-
}
|
|
9030
|
-
function reciprocalRankFusion(lists) {
|
|
9031
|
-
const scores = new Map;
|
|
9032
|
-
for (const list of lists) {
|
|
9033
|
-
const seen = new Set;
|
|
9034
|
-
for (const [index, key] of list.entries()) {
|
|
9035
|
-
if (seen.has(key))
|
|
9036
|
-
continue;
|
|
9037
|
-
seen.add(key);
|
|
9038
|
-
scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K + index + 1));
|
|
9039
|
-
}
|
|
9040
|
-
}
|
|
9041
|
-
return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
|
|
9042
|
-
}
|
|
9043
|
-
var RRF_K = 60;
|
|
9044
|
-
|
|
9045
|
-
// src/datasource/sidecar/index.ts
|
|
9046
|
-
import { createHash as createHash6 } from "node:crypto";
|
|
9047
|
-
import { mkdirSync as mkdirSync8, statSync } from "node:fs";
|
|
9048
|
-
import { dirname as dirname12 } from "node:path";
|
|
9049
|
-
import { DatabaseSync } from "node:sqlite";
|
|
9050
|
-
import { pathToFileURL } from "node:url";
|
|
9051
|
-
|
|
9052
|
-
class CrawlerSidecar {
|
|
9053
|
-
descriptor;
|
|
9054
|
-
options;
|
|
9055
|
-
select;
|
|
9056
|
-
where;
|
|
9057
|
-
opened;
|
|
9058
|
-
attached;
|
|
9059
|
-
lastSemantic;
|
|
9060
|
-
constructor(options) {
|
|
9061
|
-
this.options = options;
|
|
9062
|
-
const { descriptor } = options;
|
|
9063
|
-
this.descriptor = descriptor;
|
|
9064
|
-
const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
|
|
9065
|
-
this.select = [
|
|
9066
|
-
`${descriptor.keyExpression} as k`,
|
|
9067
|
-
`${descriptor.textExpression} as content`,
|
|
9068
|
-
`${descriptor.titleExpression ?? "null"} as title`,
|
|
9069
|
-
`${descriptor.epochSecondsExpression ?? "null"} as epoch`,
|
|
9070
|
-
...metadataColumns
|
|
9071
|
-
].join(", ");
|
|
9072
|
-
const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
|
|
9073
|
-
const allowlist = [
|
|
9074
|
-
...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
|
|
9075
|
-
...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
|
|
9076
|
-
];
|
|
9077
|
-
this.where = [
|
|
9078
|
-
descriptor.liveWhere,
|
|
9079
|
-
...scope ? [`${descriptor.scopeExpression} = ?`] : [],
|
|
9080
|
-
...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
|
|
9081
|
-
].join(" and ");
|
|
9082
|
-
}
|
|
9083
|
-
get db() {
|
|
9084
|
-
if (this.opened !== undefined)
|
|
9085
|
-
return this.opened;
|
|
9086
|
-
mkdirSync8(dirname12(this.options.sidecarDbPath), { recursive: true });
|
|
9087
|
-
const db = new DatabaseSync(this.options.sidecarDbPath);
|
|
9088
|
-
db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
9089
|
-
db.exec("pragma journal_mode = wal");
|
|
9090
|
-
db.exec(`
|
|
9091
|
-
create table if not exists sidecar_meta (k text primary key, v text not null);
|
|
9092
|
-
create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
|
|
9093
|
-
`);
|
|
9094
|
-
this.opened = db;
|
|
9095
|
-
return db;
|
|
9096
|
-
}
|
|
9097
|
-
attachCrawler() {
|
|
9098
|
-
let file;
|
|
9099
|
-
try {
|
|
9100
|
-
const { dev, ino } = statSync(this.options.crawlerDbPath);
|
|
9101
|
-
file = { dev, ino };
|
|
9102
|
-
} catch {
|
|
9103
|
-
this.detachCrawler();
|
|
9104
|
-
return false;
|
|
9105
|
-
}
|
|
9106
|
-
if (this.attached !== undefined) {
|
|
9107
|
-
if (this.attached.dev === file.dev && this.attached.ino === file.ino)
|
|
9108
|
-
return true;
|
|
9109
|
-
this.detachCrawler();
|
|
9110
|
-
}
|
|
9111
|
-
const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
|
|
9112
|
-
try {
|
|
9113
|
-
this.db.exec(`attach database '${crawlerUri}' as crawler`);
|
|
9114
|
-
} catch {
|
|
9115
|
-
return false;
|
|
9116
|
-
}
|
|
9117
|
-
const { descriptor } = this.options;
|
|
9118
|
-
try {
|
|
9119
|
-
this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
|
|
9120
|
-
this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
|
|
9121
|
-
} catch {
|
|
9122
|
-
this.db.exec("detach database crawler");
|
|
9123
|
-
return false;
|
|
9124
|
-
}
|
|
9125
|
-
this.attached = file;
|
|
9126
|
-
this.lastSemantic = undefined;
|
|
9127
|
-
return true;
|
|
9128
|
-
}
|
|
9129
|
-
detachCrawler() {
|
|
9130
|
-
if (this.attached === undefined)
|
|
9131
|
-
return;
|
|
9132
|
-
this.attached = undefined;
|
|
9133
|
-
this.lastSemantic = undefined;
|
|
9134
|
-
try {
|
|
9135
|
-
this.opened?.exec("detach database crawler");
|
|
9136
|
-
} catch {}
|
|
9137
|
-
}
|
|
9138
|
-
get semanticEnabled() {
|
|
9139
|
-
return this.options.embedder !== undefined;
|
|
9140
|
-
}
|
|
9141
|
-
close() {
|
|
9142
|
-
this.opened?.close();
|
|
9143
|
-
this.opened = undefined;
|
|
9144
|
-
this.attached = undefined;
|
|
9145
|
-
this.lastSemantic = undefined;
|
|
9146
|
-
}
|
|
9147
|
-
whereParams() {
|
|
9148
|
-
const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
|
|
9149
|
-
return [
|
|
9150
|
-
...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
|
|
9151
|
-
...channelIds,
|
|
9152
|
-
...channelNames
|
|
9153
|
-
];
|
|
9154
|
-
}
|
|
9155
|
-
searchLexical(query, topK) {
|
|
9156
|
-
const match = toMatchQuery(query);
|
|
9157
|
-
if (match === undefined || topK <= 0 || !this.attachCrawler())
|
|
9158
|
-
return [];
|
|
9159
|
-
const { descriptor } = this.options;
|
|
9160
|
-
const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
|
|
9161
|
-
from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
|
|
9162
|
-
where ${descriptor.fts.table} match ? and ${this.where}
|
|
9163
|
-
order by rank limit ?`).all(match, ...this.whereParams(), topK);
|
|
9164
|
-
return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
|
|
9165
|
-
}
|
|
9166
|
-
async searchSemantic(query, topK) {
|
|
9167
|
-
const trimmed = query.trim();
|
|
9168
|
-
if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
|
|
9169
|
-
return [];
|
|
9170
|
-
const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
|
|
9171
|
-
if (ranked.length === 0)
|
|
9172
|
-
return [];
|
|
9173
|
-
const live = this.hydrate(ranked.map((entry) => entry.key));
|
|
9174
|
-
const hits = [];
|
|
9175
|
-
for (const entry of ranked) {
|
|
9176
|
-
const row = live.get(entry.key);
|
|
9177
|
-
if (row !== undefined)
|
|
9178
|
-
hits.push(this.toHit(row, entry.score));
|
|
9179
|
-
if (hits.length >= topK)
|
|
9180
|
-
break;
|
|
9181
|
-
}
|
|
9182
|
-
return hits;
|
|
9183
|
-
}
|
|
9184
|
-
async rankSemantic(query, pool) {
|
|
9185
|
-
const now = Date.now();
|
|
9186
|
-
let cached = this.lastSemantic;
|
|
9187
|
-
if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
|
|
9188
|
-
const embedder = this.options.embedder;
|
|
9189
|
-
if (embedder === undefined)
|
|
9190
|
-
return [];
|
|
9191
|
-
cached = {
|
|
9192
|
-
query,
|
|
9193
|
-
at: now,
|
|
9194
|
-
vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
|
|
9195
|
-
return;
|
|
9196
|
-
}),
|
|
9197
|
-
ranked: undefined
|
|
9198
|
-
};
|
|
9199
|
-
this.lastSemantic = cached;
|
|
9200
|
-
}
|
|
9201
|
-
const vector = await cached.vector;
|
|
9202
|
-
if (vector === undefined) {
|
|
9203
|
-
if (this.lastSemantic === cached)
|
|
9204
|
-
this.lastSemantic = undefined;
|
|
9205
|
-
return [];
|
|
9206
|
-
}
|
|
9207
|
-
if (cached.ranked === undefined || cached.ranked.pool < pool) {
|
|
9208
|
-
const wide = pool * POOL_FACTOR;
|
|
9209
|
-
cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
|
|
9210
|
-
}
|
|
9211
|
-
return cached.ranked.keys.slice(0, pool);
|
|
9212
|
-
}
|
|
9213
|
-
async searchHybrid(query, topK) {
|
|
9214
|
-
if (topK <= 0)
|
|
9215
|
-
return [];
|
|
9216
|
-
const pool = topK * POOL_FACTOR;
|
|
9217
|
-
const lexical = this.searchLexical(query, pool);
|
|
9218
|
-
const semantic = await this.searchSemantic(query, pool);
|
|
9219
|
-
if (semantic.length === 0)
|
|
9220
|
-
return lexical.slice(0, topK);
|
|
9221
|
-
const byKey = new Map;
|
|
9222
|
-
for (const hit of [...semantic, ...lexical])
|
|
9223
|
-
byKey.set(hit.key, hit);
|
|
9224
|
-
const fused = [];
|
|
9225
|
-
for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
|
|
9226
|
-
const hit = byKey.get(entry.key);
|
|
9227
|
-
if (hit !== undefined)
|
|
9228
|
-
fused.push({ ...hit, score: entry.score });
|
|
9229
|
-
if (fused.length >= topK)
|
|
9230
|
-
break;
|
|
9231
|
-
}
|
|
9232
|
-
return fused;
|
|
9233
|
-
}
|
|
9234
|
-
async sync() {
|
|
9235
|
-
const { embedder } = this.options;
|
|
9236
|
-
if (embedder === undefined)
|
|
9237
|
-
return { ok: false, message: "no embedder configured" };
|
|
9238
|
-
if (!this.attachCrawler())
|
|
9239
|
-
return { ok: false, message: "crawler archive not available yet" };
|
|
9240
|
-
let identity;
|
|
9241
|
-
try {
|
|
9242
|
-
identity = await embedder.identity();
|
|
9243
|
-
} catch (error) {
|
|
9244
|
-
return { ok: false, message: errorMessage(error) };
|
|
9245
|
-
}
|
|
9246
|
-
this.adoptIdentity(identity);
|
|
9247
|
-
const pruned = this.prune();
|
|
9248
|
-
const pending = this.pendingKeys();
|
|
9249
|
-
const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
|
|
9250
|
-
const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
|
|
9251
|
-
const target = Math.min(pending.length, budget);
|
|
9252
|
-
const embed = async (rows) => {
|
|
9253
|
-
const vectors = await embedder.embed(rows.map((row) => row.input));
|
|
9254
|
-
if (vectors.length !== rows.length) {
|
|
9255
|
-
throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
|
|
9256
|
-
}
|
|
9257
|
-
const wrong = vectors.find((vector) => vector.length !== identity.dimension);
|
|
9258
|
-
if (wrong !== undefined) {
|
|
9259
|
-
throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
|
|
9260
|
-
}
|
|
9261
|
-
return vectors;
|
|
9262
|
-
};
|
|
9263
|
-
let embedded = 0;
|
|
9264
|
-
let skipped = 0;
|
|
9265
|
-
for (let offset = 0;offset < target; offset += batchSize) {
|
|
9266
|
-
const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
|
|
9267
|
-
if (rows.length === 0)
|
|
9268
|
-
continue;
|
|
9269
|
-
try {
|
|
9270
|
-
this.store(rows, await embed(rows));
|
|
9271
|
-
embedded += rows.length;
|
|
9272
|
-
} catch (batchError) {
|
|
9273
|
-
const rejected = [];
|
|
9274
|
-
let lastError = batchError;
|
|
9275
|
-
for (const row of rows) {
|
|
9276
|
-
try {
|
|
9277
|
-
this.store([row], await embed([row]));
|
|
9278
|
-
embedded += 1;
|
|
9279
|
-
} catch (error) {
|
|
9280
|
-
rejected.push(row);
|
|
9281
|
-
lastError = error;
|
|
9282
|
-
}
|
|
9283
|
-
}
|
|
9284
|
-
if (rejected.length === 0)
|
|
9285
|
-
continue;
|
|
9286
|
-
if (embedded === 0)
|
|
9287
|
-
return { ok: false, message: errorMessage(lastError) };
|
|
9288
|
-
this.store(rejected, undefined);
|
|
9289
|
-
skipped += rejected.length;
|
|
9290
|
-
}
|
|
9291
|
-
}
|
|
9292
|
-
return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
|
|
9293
|
-
}
|
|
9294
|
-
adoptIdentity(identity) {
|
|
9295
|
-
const serialized = JSON.stringify(identity);
|
|
9296
|
-
const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
|
|
9297
|
-
if (current?.v === serialized)
|
|
9298
|
-
return;
|
|
9299
|
-
this.transaction(() => {
|
|
9300
|
-
this.db.exec("delete from sidecar_vec");
|
|
9301
|
-
this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
|
|
9302
|
-
});
|
|
9303
|
-
this.lastSemantic = undefined;
|
|
9304
|
-
}
|
|
9305
|
-
prune() {
|
|
9306
|
-
const { descriptor } = this.options;
|
|
9307
|
-
const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
|
|
9308
|
-
select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
|
|
9309
|
-
)`).run(...this.whereParams());
|
|
9310
|
-
if (Number(result.changes) > 0)
|
|
9311
|
-
this.lastSemantic = undefined;
|
|
9312
|
-
return Number(result.changes);
|
|
9313
|
-
}
|
|
9314
|
-
pendingKeys() {
|
|
9315
|
-
const { descriptor } = this.options;
|
|
9316
|
-
const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
|
|
9317
|
-
from ${descriptor.fromClause}
|
|
9318
|
-
left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
|
|
9319
|
-
where ${this.where}`).iterate(...this.whereParams());
|
|
9320
|
-
const pending = [];
|
|
9321
|
-
for (const row of rows) {
|
|
9322
|
-
if (row.stored_hash !== this.toRow(row).hash)
|
|
9323
|
-
pending.push(String(row.k));
|
|
9324
|
-
}
|
|
9325
|
-
return pending;
|
|
9326
|
-
}
|
|
9327
|
-
hydrate(keys) {
|
|
9328
|
-
const { descriptor } = this.options;
|
|
9329
|
-
const out = new Map;
|
|
9330
|
-
if (keys.length === 0)
|
|
9331
|
-
return out;
|
|
9332
|
-
const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
|
|
9333
|
-
where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
|
|
9334
|
-
const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
|
|
9335
|
-
for (const key of keys) {
|
|
9336
|
-
const row = byKey.get(key);
|
|
9337
|
-
if (row !== undefined)
|
|
9338
|
-
out.set(key, row);
|
|
9339
|
-
}
|
|
9340
|
-
return out;
|
|
9341
|
-
}
|
|
9342
|
-
store(rows, vectors) {
|
|
9343
|
-
const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
|
|
9344
|
-
this.transaction(() => {
|
|
9345
|
-
for (const [index, row] of rows.entries()) {
|
|
9346
|
-
upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
|
|
9347
|
-
}
|
|
9348
|
-
});
|
|
9349
|
-
this.lastSemantic = undefined;
|
|
9350
|
-
}
|
|
9351
|
-
*storedVectors() {
|
|
9352
|
-
const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
|
|
9353
|
-
for (const row of rows) {
|
|
9354
|
-
const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
|
|
9355
|
-
if (vector !== undefined)
|
|
9356
|
-
yield { key: String(row.message_key), vector };
|
|
9357
|
-
}
|
|
9358
|
-
}
|
|
9359
|
-
transaction(work) {
|
|
9360
|
-
this.db.exec("begin immediate");
|
|
9361
|
-
try {
|
|
9362
|
-
work();
|
|
9363
|
-
this.db.exec("commit");
|
|
9364
|
-
} catch (error) {
|
|
9365
|
-
this.db.exec("rollback");
|
|
9366
|
-
throw error;
|
|
9367
|
-
}
|
|
9368
|
-
}
|
|
9369
|
-
toRow(row) {
|
|
9370
|
-
const metadata = {};
|
|
9371
|
-
for (const [index, column] of this.options.descriptor.metadata.entries()) {
|
|
9372
|
-
const value = row[`m${index}`];
|
|
9373
|
-
if (typeof value === "string" && value.length > 0 || typeof value === "number")
|
|
9374
|
-
metadata[column.key] = String(value);
|
|
9375
|
-
}
|
|
9376
|
-
const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
|
|
9377
|
-
const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
|
|
9378
|
-
const content = String(row.content ?? "");
|
|
9379
|
-
const input = `${title ?? ""}
|
|
9380
|
-
${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
|
|
9381
|
-
return {
|
|
9382
|
-
key: String(row.k),
|
|
9383
|
-
content,
|
|
9384
|
-
title,
|
|
9385
|
-
epoch,
|
|
9386
|
-
metadata,
|
|
9387
|
-
input,
|
|
9388
|
-
hash: createHash6("sha1").update(input).digest("hex")
|
|
9389
|
-
};
|
|
9390
|
-
}
|
|
9391
|
-
toHit(row, score) {
|
|
9392
|
-
return {
|
|
9393
|
-
key: row.key,
|
|
9394
|
-
content: row.content,
|
|
9395
|
-
score,
|
|
9396
|
-
...row.title !== undefined ? { title: row.title } : {},
|
|
9397
|
-
...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
|
|
9398
|
-
metadata: row.metadata
|
|
9399
|
-
};
|
|
9400
|
-
}
|
|
9401
|
-
}
|
|
9402
|
-
function toMatchQuery(text) {
|
|
9403
|
-
const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
|
|
9404
|
-
return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
|
|
9405
|
-
}
|
|
9406
|
-
function errorMessage(error) {
|
|
9407
|
-
return error instanceof Error ? error.message : String(error);
|
|
9408
|
-
}
|
|
9409
|
-
var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
|
|
9410
|
-
var init_sidecar = () => {};
|
|
9411
|
-
|
|
9412
|
-
// src/datasource/sidecar/methods.ts
|
|
9413
|
-
function sidecarMethodName(datasourceId, mode) {
|
|
9414
|
-
return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
|
|
9415
|
-
}
|
|
9416
|
-
|
|
9417
|
-
class SidecarRetrievalMethod {
|
|
9418
|
-
options;
|
|
9419
|
-
constructor(options) {
|
|
9420
|
-
this.options = options;
|
|
9421
|
-
}
|
|
9422
|
-
describe() {
|
|
9423
|
-
const { datasourceId, skillType, mode, tags } = this.options;
|
|
9424
|
-
const { type, text, capabilities } = DESCRIPTIONS[mode];
|
|
9425
|
-
return {
|
|
9426
|
-
name: sidecarMethodName(datasourceId, mode),
|
|
9427
|
-
type,
|
|
9428
|
-
description: `${text} over ${skillType} via a read-only sidecar index`,
|
|
9429
|
-
status: "active",
|
|
9430
|
-
capabilities: [...capabilities, "scoped", "path-opaque-sources"],
|
|
9431
|
-
datasourceId,
|
|
9432
|
-
tags
|
|
9433
|
-
};
|
|
9434
|
-
}
|
|
9435
|
-
async retrieve(query, options) {
|
|
9436
|
-
const trimmed = query.trim();
|
|
9437
|
-
if (trimmed.length === 0)
|
|
9438
|
-
return [];
|
|
9439
|
-
const topK = options.topK ?? DEFAULT_TOP_K5;
|
|
9440
|
-
const { sidecar, mode } = this.options;
|
|
9441
|
-
const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
|
|
9442
|
-
const results = [];
|
|
9443
|
-
for (const hit of hits) {
|
|
9444
|
-
const result = this.toResult(hit, options);
|
|
9445
|
-
if (result !== undefined)
|
|
9446
|
-
results.push(result);
|
|
9447
|
-
}
|
|
9448
|
-
return results;
|
|
9449
|
-
}
|
|
9450
|
-
toResult(hit, options) {
|
|
9451
|
-
const { descriptor, datasourceId, instanceId, mode } = this.options;
|
|
9452
|
-
const chunkId = descriptor.chunkId(hit.key, hit.metadata);
|
|
9453
|
-
const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
|
|
9454
|
-
if (!matchesDatasourceScope(source, options.scope))
|
|
9455
|
-
return;
|
|
9456
|
-
const method = sidecarMethodName(datasourceId, mode);
|
|
9457
|
-
return {
|
|
9458
|
-
id: `${datasourceId}:${instanceId}:${chunkId}`,
|
|
9459
|
-
content: hit.content,
|
|
9460
|
-
source,
|
|
9461
|
-
score: hit.score,
|
|
9462
|
-
metadata: {
|
|
9463
|
-
...hit.metadata,
|
|
9464
|
-
method,
|
|
9465
|
-
datasourceId,
|
|
9466
|
-
instanceId,
|
|
9467
|
-
backend: "sidecar",
|
|
9468
|
-
...hit.title !== undefined ? { title: hit.title } : {},
|
|
9469
|
-
hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
|
|
9470
|
-
...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
|
|
9471
|
-
}
|
|
9472
|
-
};
|
|
9473
|
-
}
|
|
9474
|
-
}
|
|
9475
|
-
var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
|
|
9476
|
-
var init_methods5 = __esm(() => {
|
|
9477
|
-
init_scope2();
|
|
9478
|
-
DESCRIPTIONS = {
|
|
9479
|
-
bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
|
|
9480
|
-
semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
|
|
9481
|
-
hybrid: {
|
|
9482
|
-
type: "hybrid",
|
|
9483
|
-
text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
|
|
9484
|
-
capabilities: ["lexical", "semantic", "hybrid"]
|
|
9485
|
-
}
|
|
9486
|
-
};
|
|
9487
|
-
});
|
|
9488
|
-
|
|
9489
9603
|
// src/datasource/crawler-skill.ts
|
|
9490
9604
|
class CrawlerDatasourceSkill {
|
|
9491
9605
|
definition;
|
|
@@ -9766,7 +9880,7 @@ var init_crawler_skill = __esm(() => {
|
|
|
9766
9880
|
init_errors2();
|
|
9767
9881
|
init_scope2();
|
|
9768
9882
|
init_sidecar();
|
|
9769
|
-
|
|
9883
|
+
init_methods4();
|
|
9770
9884
|
DEFAULT_POLLING_INTERVAL_MS4 = 15 * 60 * 1000;
|
|
9771
9885
|
});
|
|
9772
9886
|
|
|
@@ -11555,14 +11669,27 @@ var init_factory = __esm(() => {
|
|
|
11555
11669
|
...config.channels?.ids !== undefined ? { chatIds: config.channels.ids } : {},
|
|
11556
11670
|
...larkConnectorOptions(config.connector)
|
|
11557
11671
|
}),
|
|
11558
|
-
kakao: (config,
|
|
11559
|
-
|
|
11560
|
-
|
|
11561
|
-
|
|
11562
|
-
|
|
11563
|
-
|
|
11564
|
-
|
|
11565
|
-
|
|
11672
|
+
kakao: (config, workspaceRoot, registrationName) => {
|
|
11673
|
+
const connector = config.connector ?? {};
|
|
11674
|
+
const dataDir = typeof connector.workspacePath === "string" && connector.workspacePath.length > 0 ? connector.workspacePath : join16(homedir4(), "Library", "Application Support", "katok");
|
|
11675
|
+
const sidecar = resolveSidecarOptions({
|
|
11676
|
+
descriptor: CRAWLER_DESCRIPTORS.kakao,
|
|
11677
|
+
connector: config.connector,
|
|
11678
|
+
workspaceRoot,
|
|
11679
|
+
registrationName,
|
|
11680
|
+
instanceId: config.instanceId ?? "default",
|
|
11681
|
+
defaultCrawlerDb: join16(dataDir, "archive.sqlite3")
|
|
11682
|
+
});
|
|
11683
|
+
return new LazykatokSkill({
|
|
11684
|
+
client: new LazykatokClient(connector),
|
|
11685
|
+
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
11686
|
+
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
11687
|
+
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
11688
|
+
channelIds: config.channels?.ids,
|
|
11689
|
+
channelNames: config.channels?.names,
|
|
11690
|
+
...sidecar === undefined ? {} : { sidecar }
|
|
11691
|
+
});
|
|
11692
|
+
},
|
|
11566
11693
|
github: (config, workspaceRoot, registrationName) => new GitHubSkill({
|
|
11567
11694
|
...common(config, workspaceRoot),
|
|
11568
11695
|
skillName: registrationName,
|
|
@@ -12227,7 +12354,7 @@ function migrateLegacyConfig(configPath, legacyPath) {
|
|
|
12227
12354
|
const legacy = readConfigFile(legacyPath, true);
|
|
12228
12355
|
const legacyBytes = readFileSync14(legacyPath);
|
|
12229
12356
|
const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname13(legacyPath));
|
|
12230
|
-
const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
|
|
12357
|
+
const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths) && JSON.stringify(legacy?.excludePaths) === JSON.stringify(migrated.excludePaths);
|
|
12231
12358
|
const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
|
|
12232
12359
|
`;
|
|
12233
12360
|
mkdirSync11(dirname13(configPath), { recursive: true });
|
|
@@ -12265,15 +12392,25 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
|
|
|
12265
12392
|
function resolveSearchPaths(searchPaths, origin) {
|
|
12266
12393
|
return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
|
|
12267
12394
|
}
|
|
12395
|
+
function resolveExcludePaths(value, origin) {
|
|
12396
|
+
if (value === undefined)
|
|
12397
|
+
return;
|
|
12398
|
+
if (!Array.isArray(value) || value.some((path) => typeof path !== "string" || path.length === 0)) {
|
|
12399
|
+
throw new ConfigError("Config field 'excludePaths' must be a non-empty string array");
|
|
12400
|
+
}
|
|
12401
|
+
return resolveSearchPaths(value, origin);
|
|
12402
|
+
}
|
|
12268
12403
|
function resolvePersistedPath(path, origin) {
|
|
12269
12404
|
return isAbsolute5(path) ? path : resolve7(origin, path);
|
|
12270
12405
|
}
|
|
12271
12406
|
function normalizeLegacyConfigPaths(partial, origin) {
|
|
12272
12407
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", origin);
|
|
12408
|
+
const excludePaths = resolveExcludePaths(partial.excludePaths, workspacePath);
|
|
12273
12409
|
return {
|
|
12274
12410
|
...partial,
|
|
12275
12411
|
workspacePath,
|
|
12276
12412
|
...partial.searchPaths === undefined ? {} : { searchPaths: resolveSearchPaths(partial.searchPaths, workspacePath) },
|
|
12413
|
+
...excludePaths === undefined ? {} : { excludePaths },
|
|
12277
12414
|
...partial.memoryPath === undefined ? {} : { memoryPath: resolvePersistedPath(partial.memoryPath, workspacePath) }
|
|
12278
12415
|
};
|
|
12279
12416
|
}
|
|
@@ -12728,10 +12865,7 @@ function resolveConfig(input) {
|
|
|
12728
12865
|
const configOrigin = dirname13(resolve7(configPath));
|
|
12729
12866
|
const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
|
|
12730
12867
|
const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
|
|
12731
|
-
|
|
12732
|
-
throw new ConfigError("Config field 'excludePaths' must be a non-empty string array");
|
|
12733
|
-
}
|
|
12734
|
-
const fileExcludePaths = file.excludePaths ? resolveSearchPaths(file.excludePaths, fileWorkspacePath ?? configOrigin) : undefined;
|
|
12868
|
+
const fileExcludePaths = resolveExcludePaths(file.excludePaths, fileWorkspacePath ?? configOrigin);
|
|
12735
12869
|
const searchPaths = flagSearchPaths ?? envSearchPaths ?? fileSearchPaths ?? defaultSearchPaths;
|
|
12736
12870
|
const flagWorkspacePath = flagString(flags, "workspace");
|
|
12737
12871
|
const envWorkspacePath = envString(env, "AUTORAG_WORKSPACE");
|
|
@@ -13467,7 +13601,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
13467
13601
|
full.dupey = partial.dupey ?? { enabled: true };
|
|
13468
13602
|
full.excludeExactDuplicates = partial.excludeExactDuplicates ?? true;
|
|
13469
13603
|
if (partial.excludePaths !== undefined)
|
|
13470
|
-
full.excludePaths =
|
|
13604
|
+
full.excludePaths = resolveExcludePaths(partial.excludePaths, workspacePath);
|
|
13471
13605
|
if (partial.limits !== undefined)
|
|
13472
13606
|
full.limits = normalizeLimitsConfig(partial.limits);
|
|
13473
13607
|
if (partial.refresh !== undefined)
|
|
@@ -13611,7 +13745,7 @@ var init_config4 = __esm(() => {
|
|
|
13611
13745
|
|
|
13612
13746
|
// src/cli/commands/init.ts
|
|
13613
13747
|
import { existsSync as existsSync11, readFileSync as readFileSync15 } from "node:fs";
|
|
13614
|
-
import { dirname as dirname14 } from "node:path";
|
|
13748
|
+
import { dirname as dirname14, resolve as resolve8 } from "node:path";
|
|
13615
13749
|
async function runInit(ctx) {
|
|
13616
13750
|
const flags = ctx.flags;
|
|
13617
13751
|
const partial = {};
|
|
@@ -13619,6 +13753,14 @@ async function runInit(ctx) {
|
|
|
13619
13753
|
if (searchPathsFlag) {
|
|
13620
13754
|
partial.searchPaths = searchPathsFlag.split(",").map((entry) => entry.trim()).filter((entry) => entry.length > 0);
|
|
13621
13755
|
}
|
|
13756
|
+
const excludedPaths = ctx.repeatedFlags?.exclude ?? (typeof flags.exclude === "string" ? [flags.exclude] : undefined);
|
|
13757
|
+
if (flags.exclude !== undefined || excludedPaths !== undefined) {
|
|
13758
|
+
if (excludedPaths === undefined || excludedPaths.length === 0 || excludedPaths.some((path) => path.trim().length === 0)) {
|
|
13759
|
+
ctx.stderr(renderError(new ConfigError("--exclude requires a non-empty path"), { json: ctx.json, debug: ctx.debug }));
|
|
13760
|
+
return 2;
|
|
13761
|
+
}
|
|
13762
|
+
partial.excludePaths = excludedPaths.map((path) => resolve8(ctx.cwd, path));
|
|
13763
|
+
}
|
|
13622
13764
|
if (typeof flags.workspace === "string")
|
|
13623
13765
|
partial.workspacePath = flags.workspace;
|
|
13624
13766
|
if (typeof flags["memory-path"] === "string")
|
|
@@ -13683,6 +13825,7 @@ async function runInit(ctx) {
|
|
|
13683
13825
|
const legacy = JSON.parse(readFileSync15(legacyPath, "utf8"));
|
|
13684
13826
|
const normalizedLegacy = normalizeLegacyConfigPaths(legacy, dirname14(legacyPath));
|
|
13685
13827
|
partial.searchPaths ??= normalizedLegacy.searchPaths;
|
|
13828
|
+
partial.excludePaths ??= normalizedLegacy.excludePaths;
|
|
13686
13829
|
partial.workspacePath ??= normalizedLegacy.workspacePath;
|
|
13687
13830
|
partial.memoryPath ??= normalizedLegacy.memoryPath;
|
|
13688
13831
|
partial.model ??= legacy.model;
|
|
@@ -14265,9 +14408,9 @@ var init_retry = __esm(() => {
|
|
|
14265
14408
|
|
|
14266
14409
|
// src/dupey/cli.ts
|
|
14267
14410
|
import { spawn as spawn10 } from "node:child_process";
|
|
14268
|
-
import { resolve as
|
|
14411
|
+
import { resolve as resolve9 } from "node:path";
|
|
14269
14412
|
async function scanWithDupey(dir, options = {}) {
|
|
14270
|
-
const absoluteDir =
|
|
14413
|
+
const absoluteDir = resolve9(dir);
|
|
14271
14414
|
const stdout = options.run ? await options.run(["scan", absoluteDir, "--json"]) : await spawnDupey(options.executable ?? "dupey", ["scan", absoluteDir, "--json"], options);
|
|
14272
14415
|
let parsed;
|
|
14273
14416
|
try {
|
|
@@ -14337,13 +14480,13 @@ var init_cli = __esm(() => {
|
|
|
14337
14480
|
|
|
14338
14481
|
// src/dupey/filter.ts
|
|
14339
14482
|
import { stat as stat5 } from "node:fs/promises";
|
|
14340
|
-
import { resolve as
|
|
14483
|
+
import { resolve as resolve10 } from "node:path";
|
|
14341
14484
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable) {
|
|
14342
14485
|
const byHash = new Map;
|
|
14343
14486
|
for (const file of scan.files) {
|
|
14344
14487
|
if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
|
|
14345
14488
|
continue;
|
|
14346
|
-
const path =
|
|
14489
|
+
const path = resolve10(root, file.path);
|
|
14347
14490
|
const group = byHash.get(file.content_hash) ?? [];
|
|
14348
14491
|
group.push(path);
|
|
14349
14492
|
byHash.set(file.content_hash, group);
|
|
@@ -14380,7 +14523,7 @@ function dupeyExactRanking(scan) {
|
|
|
14380
14523
|
return;
|
|
14381
14524
|
const path = entry.path;
|
|
14382
14525
|
if (typeof path === "string")
|
|
14383
|
-
ranking.set(
|
|
14526
|
+
ranking.set(resolve10(path), index);
|
|
14384
14527
|
});
|
|
14385
14528
|
}
|
|
14386
14529
|
return ranking;
|
|
@@ -14536,14 +14679,33 @@ function errorMessage2(error) {
|
|
|
14536
14679
|
var MANIFEST_FILENAME = "manifest.json";
|
|
14537
14680
|
var init_bundle = () => {};
|
|
14538
14681
|
|
|
14682
|
+
// src/filesystem/path-exclusions.ts
|
|
14683
|
+
import { posix as posix2, win32 as win322 } from "node:path";
|
|
14684
|
+
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
14685
|
+
const paths = platform === "win32" ? win322 : posix2;
|
|
14686
|
+
const normalize = (path) => {
|
|
14687
|
+
const absolute = paths.resolve(path);
|
|
14688
|
+
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
14689
|
+
};
|
|
14690
|
+
const excluded = excludedPaths.map((path) => {
|
|
14691
|
+
const absolute = normalize(path);
|
|
14692
|
+
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
14693
|
+
});
|
|
14694
|
+
return (path) => {
|
|
14695
|
+
const absolute = normalize(path);
|
|
14696
|
+
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
14697
|
+
};
|
|
14698
|
+
}
|
|
14699
|
+
var init_path_exclusions = () => {};
|
|
14700
|
+
|
|
14539
14701
|
// src/everything/client.ts
|
|
14540
14702
|
import { spawn as spawn11 } from "node:child_process";
|
|
14541
14703
|
import { createHash as createHash9 } from "node:crypto";
|
|
14542
14704
|
import { mkdirSync as mkdirSync13, writeFileSync as writeFileSync11 } from "node:fs";
|
|
14543
|
-
import { join as join22, resolve as
|
|
14705
|
+
import { join as join22, resolve as resolve11 } from "node:path";
|
|
14544
14706
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
14545
14707
|
function everythingInstanceName(root) {
|
|
14546
|
-
return `autorag-${createHash9("sha256").update(
|
|
14708
|
+
return `autorag-${createHash9("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
14547
14709
|
}
|
|
14548
14710
|
function buildEverythingIni(input) {
|
|
14549
14711
|
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
@@ -14653,6 +14815,7 @@ class EverythingClient {
|
|
|
14653
14815
|
run;
|
|
14654
14816
|
launch;
|
|
14655
14817
|
stateDir;
|
|
14818
|
+
isExcludedPath;
|
|
14656
14819
|
binaries;
|
|
14657
14820
|
operation = Promise.resolve();
|
|
14658
14821
|
constructor(options) {
|
|
@@ -14662,6 +14825,7 @@ class EverythingClient {
|
|
|
14662
14825
|
this.run = options.run ?? runProcess;
|
|
14663
14826
|
this.launch = options.launch ?? launchDetached;
|
|
14664
14827
|
this.stateDir = join22(options.root, ".autorag", "everything");
|
|
14828
|
+
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
14665
14829
|
}
|
|
14666
14830
|
isSupported() {
|
|
14667
14831
|
return this.platform === "win32";
|
|
@@ -14681,6 +14845,9 @@ class EverythingClient {
|
|
|
14681
14845
|
}
|
|
14682
14846
|
const es = { ok: true, esPath: binaries.esPath };
|
|
14683
14847
|
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
14848
|
+
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
14849
|
+
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
14850
|
+
}
|
|
14684
14851
|
const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
|
|
14685
14852
|
const result = await this.run(es.esPath, args, timeoutMs + 5000);
|
|
14686
14853
|
if (result.code !== 0)
|
|
@@ -14692,6 +14859,50 @@ class EverythingClient {
|
|
|
14692
14859
|
}
|
|
14693
14860
|
});
|
|
14694
14861
|
}
|
|
14862
|
+
async searchWithExclusions(esPath, request, timeoutMs) {
|
|
14863
|
+
const deadline = Date.now() + timeoutMs;
|
|
14864
|
+
const offset = request.offset ?? 0;
|
|
14865
|
+
const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
14866
|
+
const results = [];
|
|
14867
|
+
let skipped = 0;
|
|
14868
|
+
let nativeOffset = 0;
|
|
14869
|
+
const timedOut = () => ({
|
|
14870
|
+
ok: false,
|
|
14871
|
+
reason: "search-failed",
|
|
14872
|
+
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
14873
|
+
});
|
|
14874
|
+
for (;; ) {
|
|
14875
|
+
const remainingMs = deadline - Date.now();
|
|
14876
|
+
if (remainingMs <= 0)
|
|
14877
|
+
return timedOut();
|
|
14878
|
+
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
14879
|
+
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
14880
|
+
if (response.code !== 0)
|
|
14881
|
+
return this.failure("search-failed", esPath, args, response);
|
|
14882
|
+
if (Date.now() >= deadline)
|
|
14883
|
+
return timedOut();
|
|
14884
|
+
let entries;
|
|
14885
|
+
try {
|
|
14886
|
+
entries = parseEverythingJson(response.stdout);
|
|
14887
|
+
} catch (error) {
|
|
14888
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
14889
|
+
}
|
|
14890
|
+
for (const entry of entries) {
|
|
14891
|
+
if (this.isExcludedPath(entry.path))
|
|
14892
|
+
continue;
|
|
14893
|
+
if (skipped < offset) {
|
|
14894
|
+
skipped += 1;
|
|
14895
|
+
continue;
|
|
14896
|
+
}
|
|
14897
|
+
results.push(entry);
|
|
14898
|
+
if (results.length >= maxResults)
|
|
14899
|
+
return { ok: true, results };
|
|
14900
|
+
}
|
|
14901
|
+
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
14902
|
+
return { ok: true, results };
|
|
14903
|
+
nativeOffset += entries.length;
|
|
14904
|
+
}
|
|
14905
|
+
}
|
|
14695
14906
|
index() {
|
|
14696
14907
|
return this.serialize(async () => {
|
|
14697
14908
|
const es = await this.ensureRunning(true);
|
|
@@ -14765,7 +14976,8 @@ class EverythingClient {
|
|
|
14765
14976
|
excludeFolders: [
|
|
14766
14977
|
...new Set([
|
|
14767
14978
|
join22(this.options.root, ".autorag"),
|
|
14768
|
-
...this.options.folders.map((folder) => join22(folder, ".autorag"))
|
|
14979
|
+
...this.options.folders.map((folder) => join22(folder, ".autorag")),
|
|
14980
|
+
...this.options.excludeFolders ?? []
|
|
14769
14981
|
])
|
|
14770
14982
|
]
|
|
14771
14983
|
}));
|
|
@@ -14859,8 +15071,9 @@ function launchDetached(command, args) {
|
|
|
14859
15071
|
child.on("error", () => {});
|
|
14860
15072
|
child.unref();
|
|
14861
15073
|
}
|
|
14862
|
-
var DEFAULT_TIMEOUT_MS6 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS2, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
15074
|
+
var DEFAULT_TIMEOUT_MS6 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS2, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
14863
15075
|
var init_client7 = __esm(() => {
|
|
15076
|
+
init_path_exclusions();
|
|
14864
15077
|
init_bundle();
|
|
14865
15078
|
DEFAULT_INDEX_TIMEOUT_MS2 = 10 * 60000;
|
|
14866
15079
|
MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
|
|
@@ -14880,6 +15093,7 @@ function compare(a, b) {
|
|
|
14880
15093
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
14881
15094
|
}
|
|
14882
15095
|
async function walkFileSearch(folders, request, options = {}) {
|
|
15096
|
+
const isExcluded = createPathExclusionMatcher(options.excludeFolders ?? []);
|
|
14883
15097
|
const maxVisited = options.maxVisited ?? DEFAULT_MAX_VISITED;
|
|
14884
15098
|
const maxMatches = options.maxMatches ?? DEFAULT_MAX_MATCHES;
|
|
14885
15099
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
@@ -14902,6 +15116,8 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
14902
15116
|
outer:
|
|
14903
15117
|
while (pending.length > 0) {
|
|
14904
15118
|
const dir = pending.pop();
|
|
15119
|
+
if (isExcluded(dir))
|
|
15120
|
+
continue;
|
|
14905
15121
|
if (Date.now() >= deadline) {
|
|
14906
15122
|
truncated = true;
|
|
14907
15123
|
break;
|
|
@@ -14919,6 +15135,8 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
14919
15135
|
}
|
|
14920
15136
|
visited += 1;
|
|
14921
15137
|
const path = join23(dir, dirent.name);
|
|
15138
|
+
if (isExcluded(path))
|
|
15139
|
+
continue;
|
|
14922
15140
|
if (dirent.isDirectory()) {
|
|
14923
15141
|
if (dirent.name === ".autorag")
|
|
14924
15142
|
continue;
|
|
@@ -14953,6 +15171,8 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
14953
15171
|
}
|
|
14954
15172
|
}
|
|
14955
15173
|
for (const folder of folders) {
|
|
15174
|
+
if (isExcluded(folder))
|
|
15175
|
+
continue;
|
|
14956
15176
|
if (!underPath(folder) || !matches(folder))
|
|
14957
15177
|
continue;
|
|
14958
15178
|
if (request.kind === "files")
|
|
@@ -14979,6 +15199,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
14979
15199
|
}
|
|
14980
15200
|
var DEFAULT_MAX_VISITED = 500000, DEFAULT_DEADLINE_MS = 20000, DEFAULT_MAX_MATCHES = 1e4, SORT_ORDERS;
|
|
14981
15201
|
var init_walk = __esm(() => {
|
|
15202
|
+
init_path_exclusions();
|
|
14982
15203
|
SORT_ORDERS = {
|
|
14983
15204
|
"name-ascending": (a, b) => compare(a.name, b.name),
|
|
14984
15205
|
"name-descending": (a, b) => compare(b.name, a.name),
|
|
@@ -14995,14 +15216,14 @@ var init_walk = __esm(() => {
|
|
|
14995
15216
|
import { spawn as spawn12 } from "node:child_process";
|
|
14996
15217
|
import { createHash as createHash10 } from "node:crypto";
|
|
14997
15218
|
import { existsSync as existsSync15, mkdirSync as mkdirSync14, readFileSync as readFileSync18, realpathSync as realpathSync2, rmSync as rmSync4, writeFileSync as writeFileSync12 } from "node:fs";
|
|
14998
|
-
import { basename as basename9, join as join24, resolve as
|
|
15219
|
+
import { basename as basename9, join as join24, resolve as resolve12, sep as sep4 } from "node:path";
|
|
14999
15220
|
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
15000
15221
|
function fsearchSocketPath(dbPath) {
|
|
15001
15222
|
let canonical;
|
|
15002
15223
|
try {
|
|
15003
15224
|
canonical = realpathSync2(dbPath);
|
|
15004
15225
|
} catch {
|
|
15005
|
-
canonical =
|
|
15226
|
+
canonical = resolve12(dbPath);
|
|
15006
15227
|
}
|
|
15007
15228
|
const hash = createHash10("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
15008
15229
|
return `/tmp/autorag-fsearch-${process.getuid?.() ?? 0}-${hash}.sock`;
|
|
@@ -15088,6 +15309,7 @@ class FSearchClient {
|
|
|
15088
15309
|
launch;
|
|
15089
15310
|
isAlive;
|
|
15090
15311
|
kill;
|
|
15312
|
+
isExcludedPath;
|
|
15091
15313
|
stateDir;
|
|
15092
15314
|
dbPath;
|
|
15093
15315
|
pidPath;
|
|
@@ -15100,6 +15322,7 @@ class FSearchClient {
|
|
|
15100
15322
|
this.launch = options.launch ?? launchDetached2;
|
|
15101
15323
|
this.isAlive = options.isProcessAlive ?? defaultIsProcessAlive;
|
|
15102
15324
|
this.kill = options.killProcess ?? ((pid) => process.kill(pid, "SIGTERM"));
|
|
15325
|
+
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? []);
|
|
15103
15326
|
this.stateDir = join24(options.root, ".autorag", "fsearch");
|
|
15104
15327
|
this.dbPath = join24(this.stateDir, "fsearch.db");
|
|
15105
15328
|
this.pidPath = join24(this.stateDir, "watch.pid");
|
|
@@ -15117,6 +15340,7 @@ class FSearchClient {
|
|
|
15117
15340
|
if (unavailable !== undefined || !binary.ok) {
|
|
15118
15341
|
try {
|
|
15119
15342
|
const walked = await walkFileSearch(this.options.folders, request, {
|
|
15343
|
+
excludeFolders: this.options.excludeFolders,
|
|
15120
15344
|
maxVisited: this.options.walkMaxVisited,
|
|
15121
15345
|
deadlineMs: this.options.walkDeadlineMs
|
|
15122
15346
|
});
|
|
@@ -15131,7 +15355,9 @@ class FSearchClient {
|
|
|
15131
15355
|
return { ok: false, reason: "search-failed", message: error.message };
|
|
15132
15356
|
}
|
|
15133
15357
|
}
|
|
15134
|
-
const
|
|
15358
|
+
const hasExclusions = (this.options.excludeFolders?.length ?? 0) > 0;
|
|
15359
|
+
const nativeRequest = hasExclusions ? { ...request, offset: 0, maxResults: MAX_RESULTS_CAP } : request;
|
|
15360
|
+
const args = buildFSearchSearchArgs(this.dbPath, this.socketPath(), nativeRequest);
|
|
15135
15361
|
const result = await this.run(binary.path, args, this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
15136
15362
|
if (result.code !== 0)
|
|
15137
15363
|
return this.failure("search-failed", binary.path, args, result);
|
|
@@ -15142,6 +15368,10 @@ class FSearchClient {
|
|
|
15142
15368
|
return { ok: false, reason: "search-failed", message: error.message };
|
|
15143
15369
|
}
|
|
15144
15370
|
let entries = parsed.entries;
|
|
15371
|
+
if (hasExclusions) {
|
|
15372
|
+
const isExcluded = this.isExcludedPath;
|
|
15373
|
+
entries = entries.filter((entry) => !isExcluded(entry.path));
|
|
15374
|
+
}
|
|
15145
15375
|
if (request.path !== undefined) {
|
|
15146
15376
|
const prefix = request.path.endsWith(sep4) ? request.path : `${request.path}${sep4}`;
|
|
15147
15377
|
entries = entries.filter((entry) => entry.path === request.path || entry.path.startsWith(prefix));
|
|
@@ -15151,7 +15381,16 @@ class FSearchClient {
|
|
|
15151
15381
|
const maxResults = Math.min(request.maxResults ?? DEFAULT_MAX_RESULTS2, MAX_RESULTS_CAP);
|
|
15152
15382
|
if (entries.length > maxResults)
|
|
15153
15383
|
entries = entries.slice(0, maxResults);
|
|
15154
|
-
return {
|
|
15384
|
+
return {
|
|
15385
|
+
ok: true,
|
|
15386
|
+
backend: "fsearch-cli",
|
|
15387
|
+
results: entries,
|
|
15388
|
+
total: parsed.total,
|
|
15389
|
+
...hasExclusions && parsed.total !== undefined && parsed.total > parsed.entries.length && entries.length < maxResults ? {
|
|
15390
|
+
truncated: true,
|
|
15391
|
+
note: `FSearch checked ${parsed.entries.length} of ${parsed.total} native matches before exclusions (candidate limit ${MAX_RESULTS_CAP}); results may be incomplete. Narrow the query.`
|
|
15392
|
+
} : {}
|
|
15393
|
+
};
|
|
15155
15394
|
});
|
|
15156
15395
|
}
|
|
15157
15396
|
index() {
|
|
@@ -15345,6 +15584,7 @@ function defaultIsProcessAlive(pid) {
|
|
|
15345
15584
|
}
|
|
15346
15585
|
var DEFAULT_TIMEOUT_MS7 = 30000, DEFAULT_STARTUP_TIMEOUT_MS2 = 1e4, STOP_TIMEOUT_MS = 2000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS2 = 250, DEFAULT_MAX_RESULTS2 = 100, MAX_RESULTS_CAP = 1000, MAX_OUTPUT_BYTES2, SORT_KEYS;
|
|
15347
15586
|
var init_client8 = __esm(() => {
|
|
15587
|
+
init_path_exclusions();
|
|
15348
15588
|
init_walk();
|
|
15349
15589
|
DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
|
|
15350
15590
|
MAX_OUTPUT_BYTES2 = 8 * 1024 * 1024;
|
|
@@ -15416,7 +15656,7 @@ var init_diagnostics = __esm(() => {
|
|
|
15416
15656
|
|
|
15417
15657
|
// src/jikji/answer-pack.ts
|
|
15418
15658
|
import { realpathSync as realpathSync3 } from "node:fs";
|
|
15419
|
-
import { isAbsolute as isAbsolute7, join as join25, relative as relative3, resolve as
|
|
15659
|
+
import { isAbsolute as isAbsolute7, join as join25, relative as relative3, resolve as resolve13 } from "node:path";
|
|
15420
15660
|
function isRecord10(value) {
|
|
15421
15661
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
15422
15662
|
}
|
|
@@ -15546,7 +15786,7 @@ function normalizeJikjiAnswerPath(rawPath, sourceRoots) {
|
|
|
15546
15786
|
} else {
|
|
15547
15787
|
if (normalized.split("/").includes(".."))
|
|
15548
15788
|
continue;
|
|
15549
|
-
candidate =
|
|
15789
|
+
candidate = resolve13(realRoot, normalized);
|
|
15550
15790
|
}
|
|
15551
15791
|
let realCandidate;
|
|
15552
15792
|
try {
|
|
@@ -15983,7 +16223,7 @@ var init_client9 = __esm(() => {
|
|
|
15983
16223
|
});
|
|
15984
16224
|
|
|
15985
16225
|
// src/jikji/path-map.ts
|
|
15986
|
-
import { isAbsolute as isAbsolute8, relative as relative4, resolve as
|
|
16226
|
+
import { isAbsolute as isAbsolute8, relative as relative4, resolve as resolve14, sep as sep5 } from "node:path";
|
|
15987
16227
|
function planJikjiSourceRoots(searchPaths) {
|
|
15988
16228
|
return planSourceRoots(searchPaths).map((sourceRoot) => ({
|
|
15989
16229
|
...sourceRoot,
|
|
@@ -15991,7 +16231,7 @@ function planJikjiSourceRoots(searchPaths) {
|
|
|
15991
16231
|
}));
|
|
15992
16232
|
}
|
|
15993
16233
|
function outermostJikjiRoots(searchPaths) {
|
|
15994
|
-
const unique = [...new Set(searchPaths.map((searchPath) =>
|
|
16234
|
+
const unique = [...new Set(searchPaths.map((searchPath) => resolve14(searchPath)))];
|
|
15995
16235
|
return unique.filter((candidate) => !unique.some((other) => {
|
|
15996
16236
|
if (other === candidate)
|
|
15997
16237
|
return false;
|
|
@@ -16012,13 +16252,13 @@ function resolveReturnedPath(rootPath, returnedPath) {
|
|
|
16012
16252
|
return;
|
|
16013
16253
|
if (normalized.split("/").includes(".."))
|
|
16014
16254
|
return;
|
|
16015
|
-
return
|
|
16255
|
+
return resolve14(rootPath, normalized);
|
|
16016
16256
|
}
|
|
16017
16257
|
function isUnsafeReturnedPath(normalizedPath) {
|
|
16018
16258
|
return normalizedPath.startsWith("/") || /^[A-Za-z]:\//.test(normalizedPath) || normalizedPath.startsWith("//") || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(normalizedPath);
|
|
16019
16259
|
}
|
|
16020
16260
|
function isWithinRoot(rootPath, sourcePath) {
|
|
16021
|
-
const rel = relative4(
|
|
16261
|
+
const rel = relative4(resolve14(rootPath), resolve14(sourcePath));
|
|
16022
16262
|
return rel === "" || !rel.startsWith("..") && !rel.includes(`..${sep5}`) && !isAbsolute8(rel);
|
|
16023
16263
|
}
|
|
16024
16264
|
var init_path_map = __esm(() => {
|
|
@@ -18314,7 +18554,7 @@ var init_isolated = __esm(() => {
|
|
|
18314
18554
|
// src/mirror/sync.ts
|
|
18315
18555
|
import { existsSync as existsSync24, rmSync as rmSync8 } from "node:fs";
|
|
18316
18556
|
import { opendir, readFile as readFile5, stat as stat7 } from "node:fs/promises";
|
|
18317
|
-
import { extname as extname5, resolve as
|
|
18557
|
+
import { extname as extname5, resolve as resolve15, sep as sep7 } from "node:path";
|
|
18318
18558
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
18319
18559
|
function errorCode(error) {
|
|
18320
18560
|
if (!(error instanceof Error))
|
|
@@ -18832,7 +19072,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
18832
19072
|
continue;
|
|
18833
19073
|
if (PRODUCT_ARTIFACT_FILE_NAMES.has(entry.name))
|
|
18834
19074
|
continue;
|
|
18835
|
-
const sourcePath =
|
|
19075
|
+
const sourcePath = resolve15(directory, entry.name);
|
|
18836
19076
|
if (entry.isDirectory()) {
|
|
18837
19077
|
if (depth + 1 > state.limits.maxDepth) {
|
|
18838
19078
|
listing.unreliablePrefixes.push(sourceIdentifier(sourceRoot, sourcePath));
|
|
@@ -20015,12 +20255,43 @@ function digitsEndAt(text, from) {
|
|
|
20015
20255
|
end += 1;
|
|
20016
20256
|
return end;
|
|
20017
20257
|
}
|
|
20018
|
-
function continuesAsProse(answer, from) {
|
|
20019
|
-
|
|
20258
|
+
function continuesAsProse(answer, openIndex, from) {
|
|
20259
|
+
let cursor = from;
|
|
20260
|
+
let leadingEmphasis = 0;
|
|
20261
|
+
while (openIndex - 1 - leadingEmphasis >= 0 && (answer[openIndex - 1 - leadingEmphasis] === "*" || answer[openIndex - 1 - leadingEmphasis] === "_")) {
|
|
20262
|
+
leadingEmphasis += 1;
|
|
20263
|
+
}
|
|
20264
|
+
if (leadingEmphasis > 0) {
|
|
20265
|
+
let trailingEmphasis = 0;
|
|
20266
|
+
while (cursor + trailingEmphasis < answer.length && trailingEmphasis < leadingEmphasis && answer[cursor + trailingEmphasis] === answer[openIndex - 1 - trailingEmphasis]) {
|
|
20267
|
+
trailingEmphasis += 1;
|
|
20268
|
+
}
|
|
20269
|
+
if (trailingEmphasis === leadingEmphasis)
|
|
20270
|
+
cursor += trailingEmphasis;
|
|
20271
|
+
}
|
|
20272
|
+
const tokenStart = openIndex - leadingEmphasis;
|
|
20273
|
+
const linePrefix = answer.slice(answer.lastIndexOf(`
|
|
20274
|
+
`, tokenStart - 1) + 1, tokenStart);
|
|
20275
|
+
const atHeadingStart = /^[ \t>*-]*$/u.test(linePrefix);
|
|
20276
|
+
if (atHeadingStart && answer[cursor] === ":")
|
|
20277
|
+
cursor += 1;
|
|
20278
|
+
if (answer[cursor] !== " " && answer[cursor] !== " ")
|
|
20020
20279
|
return false;
|
|
20021
|
-
|
|
20280
|
+
cursor += 1;
|
|
20281
|
+
while (cursor < answer.length && (answer[cursor] === "*" || answer[cursor] === "_" || answer[cursor] === '"' || answer[cursor] === "'" || answer[cursor] === "“" || answer[cursor] === "‘")) {
|
|
20282
|
+
cursor += 1;
|
|
20283
|
+
}
|
|
20284
|
+
const next = answer.codePointAt(cursor);
|
|
20022
20285
|
return next !== undefined && /[\p{L}\p{N}]/u.test(String.fromCodePoint(next));
|
|
20023
20286
|
}
|
|
20287
|
+
function isLegalPinpoint(answer, openIndex, afterCloseIndex) {
|
|
20288
|
+
if (PINPOINT_RANGE_AHEAD.test(answer.slice(afterCloseIndex, afterCloseIndex + 16)))
|
|
20289
|
+
return true;
|
|
20290
|
+
if (PINPOINT_RANGE_BEHIND.test(answer.slice(Math.max(0, openIndex - 16), openIndex)))
|
|
20291
|
+
return true;
|
|
20292
|
+
const prefixWindow = answer.slice(Math.max(0, openIndex - 120), openIndex);
|
|
20293
|
+
return PARA_PINPOINT_PREFIX.test(prefixWindow) || CLAUSE_START_AT_PREFIX.test(prefixWindow) || CASE_CITATION_AT_PREFIX.test(prefixWindow);
|
|
20294
|
+
}
|
|
20024
20295
|
function evidenceMarkerAt(answer, index) {
|
|
20025
20296
|
const ids = [];
|
|
20026
20297
|
let cursor = index + 1;
|
|
@@ -20086,7 +20357,7 @@ function answerMarkers(answer, isFile = () => false) {
|
|
|
20086
20357
|
}
|
|
20087
20358
|
if (char === "[") {
|
|
20088
20359
|
const digitsEnd = digitsEndAt(answer, index + 1);
|
|
20089
|
-
if (digitsEnd > index + 1 && answer[digitsEnd] === "]" && answer[digitsEnd + 1] !== "(" && !continuesAsProse(answer, digitsEnd + 1)) {
|
|
20360
|
+
if (digitsEnd > index + 1 && answer[digitsEnd] === "]" && answer[digitsEnd + 1] !== "(" && !continuesAsProse(answer, index, digitsEnd + 1) && !isLegalPinpoint(answer, index, digitsEnd + 1)) {
|
|
20090
20361
|
markers.push({
|
|
20091
20362
|
kind: "number",
|
|
20092
20363
|
start: leadingStart(index),
|
|
@@ -20158,7 +20429,14 @@ function assertResultsMappingOneToOne(label, results, mapping) {
|
|
|
20158
20429
|
].filter((note) => note !== undefined);
|
|
20159
20430
|
throw new Error(`${label}: result numbers and mapping numbers must be one-to-one, but results contain ${formatCitationList(resultNumbers)} ` + `and mapping contains ${formatCitationList(mappingNumbers)}` + (duplicateNotes.length > 0 ? ` (${duplicateNotes.join("; ")})` : "") + ". Give every result a unique number with exactly one mapping entry carrying the same number and no mapping entry without a result.");
|
|
20160
20431
|
}
|
|
20161
|
-
var MAX_FILE_MARKER_CLOSERS = 8, FILE_MARKER_PREFIX = "[file:";
|
|
20432
|
+
var MAX_FILE_MARKER_CLOSERS = 8, FILE_MARKER_PREFIX = "[file:", PARA_PINPOINT_PREFIX, CLAUSE_START_AT_PREFIX, CASE_CITATION_AT_PREFIX, PINPOINT_RANGE_AHEAD, PINPOINT_RANGE_BEHIND;
|
|
20433
|
+
var init_citations = __esm(() => {
|
|
20434
|
+
PARA_PINPOINT_PREFIX = /(?:^|[^\p{L}\p{N}])(?:paras?\.?|paragraphs?)(?:[ \t\u00a0]+\[\d+\][ \t\u00a0]*(?:[-–—]|,(?:[ \t\u00a0]*(?:and|or)\b)?|and\b|or\b))*[ \t\u00a0]*$/iu;
|
|
20435
|
+
CLAUSE_START_AT_PREFIX = /(?:^|[\n.!?؛;:()]|[>*-])[ \t\u00a0]*At(?:[ \t\u00a0]+\[\d+\][ \t\u00a0]*(?:[-–—]|,(?:[ \t\u00a0]*(?:and|or)\b)?|and\b|or\b))*[ \t\u00a0]+$/u;
|
|
20436
|
+
CASE_CITATION_AT_PREFIX = /(?:\[\d{4}\][^\n.;]*?|\b(?:v|vs\.?)\s+[^\n.;]{1,60}?|\bs(?:ection)?[ \t\u00a0]+\d+[A-Za-z]?(?:\(\d+\))?[^\n.;]{0,30}?)[ \t\u00a0]+at(?:[ \t\u00a0]+\[\d+\][ \t\u00a0]*(?:[-–—]|,(?:[ \t\u00a0]*(?:and|or)\b)?|and\b|or\b))*[ \t\u00a0]+$/iu;
|
|
20437
|
+
PINPOINT_RANGE_AHEAD = /^[ \t\u00a0]*[-–—][ \t\u00a0]*\[\d+\]/u;
|
|
20438
|
+
PINPOINT_RANGE_BEHIND = /\[\d+\][ \t\u00a0]*[-–—][ \t\u00a0]*$/u;
|
|
20439
|
+
});
|
|
20162
20440
|
|
|
20163
20441
|
// src/agent/evidence-ledger.ts
|
|
20164
20442
|
import { createHash as createHash15 } from "node:crypto";
|
|
@@ -20442,6 +20720,7 @@ function deriveResultsFromAnswer(text, ledger, options) {
|
|
|
20442
20720
|
}
|
|
20443
20721
|
var MAX_EXCERPT_CHARS = 400, MAX_CLAIM_CHARS = 280, CLAIM_LOOKBACK_CHARS = 600, SENTENCE_END, LEADING_LIST_MARK, EVIDENCE_MARKER_TEXT, LEADING_BLANK;
|
|
20444
20722
|
var init_answer_citations = __esm(() => {
|
|
20723
|
+
init_citations();
|
|
20445
20724
|
init_evidence_ledger();
|
|
20446
20725
|
SENTENCE_END = /[.!?。!?]/u;
|
|
20447
20726
|
LEADING_LIST_MARK = /^[\s\-*•>]+/u;
|
|
@@ -20450,7 +20729,7 @@ var init_answer_citations = __esm(() => {
|
|
|
20450
20729
|
});
|
|
20451
20730
|
|
|
20452
20731
|
// src/agent/answer-guidelines.ts
|
|
20453
|
-
var ANSWER_IMAGE_EMBED_RULE = "**Images**: when a retrieved result is itself an image file (png, jpg, jpeg, gif, webp, svg, bmp, avif, heic, tiff) and showing it answers the question, embed it in the answer as a markdown image on its own line: ``, followed by its citation, e.g. ` [e2]`. Use only the real source path of retrieved evidence. Never invent a path, use relative paths, or embed remote URLs. Apart from these image embeds, the no-file-paths rule stands.", ANSWER_IMAGE_DELTA_RULE = "An image already embedded in the first answer must not be embedded again.", ANSWER_CITATION_RULE = "**Citations**: cite supporting evidence inline with the evidence id shown next to each retrieved result, in square brackets (e.g. [e3], or [e3][e7] for two). Copy ids exactly; never invent one. For a local file you opened yourself with `bash`, cite `[file:<absolute path>]`. Cite the evidence that backs a sentence at the end of that sentence. Do not cite numbers, paths, or anything else.";
|
|
20732
|
+
var ANSWER_IMAGE_EMBED_RULE = "**Images**: when a retrieved result is itself an image file (png, jpg, jpeg, gif, webp, svg, bmp, avif, heic, tiff) and showing it answers the question, embed it in the answer as a markdown image on its own line: ``, followed by its citation, e.g. ` [e2]`. Use only the real source path of retrieved evidence. Never invent a path, use relative paths, or embed remote URLs. Apart from these image embeds, the no-file-paths rule stands.", ANSWER_IMAGE_DELTA_RULE = "An image already embedded in the first answer must not be embedded again.", ANSWER_CITATION_RULE = "**Citations**: cite supporting evidence inline with the evidence id shown next to each retrieved result, in square brackets (e.g. [e3], or [e3][e7] for two). Copy ids exactly; never invent one. For a local file you opened yourself with `bash`, cite `[file:<absolute path>]`. Cite the evidence that backs a sentence at the end of that sentence. Do not cite numbers, paths, or anything else.", ANSWER_DELIVERABLE_RULE = '**Deliverables and artifacts**: if the user asks you to draft, write, generate, or prepare a deliverable or artifact (a reply email, letter, legal opinion, summary document, standard-format case citation block, checklist, report, or workflow document — including requests phrased as "how should I reply..." to a specific message), produce the finished, ready-to-use artifact as a file (using `write`, `edit`, or `bash` in local sessions, without overwriting existing source files) in a file format, extension, and structure aligned with the context and domain (e.g. `.eml` or `.md`/`.txt` with headers for an email reply; `.md`, `.txt`, `.docx`, `.hwp`, or `.pdf` matching workspace conventions for a legal opinion, insertable case-citation block, or summary document; `.csv`, `.xlsx`, or `.json` for structured tabular data), containing the complete unabridged artifact (including required headers, precedent passages, pinpoint paragraphs, and standard-format citations) inside the file instead of merely describing how to produce it. Do NOT dump the full artifact text into your reply: unless the user explicitly requests otherwise, keep your reply concise within the at-most-5-bullet direct-answer format, clearly stating the created file\'s exact path/location and briefly summarizing what was generated with inline citations to the underlying source evidence (`[eN]` or `[file:<source path>]`); never cite the newly created deliverable file itself as `[file:...]`.', ANSWER_DELIVERABLE_DELTA_RULE = "If a deliverable or artifact was requested and the first answer could not write the file yet (the fast phase runs without tool calls) or only gave advice about it, create the complete context-aligned deliverable file now during verification and report the created file's location and a brief summary concisely.";
|
|
20454
20733
|
|
|
20455
20734
|
// src/agent/datasource-skill.ts
|
|
20456
20735
|
import { Type as Type3 } from "typebox";
|
|
@@ -22651,7 +22930,7 @@ var init_egress_gate = __esm(() => {
|
|
|
22651
22930
|
|
|
22652
22931
|
// src/p2p/policy.ts
|
|
22653
22932
|
import { existsSync as existsSync26, mkdirSync as mkdirSync21, readFileSync as readFileSync23, writeFileSync as writeFileSync17 } from "node:fs";
|
|
22654
|
-
import { basename as basename13, dirname as dirname23, join as join36, resolve as
|
|
22933
|
+
import { basename as basename13, dirname as dirname23, join as join36, resolve as resolve16 } from "node:path";
|
|
22655
22934
|
import { parse as parseToml2 } from "smol-toml";
|
|
22656
22935
|
function isRecord17(value) {
|
|
22657
22936
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -22934,7 +23213,7 @@ class PolicyStore {
|
|
|
22934
23213
|
if (typeof normalizedOptions.workspacePath !== "string" || normalizedOptions.workspacePath.length === 0) {
|
|
22935
23214
|
throw new PolicyError("workspacePath must be a non-empty path");
|
|
22936
23215
|
}
|
|
22937
|
-
this.workspacePath =
|
|
23216
|
+
this.workspacePath = resolve16(normalizedOptions.workspacePath);
|
|
22938
23217
|
this.sourceRoots = normalizedOptions.sourceRoots;
|
|
22939
23218
|
const globalConfigPath = normalizedOptions.globalConfigPath ?? resolveGlobalConfigPath(normalizedOptions.homePath);
|
|
22940
23219
|
const workspacePolicyPath = normalizedOptions.workspacePolicyPath ?? join36(this.workspacePath, ".autorag", "p2p", "policy.toml");
|
|
@@ -24365,6 +24644,7 @@ function recordStructuredResultsSession(sessionId, query, details, memory, compo
|
|
|
24365
24644
|
}
|
|
24366
24645
|
var init_search_documents = __esm(() => {
|
|
24367
24646
|
init_memory();
|
|
24647
|
+
init_citations();
|
|
24368
24648
|
});
|
|
24369
24649
|
|
|
24370
24650
|
// src/agent/search-all-tool.ts
|
|
@@ -24668,6 +24948,10 @@ function buildSystemPrompt(config) {
|
|
|
24668
24948
|
const modelId = config.modelId ?? "the configured model";
|
|
24669
24949
|
const toolLines = [
|
|
24670
24950
|
toolLine(config, "bash", "read and inspect configured document collections with ls, find, grep, cat, and similar tools, and — in local sessions — organize local files when the user asks (rename, move, copy, create folders, move to trash)"),
|
|
24951
|
+
...config.remoteSession ? [] : [
|
|
24952
|
+
toolLine(config, "write", "create a new file in local sessions — use this to produce requested deliverables and artifacts in a file format aligned with the request and context without overwriting existing source files"),
|
|
24953
|
+
toolLine(config, "edit", "surgically update an existing deliverable or config file when the user asks (local sessions only)")
|
|
24954
|
+
],
|
|
24671
24955
|
toolLine(config, "jikji_find", "local discovery through Jikji answer packs"),
|
|
24672
24956
|
toolLine(config, "everything_search", "instant Windows file/folder name search (voidtools Everything) across the configured search folders"),
|
|
24673
24957
|
toolLine(config, "fsearch_search", "instant macOS/Linux file/folder name search (fsearch-cli/FSearch) across the configured search folders, with a slow filesystem-walk fallback"),
|
|
@@ -24684,6 +24968,12 @@ function buildSystemPrompt(config) {
|
|
|
24684
24968
|
...config.toolNames.filter((name) => name.startsWith("search_datasource_")).map((name) => `- **${name}**: search only the ${name.slice("search_datasource_".length).replace(/_/g, "-")} datasource connection, spawning no other datasource CLIs`),
|
|
24685
24969
|
...config.toolNames.filter((name) => ![
|
|
24686
24970
|
"bash",
|
|
24971
|
+
"read",
|
|
24972
|
+
"write",
|
|
24973
|
+
"edit",
|
|
24974
|
+
"grep",
|
|
24975
|
+
"find",
|
|
24976
|
+
"ls",
|
|
24687
24977
|
"jikji_find",
|
|
24688
24978
|
"everything_search",
|
|
24689
24979
|
"fsearch_search",
|
|
@@ -24816,8 +25106,9 @@ Your answer is your final reply, written as plain text, never as a tool call. Ci
|
|
|
24816
25106
|
## Answer Guidelines
|
|
24817
25107
|
|
|
24818
25108
|
- **Complete vs. delta answer**: When no first answer reached the caller, give the complete core answer. When a first answer was already delivered, return only the delta against it — corrections and newly verified findings — and never repeat its unchanged content; if nothing changed, confirm the first answer in one short line.
|
|
24819
|
-
- **Bullet-point core answer**:
|
|
24820
|
-
- **Direct answer only**:
|
|
25109
|
+
- **Bullet-point core answer**: Unless the user explicitly requests a different format or length, provide the core answer to the user's question in at most 5 bullet points. If additional explanation or context is necessary, append it after the bullet points.
|
|
25110
|
+
- **Direct answer only**: Unless the user explicitly requests otherwise, the caller only needs the direct answer to their question. Never include specific file paths, datasource descriptions, or retrieval mechanics/principles in the answer (source paths are attached to each citation from the evidence ids you cite). Do not quote raw chunk text.
|
|
25111
|
+
- ${ANSWER_DELIVERABLE_RULE} ${ANSWER_DELIVERABLE_DELTA_RULE}
|
|
24821
25112
|
- ${ANSWER_CITATION_RULE}
|
|
24822
25113
|
- ${ANSWER_IMAGE_EMBED_RULE} ${ANSWER_IMAGE_DELTA_RULE}
|
|
24823
25114
|
- **No per-source negative reports**: Never report individual negative findings per source (e.g. "no information found in Slack" or "checked Drive but found nothing"). Simply omit unproductive sources from the answer and focus on what was found or provide a concise overall conclusion.
|
|
@@ -30473,7 +30764,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
30473
30764
|
// src/agent/agent.ts
|
|
30474
30765
|
import { randomUUID as randomUUID8 } from "node:crypto";
|
|
30475
30766
|
import { existsSync as existsSync31, watch as fsWatch, mkdirSync as mkdirSync25, realpathSync as realpathSync5, statSync as statSync3, writeFileSync as writeFileSync22 } from "node:fs";
|
|
30476
|
-
import { basename as basename14, dirname as dirname27, join as join41, resolve as
|
|
30767
|
+
import { basename as basename14, dirname as dirname27, join as join41, resolve as resolve17 } from "node:path";
|
|
30477
30768
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
30478
30769
|
import { getAgentDir as getAgentDir4 } from "@earendil-works/pi-coding-agent";
|
|
30479
30770
|
function positiveLimit(value, fallback, path) {
|
|
@@ -30633,7 +30924,7 @@ class AutoRAGAgent {
|
|
|
30633
30924
|
this.datasourceVirtualScopePrefixes = this.datasourceSkills.map((skill) => normalizeVirtualPath2(`/${skill.describe().name}`));
|
|
30634
30925
|
this.startupDiagnostics = options.startupDiagnostics ?? [];
|
|
30635
30926
|
this.datasourceAgentSkills = this.buildDatasourceAgentSkills();
|
|
30636
|
-
this.configuredSearchPaths = options.searchPaths.map((searchPath) =>
|
|
30927
|
+
this.configuredSearchPaths = options.searchPaths.map((searchPath) => resolve17(searchPath));
|
|
30637
30928
|
this.languages = options.languages ?? DEFAULT_LANGUAGES;
|
|
30638
30929
|
this.searchPaths = options.searchPaths.map(pinSearchRoot);
|
|
30639
30930
|
this.jikjiRoots = outermostJikjiRoots(this.searchPaths);
|
|
@@ -30680,7 +30971,8 @@ class AutoRAGAgent {
|
|
|
30680
30971
|
const everythingClient = new EverythingClient({
|
|
30681
30972
|
...options.everything ?? {},
|
|
30682
30973
|
root: this.workspaceProjectRoot,
|
|
30683
|
-
folders: this.searchPaths
|
|
30974
|
+
folders: this.searchPaths,
|
|
30975
|
+
excludeFolders: this.excludePaths
|
|
30684
30976
|
});
|
|
30685
30977
|
if (everythingClient.isSupported())
|
|
30686
30978
|
this.everythingClient = everythingClient;
|
|
@@ -30786,8 +31078,9 @@ class AutoRAGAgent {
|
|
|
30786
31078
|
return true;
|
|
30787
31079
|
});
|
|
30788
31080
|
this.tools = tools;
|
|
31081
|
+
const builtinToolNames = this.remoteSession ? PI_BUILTIN_TOOL_NAMES.filter((name) => name !== "write" && name !== "edit") : PI_BUILTIN_TOOL_NAMES;
|
|
30789
31082
|
const toolNames = [
|
|
30790
|
-
...
|
|
31083
|
+
...builtinToolNames,
|
|
30791
31084
|
...tools.map((tool) => tool.name),
|
|
30792
31085
|
...this.jevExtension !== undefined ? [JEV_TOOL_NAME] : []
|
|
30793
31086
|
];
|
|
@@ -30803,7 +31096,7 @@ class AutoRAGAgent {
|
|
|
30803
31096
|
this.innerAgent = {
|
|
30804
31097
|
state: {
|
|
30805
31098
|
systemPrompt,
|
|
30806
|
-
tools: [...
|
|
31099
|
+
tools: [...builtinToolNames.map((name) => ({ name })), ...tools],
|
|
30807
31100
|
messages: []
|
|
30808
31101
|
},
|
|
30809
31102
|
transformContext: (messages) => this.withMemoryContext(messages)
|
|
@@ -31120,6 +31413,7 @@ ${context.text}
|
|
|
31120
31413
|
...this.jevExtension !== undefined ? [this.jevExtension] : []
|
|
31121
31414
|
],
|
|
31122
31415
|
...this.jevExtension !== undefined ? { extensionToolNames: [JEV_TOOL_NAME] } : {},
|
|
31416
|
+
remoteSession: this.remoteSession,
|
|
31123
31417
|
...this.updateNotice === undefined ? {} : { updateNotice: this.updateNotice }
|
|
31124
31418
|
});
|
|
31125
31419
|
this.boundPiRuntime = runtime.runtime;
|
|
@@ -31574,7 +31868,7 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
31574
31868
|
result.summary,
|
|
31575
31869
|
...result.evidence.map((evidence) => evidence.excerpt)
|
|
31576
31870
|
])
|
|
31577
|
-
], [this.workspaceProjectRoot, ...this.searchPaths].map((root) =>
|
|
31871
|
+
], [this.workspaceProjectRoot, ...this.searchPaths].map((root) => resolve17(root)));
|
|
31578
31872
|
if (!scan.ok)
|
|
31579
31873
|
throw new RemoteSessionRejectedError(scan.code);
|
|
31580
31874
|
}
|
|
@@ -32046,8 +32340,9 @@ ${baseline}
|
|
|
32046
32340
|
` + `Produce the best complete, self-contained answer you can RIGHT NOW from this evidence. Do NOT call any search, retrieval, or file-reading tools and do NOT wait for more evidence. ` + `If the query is answerable from general knowledge alone, answer directly.
|
|
32047
32341
|
|
|
32048
32342
|
` + `Formatting and content rules for the answer:
|
|
32049
|
-
` + `-
|
|
32050
|
-
` + `-
|
|
32343
|
+
` + `- Unless the user explicitly requests a different format or length, provide the core answer to the user's question in at most 5 bullet points. If additional explanation is necessary, append it after the bullet points.
|
|
32344
|
+
` + `- Unless the user explicitly requests otherwise, answer the question directly. Do not include specific file paths, datasource descriptions, or retrieval mechanics in the answer text.
|
|
32345
|
+
` + `- ${ANSWER_DELIVERABLE_RULE}
|
|
32051
32346
|
` + `- ${ANSWER_CITATION_RULE}
|
|
32052
32347
|
` + `- ${ANSWER_IMAGE_EMBED_RULE}
|
|
32053
32348
|
` + `- Do not report per-source negative findings (e.g. "no information found in Slack" or "checked Drive but found nothing").
|
|
@@ -32086,12 +32381,14 @@ ${baseline}
|
|
|
32086
32381
|
` + `- NEVER restate or re-list first-answer facts that remain correct, and never repeat its bullet list.
|
|
32087
32382
|
` + `- Mark each item clearly as a correction or as a new finding.
|
|
32088
32383
|
` + `- If verification changed nothing and found nothing new, say so in one short line (the first answer is confirmed as-is) instead of restating it.
|
|
32384
|
+
` + `- ${ANSWER_DELIVERABLE_RULE} ${ANSWER_DELIVERABLE_DELTA_RULE}
|
|
32089
32385
|
` + `- ${ANSWER_CITATION_RULE} A correction or new finding that relies on evidence the first answer cited must cite that evidence's id again.
|
|
32090
32386
|
` + `- ${ANSWER_IMAGE_EMBED_RULE} ${ANSWER_IMAGE_DELTA_RULE}
|
|
32091
32387
|
` + `- Do not report per-source negative findings (e.g. "no information found in Slack").
|
|
32092
32388
|
` + `- When evidence conflicts, treat the freshest (most recent) information as the correct source of truth.` : `Formatting and content rules for the final answer (COMPLETE — no first answer reached the caller):
|
|
32093
|
-
` + `-
|
|
32094
|
-
` + `-
|
|
32389
|
+
` + `- Unless the user explicitly requests a different format or length, provide the core answer to the user's question in at most 5 bullet points. If additional explanation is necessary, append it after the bullet points.
|
|
32390
|
+
` + `- Unless the user explicitly requests otherwise, answer the question directly. Do not include specific file paths, datasource descriptions, or retrieval mechanics in the answer text.
|
|
32391
|
+
` + `- ${ANSWER_DELIVERABLE_RULE}${fastAnswer !== undefined ? ` ${ANSWER_DELIVERABLE_DELTA_RULE}` : ""}
|
|
32095
32392
|
` + `- ${ANSWER_CITATION_RULE}
|
|
32096
32393
|
` + `- ${ANSWER_IMAGE_EMBED_RULE}
|
|
32097
32394
|
` + `- Do not report per-source negative findings (e.g. "no information found in Slack").
|
|
@@ -32119,8 +32416,9 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
32119
32416
|
` + `Treat candidates as unverified evidence, verify important claims against source files when needed, and use additional tools when needed. ` + `Judge relevance, conflicts, freshness, and sufficiency in this agent loop.
|
|
32120
32417
|
|
|
32121
32418
|
` + `Formatting and content rules for the answer:
|
|
32122
|
-
` + `-
|
|
32123
|
-
` + `-
|
|
32419
|
+
` + `- Unless the user explicitly requests a different format or length, provide the core answer to the user's question in at most 5 bullet points. If additional explanation is necessary, append it after the bullet points.
|
|
32420
|
+
` + `- Unless the user explicitly requests otherwise, answer the question directly. Do not include specific file paths, datasource descriptions, or retrieval mechanics in the answer text.
|
|
32421
|
+
` + `- ${ANSWER_DELIVERABLE_RULE}
|
|
32124
32422
|
` + `- ${ANSWER_CITATION_RULE}
|
|
32125
32423
|
` + `- ${ANSWER_IMAGE_EMBED_RULE}
|
|
32126
32424
|
` + `- Do not report per-source negative findings (e.g. "no information found in Slack").
|
|
@@ -32405,7 +32703,7 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
32405
32703
|
}
|
|
32406
32704
|
startWatchRefresh(options = {}) {
|
|
32407
32705
|
this.refreshState = { ...this.refreshState, watchLimited: false, watchFailed: false };
|
|
32408
|
-
const dirs = this.searchPaths.map((searchPath) =>
|
|
32706
|
+
const dirs = this.searchPaths.map((searchPath) => resolve17(searchPath));
|
|
32409
32707
|
const watcherFactory = options.watcherFactory ?? this.defaultWatcherFactory();
|
|
32410
32708
|
return createWatchRefresh({
|
|
32411
32709
|
dirs,
|
|
@@ -32582,7 +32880,7 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
32582
32880
|
const evidencePack = [];
|
|
32583
32881
|
const allPaths = [];
|
|
32584
32882
|
for (const entry of entries) {
|
|
32585
|
-
const originRoot = sourceRoots.find((sr) => sr.rootPath ===
|
|
32883
|
+
const originRoot = sourceRoots.find((sr) => sr.rootPath === resolve17(entry.root));
|
|
32586
32884
|
if (originRoot === undefined)
|
|
32587
32885
|
continue;
|
|
32588
32886
|
const originRoots = [originRoot];
|
|
@@ -32995,7 +33293,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
32995
33293
|
};
|
|
32996
33294
|
}
|
|
32997
33295
|
function pinExcludedPath(path) {
|
|
32998
|
-
const resolvedPath =
|
|
33296
|
+
const resolvedPath = resolve17(path);
|
|
32999
33297
|
try {
|
|
33000
33298
|
return join41(realpathSync5(dirname27(resolvedPath)), basename14(resolvedPath));
|
|
33001
33299
|
} catch {
|
|
@@ -33003,7 +33301,7 @@ function pinExcludedPath(path) {
|
|
|
33003
33301
|
}
|
|
33004
33302
|
}
|
|
33005
33303
|
function pinSearchRoot(searchPath) {
|
|
33006
|
-
const resolvedPath =
|
|
33304
|
+
const resolvedPath = resolve17(searchPath);
|
|
33007
33305
|
let canonicalPath;
|
|
33008
33306
|
try {
|
|
33009
33307
|
canonicalPath = realpathSync5(resolvedPath);
|
|
@@ -33456,7 +33754,7 @@ var init_memory2 = __esm(() => {
|
|
|
33456
33754
|
|
|
33457
33755
|
// src/cli/commands/index.ts
|
|
33458
33756
|
import { existsSync as existsSync32, lstatSync as lstatSync3, rmSync as rmSync11 } from "node:fs";
|
|
33459
|
-
import { join as join42, resolve as
|
|
33757
|
+
import { join as join42, resolve as resolve18, sep as sep8 } from "node:path";
|
|
33460
33758
|
async function runIndex2(ctx) {
|
|
33461
33759
|
const sub = ctx.positionals[0];
|
|
33462
33760
|
if (sub !== "reset" && sub !== "rebuild") {
|
|
@@ -33472,8 +33770,8 @@ async function runIndex2(ctx) {
|
|
|
33472
33770
|
}
|
|
33473
33771
|
const methods = parseMethodFlag(ctx.flags.method);
|
|
33474
33772
|
const { targetNames, targetSubdirs, refreshMethods } = resolveResetScope(methods);
|
|
33475
|
-
const autoragDir =
|
|
33476
|
-
const targets = targetSubdirs.map((subdir) =>
|
|
33773
|
+
const autoragDir = resolve18(config.workspacePath, ".autorag");
|
|
33774
|
+
const targets = targetSubdirs.map((subdir) => resolve18(config.workspacePath, subdir));
|
|
33477
33775
|
for (const dir of [autoragDir, ...targets]) {
|
|
33478
33776
|
if (existsSync32(dir) && lstatSync3(dir).isSymbolicLink()) {
|
|
33479
33777
|
ctx.stderr(renderError(new Error("Refusing to reset: an index path is a symlink, not a real directory."), {
|
|
@@ -33835,12 +34133,12 @@ var init_health = __esm(() => {
|
|
|
33835
34133
|
});
|
|
33836
34134
|
|
|
33837
34135
|
// src/cli/commands/duplicates.ts
|
|
33838
|
-
import { resolve as
|
|
34136
|
+
import { resolve as resolve19 } from "node:path";
|
|
33839
34137
|
async function runDuplicates(ctx, scanner = (path) => scanWithDupey(path)) {
|
|
33840
34138
|
try {
|
|
33841
34139
|
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
33842
34140
|
const requested = ctx.positionals[0];
|
|
33843
|
-
const roots = requested ? [
|
|
34141
|
+
const roots = requested ? [resolve19(ctx.cwd, requested)] : config.searchPaths.map((path) => resolve19(path));
|
|
33844
34142
|
const scans = await Promise.all(roots.map((root) => scanner(root)));
|
|
33845
34143
|
const exactGroups = scans.flatMap(exactGroupsFromScan);
|
|
33846
34144
|
const extractionErrors = scans.flatMap((scan) => scan.errors);
|
|
@@ -35136,7 +35434,7 @@ var init_results = __esm(() => {
|
|
|
35136
35434
|
confidence: Type18.Optional(Type18.Number({ description: "Evidence confidence, 0..1", minimum: 0, maximum: 1 }))
|
|
35137
35435
|
});
|
|
35138
35436
|
answerSchema = Type18.String({
|
|
35139
|
-
description: `Answer for the caller.
|
|
35437
|
+
description: `Answer for the caller. Unless the user explicitly requests otherwise, provide at most 5 bullet points (plus optional explanation) and answer directly; reference results by bracketed number (e.g. [1], [2]) without file paths or raw chunk text, except the image-embed exception below. ${ANSWER_DELIVERABLE_RULE} ${ANSWER_CITATION_RULE} ${ANSWER_IMAGE_EMBED_RULE}`
|
|
35140
35438
|
});
|
|
35141
35439
|
warningsSchema = Type18.Optional(Type18.Array(Type18.String(), { description: "Optional warnings about this result set" }));
|
|
35142
35440
|
curatedResultFields = {
|
|
@@ -35313,7 +35611,8 @@ Usage: autorag lite <subcommand> [args] [flags]
|
|
|
35313
35611
|
Subcommands:
|
|
35314
35612
|
autorag lite init
|
|
35315
35613
|
Write the configured model-free lifecycle config
|
|
35316
|
-
(--search-paths PATHS --workspace DIR --memory-path FILE --force)
|
|
35614
|
+
(--search-paths PATHS --exclude PATH --workspace DIR --memory-path FILE --force)
|
|
35615
|
+
--exclude may be repeated; relative paths use the current directory
|
|
35317
35616
|
autorag lite refresh Run an incremental index refresh (no model required)
|
|
35318
35617
|
(--full --force --method minsync,parsed,datasources,jikji,everything,fsearch,all)
|
|
35319
35618
|
autorag lite watch Watch configured roots (or --once for one refresh tick)
|
|
@@ -35389,6 +35688,7 @@ var COMMAND_USAGE = {
|
|
|
35389
35688
|
"",
|
|
35390
35689
|
"Flags:",
|
|
35391
35690
|
" --search-paths <csv> Folders to index/search",
|
|
35691
|
+
" --exclude <path> Exclude a file or folder (repeatable; relative to the current directory)",
|
|
35392
35692
|
" --languages <csv> OCR/parser languages",
|
|
35393
35693
|
" --workspace <dir> Workspace that owns .autorag state",
|
|
35394
35694
|
" --memory-path <file> Retrieval memory file",
|
|
@@ -35657,6 +35957,7 @@ var BOOLEAN_FLAGS = new Set([
|
|
|
35657
35957
|
var VALUE_FLAGS = new Set([
|
|
35658
35958
|
"config",
|
|
35659
35959
|
"search-paths",
|
|
35960
|
+
"exclude",
|
|
35660
35961
|
"languages",
|
|
35661
35962
|
"workspace",
|
|
35662
35963
|
"memory-path",
|
|
@@ -35786,6 +36087,7 @@ Run: autorag <command> --help for command-specific usage.
|
|
|
35786
36087
|
function parseArgs(argv) {
|
|
35787
36088
|
const positionals = [];
|
|
35788
36089
|
const flags = {};
|
|
36090
|
+
const repeatedFlags = {};
|
|
35789
36091
|
for (let i = 0;i < argv.length; i++) {
|
|
35790
36092
|
const token = argv[i];
|
|
35791
36093
|
if (token === "-h") {
|
|
@@ -35806,22 +36108,27 @@ function parseArgs(argv) {
|
|
|
35806
36108
|
if (!BOOLEAN_FLAGS.has(key) && !VALUE_FLAGS.has(key)) {
|
|
35807
36109
|
return { error: `Unknown flag: --${key}` };
|
|
35808
36110
|
}
|
|
35809
|
-
if (eq
|
|
35810
|
-
flags[key] = body.slice(eq + 1);
|
|
35811
|
-
continue;
|
|
35812
|
-
}
|
|
35813
|
-
if (BOOLEAN_FLAGS.has(key)) {
|
|
36111
|
+
if (eq === -1 && BOOLEAN_FLAGS.has(key)) {
|
|
35814
36112
|
flags[key] = true;
|
|
35815
36113
|
continue;
|
|
35816
36114
|
}
|
|
35817
|
-
const
|
|
35818
|
-
if (
|
|
36115
|
+
const value = eq === -1 ? argv[++i] : body.slice(eq + 1);
|
|
36116
|
+
if (value === undefined || eq === -1 && value.startsWith("--")) {
|
|
35819
36117
|
return { error: `Flag --${key} requires a value` };
|
|
35820
36118
|
}
|
|
35821
|
-
|
|
35822
|
-
|
|
36119
|
+
if (key === "exclude") {
|
|
36120
|
+
if (value.trim().length === 0)
|
|
36121
|
+
return { error: "Flag --exclude requires a non-empty path" };
|
|
36122
|
+
repeatedFlags[key] ??= [];
|
|
36123
|
+
repeatedFlags[key].push(value);
|
|
36124
|
+
}
|
|
36125
|
+
flags[key] = value;
|
|
35823
36126
|
}
|
|
35824
|
-
return {
|
|
36127
|
+
return {
|
|
36128
|
+
positionals,
|
|
36129
|
+
flags,
|
|
36130
|
+
...Object.keys(repeatedFlags).length > 0 ? { repeatedFlags } : {}
|
|
36131
|
+
};
|
|
35825
36132
|
}
|
|
35826
36133
|
function promptYesNo(question) {
|
|
35827
36134
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
@@ -35943,6 +36250,7 @@ ${USAGE2}`);
|
|
|
35943
36250
|
const ctx = {
|
|
35944
36251
|
positionals: positionals.slice(1),
|
|
35945
36252
|
flags,
|
|
36253
|
+
...parsed.repeatedFlags === undefined ? {} : { repeatedFlags: parsed.repeatedFlags },
|
|
35946
36254
|
json,
|
|
35947
36255
|
debug,
|
|
35948
36256
|
cwd: process.cwd(),
|