@autorag/librarian 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -1
- package/dist/agent/search-documents.d.ts +1 -0
- package/dist/cli/config.d.ts +6 -0
- package/dist/cli/index.js +3423 -306
- package/dist/datasource/chunk-store.d.ts +67 -0
- package/dist/datasource/connector-skill.d.ts +107 -0
- package/dist/datasource/connector.d.ts +72 -0
- package/dist/datasource/http.d.ts +49 -0
- package/dist/datasource/index.d.ts +14 -0
- package/dist/datasource/skills/discord/connector.d.ts +30 -0
- package/dist/datasource/skills/discord/index.d.ts +2 -0
- package/dist/datasource/skills/discord/skill.d.ts +20 -0
- package/dist/datasource/skills/factory.d.ts +35 -0
- package/dist/datasource/skills/gdrive/connector.d.ts +30 -0
- package/dist/datasource/skills/gdrive/index.d.ts +3 -0
- package/dist/datasource/skills/gdrive/rclone-connector.d.ts +51 -0
- package/dist/datasource/skills/gdrive/skill.d.ts +24 -0
- package/dist/datasource/skills/github/connector.d.ts +28 -0
- package/dist/datasource/skills/github/index.d.ts +2 -0
- package/dist/datasource/skills/github/skill.d.ts +20 -0
- package/dist/datasource/skills/gmail/connector.d.ts +30 -0
- package/dist/datasource/skills/gmail/himalaya-connector.d.ts +40 -0
- package/dist/datasource/skills/gmail/index.d.ts +3 -0
- package/dist/datasource/skills/gmail/skill.d.ts +24 -0
- package/dist/datasource/skills/mail-export/connector.d.ts +21 -0
- package/dist/datasource/skills/mail-export/index.d.ts +2 -0
- package/dist/datasource/skills/mail-export/skill.d.ts +20 -0
- package/dist/datasource/skills/notion/connector.d.ts +26 -0
- package/dist/datasource/skills/notion/index.d.ts +2 -0
- package/dist/datasource/skills/notion/skill.d.ts +20 -0
- package/dist/datasource/skills/obsidian/connector.d.ts +22 -0
- package/dist/datasource/skills/obsidian/index.d.ts +2 -0
- package/dist/datasource/skills/obsidian/skill.d.ts +20 -0
- package/dist/datasource/skills/rss/connector.d.ts +28 -0
- package/dist/datasource/skills/rss/index.d.ts +2 -0
- package/dist/datasource/skills/rss/skill.d.ts +22 -0
- package/dist/datasource/skills/slack/connector.d.ts +28 -0
- package/dist/datasource/skills/slack/index.d.ts +2 -0
- package/dist/datasource/skills/slack/skill.d.ts +20 -0
- package/dist/datasource/skills/spotlight/connector.d.ts +59 -0
- package/dist/datasource/skills/spotlight/index.d.ts +2 -0
- package/dist/datasource/skills/spotlight/skill.d.ts +20 -0
- package/dist/index.js +3267 -152
- package/dist/parser/defaults.d.ts +2 -0
- package/dist/parser/hwp-markdown.d.ts +25 -0
- package/dist/parser/hwp.d.ts +7 -0
- package/dist/parser/index.d.ts +2 -1
- package/dist/parser/rhwp-adapter.d.ts +30 -0
- package/package.json +3 -2
- package/skills/autorag-setup/SKILL.md +52 -0
package/dist/cli/index.js
CHANGED
|
@@ -135,6 +135,8 @@ function searchEnvelope(resp, debug) {
|
|
|
135
135
|
title: r.title,
|
|
136
136
|
summary: r.summary
|
|
137
137
|
};
|
|
138
|
+
if (r.source !== undefined)
|
|
139
|
+
base.source = r.source;
|
|
138
140
|
if (debug) {
|
|
139
141
|
base.confidence = r.confidence;
|
|
140
142
|
base.evidence = r.evidence.map((e) => {
|
|
@@ -169,6 +171,8 @@ function renderSearchHuman(resp, debug) {
|
|
|
169
171
|
for (const r of resp.results) {
|
|
170
172
|
const header = `${r.number}. ${r.title}`;
|
|
171
173
|
lines.push(header);
|
|
174
|
+
if (r.source !== undefined)
|
|
175
|
+
lines.push(` source: ${r.source}`);
|
|
172
176
|
if (r.summary)
|
|
173
177
|
lines.push(` ${r.summary}`);
|
|
174
178
|
if (debug) {
|
|
@@ -356,20 +360,2671 @@ function resolveAutoRAGHome(env = process.env) {
|
|
|
356
360
|
var AUTORAG_HOME_ENV = "AUTORAG_HOME";
|
|
357
361
|
var init_home = () => {};
|
|
358
362
|
|
|
363
|
+
// src/datasource/http.ts
|
|
364
|
+
function combineSignals(timeoutMs, outer) {
|
|
365
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
366
|
+
return outer === undefined ? timeout : AbortSignal.any([timeout, outer]);
|
|
367
|
+
}
|
|
368
|
+
function statusToReason(status) {
|
|
369
|
+
if (status === 401)
|
|
370
|
+
return "auth";
|
|
371
|
+
if (status === 403)
|
|
372
|
+
return "permission";
|
|
373
|
+
if (status === 429)
|
|
374
|
+
return "rate-limited";
|
|
375
|
+
return "api-error";
|
|
376
|
+
}
|
|
377
|
+
async function requestText(url, options) {
|
|
378
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
379
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
380
|
+
let response;
|
|
381
|
+
try {
|
|
382
|
+
response = await fetchImpl(url, {
|
|
383
|
+
method: options.method ?? "GET",
|
|
384
|
+
headers: options.headers,
|
|
385
|
+
body: options.body,
|
|
386
|
+
signal: combineSignals(timeoutMs, options.signal)
|
|
387
|
+
});
|
|
388
|
+
} catch {
|
|
389
|
+
return { ok: false, reason: "unavailable", message: "request failed or timed out" };
|
|
390
|
+
}
|
|
391
|
+
if (!response.ok) {
|
|
392
|
+
return { ok: false, reason: statusToReason(response.status), message: `http-${response.status}` };
|
|
393
|
+
}
|
|
394
|
+
let text;
|
|
395
|
+
try {
|
|
396
|
+
text = await response.text();
|
|
397
|
+
} catch {
|
|
398
|
+
return { ok: false, reason: "invalid-data", message: "response body unreadable" };
|
|
399
|
+
}
|
|
400
|
+
return { ok: true, status: response.status, text };
|
|
401
|
+
}
|
|
402
|
+
function httpText(url, options = {}) {
|
|
403
|
+
return requestText(url, options);
|
|
404
|
+
}
|
|
405
|
+
async function httpJson(url, options = {}) {
|
|
406
|
+
const result = await requestText(url, options);
|
|
407
|
+
if (!result.ok)
|
|
408
|
+
return result;
|
|
409
|
+
try {
|
|
410
|
+
return { ok: true, status: result.status, json: JSON.parse(result.text) };
|
|
411
|
+
} catch {
|
|
412
|
+
return { ok: false, reason: "invalid-data", message: "response was not valid JSON" };
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
function asRecord(value) {
|
|
416
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
|
|
417
|
+
}
|
|
418
|
+
function asArray(value) {
|
|
419
|
+
return Array.isArray(value) ? value : [];
|
|
420
|
+
}
|
|
421
|
+
function asString(value) {
|
|
422
|
+
return typeof value === "string" ? value : undefined;
|
|
423
|
+
}
|
|
424
|
+
function asNumber(value) {
|
|
425
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
426
|
+
}
|
|
427
|
+
function parseEpochMs(value) {
|
|
428
|
+
if (typeof value !== "string" || value.length === 0)
|
|
429
|
+
return;
|
|
430
|
+
const parsed = Date.parse(value);
|
|
431
|
+
return Number.isNaN(parsed) ? undefined : parsed;
|
|
432
|
+
}
|
|
433
|
+
function resolveToken(token, tokenEnv) {
|
|
434
|
+
if (token !== undefined && token.length > 0)
|
|
435
|
+
return token;
|
|
436
|
+
if (tokenEnv !== undefined && tokenEnv.length > 0) {
|
|
437
|
+
const fromEnv = process.env[tokenEnv];
|
|
438
|
+
if (fromEnv !== undefined && fromEnv.length > 0)
|
|
439
|
+
return fromEnv;
|
|
440
|
+
}
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
var DEFAULT_TIMEOUT_MS = 15000;
|
|
444
|
+
|
|
445
|
+
// src/datasource/skills/discord/connector.ts
|
|
446
|
+
class DiscordConnector {
|
|
447
|
+
options;
|
|
448
|
+
constructor(options = {}) {
|
|
449
|
+
this.options = options;
|
|
450
|
+
}
|
|
451
|
+
async fetch(signal) {
|
|
452
|
+
const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV);
|
|
453
|
+
if (token === undefined)
|
|
454
|
+
return { ok: false, reason: "not-configured", message: "auth token not configured" };
|
|
455
|
+
const guildId = this.options.guildId;
|
|
456
|
+
if (guildId === undefined || guildId.length === 0) {
|
|
457
|
+
return { ok: false, reason: "not-configured", message: "guild id not configured" };
|
|
458
|
+
}
|
|
459
|
+
const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL;
|
|
460
|
+
const request = {
|
|
461
|
+
headers: { Authorization: `Bot ${token}` },
|
|
462
|
+
timeoutMs: this.options.timeoutMs,
|
|
463
|
+
fetchImpl: this.options.fetchImpl,
|
|
464
|
+
signal
|
|
465
|
+
};
|
|
466
|
+
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES;
|
|
467
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS;
|
|
468
|
+
const listResult = await httpJson(`${baseUrl}/guilds/${guildId}/channels`, request);
|
|
469
|
+
if (!listResult.ok) {
|
|
470
|
+
return { ok: false, reason: listResult.reason, message: `channel list failed: ${listResult.message}` };
|
|
471
|
+
}
|
|
472
|
+
const channels = [];
|
|
473
|
+
for (const raw of asArray(listResult.json)) {
|
|
474
|
+
const channel = asRecord(raw);
|
|
475
|
+
const id = asString(channel?.id);
|
|
476
|
+
const name = asString(channel?.name);
|
|
477
|
+
const type = asNumber(channel?.type);
|
|
478
|
+
if (id === undefined || name === undefined || type === undefined)
|
|
479
|
+
continue;
|
|
480
|
+
if (!TEXT_CHANNEL_TYPES.has(type))
|
|
481
|
+
continue;
|
|
482
|
+
channels.push({ id, name });
|
|
483
|
+
}
|
|
484
|
+
const filter = this.options.channels;
|
|
485
|
+
const selected = filter === undefined || filter.length === 0 ? channels : channels.filter((channel) => filter.includes(channel.id) || filter.includes(channel.name));
|
|
486
|
+
const documents = [];
|
|
487
|
+
const warnings = [];
|
|
488
|
+
for (const channel of selected) {
|
|
489
|
+
if (documents.length >= maxDocuments)
|
|
490
|
+
break;
|
|
491
|
+
let before;
|
|
492
|
+
for (let page = 0;page < maxPages && documents.length < maxDocuments; page += 1) {
|
|
493
|
+
const url = new URL(`${baseUrl}/channels/${channel.id}/messages`);
|
|
494
|
+
url.searchParams.set("limit", "100");
|
|
495
|
+
if (before !== undefined)
|
|
496
|
+
url.searchParams.set("before", before);
|
|
497
|
+
const result = await httpJson(url.toString(), request);
|
|
498
|
+
if (!result.ok) {
|
|
499
|
+
if (result.reason === "rate-limited") {
|
|
500
|
+
return { ok: false, reason: "rate-limited", message: "discord: rate limited" };
|
|
501
|
+
}
|
|
502
|
+
if (result.reason === "permission") {
|
|
503
|
+
warnings.push(`channel ${channel.name} skipped: missing permission`);
|
|
504
|
+
} else {
|
|
505
|
+
warnings.push(`channel ${channel.name} history failed: ${result.message}`);
|
|
506
|
+
}
|
|
507
|
+
break;
|
|
508
|
+
}
|
|
509
|
+
const messages = asArray(result.json);
|
|
510
|
+
if (messages.length === 0)
|
|
511
|
+
break;
|
|
512
|
+
let lastId;
|
|
513
|
+
for (const raw of messages) {
|
|
514
|
+
const message = asRecord(raw);
|
|
515
|
+
const id = asString(message?.id);
|
|
516
|
+
if (id !== undefined)
|
|
517
|
+
lastId = id;
|
|
518
|
+
if (documents.length >= maxDocuments)
|
|
519
|
+
continue;
|
|
520
|
+
const content = asString(message?.content);
|
|
521
|
+
if (message === undefined || id === undefined || content === undefined || content.length === 0)
|
|
522
|
+
continue;
|
|
523
|
+
const author = asString(asRecord(message.author)?.username) ?? "unknown";
|
|
524
|
+
documents.push({
|
|
525
|
+
docId: `${channel.id}-${id}`,
|
|
526
|
+
hierarchy: ["channels", channel.name],
|
|
527
|
+
title: `#${channel.name}`,
|
|
528
|
+
content: `${author}: ${content}`,
|
|
529
|
+
publishedAt: parseEpochMs(message.timestamp),
|
|
530
|
+
metadata: { channelId: channel.id, messageId: id, author }
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
if (lastId === undefined || messages.length < 100)
|
|
534
|
+
break;
|
|
535
|
+
before = lastId;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
var DEFAULT_BASE_URL = "https://discord.com/api/v10", DEFAULT_TOKEN_ENV = "DISCORD_BOT_TOKEN", DEFAULT_MAX_PAGES = 10, DEFAULT_MAX_DOCUMENTS = 500, TEXT_CHANNEL_TYPES;
|
|
542
|
+
var init_connector = __esm(() => {
|
|
543
|
+
TEXT_CHANNEL_TYPES = new Set([0, 5]);
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
// src/datasource/connector.ts
|
|
547
|
+
import { createHash } from "node:crypto";
|
|
548
|
+
function connectorFailureToDiagnosticCode(reason) {
|
|
549
|
+
switch (reason) {
|
|
550
|
+
case "auth":
|
|
551
|
+
return "datasource-auth-error";
|
|
552
|
+
case "permission":
|
|
553
|
+
return "datasource-permission-denied";
|
|
554
|
+
case "rate-limited":
|
|
555
|
+
return "datasource-rate-limited";
|
|
556
|
+
case "unavailable":
|
|
557
|
+
case "not-configured":
|
|
558
|
+
return "datasource-unavailable";
|
|
559
|
+
case "empty":
|
|
560
|
+
return "datasource-empty";
|
|
561
|
+
case "api-error":
|
|
562
|
+
case "invalid-data":
|
|
563
|
+
return "datasource-index-failed";
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
function boundDiagnosticText(value) {
|
|
567
|
+
const trimmed = value.trim();
|
|
568
|
+
return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
|
|
569
|
+
}
|
|
570
|
+
function sanitizeIdSegment(value) {
|
|
571
|
+
const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
|
|
572
|
+
let start = 0;
|
|
573
|
+
while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
|
|
574
|
+
start += 1;
|
|
575
|
+
let end = normalized.length;
|
|
576
|
+
while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
|
|
577
|
+
end -= 1;
|
|
578
|
+
const cleaned = normalized.slice(start, end);
|
|
579
|
+
if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
|
|
580
|
+
return cleaned;
|
|
581
|
+
const hash = createHash("sha256").update(value).digest("hex").slice(0, 8);
|
|
582
|
+
const stem = cleaned.slice(0, 60);
|
|
583
|
+
return stem.length > 0 ? `${stem}-${hash}` : hash;
|
|
584
|
+
}
|
|
585
|
+
var init_connector2 = () => {};
|
|
586
|
+
|
|
587
|
+
// src/datasource/chunk-store.ts
|
|
588
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
589
|
+
import { dirname, join as join2 } from "node:path";
|
|
590
|
+
function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
|
|
591
|
+
return join2(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
class DatasourceChunkStore {
|
|
595
|
+
skillName;
|
|
596
|
+
instanceId;
|
|
597
|
+
storePath;
|
|
598
|
+
maxChunkChars;
|
|
599
|
+
chunkList = [];
|
|
600
|
+
loaded = false;
|
|
601
|
+
constructor(options) {
|
|
602
|
+
this.skillName = options.skillName;
|
|
603
|
+
this.instanceId = options.instanceId;
|
|
604
|
+
this.maxChunkChars = options.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS;
|
|
605
|
+
this.storePath = options.workspaceRoot === undefined ? undefined : datasourceChunkStorePath(options.workspaceRoot, options.skillName, options.instanceId);
|
|
606
|
+
}
|
|
607
|
+
replaceDocuments(documents) {
|
|
608
|
+
const chunks = [];
|
|
609
|
+
const seenIds = new Set;
|
|
610
|
+
for (const document of documents) {
|
|
611
|
+
const docSegment = sanitizeIdSegment(document.docId);
|
|
612
|
+
const hierarchy = (document.hierarchy ?? []).map(sanitizeIdSegment).filter((segment) => segment.length > 0);
|
|
613
|
+
const parts = splitContent(document.content, this.maxChunkChars);
|
|
614
|
+
for (const [index, part] of parts.entries()) {
|
|
615
|
+
let chunkId = parts.length === 1 ? docSegment : `${docSegment}-p${index + 1}`;
|
|
616
|
+
while (seenIds.has(chunkId))
|
|
617
|
+
chunkId = `${chunkId}x`;
|
|
618
|
+
seenIds.add(chunkId);
|
|
619
|
+
chunks.push({
|
|
620
|
+
chunkId,
|
|
621
|
+
docId: document.docId,
|
|
622
|
+
hierarchy,
|
|
623
|
+
...document.title !== undefined ? { title: document.title } : {},
|
|
624
|
+
content: part,
|
|
625
|
+
...document.publishedAt !== undefined ? { publishedAt: document.publishedAt } : {},
|
|
626
|
+
metadata: { ...document.metadata ?? {} }
|
|
627
|
+
});
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
this.chunkList = chunks;
|
|
631
|
+
this.loaded = true;
|
|
632
|
+
this.persist();
|
|
633
|
+
return chunks.length;
|
|
634
|
+
}
|
|
635
|
+
load() {
|
|
636
|
+
if (this.storePath === undefined)
|
|
637
|
+
return false;
|
|
638
|
+
try {
|
|
639
|
+
if (!existsSync(this.storePath))
|
|
640
|
+
return false;
|
|
641
|
+
const parsed = JSON.parse(readFileSync(this.storePath, "utf8"));
|
|
642
|
+
if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
|
|
643
|
+
return false;
|
|
644
|
+
this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
|
|
645
|
+
this.loaded = true;
|
|
646
|
+
return true;
|
|
647
|
+
} catch {
|
|
648
|
+
return false;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
ensureLoaded() {
|
|
652
|
+
if (!this.loaded) {
|
|
653
|
+
this.load();
|
|
654
|
+
this.loaded = true;
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
chunks() {
|
|
658
|
+
this.ensureLoaded();
|
|
659
|
+
return this.chunkList;
|
|
660
|
+
}
|
|
661
|
+
hierarchies() {
|
|
662
|
+
this.ensureLoaded();
|
|
663
|
+
const seen = new Map;
|
|
664
|
+
for (const chunk of this.chunkList) {
|
|
665
|
+
seen.set(chunk.hierarchy.join("/"), chunk.hierarchy);
|
|
666
|
+
}
|
|
667
|
+
return [...seen.values()];
|
|
668
|
+
}
|
|
669
|
+
search(query, topK) {
|
|
670
|
+
this.ensureLoaded();
|
|
671
|
+
const queryTerms = [...new Set(tokenize(query))];
|
|
672
|
+
if (queryTerms.length === 0 || this.chunkList.length === 0 || topK <= 0)
|
|
673
|
+
return [];
|
|
674
|
+
const docCount = this.chunkList.length;
|
|
675
|
+
const tokenized = this.chunkList.map((chunk) => tokenize(`${chunk.title ?? ""} ${chunk.content}`));
|
|
676
|
+
const avgLength = tokenized.reduce((sum, tokens) => sum + tokens.length, 0) / docCount || 1;
|
|
677
|
+
const termFrequency = (term, tokens) => {
|
|
678
|
+
let tf = 0;
|
|
679
|
+
for (const token of tokens) {
|
|
680
|
+
if (token === term)
|
|
681
|
+
tf += 1;
|
|
682
|
+
else if (term.length >= 2 && token.startsWith(term))
|
|
683
|
+
tf += 0.75;
|
|
684
|
+
}
|
|
685
|
+
return tf;
|
|
686
|
+
};
|
|
687
|
+
const frequencies = queryTerms.map((term) => tokenized.map((tokens) => termFrequency(term, tokens)));
|
|
688
|
+
const documentFrequency = frequencies.map((perDoc) => perDoc.reduce((count, tf) => count + (tf > 0 ? 1 : 0), 0));
|
|
689
|
+
const k1 = 1.2;
|
|
690
|
+
const b = 0.75;
|
|
691
|
+
const scored = [];
|
|
692
|
+
for (const [index, chunk] of this.chunkList.entries()) {
|
|
693
|
+
const tokens = tokenized[index] ?? [];
|
|
694
|
+
let score = 0;
|
|
695
|
+
for (const [termIndex] of queryTerms.entries()) {
|
|
696
|
+
const df = documentFrequency[termIndex] ?? 0;
|
|
697
|
+
if (df === 0)
|
|
698
|
+
continue;
|
|
699
|
+
const tf = frequencies[termIndex]?.[index] ?? 0;
|
|
700
|
+
if (tf === 0)
|
|
701
|
+
continue;
|
|
702
|
+
const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
|
|
703
|
+
score += idf * tf * (k1 + 1) / (tf + k1 * (1 - b + b * tokens.length / avgLength));
|
|
704
|
+
}
|
|
705
|
+
if (score > 0)
|
|
706
|
+
scored.push({ chunk, score });
|
|
707
|
+
}
|
|
708
|
+
scored.sort((a, b2) => b2.score - a.score || a.chunk.chunkId.localeCompare(b2.chunk.chunkId));
|
|
709
|
+
return scored.slice(0, topK);
|
|
710
|
+
}
|
|
711
|
+
persist() {
|
|
712
|
+
if (this.storePath === undefined)
|
|
713
|
+
return;
|
|
714
|
+
try {
|
|
715
|
+
mkdirSync(dirname(this.storePath), { recursive: true });
|
|
716
|
+
const payload = {
|
|
717
|
+
version: STORE_VERSION,
|
|
718
|
+
skill: this.skillName,
|
|
719
|
+
instance: this.instanceId,
|
|
720
|
+
updatedAt: Date.now(),
|
|
721
|
+
chunks: this.chunkList
|
|
722
|
+
};
|
|
723
|
+
writeFileSync(this.storePath, `${JSON.stringify(payload)}
|
|
724
|
+
`, "utf8");
|
|
725
|
+
} catch {}
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
function splitContent(content, maxChars) {
|
|
729
|
+
const trimmed = content.trim();
|
|
730
|
+
if (trimmed.length === 0)
|
|
731
|
+
return [];
|
|
732
|
+
if (trimmed.length <= maxChars)
|
|
733
|
+
return [trimmed];
|
|
734
|
+
const parts = [];
|
|
735
|
+
const paragraphs = trimmed.split(/\n{2,}/u);
|
|
736
|
+
let current = "";
|
|
737
|
+
for (const paragraph of paragraphs) {
|
|
738
|
+
if (current.length > 0 && current.length + paragraph.length + 2 > maxChars) {
|
|
739
|
+
parts.push(current);
|
|
740
|
+
current = "";
|
|
741
|
+
}
|
|
742
|
+
if (paragraph.length > maxChars) {
|
|
743
|
+
if (current.length > 0) {
|
|
744
|
+
parts.push(current);
|
|
745
|
+
current = "";
|
|
746
|
+
}
|
|
747
|
+
for (let offset = 0;offset < paragraph.length; offset += maxChars) {
|
|
748
|
+
parts.push(paragraph.slice(offset, offset + maxChars));
|
|
749
|
+
}
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
752
|
+
current = current.length === 0 ? paragraph : `${current}
|
|
753
|
+
|
|
754
|
+
${paragraph}`;
|
|
755
|
+
}
|
|
756
|
+
if (current.length > 0)
|
|
757
|
+
parts.push(current);
|
|
758
|
+
return parts.filter((part) => part.trim().length > 0);
|
|
759
|
+
}
|
|
760
|
+
function tokenize(text) {
|
|
761
|
+
return text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length > 0);
|
|
762
|
+
}
|
|
763
|
+
var STORE_VERSION = 1, DEFAULT_MAX_CHUNK_CHARS = 2000;
|
|
764
|
+
var init_chunk_store = __esm(() => {
|
|
765
|
+
init_connector2();
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
// src/retrieval/scope.ts
|
|
769
|
+
function normalizeVirtualPath(value) {
|
|
770
|
+
const normalized = value.replace(/\\/g, "/").replace(/\/+/g, "/").trim();
|
|
771
|
+
if (normalized.length === 0 || normalized === "/")
|
|
772
|
+
return "/";
|
|
773
|
+
const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`;
|
|
774
|
+
return withLeadingSlash.length > 1 ? withLeadingSlash.replace(/\/+$/g, "") : withLeadingSlash;
|
|
775
|
+
}
|
|
776
|
+
function normalizeVirtualPathScope(scope) {
|
|
777
|
+
if (scope === undefined)
|
|
778
|
+
return;
|
|
779
|
+
const normalized = normalizeVirtualPath(scope);
|
|
780
|
+
return normalized === "/" ? undefined : normalized;
|
|
781
|
+
}
|
|
782
|
+
function matchesVirtualPathScope(virtualPath, scope) {
|
|
783
|
+
const normalizedScope = normalizeVirtualPathScope(scope);
|
|
784
|
+
if (normalizedScope === undefined)
|
|
785
|
+
return true;
|
|
786
|
+
return virtualPathScopeToRegExp(normalizedScope).test(normalizeVirtualPath(virtualPath));
|
|
787
|
+
}
|
|
788
|
+
function virtualPathScopeToRegExp(scope) {
|
|
789
|
+
const normalized = normalizeVirtualPath(scope);
|
|
790
|
+
const scoped = hasGlob(normalized) || looksLikeFileScope(normalized) ? normalized : `${normalized}/**`;
|
|
791
|
+
const pattern = scoped.split("/").map((segment, index) => {
|
|
792
|
+
if (index === 0)
|
|
793
|
+
return "";
|
|
794
|
+
if (segment === "**")
|
|
795
|
+
return "(?:.*)";
|
|
796
|
+
return escapeRegExp(segment).replace(/\\\*/g, "[^/]*");
|
|
797
|
+
}).join("/").replace(/\/\(\?:\.\*\)$/u, "(?:/.*)?");
|
|
798
|
+
return new RegExp(`^${pattern}$`);
|
|
799
|
+
}
|
|
800
|
+
function hasGlob(scope) {
|
|
801
|
+
return scope.includes("*");
|
|
802
|
+
}
|
|
803
|
+
function looksLikeFileScope(scope) {
|
|
804
|
+
const finalSegment = scope.split("/").at(-1) ?? "";
|
|
805
|
+
return /\.[^/.]+$/u.test(finalSegment);
|
|
806
|
+
}
|
|
807
|
+
function escapeRegExp(literal) {
|
|
808
|
+
return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// src/datasource/scope.ts
|
|
812
|
+
function buildDatasourceInstanceSource(skillName, instanceId) {
|
|
813
|
+
return normalizeVirtualPath(`/${skillName}/${instanceId}`);
|
|
814
|
+
}
|
|
815
|
+
function buildDatasourceChunkSource(skillName, instanceId, chunkId) {
|
|
816
|
+
return normalizeVirtualPath(`/${skillName}/${instanceId}/${DATASOURCE_CHUNKS_SEGMENT}/${chunkId}`);
|
|
817
|
+
}
|
|
818
|
+
function datasourceSourcePath(skillName, instanceId, chunkId) {
|
|
819
|
+
return chunkId === undefined ? buildDatasourceInstanceSource(skillName, instanceId) : buildDatasourceChunkSource(skillName, instanceId, chunkId);
|
|
820
|
+
}
|
|
821
|
+
function datasourceSourceHasFragment(source) {
|
|
822
|
+
return typeof source === "string" && source.includes("#");
|
|
823
|
+
}
|
|
824
|
+
function isDatasourceSource(source) {
|
|
825
|
+
if (typeof source !== "string" || source.length === 0)
|
|
826
|
+
return false;
|
|
827
|
+
if (datasourceSourceHasFragment(source))
|
|
828
|
+
return false;
|
|
829
|
+
const normalized = normalizeVirtualPath(source);
|
|
830
|
+
return normalized !== "/";
|
|
831
|
+
}
|
|
832
|
+
function matchesDatasourceScope(source, scope) {
|
|
833
|
+
if (datasourceSourceHasFragment(source))
|
|
834
|
+
return false;
|
|
835
|
+
return matchesVirtualPathScope(source, scope);
|
|
836
|
+
}
|
|
837
|
+
var DATASOURCE_CHUNKS_SEGMENT = "chunks";
|
|
838
|
+
var init_scope = () => {};
|
|
839
|
+
|
|
840
|
+
// src/datasource/connector-skill.ts
|
|
841
|
+
class ConnectorDatasourceSkill {
|
|
842
|
+
definition;
|
|
843
|
+
connector;
|
|
844
|
+
instanceId;
|
|
845
|
+
instances;
|
|
846
|
+
pollingIntervalMs;
|
|
847
|
+
tags;
|
|
848
|
+
store;
|
|
849
|
+
dedupeWindowMs;
|
|
850
|
+
lastIndexedAt;
|
|
851
|
+
lastPolledAt;
|
|
852
|
+
lastError;
|
|
853
|
+
seenDocs = new Map;
|
|
854
|
+
constructor(definition, options) {
|
|
855
|
+
this.definition = definition;
|
|
856
|
+
this.connector = options.connector;
|
|
857
|
+
this.instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID;
|
|
858
|
+
this.instances = options.instances !== undefined && options.instances.length > 0 ? options.instances : [this.instanceId];
|
|
859
|
+
this.pollingIntervalMs = options.pollingIntervalMs ?? DEFAULT_POLLING_INTERVAL_MS;
|
|
860
|
+
this.tags = options.tags ?? definition.defaultTags;
|
|
861
|
+
this.lastIndexedAt = options.lastIndexedAt;
|
|
862
|
+
this.dedupeWindowMs = options.dedupeWindowMs ?? 0;
|
|
863
|
+
this.store = new DatasourceChunkStore({
|
|
864
|
+
skillName: definition.skillName,
|
|
865
|
+
instanceId: this.instanceId,
|
|
866
|
+
workspaceRoot: options.workspaceRoot,
|
|
867
|
+
maxChunkChars: options.maxChunkChars
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
describe() {
|
|
871
|
+
return {
|
|
872
|
+
name: this.definition.skillName,
|
|
873
|
+
id: this.definition.skillName,
|
|
874
|
+
type: this.definition.skillType,
|
|
875
|
+
description: this.definition.description,
|
|
876
|
+
capabilities: this.definition.capabilities,
|
|
877
|
+
tags: this.tags,
|
|
878
|
+
status: "active",
|
|
879
|
+
datasourceId: this.definition.skillName,
|
|
880
|
+
instanceId: this.instanceId,
|
|
881
|
+
instances: this.instances
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
polling() {
|
|
885
|
+
return {
|
|
886
|
+
mode: this.pollingIntervalMs > 0 ? "poll" : "none",
|
|
887
|
+
...this.pollingIntervalMs > 0 ? { intervalMs: this.pollingIntervalMs } : {},
|
|
888
|
+
lastIndexedAt: this.lastIndexedAt,
|
|
889
|
+
...this.lastPolledAt !== undefined ? { lastPolledAt: this.lastPolledAt } : {},
|
|
890
|
+
...this.lastError !== undefined ? { lastError: this.lastError } : {}
|
|
891
|
+
};
|
|
892
|
+
}
|
|
893
|
+
async index() {
|
|
894
|
+
this.lastPolledAt = Date.now();
|
|
895
|
+
let fetched;
|
|
896
|
+
try {
|
|
897
|
+
fetched = await this.connector.fetch();
|
|
898
|
+
} catch {
|
|
899
|
+
return this.fail("datasource-unavailable", "connector failed unexpectedly");
|
|
900
|
+
}
|
|
901
|
+
if (!fetched.ok) {
|
|
902
|
+
const code = connectorFailureToDiagnosticCode(fetched.reason);
|
|
903
|
+
const message = boundDiagnosticText(fetched.message ?? fetched.reason);
|
|
904
|
+
return this.fail(code, `${fetched.reason}: ${message}`);
|
|
905
|
+
}
|
|
906
|
+
const documents = this.applyDedupeWindow(fetched.documents);
|
|
907
|
+
const chunkCount = this.store.replaceDocuments(documents);
|
|
908
|
+
this.lastIndexedAt = Date.now();
|
|
909
|
+
this.lastError = undefined;
|
|
910
|
+
const diagnostics = (fetched.warnings ?? []).map((warning) => ({
|
|
911
|
+
code: "datasource-index-failed",
|
|
912
|
+
severity: "warning",
|
|
913
|
+
message: boundDiagnosticText(warning),
|
|
914
|
+
instanceId: this.instanceId,
|
|
915
|
+
source: this.definition.skillName
|
|
916
|
+
}));
|
|
917
|
+
if (chunkCount === 0) {
|
|
918
|
+
diagnostics.push({
|
|
919
|
+
code: "datasource-empty",
|
|
920
|
+
severity: "info",
|
|
921
|
+
message: "datasource fetch returned no indexable documents",
|
|
922
|
+
instanceId: this.instanceId,
|
|
923
|
+
source: this.definition.skillName
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
return {
|
|
927
|
+
ok: true,
|
|
928
|
+
instanceId: this.instanceId,
|
|
929
|
+
skill: this.definition.skillName,
|
|
930
|
+
chunkCount,
|
|
931
|
+
indexedAt: this.lastIndexedAt,
|
|
932
|
+
diagnostics
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
retrievalMethods() {
|
|
936
|
+
return [
|
|
937
|
+
new ConnectorLexicalMethod({
|
|
938
|
+
skillName: this.definition.skillName,
|
|
939
|
+
skillType: this.definition.skillType,
|
|
940
|
+
instanceId: this.instanceId,
|
|
941
|
+
tags: this.tags,
|
|
942
|
+
store: this.store
|
|
943
|
+
})
|
|
944
|
+
];
|
|
945
|
+
}
|
|
946
|
+
describeSources() {
|
|
947
|
+
const out = this.instances.map((instanceId) => ({
|
|
948
|
+
source: datasourceSourcePath(this.definition.skillName, instanceId),
|
|
949
|
+
datasourceId: this.definition.skillName,
|
|
950
|
+
skill: this.definition.skillName,
|
|
951
|
+
instanceId,
|
|
952
|
+
contentType: this.definition.contentType,
|
|
953
|
+
metadata: {
|
|
954
|
+
datasourceId: this.definition.skillName,
|
|
955
|
+
instanceId,
|
|
956
|
+
tags: this.tags
|
|
957
|
+
}
|
|
958
|
+
}));
|
|
959
|
+
for (const hierarchy of this.store.hierarchies()) {
|
|
960
|
+
if (hierarchy.length === 0)
|
|
961
|
+
continue;
|
|
962
|
+
out.push({
|
|
963
|
+
source: `${datasourceSourcePath(this.definition.skillName, this.instanceId)}/${hierarchy.join("/")}`,
|
|
964
|
+
datasourceId: this.definition.skillName,
|
|
965
|
+
skill: this.definition.skillName,
|
|
966
|
+
instanceId: this.instanceId,
|
|
967
|
+
contentType: this.definition.contentType,
|
|
968
|
+
metadata: {
|
|
969
|
+
datasourceId: this.definition.skillName,
|
|
970
|
+
instanceId: this.instanceId,
|
|
971
|
+
hierarchy
|
|
972
|
+
}
|
|
973
|
+
});
|
|
974
|
+
}
|
|
975
|
+
return out;
|
|
976
|
+
}
|
|
977
|
+
skillManifest() {
|
|
978
|
+
const skillName = this.definition.skillName;
|
|
979
|
+
const instanceScopes = this.instances.map((instanceId) => `- \`${datasourceSourcePath(skillName, instanceId)}\``).join(`
|
|
980
|
+
`);
|
|
981
|
+
const cadence = this.pollingIntervalMs > 0 ? `roughly every ${Math.round(this.pollingIntervalMs / 60000)} minute(s) when auto-refresh runs` : "on manual refresh only";
|
|
982
|
+
return {
|
|
983
|
+
name: `datasource-${skillName}`,
|
|
984
|
+
description: this.definition.manifestDescription,
|
|
985
|
+
content: [
|
|
986
|
+
`# ${this.definition.description} (${this.definition.skillType})`,
|
|
987
|
+
"",
|
|
988
|
+
"## When to use",
|
|
989
|
+
this.definition.manifestDescription,
|
|
990
|
+
"",
|
|
991
|
+
"## Indexing",
|
|
992
|
+
`Indexing is server-managed and refreshed ${cadence}. You do not trigger indexing; just search.`,
|
|
993
|
+
"",
|
|
994
|
+
"## How to search",
|
|
995
|
+
"Call `search_datasource_documents` with a natural-language `query`. Optionally pass `topK` and a narrowing `scope`. Available authorized scopes:",
|
|
996
|
+
instanceScopes.length > 0 ? instanceScopes : "- (no authorized instances)",
|
|
997
|
+
"",
|
|
998
|
+
"`scope` can only narrow within already-authorized scopes; it can never widen access.",
|
|
999
|
+
"",
|
|
1000
|
+
"## Output rules",
|
|
1001
|
+
`Datasource source identifiers such as \`/${skillName}/<instance>/chunks/<id>\` are stable and traceable. Result metadata may carry real file paths or account identifiers; you may cite them in the visible answer when they help the user locate the underlying item. Privacy is the operator's responsibility: run AutoRAG with a local LLM if results must not leave this machine.`,
|
|
1002
|
+
...this.definition.manifestNotes !== undefined && this.definition.manifestNotes.length > 0 ? ["", ...this.definition.manifestNotes] : []
|
|
1003
|
+
].join(`
|
|
1004
|
+
`)
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
applyDedupeWindow(documents) {
|
|
1008
|
+
if (this.dedupeWindowMs <= 0)
|
|
1009
|
+
return documents;
|
|
1010
|
+
const now = Date.now();
|
|
1011
|
+
for (const [docId, seenAt] of this.seenDocs) {
|
|
1012
|
+
if (now - seenAt > this.dedupeWindowMs)
|
|
1013
|
+
this.seenDocs.delete(docId);
|
|
1014
|
+
}
|
|
1015
|
+
const out = [];
|
|
1016
|
+
const inBatch = new Set;
|
|
1017
|
+
for (const document of documents) {
|
|
1018
|
+
if (inBatch.has(document.docId))
|
|
1019
|
+
continue;
|
|
1020
|
+
inBatch.add(document.docId);
|
|
1021
|
+
this.seenDocs.set(document.docId, now);
|
|
1022
|
+
out.push(document);
|
|
1023
|
+
}
|
|
1024
|
+
return out;
|
|
1025
|
+
}
|
|
1026
|
+
fail(code, message) {
|
|
1027
|
+
const sanitized = boundDiagnosticText(message);
|
|
1028
|
+
this.lastError = sanitized;
|
|
1029
|
+
const diagnostic = {
|
|
1030
|
+
code,
|
|
1031
|
+
severity: code === "datasource-unavailable" || code === "datasource-empty" ? "warning" : "error",
|
|
1032
|
+
message: sanitized,
|
|
1033
|
+
instanceId: this.instanceId,
|
|
1034
|
+
source: this.definition.skillName
|
|
1035
|
+
};
|
|
1036
|
+
return {
|
|
1037
|
+
ok: false,
|
|
1038
|
+
instanceId: this.instanceId,
|
|
1039
|
+
skill: this.definition.skillName,
|
|
1040
|
+
indexedAt: Date.now(),
|
|
1041
|
+
diagnostics: [diagnostic],
|
|
1042
|
+
error: sanitized,
|
|
1043
|
+
code,
|
|
1044
|
+
message: sanitized
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
class ConnectorLexicalMethod {
|
|
1050
|
+
options;
|
|
1051
|
+
constructor(options) {
|
|
1052
|
+
this.options = options;
|
|
1053
|
+
}
|
|
1054
|
+
describe() {
|
|
1055
|
+
const { skillName, skillType, tags } = this.options;
|
|
1056
|
+
return {
|
|
1057
|
+
name: `${skillName}-lexical`,
|
|
1058
|
+
type: "bm25",
|
|
1059
|
+
description: `Lexical retrieval over indexed ${skillType} chunks`,
|
|
1060
|
+
status: "active",
|
|
1061
|
+
capabilities: ["lexical", "scoped", "path-opaque-sources"],
|
|
1062
|
+
datasourceId: skillName,
|
|
1063
|
+
tags: [...tags]
|
|
1064
|
+
};
|
|
1065
|
+
}
|
|
1066
|
+
async retrieve(query, options) {
|
|
1067
|
+
const trimmed = query.trim();
|
|
1068
|
+
if (trimmed.length === 0)
|
|
1069
|
+
return [];
|
|
1070
|
+
const topK = options.topK ?? DEFAULT_TOP_K;
|
|
1071
|
+
let hits;
|
|
1072
|
+
try {
|
|
1073
|
+
hits = this.options.store.search(trimmed, Math.max(topK * 3, topK));
|
|
1074
|
+
} catch {
|
|
1075
|
+
return [];
|
|
1076
|
+
}
|
|
1077
|
+
const { skillName, instanceId } = this.options;
|
|
1078
|
+
const mapped = [];
|
|
1079
|
+
for (const { chunk, score } of hits) {
|
|
1080
|
+
const source = datasourceSourcePath(skillName, instanceId, chunk.chunkId);
|
|
1081
|
+
if (!this.matchesScope(source, options.scope, options.allowedScopes))
|
|
1082
|
+
continue;
|
|
1083
|
+
mapped.push({
|
|
1084
|
+
id: `${skillName}:${instanceId}:${chunk.chunkId}`,
|
|
1085
|
+
content: chunk.content,
|
|
1086
|
+
source,
|
|
1087
|
+
score,
|
|
1088
|
+
metadata: {
|
|
1089
|
+
...chunk.metadata,
|
|
1090
|
+
method: `${skillName}-lexical`,
|
|
1091
|
+
datasourceId: skillName,
|
|
1092
|
+
instanceId,
|
|
1093
|
+
chunkId: chunk.chunkId,
|
|
1094
|
+
...chunk.title !== undefined ? { title: chunk.title } : {},
|
|
1095
|
+
...chunk.hierarchy.length > 0 ? { hierarchy: chunk.hierarchy.join("/") } : {},
|
|
1096
|
+
...chunk.publishedAt !== undefined ? { publishedAt: chunk.publishedAt } : {}
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
if (mapped.length >= topK)
|
|
1100
|
+
break;
|
|
1101
|
+
}
|
|
1102
|
+
return mapped;
|
|
1103
|
+
}
|
|
1104
|
+
matchesScope(source, scope, allowedScopes) {
|
|
1105
|
+
if (!matchesDatasourceScope(source, scope))
|
|
1106
|
+
return false;
|
|
1107
|
+
if (allowedScopes === undefined || allowedScopes.length === 0)
|
|
1108
|
+
return true;
|
|
1109
|
+
return allowedScopes.some((entry) => matchesDatasourceScope(source, entry));
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
var DEFAULT_POLLING_INTERVAL_MS, DEFAULT_INSTANCE_ID = "default", DEFAULT_TOP_K = 20;
|
|
1113
|
+
var init_connector_skill = __esm(() => {
|
|
1114
|
+
init_chunk_store();
|
|
1115
|
+
init_connector2();
|
|
1116
|
+
init_scope();
|
|
1117
|
+
DEFAULT_POLLING_INTERVAL_MS = 15 * 60 * 1000;
|
|
1118
|
+
});
|
|
1119
|
+
|
|
1120
|
+
// src/datasource/skills/discord/skill.ts
|
|
1121
|
+
var DISCORD_SKILL_DEFINITION, DiscordSkill;
|
|
1122
|
+
var init_skill = __esm(() => {
|
|
1123
|
+
init_connector_skill();
|
|
1124
|
+
init_connector();
|
|
1125
|
+
DISCORD_SKILL_DEFINITION = {
|
|
1126
|
+
skillName: "discord",
|
|
1127
|
+
skillType: "discord-guild",
|
|
1128
|
+
description: "Discord guild datasource",
|
|
1129
|
+
capabilities: ["chat", "api", "polling"],
|
|
1130
|
+
defaultTags: ["discord", "chat", "pii"],
|
|
1131
|
+
contentType: "chat",
|
|
1132
|
+
manifestDescription: "Search indexed Discord messages across authorized guild channels and threads. Use for questions about Discord conversations or community discussions.",
|
|
1133
|
+
manifestNotes: ["Channel visibility and history windows are bounded by the server-configured bot permissions."]
|
|
1134
|
+
};
|
|
1135
|
+
DiscordSkill = class DiscordSkill extends ConnectorDatasourceSkill {
|
|
1136
|
+
constructor(options = {}) {
|
|
1137
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
1138
|
+
super(DISCORD_SKILL_DEFINITION, {
|
|
1139
|
+
...rest,
|
|
1140
|
+
connector: connector ?? new DiscordConnector(connectorOptions ?? {})
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
};
|
|
1144
|
+
});
|
|
1145
|
+
|
|
1146
|
+
// src/datasource/skills/discord/index.ts
|
|
1147
|
+
var init_discord = __esm(() => {
|
|
1148
|
+
init_connector();
|
|
1149
|
+
init_skill();
|
|
1150
|
+
});
|
|
1151
|
+
|
|
1152
|
+
// src/datasource/skills/gdrive/connector.ts
|
|
1153
|
+
class GDriveConnector {
|
|
1154
|
+
options;
|
|
1155
|
+
constructor(options = {}) {
|
|
1156
|
+
this.options = options;
|
|
1157
|
+
}
|
|
1158
|
+
async fetch(signal) {
|
|
1159
|
+
const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV2);
|
|
1160
|
+
if (token === undefined)
|
|
1161
|
+
return { ok: false, reason: "not-configured", message: "auth token not configured" };
|
|
1162
|
+
const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL2;
|
|
1163
|
+
const request = {
|
|
1164
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
1165
|
+
timeoutMs: this.options.timeoutMs,
|
|
1166
|
+
fetchImpl: this.options.fetchImpl,
|
|
1167
|
+
signal
|
|
1168
|
+
};
|
|
1169
|
+
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES2;
|
|
1170
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS2;
|
|
1171
|
+
const files = [];
|
|
1172
|
+
let pageToken;
|
|
1173
|
+
for (let page = 0;page < maxPages; page += 1) {
|
|
1174
|
+
const url = new URL(`${baseUrl}/files`);
|
|
1175
|
+
url.searchParams.set("pageSize", "100");
|
|
1176
|
+
url.searchParams.set("fields", "nextPageToken,files(id,name,mimeType,modifiedTime,parents)");
|
|
1177
|
+
const query = this.options.folderId !== undefined ? `trashed=false and '${this.options.folderId}' in parents` : "trashed=false";
|
|
1178
|
+
url.searchParams.set("q", query);
|
|
1179
|
+
if (this.options.includeSharedDrives === true) {
|
|
1180
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
1181
|
+
url.searchParams.set("includeItemsFromAllDrives", "true");
|
|
1182
|
+
url.searchParams.set("corpora", "allDrives");
|
|
1183
|
+
}
|
|
1184
|
+
if (pageToken !== undefined)
|
|
1185
|
+
url.searchParams.set("pageToken", pageToken);
|
|
1186
|
+
const result = await httpJson(url.toString(), request);
|
|
1187
|
+
if (!result.ok)
|
|
1188
|
+
return { ok: false, reason: result.reason, message: `file list failed: ${result.message}` };
|
|
1189
|
+
const envelope = asRecord(result.json);
|
|
1190
|
+
if (envelope === undefined)
|
|
1191
|
+
return { ok: false, reason: "invalid-data", message: "file list malformed" };
|
|
1192
|
+
for (const raw of asArray(envelope.files)) {
|
|
1193
|
+
const file = asRecord(raw);
|
|
1194
|
+
const id = asString(file?.id);
|
|
1195
|
+
const name = asString(file?.name);
|
|
1196
|
+
const mimeType = asString(file?.mimeType);
|
|
1197
|
+
if (id === undefined || name === undefined || mimeType === undefined)
|
|
1198
|
+
continue;
|
|
1199
|
+
files.push({ id, name, mimeType, modifiedTime: asString(file?.modifiedTime) });
|
|
1200
|
+
}
|
|
1201
|
+
pageToken = asString(envelope.nextPageToken);
|
|
1202
|
+
if (pageToken === undefined || pageToken.length === 0)
|
|
1203
|
+
break;
|
|
1204
|
+
}
|
|
1205
|
+
const documents = [];
|
|
1206
|
+
const warnings = [];
|
|
1207
|
+
for (const file of files) {
|
|
1208
|
+
if (documents.length >= maxDocuments)
|
|
1209
|
+
break;
|
|
1210
|
+
const exportMime = EXPORT_MIME[file.mimeType];
|
|
1211
|
+
let contentUrl;
|
|
1212
|
+
if (exportMime !== undefined) {
|
|
1213
|
+
contentUrl = `${baseUrl}/files/${file.id}/export?mimeType=${encodeURIComponent(exportMime)}`;
|
|
1214
|
+
} else if (DOWNLOAD_MIME.has(file.mimeType)) {
|
|
1215
|
+
contentUrl = `${baseUrl}/files/${file.id}?alt=media`;
|
|
1216
|
+
} else {
|
|
1217
|
+
continue;
|
|
1218
|
+
}
|
|
1219
|
+
const contentResult = await httpText(contentUrl, request);
|
|
1220
|
+
if (!contentResult.ok) {
|
|
1221
|
+
warnings.push(`file export failed: ${contentResult.message}`);
|
|
1222
|
+
continue;
|
|
1223
|
+
}
|
|
1224
|
+
const content = contentResult.text.trim().slice(0, MAX_CONTENT_CHARS);
|
|
1225
|
+
if (content.length === 0)
|
|
1226
|
+
continue;
|
|
1227
|
+
documents.push({
|
|
1228
|
+
docId: file.id,
|
|
1229
|
+
hierarchy: ["files"],
|
|
1230
|
+
title: file.name,
|
|
1231
|
+
content,
|
|
1232
|
+
publishedAt: parseEpochMs(file.modifiedTime),
|
|
1233
|
+
metadata: {
|
|
1234
|
+
mimeType: file.mimeType,
|
|
1235
|
+
...file.modifiedTime !== undefined ? { modifiedTime: file.modifiedTime } : {}
|
|
1236
|
+
}
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1239
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
var DEFAULT_BASE_URL2 = "https://www.googleapis.com/drive/v3", DEFAULT_TOKEN_ENV2 = "GDRIVE_ACCESS_TOKEN", DEFAULT_MAX_PAGES2 = 10, DEFAULT_MAX_DOCUMENTS2 = 500, MAX_CONTENT_CHARS = 1e5, EXPORT_MIME, DOWNLOAD_MIME;
|
|
1243
|
+
var init_connector3 = __esm(() => {
|
|
1244
|
+
EXPORT_MIME = {
|
|
1245
|
+
"application/vnd.google-apps.document": "text/plain",
|
|
1246
|
+
"application/vnd.google-apps.spreadsheet": "text/csv"
|
|
1247
|
+
};
|
|
1248
|
+
DOWNLOAD_MIME = new Set(["text/plain", "text/markdown", "text/csv", "application/json"]);
|
|
1249
|
+
});
|
|
1250
|
+
|
|
1251
|
+
// src/datasource/skills/gdrive/rclone-connector.ts
|
|
1252
|
+
import { spawn } from "node:child_process";
|
|
1253
|
+
|
|
1254
|
+
class RcloneConnector {
|
|
1255
|
+
options;
|
|
1256
|
+
runner;
|
|
1257
|
+
constructor(options = {}) {
|
|
1258
|
+
this.options = options;
|
|
1259
|
+
this.runner = options.runner ?? ((args, timeoutMs) => runBinary(options.binaryPath ?? DEFAULT_BINARY, args, timeoutMs));
|
|
1260
|
+
}
|
|
1261
|
+
async fetch() {
|
|
1262
|
+
const remote = this.options.remote;
|
|
1263
|
+
if (remote === undefined || remote.length === 0) {
|
|
1264
|
+
return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
|
|
1265
|
+
}
|
|
1266
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
|
|
1267
|
+
const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
|
|
1268
|
+
const extensions = this.options.extensions ?? DEFAULT_EXTENSIONS;
|
|
1269
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS3;
|
|
1270
|
+
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE;
|
|
1271
|
+
let listResult;
|
|
1272
|
+
try {
|
|
1273
|
+
listResult = await this.runner(["lsjson", remote, "--recursive", "--files-only", "--drive-export-formats", exportFormats], timeoutMs);
|
|
1274
|
+
} catch {
|
|
1275
|
+
return { ok: false, reason: "unavailable", message: "rclone binary not found or failed to spawn" };
|
|
1276
|
+
}
|
|
1277
|
+
if (!listResult.ok) {
|
|
1278
|
+
return { ok: false, reason: classifyFailure(listResult.stderr), message: shortFailure(listResult) };
|
|
1279
|
+
}
|
|
1280
|
+
let entries;
|
|
1281
|
+
try {
|
|
1282
|
+
entries = asArray(JSON.parse(listResult.stdout));
|
|
1283
|
+
} catch {
|
|
1284
|
+
return { ok: false, reason: "invalid-data", message: "rclone listing was not valid JSON" };
|
|
1285
|
+
}
|
|
1286
|
+
const documents = [];
|
|
1287
|
+
let readFailures = 0;
|
|
1288
|
+
let skipped = 0;
|
|
1289
|
+
for (const raw of entries) {
|
|
1290
|
+
if (documents.length >= maxDocuments)
|
|
1291
|
+
break;
|
|
1292
|
+
const entry = asRecord(raw);
|
|
1293
|
+
const path = asString(entry?.Path);
|
|
1294
|
+
const name = asString(entry?.Name);
|
|
1295
|
+
if (entry === undefined || path === undefined || name === undefined)
|
|
1296
|
+
continue;
|
|
1297
|
+
const extension = name.includes(".") ? name.slice(name.lastIndexOf(".")).toLowerCase() : "";
|
|
1298
|
+
if (!extensions.includes(extension)) {
|
|
1299
|
+
skipped += 1;
|
|
1300
|
+
continue;
|
|
1301
|
+
}
|
|
1302
|
+
const size = typeof entry.Size === "number" ? entry.Size : 0;
|
|
1303
|
+
if (size > maxBytes) {
|
|
1304
|
+
skipped += 1;
|
|
1305
|
+
continue;
|
|
1306
|
+
}
|
|
1307
|
+
const target = remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
|
|
1308
|
+
let content = "";
|
|
1309
|
+
try {
|
|
1310
|
+
const catResult = await this.runner(["cat", target, "--drive-export-formats", exportFormats, "--count", String(maxBytes)], timeoutMs);
|
|
1311
|
+
if (!catResult.ok) {
|
|
1312
|
+
readFailures += 1;
|
|
1313
|
+
continue;
|
|
1314
|
+
}
|
|
1315
|
+
content = catResult.stdout.trim().slice(0, MAX_CONTENT_CHARS2);
|
|
1316
|
+
} catch {
|
|
1317
|
+
readFailures += 1;
|
|
1318
|
+
continue;
|
|
1319
|
+
}
|
|
1320
|
+
if (content.length === 0)
|
|
1321
|
+
continue;
|
|
1322
|
+
const segments = path.split("/");
|
|
1323
|
+
documents.push({
|
|
1324
|
+
docId: path,
|
|
1325
|
+
hierarchy: ["files", ...segments.slice(0, -1)],
|
|
1326
|
+
title: name,
|
|
1327
|
+
content,
|
|
1328
|
+
publishedAt: parseEpochMs(asString(entry.ModTime)),
|
|
1329
|
+
metadata: {
|
|
1330
|
+
...asString(entry.MimeType) !== undefined ? { mimeType: asString(entry.MimeType) } : {},
|
|
1331
|
+
size
|
|
1332
|
+
}
|
|
1333
|
+
});
|
|
1334
|
+
}
|
|
1335
|
+
const warnings = [];
|
|
1336
|
+
if (readFailures > 0)
|
|
1337
|
+
warnings.push(`${readFailures} file(s) failed to read`);
|
|
1338
|
+
if (skipped > 0)
|
|
1339
|
+
warnings.push(`${skipped} file(s) skipped (non-text or oversized)`);
|
|
1340
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
function classifyFailure(stderr) {
|
|
1344
|
+
const lower = stderr.toLowerCase();
|
|
1345
|
+
if (lower.includes("didn't find section") || lower.includes("config file") || lower.includes("no remotes")) {
|
|
1346
|
+
return "not-configured";
|
|
1347
|
+
}
|
|
1348
|
+
if (lower.includes("oauth") || lower.includes("token") || lower.includes("unauthorized") || lower.includes("401")) {
|
|
1349
|
+
return "auth";
|
|
1350
|
+
}
|
|
1351
|
+
if (lower.includes("permission") || lower.includes("403") || lower.includes("forbidden"))
|
|
1352
|
+
return "permission";
|
|
1353
|
+
return "api-error";
|
|
1354
|
+
}
|
|
1355
|
+
function shortFailure(result) {
|
|
1356
|
+
return `rclone exited with code ${result.code ?? "unknown"}`;
|
|
1357
|
+
}
|
|
1358
|
+
function runBinary(binary, args, timeoutMs) {
|
|
1359
|
+
return new Promise((resolvePromise) => {
|
|
1360
|
+
const child = spawn(binary, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
1361
|
+
let stdout = "";
|
|
1362
|
+
let stderr = "";
|
|
1363
|
+
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
1364
|
+
child.stdout.on("data", (chunk) => {
|
|
1365
|
+
stdout += chunk.toString("utf8");
|
|
1366
|
+
});
|
|
1367
|
+
child.stderr.on("data", (chunk) => {
|
|
1368
|
+
stderr += chunk.toString("utf8");
|
|
1369
|
+
});
|
|
1370
|
+
child.on("error", () => {
|
|
1371
|
+
clearTimeout(timer);
|
|
1372
|
+
resolvePromise({ ok: false, stdout, stderr, code: null });
|
|
1373
|
+
});
|
|
1374
|
+
child.on("close", (code) => {
|
|
1375
|
+
clearTimeout(timer);
|
|
1376
|
+
resolvePromise({ ok: code === 0, stdout, stderr, code });
|
|
1377
|
+
});
|
|
1378
|
+
});
|
|
1379
|
+
}
|
|
1380
|
+
var DEFAULT_BINARY = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS3 = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS2 = 60000, MAX_CONTENT_CHARS2 = 1e5;
|
|
1381
|
+
var init_rclone_connector = __esm(() => {
|
|
1382
|
+
DEFAULT_EXTENSIONS = [".txt", ".md", ".csv", ".json", ".html", ".rst", ".org"];
|
|
1383
|
+
DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
|
|
1384
|
+
});
|
|
1385
|
+
|
|
1386
|
+
// src/datasource/skills/gdrive/skill.ts
|
|
1387
|
+
var GDRIVE_SKILL_DEFINITION, GDriveSkill;
|
|
1388
|
+
var init_skill2 = __esm(() => {
|
|
1389
|
+
init_connector_skill();
|
|
1390
|
+
init_connector3();
|
|
1391
|
+
GDRIVE_SKILL_DEFINITION = {
|
|
1392
|
+
skillName: "gdrive",
|
|
1393
|
+
skillType: "google-drive",
|
|
1394
|
+
description: "Google Drive datasource",
|
|
1395
|
+
capabilities: ["documents", "api", "polling"],
|
|
1396
|
+
defaultTags: ["gdrive", "documents", "pii"],
|
|
1397
|
+
contentType: "document",
|
|
1398
|
+
manifestDescription: "Search indexed Google Drive documents (Docs, Sheets, plain text). Use for questions about files stored in the authorized Drive account or folder.",
|
|
1399
|
+
manifestNotes: [
|
|
1400
|
+
"Shared-drive permissions are enforced by the server-configured OAuth token; unexported binary formats are not indexed."
|
|
1401
|
+
]
|
|
1402
|
+
};
|
|
1403
|
+
GDriveSkill = class GDriveSkill extends ConnectorDatasourceSkill {
|
|
1404
|
+
constructor(options = {}) {
|
|
1405
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
1406
|
+
super(GDRIVE_SKILL_DEFINITION, {
|
|
1407
|
+
...rest,
|
|
1408
|
+
connector: connector ?? new GDriveConnector(connectorOptions ?? {})
|
|
1409
|
+
});
|
|
1410
|
+
}
|
|
1411
|
+
};
|
|
1412
|
+
});
|
|
1413
|
+
|
|
1414
|
+
// src/datasource/skills/gdrive/index.ts
|
|
1415
|
+
var init_gdrive = __esm(() => {
|
|
1416
|
+
init_connector3();
|
|
1417
|
+
init_rclone_connector();
|
|
1418
|
+
init_skill2();
|
|
1419
|
+
});
|
|
1420
|
+
|
|
1421
|
+
// src/datasource/skills/github/connector.ts
|
|
1422
|
+
class GitHubConnector {
|
|
1423
|
+
options;
|
|
1424
|
+
constructor(options = {}) {
|
|
1425
|
+
this.options = options;
|
|
1426
|
+
}
|
|
1427
|
+
async fetch(signal) {
|
|
1428
|
+
const repos = this.options.repos ?? [];
|
|
1429
|
+
if (repos.length === 0)
|
|
1430
|
+
return { ok: false, reason: "not-configured", message: "no repositories configured" };
|
|
1431
|
+
const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV3);
|
|
1432
|
+
const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL3;
|
|
1433
|
+
const request = {
|
|
1434
|
+
headers: {
|
|
1435
|
+
Accept: "application/vnd.github+json",
|
|
1436
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
1437
|
+
"User-Agent": "autorag-datasource",
|
|
1438
|
+
...token !== undefined ? { Authorization: `Bearer ${token}` } : {}
|
|
1439
|
+
},
|
|
1440
|
+
timeoutMs: this.options.timeoutMs,
|
|
1441
|
+
fetchImpl: this.options.fetchImpl,
|
|
1442
|
+
signal
|
|
1443
|
+
};
|
|
1444
|
+
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES3;
|
|
1445
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4;
|
|
1446
|
+
const documents = [];
|
|
1447
|
+
const warnings = [];
|
|
1448
|
+
for (const repo of repos) {
|
|
1449
|
+
if (documents.length >= maxDocuments)
|
|
1450
|
+
break;
|
|
1451
|
+
const [owner, name] = repo.split("/", 2);
|
|
1452
|
+
if (owner === undefined || name === undefined || owner.length === 0 || name.length === 0) {
|
|
1453
|
+
warnings.push(`repository entry ${repos.indexOf(repo) + 1} is not owner/repo`);
|
|
1454
|
+
continue;
|
|
1455
|
+
}
|
|
1456
|
+
for (let page = 1;page <= maxPages && documents.length < maxDocuments; page += 1) {
|
|
1457
|
+
const url = `${baseUrl}/repos/${owner}/${name}/issues?state=all&per_page=100&page=${page}`;
|
|
1458
|
+
const result = await httpJson(url, request);
|
|
1459
|
+
if (!result.ok) {
|
|
1460
|
+
if (result.reason === "auth")
|
|
1461
|
+
return { ok: false, reason: "auth", message: "github: unauthorized" };
|
|
1462
|
+
if (result.reason === "rate-limited" || result.reason === "permission") {
|
|
1463
|
+
return { ok: false, reason: "rate-limited", message: "github: rate limited or forbidden" };
|
|
1464
|
+
}
|
|
1465
|
+
warnings.push(`repo ${owner}-${name} fetch failed: ${result.message}`);
|
|
1466
|
+
break;
|
|
1467
|
+
}
|
|
1468
|
+
const items = asArray(result.json);
|
|
1469
|
+
if (items.length === 0)
|
|
1470
|
+
break;
|
|
1471
|
+
for (const raw of items) {
|
|
1472
|
+
if (documents.length >= maxDocuments)
|
|
1473
|
+
break;
|
|
1474
|
+
const item = asRecord(raw);
|
|
1475
|
+
const number = asNumber(item?.number);
|
|
1476
|
+
const title = asString(item?.title);
|
|
1477
|
+
if (item === undefined || number === undefined || title === undefined)
|
|
1478
|
+
continue;
|
|
1479
|
+
const isPull = asRecord(item.pull_request) !== undefined;
|
|
1480
|
+
const body = asString(item.body) ?? "";
|
|
1481
|
+
const labels = asArray(item.labels).map((label) => asString(asRecord(label)?.name)).filter((label) => label !== undefined);
|
|
1482
|
+
documents.push({
|
|
1483
|
+
docId: `${owner}-${name}-${number}`,
|
|
1484
|
+
hierarchy: [owner, name, isPull ? "pulls" : "issues"],
|
|
1485
|
+
title: `#${number} ${title}`,
|
|
1486
|
+
content: body.length > 0 ? `${title}
|
|
1487
|
+
|
|
1488
|
+
${body}` : title,
|
|
1489
|
+
publishedAt: parseEpochMs(item.updated_at),
|
|
1490
|
+
metadata: {
|
|
1491
|
+
state: asString(item.state) ?? "unknown",
|
|
1492
|
+
number,
|
|
1493
|
+
kind: isPull ? "pull" : "issue",
|
|
1494
|
+
labels
|
|
1495
|
+
}
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1498
|
+
if (items.length < 100)
|
|
1499
|
+
break;
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
var DEFAULT_BASE_URL3 = "https://api.github.com", DEFAULT_TOKEN_ENV3 = "GITHUB_TOKEN", DEFAULT_MAX_PAGES3 = 10, DEFAULT_MAX_DOCUMENTS4 = 500;
|
|
1506
|
+
var init_connector4 = () => {};
|
|
1507
|
+
|
|
1508
|
+
// src/datasource/skills/github/skill.ts
|
|
1509
|
+
var GITHUB_SKILL_DEFINITION, GitHubSkill;
|
|
1510
|
+
var init_skill3 = __esm(() => {
|
|
1511
|
+
init_connector_skill();
|
|
1512
|
+
init_connector4();
|
|
1513
|
+
GITHUB_SKILL_DEFINITION = {
|
|
1514
|
+
skillName: "github",
|
|
1515
|
+
skillType: "github-issues",
|
|
1516
|
+
description: "GitHub Issues/PRs datasource",
|
|
1517
|
+
capabilities: ["issues", "api", "polling"],
|
|
1518
|
+
defaultTags: ["github", "issues"],
|
|
1519
|
+
contentType: "issue",
|
|
1520
|
+
manifestDescription: "Search indexed GitHub issues and pull requests for the configured repositories. Use for questions about bug reports, feature requests, and review discussions.",
|
|
1521
|
+
manifestNotes: ["Repository visibility is bounded by the server-configured token scopes."]
|
|
1522
|
+
};
|
|
1523
|
+
GitHubSkill = class GitHubSkill extends ConnectorDatasourceSkill {
|
|
1524
|
+
constructor(options = {}) {
|
|
1525
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
1526
|
+
super(GITHUB_SKILL_DEFINITION, {
|
|
1527
|
+
...rest,
|
|
1528
|
+
connector: connector ?? new GitHubConnector(connectorOptions ?? {})
|
|
1529
|
+
});
|
|
1530
|
+
}
|
|
1531
|
+
};
|
|
1532
|
+
});
|
|
1533
|
+
|
|
1534
|
+
// src/datasource/skills/github/index.ts
|
|
1535
|
+
var init_github = __esm(() => {
|
|
1536
|
+
init_connector4();
|
|
1537
|
+
init_skill3();
|
|
1538
|
+
});
|
|
1539
|
+
|
|
1540
|
+
// src/datasource/skills/gmail/himalaya-connector.ts
|
|
1541
|
+
import { spawn as spawn2 } from "node:child_process";
|
|
1542
|
+
|
|
1543
|
+
class HimalayaConnector {
|
|
1544
|
+
options;
|
|
1545
|
+
runner;
|
|
1546
|
+
constructor(options = {}) {
|
|
1547
|
+
this.options = options;
|
|
1548
|
+
this.runner = options.runner ?? ((args, timeoutMs) => runBinary2(options.binaryPath ?? DEFAULT_BINARY2, args, timeoutMs));
|
|
1549
|
+
}
|
|
1550
|
+
async fetch() {
|
|
1551
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS3;
|
|
1552
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS5;
|
|
1553
|
+
const accountArgs = this.options.account !== undefined ? ["--account", this.options.account] : [];
|
|
1554
|
+
const folderArgs = this.options.folder !== undefined ? ["--folder", this.options.folder] : [];
|
|
1555
|
+
const listArgs = [
|
|
1556
|
+
"envelope",
|
|
1557
|
+
"list",
|
|
1558
|
+
...accountArgs,
|
|
1559
|
+
...folderArgs,
|
|
1560
|
+
"--page-size",
|
|
1561
|
+
String(this.options.pageSize ?? DEFAULT_PAGE_SIZE),
|
|
1562
|
+
"--output",
|
|
1563
|
+
"json"
|
|
1564
|
+
];
|
|
1565
|
+
let listResult;
|
|
1566
|
+
try {
|
|
1567
|
+
listResult = await this.runner(listArgs, timeoutMs);
|
|
1568
|
+
} catch {
|
|
1569
|
+
return { ok: false, reason: "unavailable", message: "himalaya binary not found or failed to spawn" };
|
|
1570
|
+
}
|
|
1571
|
+
if (!listResult.ok) {
|
|
1572
|
+
return { ok: false, reason: classifyFailure2(listResult.stderr), message: shortFailure2(listResult) };
|
|
1573
|
+
}
|
|
1574
|
+
let envelopes;
|
|
1575
|
+
try {
|
|
1576
|
+
envelopes = asArray(JSON.parse(listResult.stdout));
|
|
1577
|
+
} catch {
|
|
1578
|
+
return { ok: false, reason: "invalid-data", message: "envelope listing was not valid JSON" };
|
|
1579
|
+
}
|
|
1580
|
+
const account = this.options.account ?? "default";
|
|
1581
|
+
const folder = this.options.folder ?? "INBOX";
|
|
1582
|
+
const documents = [];
|
|
1583
|
+
let readFailures = 0;
|
|
1584
|
+
for (const raw of envelopes.slice(0, maxDocuments)) {
|
|
1585
|
+
const envelope = asRecord(raw);
|
|
1586
|
+
const id = asString(envelope?.id);
|
|
1587
|
+
if (envelope === undefined || id === undefined)
|
|
1588
|
+
continue;
|
|
1589
|
+
const subject = asString(envelope.subject) ?? "(no subject)";
|
|
1590
|
+
const from = asRecord(envelope.from);
|
|
1591
|
+
const fromText = [asString(from?.name), asString(from?.addr)].filter(Boolean).join(" ").trim();
|
|
1592
|
+
let body = "";
|
|
1593
|
+
try {
|
|
1594
|
+
const readResult = await this.runner(["message", "read", ...accountArgs, ...folderArgs, id], timeoutMs);
|
|
1595
|
+
if (readResult.ok)
|
|
1596
|
+
body = readResult.stdout.trim().slice(0, MAX_BODY_CHARS);
|
|
1597
|
+
else
|
|
1598
|
+
readFailures += 1;
|
|
1599
|
+
} catch {
|
|
1600
|
+
readFailures += 1;
|
|
1601
|
+
}
|
|
1602
|
+
const headerBlock = [
|
|
1603
|
+
`Subject: ${subject}`,
|
|
1604
|
+
...fromText.length > 0 ? [`From: ${fromText}`] : [],
|
|
1605
|
+
...asString(envelope.date) !== undefined ? [`Date: ${asString(envelope.date)}`] : []
|
|
1606
|
+
].join(`
|
|
1607
|
+
`);
|
|
1608
|
+
documents.push({
|
|
1609
|
+
docId: `${account}-${folder}-${id}`,
|
|
1610
|
+
hierarchy: ["accounts", account, folder],
|
|
1611
|
+
title: subject,
|
|
1612
|
+
content: body.length > 0 ? `${headerBlock}
|
|
1613
|
+
|
|
1614
|
+
${body}` : headerBlock,
|
|
1615
|
+
publishedAt: parseHimalayaDate(asString(envelope.date)),
|
|
1616
|
+
metadata: {
|
|
1617
|
+
account,
|
|
1618
|
+
folder,
|
|
1619
|
+
messageId: id,
|
|
1620
|
+
...asArray(envelope.flags).length > 0 ? {
|
|
1621
|
+
flags: asArray(envelope.flags).map((flag) => asString(flag)).filter(Boolean)
|
|
1622
|
+
} : {}
|
|
1623
|
+
}
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
const warnings = readFailures > 0 ? [`${readFailures} message(s) failed to read`] : undefined;
|
|
1627
|
+
return { ok: true, documents, ...warnings !== undefined ? { warnings } : {} };
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
function parseHimalayaDate(value) {
|
|
1631
|
+
if (value === undefined || value.length === 0)
|
|
1632
|
+
return;
|
|
1633
|
+
const parsed = Date.parse(value.replace(" ", "T"));
|
|
1634
|
+
return Number.isNaN(parsed) ? undefined : parsed;
|
|
1635
|
+
}
|
|
1636
|
+
function classifyFailure2(stderr) {
|
|
1637
|
+
const lower = stderr.toLowerCase();
|
|
1638
|
+
if (lower.includes("cannot find") && lower.includes("account"))
|
|
1639
|
+
return "not-configured";
|
|
1640
|
+
if (lower.includes("auth") || lower.includes("login") || lower.includes("credential"))
|
|
1641
|
+
return "auth";
|
|
1642
|
+
return "api-error";
|
|
1643
|
+
}
|
|
1644
|
+
function shortFailure2(result) {
|
|
1645
|
+
return `himalaya exited with code ${result.code ?? "unknown"}`;
|
|
1646
|
+
}
|
|
1647
|
+
function runBinary2(binary, args, timeoutMs) {
|
|
1648
|
+
return new Promise((resolvePromise) => {
|
|
1649
|
+
const child = spawn2(binary, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
1650
|
+
let stdout = "";
|
|
1651
|
+
let stderr = "";
|
|
1652
|
+
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
1653
|
+
child.stdout.on("data", (chunk) => {
|
|
1654
|
+
stdout += chunk.toString("utf8");
|
|
1655
|
+
});
|
|
1656
|
+
child.stderr.on("data", (chunk) => {
|
|
1657
|
+
stderr += chunk.toString("utf8");
|
|
1658
|
+
});
|
|
1659
|
+
child.on("error", () => {
|
|
1660
|
+
clearTimeout(timer);
|
|
1661
|
+
resolvePromise({ ok: false, stdout, stderr, code: null });
|
|
1662
|
+
});
|
|
1663
|
+
child.on("close", (code) => {
|
|
1664
|
+
clearTimeout(timer);
|
|
1665
|
+
resolvePromise({ ok: code === 0, stdout, stderr, code });
|
|
1666
|
+
});
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
var DEFAULT_BINARY2 = "himalaya", DEFAULT_PAGE_SIZE = 100, DEFAULT_MAX_DOCUMENTS5 = 50, DEFAULT_TIMEOUT_MS3 = 30000, MAX_BODY_CHARS = 50000;
|
|
1670
|
+
var init_himalaya_connector = () => {};
|
|
1671
|
+
|
|
1672
|
+
// src/datasource/skills/gmail/connector.ts
|
|
1673
|
+
class GmailConnector {
|
|
1674
|
+
options;
|
|
1675
|
+
constructor(options = {}) {
|
|
1676
|
+
this.options = options;
|
|
1677
|
+
}
|
|
1678
|
+
async fetch(signal) {
|
|
1679
|
+
const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV4);
|
|
1680
|
+
if (token === undefined)
|
|
1681
|
+
return { ok: false, reason: "not-configured", message: "auth token not configured" };
|
|
1682
|
+
const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL4;
|
|
1683
|
+
const request = {
|
|
1684
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
1685
|
+
timeoutMs: this.options.timeoutMs,
|
|
1686
|
+
fetchImpl: this.options.fetchImpl,
|
|
1687
|
+
signal
|
|
1688
|
+
};
|
|
1689
|
+
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES4;
|
|
1690
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS6;
|
|
1691
|
+
const messageIds = [];
|
|
1692
|
+
let pageToken;
|
|
1693
|
+
for (let page = 0;page < maxPages && messageIds.length < maxDocuments; page += 1) {
|
|
1694
|
+
const url = new URL(`${baseUrl}/users/me/messages`);
|
|
1695
|
+
url.searchParams.set("maxResults", "100");
|
|
1696
|
+
for (const labelId of this.options.labelIds ?? [])
|
|
1697
|
+
url.searchParams.append("labelIds", labelId);
|
|
1698
|
+
if (this.options.query !== undefined)
|
|
1699
|
+
url.searchParams.set("q", this.options.query);
|
|
1700
|
+
if (pageToken !== undefined)
|
|
1701
|
+
url.searchParams.set("pageToken", pageToken);
|
|
1702
|
+
const result = await httpJson(url.toString(), request);
|
|
1703
|
+
if (!result.ok)
|
|
1704
|
+
return { ok: false, reason: result.reason, message: `message list failed: ${result.message}` };
|
|
1705
|
+
const envelope = asRecord(result.json);
|
|
1706
|
+
if (envelope === undefined)
|
|
1707
|
+
return { ok: false, reason: "invalid-data", message: "message list malformed" };
|
|
1708
|
+
for (const raw of asArray(envelope.messages)) {
|
|
1709
|
+
const id = asString(asRecord(raw)?.id);
|
|
1710
|
+
if (id !== undefined)
|
|
1711
|
+
messageIds.push(id);
|
|
1712
|
+
}
|
|
1713
|
+
pageToken = asString(envelope.nextPageToken);
|
|
1714
|
+
if (pageToken === undefined || pageToken.length === 0)
|
|
1715
|
+
break;
|
|
1716
|
+
}
|
|
1717
|
+
const documents = [];
|
|
1718
|
+
let failedFetches = 0;
|
|
1719
|
+
for (const id of messageIds.slice(0, maxDocuments)) {
|
|
1720
|
+
const result = await httpJson(`${baseUrl}/users/me/messages/${id}?format=full`, request);
|
|
1721
|
+
if (!result.ok) {
|
|
1722
|
+
if (result.reason === "auth")
|
|
1723
|
+
return { ok: false, reason: "auth", message: "gmail: unauthorized" };
|
|
1724
|
+
if (result.reason === "rate-limited") {
|
|
1725
|
+
return { ok: false, reason: "rate-limited", message: "gmail: rate limited" };
|
|
1726
|
+
}
|
|
1727
|
+
failedFetches += 1;
|
|
1728
|
+
continue;
|
|
1729
|
+
}
|
|
1730
|
+
const message = asRecord(result.json);
|
|
1731
|
+
if (message === undefined) {
|
|
1732
|
+
failedFetches += 1;
|
|
1733
|
+
continue;
|
|
1734
|
+
}
|
|
1735
|
+
const payload = asRecord(message.payload);
|
|
1736
|
+
const headers = headerMap(payload);
|
|
1737
|
+
const subject = headers.get("subject") ?? "(no subject)";
|
|
1738
|
+
const body = (extractBody(payload) ?? asString(message.snippet) ?? "").slice(0, MAX_BODY_CHARS2);
|
|
1739
|
+
const labelIds = asArray(message.labelIds).map((label) => asString(label)).filter((label) => label !== undefined);
|
|
1740
|
+
const internalDate = asString(message.internalDate);
|
|
1741
|
+
const publishedAt = internalDate !== undefined && /^\d+$/u.test(internalDate) ? Number(internalDate) : undefined;
|
|
1742
|
+
const headerBlock = [
|
|
1743
|
+
`Subject: ${subject}`,
|
|
1744
|
+
...headers.has("from") ? [`From: ${headers.get("from")}`] : [],
|
|
1745
|
+
...headers.has("to") ? [`To: ${headers.get("to")}`] : [],
|
|
1746
|
+
...headers.has("date") ? [`Date: ${headers.get("date")}`] : []
|
|
1747
|
+
].join(`
|
|
1748
|
+
`);
|
|
1749
|
+
documents.push({
|
|
1750
|
+
docId: id,
|
|
1751
|
+
hierarchy: ["labels", this.options.labelIds?.[0] ?? "all"],
|
|
1752
|
+
title: subject,
|
|
1753
|
+
content: `${headerBlock}
|
|
1754
|
+
|
|
1755
|
+
${body}`.trim(),
|
|
1756
|
+
...publishedAt !== undefined ? { publishedAt } : {},
|
|
1757
|
+
metadata: {
|
|
1758
|
+
...asString(message.threadId) !== undefined ? { threadId: asString(message.threadId) } : {},
|
|
1759
|
+
labelIds
|
|
1760
|
+
}
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
const warnings = failedFetches > 0 ? [`${failedFetches} message(s) failed to fetch`] : undefined;
|
|
1764
|
+
return { ok: true, documents, ...warnings !== undefined ? { warnings } : {} };
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
function headerMap(payload) {
|
|
1768
|
+
const map = new Map;
|
|
1769
|
+
for (const raw of asArray(payload?.headers)) {
|
|
1770
|
+
const header = asRecord(raw);
|
|
1771
|
+
const name = asString(header?.name)?.toLowerCase();
|
|
1772
|
+
const value = asString(header?.value);
|
|
1773
|
+
if (name !== undefined && value !== undefined && !map.has(name))
|
|
1774
|
+
map.set(name, value);
|
|
1775
|
+
}
|
|
1776
|
+
return map;
|
|
1777
|
+
}
|
|
1778
|
+
function extractBody(payload) {
|
|
1779
|
+
if (payload === undefined)
|
|
1780
|
+
return;
|
|
1781
|
+
const mimeType = asString(payload.mimeType);
|
|
1782
|
+
const data = asString(asRecord(payload.body)?.data);
|
|
1783
|
+
if (mimeType === "text/plain" && data !== undefined)
|
|
1784
|
+
return decodeBase64Url(data);
|
|
1785
|
+
const collected = [];
|
|
1786
|
+
for (const raw of asArray(payload.parts)) {
|
|
1787
|
+
const part = asRecord(raw);
|
|
1788
|
+
const text = extractBody(part);
|
|
1789
|
+
if (text !== undefined && text.length > 0)
|
|
1790
|
+
collected.push(text);
|
|
1791
|
+
}
|
|
1792
|
+
if (collected.length > 0)
|
|
1793
|
+
return collected.join(`
|
|
1794
|
+
`);
|
|
1795
|
+
return data !== undefined ? decodeBase64Url(data) : undefined;
|
|
1796
|
+
}
|
|
1797
|
+
function decodeBase64Url(data) {
|
|
1798
|
+
try {
|
|
1799
|
+
return Buffer.from(data, "base64url").toString("utf8");
|
|
1800
|
+
} catch {
|
|
1801
|
+
return "";
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
var DEFAULT_BASE_URL4 = "https://gmail.googleapis.com/gmail/v1", DEFAULT_TOKEN_ENV4 = "GMAIL_ACCESS_TOKEN", DEFAULT_MAX_PAGES4 = 10, DEFAULT_MAX_DOCUMENTS6 = 500, MAX_BODY_CHARS2 = 50000;
|
|
1805
|
+
var init_connector5 = () => {};
|
|
1806
|
+
|
|
1807
|
+
// src/datasource/skills/gmail/skill.ts
|
|
1808
|
+
var GMAIL_SKILL_DEFINITION, GmailSkill;
|
|
1809
|
+
var init_skill4 = __esm(() => {
|
|
1810
|
+
init_connector_skill();
|
|
1811
|
+
init_connector5();
|
|
1812
|
+
GMAIL_SKILL_DEFINITION = {
|
|
1813
|
+
skillName: "gmail",
|
|
1814
|
+
skillType: "gmail-account",
|
|
1815
|
+
description: "Gmail datasource",
|
|
1816
|
+
capabilities: ["email", "api", "polling"],
|
|
1817
|
+
defaultTags: ["gmail", "email", "pii"],
|
|
1818
|
+
contentType: "email",
|
|
1819
|
+
manifestDescription: "Search indexed Gmail messages from the authorized account and labels. Use for questions about email conversations, decisions, or attachments discussed over email.",
|
|
1820
|
+
manifestNotes: ["Label and folder visibility is bounded by the server-configured OAuth token."]
|
|
1821
|
+
};
|
|
1822
|
+
GmailSkill = class GmailSkill extends ConnectorDatasourceSkill {
|
|
1823
|
+
constructor(options = {}) {
|
|
1824
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
1825
|
+
super(GMAIL_SKILL_DEFINITION, {
|
|
1826
|
+
...rest,
|
|
1827
|
+
connector: connector ?? new GmailConnector(connectorOptions ?? {})
|
|
1828
|
+
});
|
|
1829
|
+
}
|
|
1830
|
+
};
|
|
1831
|
+
});
|
|
1832
|
+
|
|
1833
|
+
// src/datasource/skills/gmail/index.ts
|
|
1834
|
+
var init_gmail = __esm(() => {
|
|
1835
|
+
init_connector5();
|
|
1836
|
+
init_himalaya_connector();
|
|
1837
|
+
init_skill4();
|
|
1838
|
+
});
|
|
1839
|
+
|
|
1840
|
+
// src/datasource/skills/mail-export/connector.ts
|
|
1841
|
+
import { readdir, readFile, stat } from "node:fs/promises";
|
|
1842
|
+
import { basename, dirname as dirname2, extname, join as join3 } from "node:path";
|
|
1843
|
+
import { simpleParser } from "mailparser";
|
|
1844
|
+
|
|
1845
|
+
class MailExportConnector {
|
|
1846
|
+
options;
|
|
1847
|
+
constructor(options = {}) {
|
|
1848
|
+
this.options = options;
|
|
1849
|
+
}
|
|
1850
|
+
async fetch() {
|
|
1851
|
+
const paths = this.options.paths ?? [];
|
|
1852
|
+
if (paths.length === 0)
|
|
1853
|
+
return { ok: false, reason: "not-configured", message: "no mail export paths configured" };
|
|
1854
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS7;
|
|
1855
|
+
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
|
|
1856
|
+
const files = [];
|
|
1857
|
+
let missingPaths = 0;
|
|
1858
|
+
for (const path of paths) {
|
|
1859
|
+
try {
|
|
1860
|
+
const info = await stat(path);
|
|
1861
|
+
if (info.isDirectory()) {
|
|
1862
|
+
const entries = await readdir(path, { recursive: true, withFileTypes: true });
|
|
1863
|
+
for (const entry of entries) {
|
|
1864
|
+
if (!entry.isFile())
|
|
1865
|
+
continue;
|
|
1866
|
+
const extension = extname(entry.name).toLowerCase();
|
|
1867
|
+
if (extension === ".eml" || extension === ".mbox") {
|
|
1868
|
+
files.push(join3(entry.parentPath, entry.name));
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
} else if (info.isFile()) {
|
|
1872
|
+
files.push(path);
|
|
1873
|
+
}
|
|
1874
|
+
} catch {
|
|
1875
|
+
missingPaths += 1;
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
if (missingPaths === paths.length) {
|
|
1879
|
+
return { ok: false, reason: "unavailable", message: "no configured mail export paths were readable" };
|
|
1880
|
+
}
|
|
1881
|
+
const documents = [];
|
|
1882
|
+
const warnings = [];
|
|
1883
|
+
if (missingPaths > 0)
|
|
1884
|
+
warnings.push(`${missingPaths} configured path(s) were unreadable`);
|
|
1885
|
+
let parseFailures = 0;
|
|
1886
|
+
let skippedLarge = 0;
|
|
1887
|
+
for (const file of files.sort()) {
|
|
1888
|
+
if (documents.length >= maxDocuments)
|
|
1889
|
+
break;
|
|
1890
|
+
let bytes;
|
|
1891
|
+
try {
|
|
1892
|
+
const info = await stat(file);
|
|
1893
|
+
if (info.size > maxBytes) {
|
|
1894
|
+
skippedLarge += 1;
|
|
1895
|
+
continue;
|
|
1896
|
+
}
|
|
1897
|
+
bytes = await readFile(file);
|
|
1898
|
+
} catch {
|
|
1899
|
+
parseFailures += 1;
|
|
1900
|
+
continue;
|
|
1901
|
+
}
|
|
1902
|
+
const stem = basename(file, extname(file));
|
|
1903
|
+
const mailboxDir = basename(dirname2(file));
|
|
1904
|
+
const hierarchy = ["mailboxes", mailboxDir, stem];
|
|
1905
|
+
const extension = extname(file).toLowerCase();
|
|
1906
|
+
const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
|
|
1907
|
+
for (const [index, raw] of rawMessages.entries()) {
|
|
1908
|
+
if (documents.length >= maxDocuments)
|
|
1909
|
+
break;
|
|
1910
|
+
try {
|
|
1911
|
+
const parsed = await simpleParser(raw);
|
|
1912
|
+
const subject = parsed.subject ?? "(no subject)";
|
|
1913
|
+
const from = Array.isArray(parsed.from) ? parsed.from[0]?.text : parsed.from?.text;
|
|
1914
|
+
const to = Array.isArray(parsed.to) ? parsed.to[0]?.text : parsed.to?.text;
|
|
1915
|
+
const body = (parsed.text ?? "").trim().slice(0, MAX_BODY_CHARS3);
|
|
1916
|
+
const headerBlock = [
|
|
1917
|
+
`Subject: ${subject}`,
|
|
1918
|
+
...from !== undefined ? [`From: ${from}`] : [],
|
|
1919
|
+
...to !== undefined ? [`To: ${to}`] : [],
|
|
1920
|
+
...parsed.date !== undefined ? [`Date: ${parsed.date.toISOString()}`] : []
|
|
1921
|
+
].join(`
|
|
1922
|
+
`);
|
|
1923
|
+
const content = `${headerBlock}
|
|
1924
|
+
|
|
1925
|
+
${body}`.trim();
|
|
1926
|
+
if (content.length === 0)
|
|
1927
|
+
continue;
|
|
1928
|
+
documents.push({
|
|
1929
|
+
docId: extension === ".mbox" ? `${stem}-${index + 1}` : stem,
|
|
1930
|
+
hierarchy,
|
|
1931
|
+
title: subject,
|
|
1932
|
+
content,
|
|
1933
|
+
...parsed.date !== undefined ? { publishedAt: parsed.date.getTime() } : {},
|
|
1934
|
+
metadata: {
|
|
1935
|
+
path: file,
|
|
1936
|
+
subject,
|
|
1937
|
+
...from !== undefined ? { from } : {},
|
|
1938
|
+
messageIndex: index
|
|
1939
|
+
}
|
|
1940
|
+
});
|
|
1941
|
+
} catch {
|
|
1942
|
+
parseFailures += 1;
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
if (skippedLarge > 0)
|
|
1947
|
+
warnings.push(`${skippedLarge} file(s) exceeded the size limit and were skipped`);
|
|
1948
|
+
if (parseFailures > 0)
|
|
1949
|
+
warnings.push(`${parseFailures} message(s) failed to parse`);
|
|
1950
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
function splitMbox(text) {
|
|
1954
|
+
const lines = text.split(/\r?\n/u);
|
|
1955
|
+
const messages = [];
|
|
1956
|
+
let current = [];
|
|
1957
|
+
for (const line of lines) {
|
|
1958
|
+
if (MBOX_SEPARATOR.test(line) && line.startsWith("From ")) {
|
|
1959
|
+
if (current.length > 0)
|
|
1960
|
+
messages.push(current.join(`
|
|
1961
|
+
`));
|
|
1962
|
+
current = [];
|
|
1963
|
+
continue;
|
|
1964
|
+
}
|
|
1965
|
+
current.push(line);
|
|
1966
|
+
}
|
|
1967
|
+
if (current.length > 0)
|
|
1968
|
+
messages.push(current.join(`
|
|
1969
|
+
`));
|
|
1970
|
+
return messages.filter((message) => message.trim().length > 0);
|
|
1971
|
+
}
|
|
1972
|
+
var DEFAULT_MAX_DOCUMENTS7 = 500, DEFAULT_MAX_BYTES_PER_FILE2, MAX_BODY_CHARS3 = 50000, MBOX_SEPARATOR;
|
|
1973
|
+
var init_connector6 = __esm(() => {
|
|
1974
|
+
DEFAULT_MAX_BYTES_PER_FILE2 = 50 * 1024 * 1024;
|
|
1975
|
+
MBOX_SEPARATOR = /^From .*\d{4}$/mu;
|
|
1976
|
+
});
|
|
1977
|
+
|
|
1978
|
+
// src/datasource/skills/mail-export/skill.ts
|
|
1979
|
+
var MAIL_EXPORT_SKILL_DEFINITION, MailExportSkill;
|
|
1980
|
+
var init_skill5 = __esm(() => {
|
|
1981
|
+
init_connector_skill();
|
|
1982
|
+
init_connector6();
|
|
1983
|
+
MAIL_EXPORT_SKILL_DEFINITION = {
|
|
1984
|
+
skillName: "mail-export",
|
|
1985
|
+
skillType: "mail-export",
|
|
1986
|
+
description: "Local mail export datasource",
|
|
1987
|
+
capabilities: ["email", "filesystem"],
|
|
1988
|
+
defaultTags: ["mail-export", "email", "pii"],
|
|
1989
|
+
contentType: "email",
|
|
1990
|
+
manifestDescription: "Search indexed local mail exports (mbox/eml archives). Use for questions about archived email conversations.",
|
|
1991
|
+
manifestNotes: [
|
|
1992
|
+
"Only the server-configured export paths are indexed; archives are read locally and never uploaded."
|
|
1993
|
+
]
|
|
1994
|
+
};
|
|
1995
|
+
MailExportSkill = class MailExportSkill extends ConnectorDatasourceSkill {
|
|
1996
|
+
constructor(options = {}) {
|
|
1997
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
1998
|
+
super(MAIL_EXPORT_SKILL_DEFINITION, {
|
|
1999
|
+
...rest,
|
|
2000
|
+
connector: connector ?? new MailExportConnector(connectorOptions ?? {})
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
};
|
|
2004
|
+
});
|
|
2005
|
+
|
|
2006
|
+
// src/datasource/skills/mail-export/index.ts
|
|
2007
|
+
var init_mail_export = __esm(() => {
|
|
2008
|
+
init_connector6();
|
|
2009
|
+
init_skill5();
|
|
2010
|
+
});
|
|
2011
|
+
|
|
2012
|
+
// src/datasource/skills/notion/connector.ts
|
|
2013
|
+
class NotionConnector {
|
|
2014
|
+
options;
|
|
2015
|
+
constructor(options = {}) {
|
|
2016
|
+
this.options = options;
|
|
2017
|
+
}
|
|
2018
|
+
async fetch(signal) {
|
|
2019
|
+
const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV5);
|
|
2020
|
+
if (token === undefined)
|
|
2021
|
+
return { ok: false, reason: "not-configured", message: "auth token not configured" };
|
|
2022
|
+
const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL5;
|
|
2023
|
+
const request = {
|
|
2024
|
+
headers: {
|
|
2025
|
+
Authorization: `Bearer ${token}`,
|
|
2026
|
+
"Notion-Version": NOTION_VERSION,
|
|
2027
|
+
"Content-Type": "application/json"
|
|
2028
|
+
},
|
|
2029
|
+
timeoutMs: this.options.timeoutMs,
|
|
2030
|
+
fetchImpl: this.options.fetchImpl,
|
|
2031
|
+
signal
|
|
2032
|
+
};
|
|
2033
|
+
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES5;
|
|
2034
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS8;
|
|
2035
|
+
const entries = [];
|
|
2036
|
+
let cursor;
|
|
2037
|
+
for (let page = 0;page < maxPages; page += 1) {
|
|
2038
|
+
const body = JSON.stringify({ page_size: 100, ...cursor !== undefined ? { start_cursor: cursor } : {} });
|
|
2039
|
+
const result = await httpJson(`${baseUrl}/search`, { ...request, method: "POST", body });
|
|
2040
|
+
if (!result.ok)
|
|
2041
|
+
return { ok: false, reason: result.reason, message: `search failed: ${result.message}` };
|
|
2042
|
+
const envelope = asRecord(result.json);
|
|
2043
|
+
if (envelope === undefined)
|
|
2044
|
+
return { ok: false, reason: "invalid-data", message: "search response malformed" };
|
|
2045
|
+
for (const raw of asArray(envelope.results)) {
|
|
2046
|
+
const entry = asRecord(raw);
|
|
2047
|
+
if (entry !== undefined)
|
|
2048
|
+
entries.push(entry);
|
|
2049
|
+
}
|
|
2050
|
+
cursor = envelope.has_more === true ? asString(envelope.next_cursor) : undefined;
|
|
2051
|
+
if (cursor === undefined)
|
|
2052
|
+
break;
|
|
2053
|
+
}
|
|
2054
|
+
const documents = [];
|
|
2055
|
+
const warnings = [];
|
|
2056
|
+
for (const entry of entries) {
|
|
2057
|
+
if (documents.length >= maxDocuments)
|
|
2058
|
+
break;
|
|
2059
|
+
const id = asString(entry.id);
|
|
2060
|
+
if (id === undefined)
|
|
2061
|
+
continue;
|
|
2062
|
+
const objectType = asString(entry.object) ?? "page";
|
|
2063
|
+
const title = extractTitle(entry);
|
|
2064
|
+
let blockText = "";
|
|
2065
|
+
if (objectType === "page") {
|
|
2066
|
+
const blocksResult = await httpJson(`${baseUrl}/blocks/${id}/children?page_size=100`, request);
|
|
2067
|
+
if (blocksResult.ok) {
|
|
2068
|
+
blockText = extractBlockText(blocksResult.json);
|
|
2069
|
+
} else {
|
|
2070
|
+
warnings.push(`page content fetch failed: ${blocksResult.message}`);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
const content = [title, blockText].filter((part) => part.length > 0).join(`
|
|
2074
|
+
|
|
2075
|
+
`);
|
|
2076
|
+
if (content.length === 0)
|
|
2077
|
+
continue;
|
|
2078
|
+
documents.push({
|
|
2079
|
+
docId: id,
|
|
2080
|
+
hierarchy: hierarchyFor(entry, objectType),
|
|
2081
|
+
...title.length > 0 ? { title } : {},
|
|
2082
|
+
content,
|
|
2083
|
+
publishedAt: parseEpochMs(entry.last_edited_time),
|
|
2084
|
+
metadata: {
|
|
2085
|
+
objectType,
|
|
2086
|
+
...asString(entry.last_edited_time) !== undefined ? { lastEditedTime: asString(entry.last_edited_time) } : {}
|
|
2087
|
+
}
|
|
2088
|
+
});
|
|
2089
|
+
}
|
|
2090
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
function hierarchyFor(entry, objectType) {
|
|
2094
|
+
if (objectType === "database")
|
|
2095
|
+
return ["databases", asString(entry.id) ?? "unknown"];
|
|
2096
|
+
const parent = asRecord(entry.parent);
|
|
2097
|
+
const parentType = asString(parent?.type);
|
|
2098
|
+
if (parentType === "database_id") {
|
|
2099
|
+
return ["databases", asString(parent?.database_id) ?? "unknown"];
|
|
2100
|
+
}
|
|
2101
|
+
if (parentType === "page_id") {
|
|
2102
|
+
return ["pages", asString(parent?.page_id) ?? "unknown"];
|
|
2103
|
+
}
|
|
2104
|
+
return ["pages"];
|
|
2105
|
+
}
|
|
2106
|
+
function extractTitle(entry) {
|
|
2107
|
+
const directTitle = plainText(entry.title);
|
|
2108
|
+
if (directTitle.length > 0)
|
|
2109
|
+
return directTitle;
|
|
2110
|
+
const properties = asRecord(entry.properties);
|
|
2111
|
+
if (properties === undefined)
|
|
2112
|
+
return "";
|
|
2113
|
+
for (const value of Object.values(properties)) {
|
|
2114
|
+
const property = asRecord(value);
|
|
2115
|
+
if (property === undefined || property.type !== "title")
|
|
2116
|
+
continue;
|
|
2117
|
+
const text = plainText(property.title);
|
|
2118
|
+
if (text.length > 0)
|
|
2119
|
+
return text;
|
|
2120
|
+
}
|
|
2121
|
+
return "";
|
|
2122
|
+
}
|
|
2123
|
+
function extractBlockText(json) {
|
|
2124
|
+
const envelope = asRecord(json);
|
|
2125
|
+
const lines = [];
|
|
2126
|
+
for (const raw of asArray(envelope?.results)) {
|
|
2127
|
+
const block = asRecord(raw);
|
|
2128
|
+
const type = asString(block?.type);
|
|
2129
|
+
if (block === undefined || type === undefined)
|
|
2130
|
+
continue;
|
|
2131
|
+
if (!TEXT_BLOCK_TYPES.includes(type))
|
|
2132
|
+
continue;
|
|
2133
|
+
const payload = asRecord(block[type]);
|
|
2134
|
+
const text = plainText(payload?.rich_text);
|
|
2135
|
+
if (text.length > 0)
|
|
2136
|
+
lines.push(text);
|
|
2137
|
+
}
|
|
2138
|
+
return lines.join(`
|
|
2139
|
+
`);
|
|
2140
|
+
}
|
|
2141
|
+
function plainText(richText) {
|
|
2142
|
+
return asArray(richText).map((raw) => asString(asRecord(raw)?.plain_text) ?? "").filter((text) => text.length > 0).join("").trim();
|
|
2143
|
+
}
|
|
2144
|
+
var DEFAULT_BASE_URL5 = "https://api.notion.com/v1", DEFAULT_TOKEN_ENV5 = "NOTION_TOKEN", NOTION_VERSION = "2022-06-28", DEFAULT_MAX_PAGES5 = 10, DEFAULT_MAX_DOCUMENTS8 = 500, TEXT_BLOCK_TYPES;
|
|
2145
|
+
var init_connector7 = __esm(() => {
|
|
2146
|
+
TEXT_BLOCK_TYPES = [
|
|
2147
|
+
"paragraph",
|
|
2148
|
+
"heading_1",
|
|
2149
|
+
"heading_2",
|
|
2150
|
+
"heading_3",
|
|
2151
|
+
"bulleted_list_item",
|
|
2152
|
+
"numbered_list_item",
|
|
2153
|
+
"to_do",
|
|
2154
|
+
"quote",
|
|
2155
|
+
"callout",
|
|
2156
|
+
"code"
|
|
2157
|
+
];
|
|
2158
|
+
});
|
|
2159
|
+
|
|
2160
|
+
// src/datasource/skills/notion/skill.ts
|
|
2161
|
+
var NOTION_SKILL_DEFINITION, NotionSkill;
|
|
2162
|
+
var init_skill6 = __esm(() => {
|
|
2163
|
+
init_connector_skill();
|
|
2164
|
+
init_connector7();
|
|
2165
|
+
NOTION_SKILL_DEFINITION = {
|
|
2166
|
+
skillName: "notion",
|
|
2167
|
+
skillType: "notion-workspace",
|
|
2168
|
+
description: "Notion workspace datasource",
|
|
2169
|
+
capabilities: ["documents", "api", "polling"],
|
|
2170
|
+
defaultTags: ["notion", "documents"],
|
|
2171
|
+
contentType: "document",
|
|
2172
|
+
manifestDescription: "Search indexed Notion pages and databases shared with the integration. Use for questions about Notion docs, wikis, and database entries.",
|
|
2173
|
+
manifestNotes: ["Only pages shared with the server-configured integration token are indexed."]
|
|
2174
|
+
};
|
|
2175
|
+
NotionSkill = class NotionSkill extends ConnectorDatasourceSkill {
|
|
2176
|
+
constructor(options = {}) {
|
|
2177
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
2178
|
+
super(NOTION_SKILL_DEFINITION, {
|
|
2179
|
+
...rest,
|
|
2180
|
+
connector: connector ?? new NotionConnector(connectorOptions ?? {})
|
|
2181
|
+
});
|
|
2182
|
+
}
|
|
2183
|
+
};
|
|
2184
|
+
});
|
|
2185
|
+
|
|
2186
|
+
// src/datasource/skills/notion/index.ts
|
|
2187
|
+
var init_notion = __esm(() => {
|
|
2188
|
+
init_connector7();
|
|
2189
|
+
init_skill6();
|
|
2190
|
+
});
|
|
2191
|
+
|
|
2192
|
+
// src/datasource/skills/obsidian/connector.ts
|
|
2193
|
+
import { readdir as readdir2, readFile as readFile2, stat as stat2 } from "node:fs/promises";
|
|
2194
|
+
import { basename as basename2, extname as extname2, join as join4, relative, sep } from "node:path";
|
|
2195
|
+
|
|
2196
|
+
class ObsidianConnector {
|
|
2197
|
+
options;
|
|
2198
|
+
constructor(options = {}) {
|
|
2199
|
+
this.options = options;
|
|
2200
|
+
}
|
|
2201
|
+
async fetch() {
|
|
2202
|
+
const vaultPath = this.options.vaultPath;
|
|
2203
|
+
if (vaultPath === undefined || vaultPath.length === 0) {
|
|
2204
|
+
return { ok: false, reason: "not-configured", message: "vault path not configured" };
|
|
2205
|
+
}
|
|
2206
|
+
try {
|
|
2207
|
+
const info = await stat2(vaultPath);
|
|
2208
|
+
if (!info.isDirectory())
|
|
2209
|
+
return { ok: false, reason: "unavailable", message: "vault path is not a directory" };
|
|
2210
|
+
} catch {
|
|
2211
|
+
return { ok: false, reason: "unavailable", message: "vault path is not readable" };
|
|
2212
|
+
}
|
|
2213
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS9;
|
|
2214
|
+
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE3;
|
|
2215
|
+
const files = [];
|
|
2216
|
+
const walk = async (dir) => {
|
|
2217
|
+
let entries;
|
|
2218
|
+
try {
|
|
2219
|
+
entries = await readdir2(dir, { withFileTypes: true });
|
|
2220
|
+
} catch {
|
|
2221
|
+
return;
|
|
2222
|
+
}
|
|
2223
|
+
for (const entry of entries) {
|
|
2224
|
+
if (entry.isDirectory()) {
|
|
2225
|
+
if (!SKIPPED_DIRS.has(entry.name))
|
|
2226
|
+
await walk(join4(dir, entry.name));
|
|
2227
|
+
} else if (entry.isFile() && extname2(entry.name).toLowerCase() === ".md") {
|
|
2228
|
+
files.push(join4(dir, entry.name));
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
};
|
|
2232
|
+
await walk(vaultPath);
|
|
2233
|
+
const documents = [];
|
|
2234
|
+
let readFailures = 0;
|
|
2235
|
+
let skippedLarge = 0;
|
|
2236
|
+
for (const file of files.sort()) {
|
|
2237
|
+
if (documents.length >= maxDocuments)
|
|
2238
|
+
break;
|
|
2239
|
+
let text;
|
|
2240
|
+
try {
|
|
2241
|
+
const info = await stat2(file);
|
|
2242
|
+
if (info.size > maxBytes) {
|
|
2243
|
+
skippedLarge += 1;
|
|
2244
|
+
continue;
|
|
2245
|
+
}
|
|
2246
|
+
text = await readFile2(file, "utf8");
|
|
2247
|
+
const relativePath = relative(vaultPath, file);
|
|
2248
|
+
const segments = relativePath.split(sep);
|
|
2249
|
+
const directorySegments = segments.slice(0, -1);
|
|
2250
|
+
const stem = basename2(file, ".md");
|
|
2251
|
+
const { body, frontmatterTags } = stripFrontmatter(text);
|
|
2252
|
+
const inlineTags = matchAll(body, INLINE_TAG_PATTERN, 1);
|
|
2253
|
+
const links = [];
|
|
2254
|
+
const embeds = [];
|
|
2255
|
+
for (const match of body.matchAll(WIKI_LINK_PATTERN)) {
|
|
2256
|
+
const target = match[2]?.trim();
|
|
2257
|
+
if (target === undefined || target.length === 0)
|
|
2258
|
+
continue;
|
|
2259
|
+
(match[1] === "!" ? embeds : links).push(target);
|
|
2260
|
+
}
|
|
2261
|
+
const heading = /^#\s+(.+)$/mu.exec(body)?.[1]?.trim();
|
|
2262
|
+
const content = body.trim().slice(0, MAX_CONTENT_CHARS3);
|
|
2263
|
+
if (content.length === 0)
|
|
2264
|
+
continue;
|
|
2265
|
+
documents.push({
|
|
2266
|
+
docId: relativePath.split(sep).join("__").replace(/\.md$/u, ""),
|
|
2267
|
+
hierarchy: ["folders", ...directorySegments],
|
|
2268
|
+
title: heading ?? stem,
|
|
2269
|
+
content,
|
|
2270
|
+
publishedAt: info.mtimeMs,
|
|
2271
|
+
metadata: {
|
|
2272
|
+
path: file,
|
|
2273
|
+
tags: [...new Set([...frontmatterTags, ...inlineTags])],
|
|
2274
|
+
links,
|
|
2275
|
+
embeds,
|
|
2276
|
+
folder: directorySegments.join("/")
|
|
2277
|
+
}
|
|
2278
|
+
});
|
|
2279
|
+
} catch {
|
|
2280
|
+
readFailures += 1;
|
|
2281
|
+
}
|
|
2282
|
+
}
|
|
2283
|
+
const warnings = [];
|
|
2284
|
+
if (skippedLarge > 0)
|
|
2285
|
+
warnings.push(`${skippedLarge} note(s) exceeded the size limit and were skipped`);
|
|
2286
|
+
if (readFailures > 0)
|
|
2287
|
+
warnings.push(`${readFailures} note(s) failed to read`);
|
|
2288
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
function stripFrontmatter(text) {
|
|
2292
|
+
if (!text.startsWith("---"))
|
|
2293
|
+
return { body: text, frontmatterTags: [] };
|
|
2294
|
+
const end = text.indexOf(`
|
|
2295
|
+
---`, 3);
|
|
2296
|
+
if (end === -1)
|
|
2297
|
+
return { body: text, frontmatterTags: [] };
|
|
2298
|
+
const frontmatter = text.slice(3, end);
|
|
2299
|
+
const body = text.slice(text.indexOf(`
|
|
2300
|
+
`, end + 1) + 1);
|
|
2301
|
+
return { body, frontmatterTags: parseFrontmatterTags(frontmatter) };
|
|
2302
|
+
}
|
|
2303
|
+
function parseFrontmatterTags(frontmatter) {
|
|
2304
|
+
const lines = frontmatter.split(`
|
|
2305
|
+
`);
|
|
2306
|
+
const tags = [];
|
|
2307
|
+
for (const [index, line] of lines.entries()) {
|
|
2308
|
+
const match = /^tags:\s*(.*)$/u.exec(line.trim());
|
|
2309
|
+
if (match === null)
|
|
2310
|
+
continue;
|
|
2311
|
+
const inline = match[1]?.trim() ?? "";
|
|
2312
|
+
if (inline.length > 0) {
|
|
2313
|
+
const cleaned = inline.replace(/^\[/u, "").replace(/\]$/u, "");
|
|
2314
|
+
for (const tag of cleaned.split(/[\s,]+/u)) {
|
|
2315
|
+
const trimmed = tag.trim().replace(/^["'#]+|["']+$/gu, "");
|
|
2316
|
+
if (trimmed.length > 0)
|
|
2317
|
+
tags.push(trimmed);
|
|
2318
|
+
}
|
|
2319
|
+
break;
|
|
2320
|
+
}
|
|
2321
|
+
for (let cursor = index + 1;cursor < lines.length; cursor += 1) {
|
|
2322
|
+
const item = /^\s*-\s*(.+)$/u.exec(lines[cursor] ?? "");
|
|
2323
|
+
if (item === null)
|
|
2324
|
+
break;
|
|
2325
|
+
const trimmed = item[1]?.trim().replace(/^["'#]+|["']+$/gu, "") ?? "";
|
|
2326
|
+
if (trimmed.length > 0)
|
|
2327
|
+
tags.push(trimmed);
|
|
2328
|
+
}
|
|
2329
|
+
break;
|
|
2330
|
+
}
|
|
2331
|
+
return tags;
|
|
2332
|
+
}
|
|
2333
|
+
function matchAll(text, pattern, group) {
|
|
2334
|
+
const out = [];
|
|
2335
|
+
for (const match of text.matchAll(pattern)) {
|
|
2336
|
+
const value = match[group]?.trim();
|
|
2337
|
+
if (value !== undefined && value.length > 0)
|
|
2338
|
+
out.push(value);
|
|
2339
|
+
}
|
|
2340
|
+
return out;
|
|
2341
|
+
}
|
|
2342
|
+
var DEFAULT_MAX_DOCUMENTS9 = 1000, DEFAULT_MAX_BYTES_PER_FILE3, MAX_CONTENT_CHARS3 = 1e5, SKIPPED_DIRS, INLINE_TAG_PATTERN, WIKI_LINK_PATTERN;
|
|
2343
|
+
var init_connector8 = __esm(() => {
|
|
2344
|
+
DEFAULT_MAX_BYTES_PER_FILE3 = 2 * 1024 * 1024;
|
|
2345
|
+
SKIPPED_DIRS = new Set([".obsidian", ".trash", ".git", "node_modules"]);
|
|
2346
|
+
INLINE_TAG_PATTERN = /(?:^|\s)#([A-Za-z0-9_/-]+)/gu;
|
|
2347
|
+
WIKI_LINK_PATTERN = /(!?)\[\[([^\]|]+)(?:\|[^\]]*)?\]\]/gu;
|
|
2348
|
+
});
|
|
2349
|
+
|
|
2350
|
+
// src/datasource/skills/obsidian/skill.ts
|
|
2351
|
+
var OBSIDIAN_SKILL_DEFINITION, ObsidianSkill;
|
|
2352
|
+
var init_skill7 = __esm(() => {
|
|
2353
|
+
init_connector_skill();
|
|
2354
|
+
init_connector8();
|
|
2355
|
+
OBSIDIAN_SKILL_DEFINITION = {
|
|
2356
|
+
skillName: "obsidian",
|
|
2357
|
+
skillType: "obsidian-vault",
|
|
2358
|
+
description: "Obsidian vault datasource",
|
|
2359
|
+
capabilities: ["notes", "filesystem"],
|
|
2360
|
+
defaultTags: ["obsidian", "notes"],
|
|
2361
|
+
contentType: "note",
|
|
2362
|
+
manifestDescription: "Search indexed Obsidian vault notes, including tags, wiki links, and embeds. Use for questions about personal notes and knowledge-base content.",
|
|
2363
|
+
manifestNotes: ["Only the server-configured vault is indexed; note metadata includes tags, links, and embeds."]
|
|
2364
|
+
};
|
|
2365
|
+
ObsidianSkill = class ObsidianSkill extends ConnectorDatasourceSkill {
|
|
2366
|
+
constructor(options = {}) {
|
|
2367
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
2368
|
+
super(OBSIDIAN_SKILL_DEFINITION, {
|
|
2369
|
+
...rest,
|
|
2370
|
+
connector: connector ?? new ObsidianConnector(connectorOptions ?? {})
|
|
2371
|
+
});
|
|
2372
|
+
}
|
|
2373
|
+
};
|
|
2374
|
+
});
|
|
2375
|
+
|
|
2376
|
+
// src/datasource/skills/obsidian/index.ts
|
|
2377
|
+
var init_obsidian = __esm(() => {
|
|
2378
|
+
init_connector8();
|
|
2379
|
+
init_skill7();
|
|
2380
|
+
});
|
|
2381
|
+
|
|
2382
|
+
// src/datasource/skills/rss/connector.ts
|
|
2383
|
+
import { XMLParser } from "fast-xml-parser";
|
|
2384
|
+
|
|
2385
|
+
class RssConnector {
|
|
2386
|
+
options;
|
|
2387
|
+
parser = new XMLParser({ ignoreAttributes: false });
|
|
2388
|
+
constructor(options = {}) {
|
|
2389
|
+
this.options = options;
|
|
2390
|
+
}
|
|
2391
|
+
async fetch(signal) {
|
|
2392
|
+
const feeds = this.options.feeds ?? [];
|
|
2393
|
+
if (feeds.length === 0)
|
|
2394
|
+
return { ok: false, reason: "not-configured", message: "no feeds configured" };
|
|
2395
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS10;
|
|
2396
|
+
const maxItemsPerFeed = this.options.maxItemsPerFeed ?? DEFAULT_MAX_ITEMS_PER_FEED;
|
|
2397
|
+
const documents = [];
|
|
2398
|
+
const warnings = [];
|
|
2399
|
+
let failedFeeds = 0;
|
|
2400
|
+
for (const [index, feed] of feeds.entries()) {
|
|
2401
|
+
if (documents.length >= maxDocuments)
|
|
2402
|
+
break;
|
|
2403
|
+
const result = await httpText(feed.url, {
|
|
2404
|
+
headers: { Accept: ACCEPT_HEADER },
|
|
2405
|
+
timeoutMs: this.options.timeoutMs ?? DEFAULT_FEED_TIMEOUT_MS,
|
|
2406
|
+
fetchImpl: this.options.fetchImpl,
|
|
2407
|
+
signal
|
|
2408
|
+
});
|
|
2409
|
+
if (!result.ok) {
|
|
2410
|
+
failedFeeds += 1;
|
|
2411
|
+
warnings.push(`feed ${index + 1} failed: ${result.message}`);
|
|
2412
|
+
continue;
|
|
2413
|
+
}
|
|
2414
|
+
let parsed;
|
|
2415
|
+
try {
|
|
2416
|
+
parsed = this.parser.parse(result.text);
|
|
2417
|
+
} catch {
|
|
2418
|
+
failedFeeds += 1;
|
|
2419
|
+
warnings.push(`feed ${index + 1} failed: unparseable XML`);
|
|
2420
|
+
continue;
|
|
2421
|
+
}
|
|
2422
|
+
const normalized = normalizeFeed(parsed);
|
|
2423
|
+
if (normalized === undefined) {
|
|
2424
|
+
failedFeeds += 1;
|
|
2425
|
+
warnings.push(`feed ${index + 1} failed: unrecognized feed shape`);
|
|
2426
|
+
continue;
|
|
2427
|
+
}
|
|
2428
|
+
const feedTitle = normalized.title ?? `feed-${index + 1}`;
|
|
2429
|
+
const hierarchy = feed.category !== undefined ? ["feeds", feed.category, feedTitle] : ["feeds", feedTitle];
|
|
2430
|
+
for (const item of normalized.items.slice(0, maxItemsPerFeed)) {
|
|
2431
|
+
if (documents.length >= maxDocuments)
|
|
2432
|
+
break;
|
|
2433
|
+
const title = item.title ?? "(untitled)";
|
|
2434
|
+
const body = stripHtml(item.body ?? "");
|
|
2435
|
+
const content = body.length > 0 ? `${title}
|
|
2436
|
+
|
|
2437
|
+
${body}` : title;
|
|
2438
|
+
documents.push({
|
|
2439
|
+
docId: item.id,
|
|
2440
|
+
hierarchy,
|
|
2441
|
+
title,
|
|
2442
|
+
content: content.slice(0, MAX_CONTENT_CHARS4),
|
|
2443
|
+
...item.publishedAt !== undefined ? { publishedAt: item.publishedAt } : {},
|
|
2444
|
+
metadata: { feedIndex: index, feedTitle, categories: item.categories }
|
|
2445
|
+
});
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
if (failedFeeds === feeds.length) {
|
|
2449
|
+
return { ok: false, reason: "unavailable", message: "all configured feeds failed" };
|
|
2450
|
+
}
|
|
2451
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
function normalizeFeed(parsed) {
|
|
2455
|
+
const root = asRecord(parsed);
|
|
2456
|
+
if (root === undefined)
|
|
2457
|
+
return;
|
|
2458
|
+
const channel = asRecord(asRecord(root.rss)?.channel);
|
|
2459
|
+
if (channel !== undefined) {
|
|
2460
|
+
const items = asArray(channel.item ?? []).concat(asRecord(channel.item) !== undefined ? [channel.item] : []);
|
|
2461
|
+
return { title: textOf(channel.title), items: items.map(normalizeRssItem).filter(present) };
|
|
2462
|
+
}
|
|
2463
|
+
const atom = asRecord(root.feed);
|
|
2464
|
+
if (atom !== undefined) {
|
|
2465
|
+
const entries = asArray(atom.entry ?? []).concat(asRecord(atom.entry) !== undefined ? [atom.entry] : []);
|
|
2466
|
+
return { title: textOf(atom.title), items: entries.map(normalizeAtomEntry).filter(present) };
|
|
2467
|
+
}
|
|
2468
|
+
return;
|
|
2469
|
+
}
|
|
2470
|
+
function normalizeRssItem(raw) {
|
|
2471
|
+
const item = asRecord(raw);
|
|
2472
|
+
if (item === undefined)
|
|
2473
|
+
return;
|
|
2474
|
+
const title = textOf(item.title);
|
|
2475
|
+
const guid = textOf(item.guid) ?? textOf(item.link) ?? title;
|
|
2476
|
+
if (guid === undefined)
|
|
2477
|
+
return;
|
|
2478
|
+
return {
|
|
2479
|
+
id: guid,
|
|
2480
|
+
title,
|
|
2481
|
+
body: textOf(item.description),
|
|
2482
|
+
publishedAt: parseEpochMs(textOf(item.pubDate)),
|
|
2483
|
+
categories: categoriesOf(item.category)
|
|
2484
|
+
};
|
|
2485
|
+
}
|
|
2486
|
+
function normalizeAtomEntry(raw) {
|
|
2487
|
+
const entry = asRecord(raw);
|
|
2488
|
+
if (entry === undefined)
|
|
2489
|
+
return;
|
|
2490
|
+
const title = textOf(entry.title);
|
|
2491
|
+
const id = textOf(entry.id) ?? title;
|
|
2492
|
+
if (id === undefined)
|
|
2493
|
+
return;
|
|
2494
|
+
return {
|
|
2495
|
+
id,
|
|
2496
|
+
title,
|
|
2497
|
+
body: textOf(entry.summary) ?? textOf(entry.content),
|
|
2498
|
+
publishedAt: parseEpochMs(textOf(entry.published) ?? textOf(entry.updated)),
|
|
2499
|
+
categories: categoriesOf(entry.category)
|
|
2500
|
+
};
|
|
2501
|
+
}
|
|
2502
|
+
function textOf(value) {
|
|
2503
|
+
const direct = asString(value);
|
|
2504
|
+
if (direct !== undefined)
|
|
2505
|
+
return direct.trim();
|
|
2506
|
+
if (typeof value === "number")
|
|
2507
|
+
return String(value);
|
|
2508
|
+
const record = asRecord(value);
|
|
2509
|
+
if (record !== undefined) {
|
|
2510
|
+
const text = asString(record["#text"]) ?? asString(record["@_term"]);
|
|
2511
|
+
if (text !== undefined)
|
|
2512
|
+
return text.trim();
|
|
2513
|
+
if (typeof record["#text"] === "number")
|
|
2514
|
+
return String(record["#text"]);
|
|
2515
|
+
}
|
|
2516
|
+
return;
|
|
2517
|
+
}
|
|
2518
|
+
function categoriesOf(value) {
|
|
2519
|
+
const values = Array.isArray(value) ? value : value !== undefined ? [value] : [];
|
|
2520
|
+
return values.map(textOf).filter((category) => category !== undefined && category.length > 0);
|
|
2521
|
+
}
|
|
2522
|
+
function stripHtml(text) {
|
|
2523
|
+
return text.replace(/<[^>]+>/gu, " ").replace(/\s+/gu, " ").trim();
|
|
2524
|
+
}
|
|
2525
|
+
function present(item) {
|
|
2526
|
+
return item !== undefined;
|
|
2527
|
+
}
|
|
2528
|
+
var DEFAULT_MAX_DOCUMENTS10 = 500, DEFAULT_MAX_ITEMS_PER_FEED = 100, DEFAULT_FEED_TIMEOUT_MS = 30000, MAX_CONTENT_CHARS4 = 20000, ACCEPT_HEADER = "application/rss+xml, application/atom+xml, application/xml, text/xml";
|
|
2529
|
+
var init_connector9 = () => {};
|
|
2530
|
+
|
|
2531
|
+
// src/datasource/skills/rss/skill.ts
|
|
2532
|
+
var RSS_SKILL_DEFINITION, DEFAULT_DEDUPE_WINDOW_MS, RssSkill;
|
|
2533
|
+
var init_skill8 = __esm(() => {
|
|
2534
|
+
init_connector_skill();
|
|
2535
|
+
init_connector9();
|
|
2536
|
+
RSS_SKILL_DEFINITION = {
|
|
2537
|
+
skillName: "rss",
|
|
2538
|
+
skillType: "rss-feeds",
|
|
2539
|
+
description: "RSS/news datasource",
|
|
2540
|
+
capabilities: ["news", "api", "polling"],
|
|
2541
|
+
defaultTags: ["rss", "news", "public"],
|
|
2542
|
+
contentType: "article",
|
|
2543
|
+
manifestDescription: "Search indexed RSS/news feed articles from the configured feeds. Use for questions about recent news items or blog posts the server subscribes to.",
|
|
2544
|
+
manifestNotes: ["Feeds are polled with a dedupe window so re-delivered items are indexed once per window."]
|
|
2545
|
+
};
|
|
2546
|
+
DEFAULT_DEDUPE_WINDOW_MS = 24 * 60 * 60 * 1000;
|
|
2547
|
+
RssSkill = class RssSkill extends ConnectorDatasourceSkill {
|
|
2548
|
+
constructor(options = {}) {
|
|
2549
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
2550
|
+
super(RSS_SKILL_DEFINITION, {
|
|
2551
|
+
...rest,
|
|
2552
|
+
connector: connector ?? new RssConnector(connectorOptions ?? {}),
|
|
2553
|
+
dedupeWindowMs: options.dedupeWindowMs ?? DEFAULT_DEDUPE_WINDOW_MS
|
|
2554
|
+
});
|
|
2555
|
+
}
|
|
2556
|
+
};
|
|
2557
|
+
});
|
|
2558
|
+
|
|
2559
|
+
// src/datasource/skills/rss/index.ts
|
|
2560
|
+
var init_rss = __esm(() => {
|
|
2561
|
+
init_connector9();
|
|
2562
|
+
init_skill8();
|
|
2563
|
+
});
|
|
2564
|
+
|
|
2565
|
+
// src/datasource/skills/slack/connector.ts
|
|
2566
|
+
class SlackConnector {
|
|
2567
|
+
options;
|
|
2568
|
+
constructor(options = {}) {
|
|
2569
|
+
this.options = options;
|
|
2570
|
+
}
|
|
2571
|
+
async fetch(signal) {
|
|
2572
|
+
const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV6);
|
|
2573
|
+
if (token === undefined)
|
|
2574
|
+
return { ok: false, reason: "not-configured", message: "auth token not configured" };
|
|
2575
|
+
const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL6;
|
|
2576
|
+
const headers = { Authorization: `Bearer ${token}` };
|
|
2577
|
+
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES6;
|
|
2578
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS11;
|
|
2579
|
+
const request = { headers, timeoutMs: this.options.timeoutMs, fetchImpl: this.options.fetchImpl, signal };
|
|
2580
|
+
const channels = [];
|
|
2581
|
+
let cursor;
|
|
2582
|
+
for (let page = 0;page < maxPages; page += 1) {
|
|
2583
|
+
const url = new URL(`${baseUrl}/conversations.list`);
|
|
2584
|
+
url.searchParams.set("limit", "200");
|
|
2585
|
+
url.searchParams.set("types", "public_channel,private_channel");
|
|
2586
|
+
if (cursor !== undefined)
|
|
2587
|
+
url.searchParams.set("cursor", cursor);
|
|
2588
|
+
const result = await httpJson(url.toString(), request);
|
|
2589
|
+
if (!result.ok)
|
|
2590
|
+
return { ok: false, reason: result.reason, message: `channel list failed: ${result.message}` };
|
|
2591
|
+
const envelope = asRecord(result.json);
|
|
2592
|
+
if (envelope === undefined)
|
|
2593
|
+
return { ok: false, reason: "invalid-data", message: "channel list malformed" };
|
|
2594
|
+
if (envelope.ok !== true) {
|
|
2595
|
+
const error = asString(envelope.error) ?? "unknown-error";
|
|
2596
|
+
if (AUTH_ERRORS.has(error))
|
|
2597
|
+
return { ok: false, reason: "auth", message: `slack: ${error}` };
|
|
2598
|
+
if (error === "ratelimited")
|
|
2599
|
+
return { ok: false, reason: "rate-limited", message: "slack: ratelimited" };
|
|
2600
|
+
return { ok: false, reason: "api-error", message: `slack: ${error}` };
|
|
2601
|
+
}
|
|
2602
|
+
for (const raw of asArray(envelope.channels)) {
|
|
2603
|
+
const channel = asRecord(raw);
|
|
2604
|
+
const id = asString(channel?.id);
|
|
2605
|
+
const name = asString(channel?.name);
|
|
2606
|
+
if (id !== undefined && name !== undefined)
|
|
2607
|
+
channels.push({ id, name });
|
|
2608
|
+
}
|
|
2609
|
+
cursor = asString(asRecord(envelope.response_metadata)?.next_cursor);
|
|
2610
|
+
if (cursor === undefined || cursor.length === 0)
|
|
2611
|
+
break;
|
|
2612
|
+
}
|
|
2613
|
+
const filter = this.options.channels;
|
|
2614
|
+
const selected = filter === undefined || filter.length === 0 ? channels : channels.filter((channel) => filter.includes(channel.id) || filter.includes(channel.name));
|
|
2615
|
+
const documents = [];
|
|
2616
|
+
const warnings = [];
|
|
2617
|
+
for (const channel of selected) {
|
|
2618
|
+
if (documents.length >= maxDocuments)
|
|
2619
|
+
break;
|
|
2620
|
+
let historyCursor;
|
|
2621
|
+
for (let page = 0;page < maxPages && documents.length < maxDocuments; page += 1) {
|
|
2622
|
+
const url = new URL(`${baseUrl}/conversations.history`);
|
|
2623
|
+
url.searchParams.set("channel", channel.id);
|
|
2624
|
+
url.searchParams.set("limit", "200");
|
|
2625
|
+
if (historyCursor !== undefined)
|
|
2626
|
+
url.searchParams.set("cursor", historyCursor);
|
|
2627
|
+
const result = await httpJson(url.toString(), request);
|
|
2628
|
+
if (!result.ok) {
|
|
2629
|
+
warnings.push(`channel ${channel.name} history failed: ${result.message}`);
|
|
2630
|
+
break;
|
|
2631
|
+
}
|
|
2632
|
+
const envelope = asRecord(result.json);
|
|
2633
|
+
if (envelope === undefined || envelope.ok !== true) {
|
|
2634
|
+
const error = asString(envelope?.error) ?? "malformed response";
|
|
2635
|
+
if (AUTH_ERRORS.has(error))
|
|
2636
|
+
return { ok: false, reason: "auth", message: `slack: ${error}` };
|
|
2637
|
+
if (error === "ratelimited")
|
|
2638
|
+
return { ok: false, reason: "rate-limited", message: "slack: ratelimited" };
|
|
2639
|
+
if (CHANNEL_SKIP_ERRORS.has(error))
|
|
2640
|
+
warnings.push(`channel ${channel.name} skipped: ${error}`);
|
|
2641
|
+
else
|
|
2642
|
+
warnings.push(`channel ${channel.name} history failed: ${error}`);
|
|
2643
|
+
break;
|
|
2644
|
+
}
|
|
2645
|
+
for (const raw of asArray(envelope.messages)) {
|
|
2646
|
+
if (documents.length >= maxDocuments)
|
|
2647
|
+
break;
|
|
2648
|
+
const message = asRecord(raw);
|
|
2649
|
+
if (message === undefined)
|
|
2650
|
+
continue;
|
|
2651
|
+
const text = asString(message.text);
|
|
2652
|
+
const ts = asString(message.ts);
|
|
2653
|
+
const subtype = asString(message.subtype);
|
|
2654
|
+
if (text === undefined || text.length === 0 || ts === undefined)
|
|
2655
|
+
continue;
|
|
2656
|
+
if (subtype === "channel_join")
|
|
2657
|
+
continue;
|
|
2658
|
+
const user = asString(message.user) ?? asString(message.username) ?? "unknown";
|
|
2659
|
+
documents.push({
|
|
2660
|
+
docId: `${channel.id}-${ts.replace(/\./gu, "-")}`,
|
|
2661
|
+
hierarchy: ["channels", channel.name],
|
|
2662
|
+
title: `#${channel.name}`,
|
|
2663
|
+
content: `${user}: ${text}`,
|
|
2664
|
+
publishedAt: Math.round(Number.parseFloat(ts) * 1000),
|
|
2665
|
+
metadata: { channelId: channel.id, ts, user }
|
|
2666
|
+
});
|
|
2667
|
+
}
|
|
2668
|
+
historyCursor = asString(asRecord(envelope.response_metadata)?.next_cursor);
|
|
2669
|
+
if (historyCursor === undefined || historyCursor.length === 0)
|
|
2670
|
+
break;
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
var DEFAULT_BASE_URL6 = "https://slack.com/api", DEFAULT_TOKEN_ENV6 = "SLACK_BOT_TOKEN", DEFAULT_MAX_PAGES6 = 10, DEFAULT_MAX_DOCUMENTS11 = 500, AUTH_ERRORS, CHANNEL_SKIP_ERRORS;
|
|
2677
|
+
var init_connector10 = __esm(() => {
|
|
2678
|
+
AUTH_ERRORS = new Set(["invalid_auth", "not_authed", "token_revoked", "account_inactive"]);
|
|
2679
|
+
CHANNEL_SKIP_ERRORS = new Set(["missing_scope", "access_denied", "channel_not_found", "not_in_channel"]);
|
|
2680
|
+
});
|
|
2681
|
+
|
|
2682
|
+
// src/datasource/skills/slack/skill.ts
|
|
2683
|
+
var SLACK_SKILL_DEFINITION, SlackSkill;
|
|
2684
|
+
var init_skill9 = __esm(() => {
|
|
2685
|
+
init_connector_skill();
|
|
2686
|
+
init_connector10();
|
|
2687
|
+
SLACK_SKILL_DEFINITION = {
|
|
2688
|
+
skillName: "slack",
|
|
2689
|
+
skillType: "slack-workspace",
|
|
2690
|
+
description: "Slack workspace datasource",
|
|
2691
|
+
capabilities: ["chat", "api", "polling"],
|
|
2692
|
+
defaultTags: ["slack", "chat", "pii"],
|
|
2693
|
+
contentType: "chat",
|
|
2694
|
+
manifestDescription: "Search indexed Slack messages across authorized workspace channels. Use for questions about Slack conversations, decisions made in channels, or who said what.",
|
|
2695
|
+
manifestNotes: [
|
|
2696
|
+
"Message history windows and channel visibility are bounded by the server-configured bot token scopes."
|
|
2697
|
+
]
|
|
2698
|
+
};
|
|
2699
|
+
SlackSkill = class SlackSkill extends ConnectorDatasourceSkill {
|
|
2700
|
+
constructor(options = {}) {
|
|
2701
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
2702
|
+
super(SLACK_SKILL_DEFINITION, {
|
|
2703
|
+
...rest,
|
|
2704
|
+
connector: connector ?? new SlackConnector(connectorOptions ?? {})
|
|
2705
|
+
});
|
|
2706
|
+
}
|
|
2707
|
+
};
|
|
2708
|
+
});
|
|
2709
|
+
|
|
2710
|
+
// src/datasource/skills/slack/index.ts
|
|
2711
|
+
var init_slack = __esm(() => {
|
|
2712
|
+
init_connector10();
|
|
2713
|
+
init_skill9();
|
|
2714
|
+
});
|
|
2715
|
+
|
|
2716
|
+
// src/datasource/skills/spotlight/connector.ts
|
|
2717
|
+
import { spawn as spawn3 } from "node:child_process";
|
|
2718
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
2719
|
+
import { readFile as fsReadFile, stat as stat3 } from "node:fs/promises";
|
|
2720
|
+
import { basename as basename3, extname as extname3 } from "node:path";
|
|
2721
|
+
|
|
2722
|
+
class SpotlightConnector {
|
|
2723
|
+
options;
|
|
2724
|
+
run;
|
|
2725
|
+
readFile;
|
|
2726
|
+
platform;
|
|
2727
|
+
constructor(options = {}) {
|
|
2728
|
+
this.options = options;
|
|
2729
|
+
this.run = options.run ?? spawnRunner;
|
|
2730
|
+
this.readFile = options.readFile ?? ((path) => fsReadFile(path, "utf8"));
|
|
2731
|
+
this.platform = options.platform ?? process.platform;
|
|
2732
|
+
}
|
|
2733
|
+
async fetch() {
|
|
2734
|
+
if (this.platform !== "darwin") {
|
|
2735
|
+
return { ok: false, reason: "unavailable", message: "spotlight datasource is supported on macOS only" };
|
|
2736
|
+
}
|
|
2737
|
+
const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
|
|
2738
|
+
if (queries.length === 0) {
|
|
2739
|
+
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
2740
|
+
}
|
|
2741
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS4;
|
|
2742
|
+
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
2743
|
+
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
2744
|
+
return {
|
|
2745
|
+
ok: false,
|
|
2746
|
+
reason: "unavailable",
|
|
2747
|
+
message: "Spotlight indexing is disabled; enable it with `sudo mdutil -i on /`"
|
|
2748
|
+
};
|
|
2749
|
+
}
|
|
2750
|
+
const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
|
|
2751
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS12;
|
|
2752
|
+
const seen = new Set;
|
|
2753
|
+
const paths = [];
|
|
2754
|
+
let mdfindMissing = false;
|
|
2755
|
+
let failedQueries = 0;
|
|
2756
|
+
for (const query of queries) {
|
|
2757
|
+
const args = this.options.onlyIn !== undefined ? ["-onlyin", this.options.onlyIn, query] : [query];
|
|
2758
|
+
const result = await this.run("mdfind", args, timeoutMs);
|
|
2759
|
+
if (!result.ok) {
|
|
2760
|
+
if (result.error === "ENOENT") {
|
|
2761
|
+
mdfindMissing = true;
|
|
2762
|
+
break;
|
|
2763
|
+
}
|
|
2764
|
+
failedQueries += 1;
|
|
2765
|
+
continue;
|
|
2766
|
+
}
|
|
2767
|
+
let kept = 0;
|
|
2768
|
+
for (const line of result.stdout.split(`
|
|
2769
|
+
`)) {
|
|
2770
|
+
const path = line.trim();
|
|
2771
|
+
if (path.length === 0 || seen.has(path))
|
|
2772
|
+
continue;
|
|
2773
|
+
if (kept >= maxPerQuery)
|
|
2774
|
+
break;
|
|
2775
|
+
seen.add(path);
|
|
2776
|
+
paths.push(path);
|
|
2777
|
+
kept += 1;
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
if (mdfindMissing) {
|
|
2781
|
+
return { ok: false, reason: "unavailable", message: "mdfind binary not found; Spotlight requires macOS" };
|
|
2782
|
+
}
|
|
2783
|
+
if (failedQueries === queries.length) {
|
|
2784
|
+
return { ok: false, reason: "unavailable", message: "every configured mdfind query failed" };
|
|
2785
|
+
}
|
|
2786
|
+
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE4;
|
|
2787
|
+
const documents = [];
|
|
2788
|
+
let permissionDenied = 0;
|
|
2789
|
+
let readFailures = 0;
|
|
2790
|
+
for (const path of paths) {
|
|
2791
|
+
if (documents.length >= maxDocuments)
|
|
2792
|
+
break;
|
|
2793
|
+
try {
|
|
2794
|
+
const info = await stat3(path);
|
|
2795
|
+
if (!info.isFile() || info.size > maxBytes)
|
|
2796
|
+
continue;
|
|
2797
|
+
const text = await this.readFile(path);
|
|
2798
|
+
const content = text.trim().slice(0, MAX_CONTENT_CHARS5);
|
|
2799
|
+
if (content.length === 0)
|
|
2800
|
+
continue;
|
|
2801
|
+
documents.push({
|
|
2802
|
+
docId: createHash2("sha256").update(path).digest("hex").slice(0, 16),
|
|
2803
|
+
hierarchy: ["files"],
|
|
2804
|
+
title: basename3(path),
|
|
2805
|
+
content,
|
|
2806
|
+
publishedAt: info.mtimeMs,
|
|
2807
|
+
metadata: { path, extension: extname3(path).toLowerCase() }
|
|
2808
|
+
});
|
|
2809
|
+
} catch (error) {
|
|
2810
|
+
const code = error.code;
|
|
2811
|
+
if (code === "EPERM" || code === "EACCES")
|
|
2812
|
+
permissionDenied += 1;
|
|
2813
|
+
else
|
|
2814
|
+
readFailures += 1;
|
|
2815
|
+
}
|
|
2816
|
+
}
|
|
2817
|
+
if (documents.length === 0 && permissionDenied > 0) {
|
|
2818
|
+
return {
|
|
2819
|
+
ok: false,
|
|
2820
|
+
reason: "permission",
|
|
2821
|
+
message: `macOS denied access to all Spotlight hits; ${FDA_GUIDANCE}`
|
|
2822
|
+
};
|
|
2823
|
+
}
|
|
2824
|
+
const warnings = [];
|
|
2825
|
+
if (failedQueries > 0)
|
|
2826
|
+
warnings.push(`${failedQueries} mdfind quer(y/ies) failed`);
|
|
2827
|
+
if (permissionDenied > 0) {
|
|
2828
|
+
warnings.push(`${permissionDenied} Spotlight hit(s) were denied by macOS privacy protections; ${FDA_GUIDANCE}`);
|
|
2829
|
+
}
|
|
2830
|
+
if (readFailures > 0)
|
|
2831
|
+
warnings.push(`${readFailures} Spotlight hit(s) failed to read`);
|
|
2832
|
+
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
2833
|
+
}
|
|
2834
|
+
}
|
|
2835
|
+
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS12 = 300, DEFAULT_MAX_BYTES_PER_FILE4, DEFAULT_TIMEOUT_MS4 = 15000, MAX_CONTENT_CHARS5 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve2) => {
|
|
2836
|
+
const child = spawn3(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
2837
|
+
let stdout = "";
|
|
2838
|
+
let stderr = "";
|
|
2839
|
+
let settled = false;
|
|
2840
|
+
const finish = (result) => {
|
|
2841
|
+
if (settled)
|
|
2842
|
+
return;
|
|
2843
|
+
settled = true;
|
|
2844
|
+
clearTimeout(timer);
|
|
2845
|
+
resolve2(result);
|
|
2846
|
+
};
|
|
2847
|
+
const timer = setTimeout(() => {
|
|
2848
|
+
child.kill("SIGKILL");
|
|
2849
|
+
finish({ ok: false, stdout, stderr: "process timed out", code: null });
|
|
2850
|
+
}, timeoutMs);
|
|
2851
|
+
child.stdout.setEncoding("utf8");
|
|
2852
|
+
child.stderr.setEncoding("utf8");
|
|
2853
|
+
child.stdout.on("data", (chunk) => {
|
|
2854
|
+
if (stdout.length < MAX_BUFFER_BYTES)
|
|
2855
|
+
stdout += chunk;
|
|
2856
|
+
});
|
|
2857
|
+
child.stderr.on("data", (chunk) => {
|
|
2858
|
+
if (stderr.length < MAX_BUFFER_BYTES)
|
|
2859
|
+
stderr += chunk;
|
|
2860
|
+
});
|
|
2861
|
+
child.on("error", (error) => {
|
|
2862
|
+
finish({ ok: false, stdout, stderr: error.message, code: null, error: error.code });
|
|
2863
|
+
});
|
|
2864
|
+
child.on("close", (code) => {
|
|
2865
|
+
finish({ ok: code === 0, stdout, stderr, code });
|
|
2866
|
+
});
|
|
2867
|
+
});
|
|
2868
|
+
var init_connector11 = __esm(() => {
|
|
2869
|
+
DEFAULT_MAX_BYTES_PER_FILE4 = 256 * 1024;
|
|
2870
|
+
MAX_BUFFER_BYTES = 4 * 1024 * 1024;
|
|
2871
|
+
});
|
|
2872
|
+
|
|
2873
|
+
// src/datasource/skills/spotlight/skill.ts
|
|
2874
|
+
var SPOTLIGHT_SKILL_DEFINITION, SpotlightSkill;
|
|
2875
|
+
var init_skill10 = __esm(() => {
|
|
2876
|
+
init_connector_skill();
|
|
2877
|
+
init_connector11();
|
|
2878
|
+
SPOTLIGHT_SKILL_DEFINITION = {
|
|
2879
|
+
skillName: "spotlight",
|
|
2880
|
+
skillType: "spotlight-search",
|
|
2881
|
+
description: "macOS Spotlight datasource",
|
|
2882
|
+
capabilities: ["files", "macos-only", "local-search"],
|
|
2883
|
+
defaultTags: ["spotlight", "files"],
|
|
2884
|
+
contentType: "document",
|
|
2885
|
+
manifestDescription: "Search files indexed by macOS Spotlight via the built-in mdfind CLI. macOS only; no external installs. Use for questions about local documents on the Mac running AutoRAG.",
|
|
2886
|
+
manifestNotes: [
|
|
2887
|
+
"## Platform requirements",
|
|
2888
|
+
"- macOS only: this datasource is unavailable on any other platform.",
|
|
2889
|
+
"- Spotlight indexing must be enabled (`sudo mdutil -i on /`; check with `mdutil -s /`).",
|
|
2890
|
+
"- **Full Disk Access**: Spotlight can find protected files (Mail, Messages, Safari data, parts of ~/Library), but reading their content fails unless the terminal or app running AutoRAG has Full Disk Access. Grant it under System Settings -> Privacy & Security -> Full Disk Access, add the host app (e.g. Terminal, iTerm, or the AutoRAG binary), then restart the app. Permission failures surface as `datasource-permission-denied`.",
|
|
2891
|
+
"",
|
|
2892
|
+
"## Operator cheat sheet",
|
|
2893
|
+
"- `mdfind <query>` — Spotlight search; `mdfind -onlyin <dir> <query>` scopes to a directory; `mdfind -name <text>` matches file names.",
|
|
2894
|
+
"- `mdls <file>` — show Spotlight metadata attributes (kMDItemDisplayName, kMDItemContentType, …) for one file.",
|
|
2895
|
+
"- `mdutil -s /` — index status; `sudo mdutil -i on /` / `-i off /` — enable/disable indexing; `sudo mdutil -E /` — erase and rebuild the index.",
|
|
2896
|
+
"",
|
|
2897
|
+
"Queries are trusted server configuration; indexing re-runs the configured mdfind queries and hydrates text content. Result metadata carries the real absolute file path (`path`) so hits are traceable back to disk; if paths must not leave this machine, run AutoRAG with a local LLM."
|
|
2898
|
+
]
|
|
2899
|
+
};
|
|
2900
|
+
SpotlightSkill = class SpotlightSkill extends ConnectorDatasourceSkill {
|
|
2901
|
+
constructor(options = {}) {
|
|
2902
|
+
const { connector, connectorOptions, ...rest } = options;
|
|
2903
|
+
super(SPOTLIGHT_SKILL_DEFINITION, {
|
|
2904
|
+
...rest,
|
|
2905
|
+
connector: connector ?? new SpotlightConnector(connectorOptions ?? {})
|
|
2906
|
+
});
|
|
2907
|
+
}
|
|
2908
|
+
};
|
|
2909
|
+
});
|
|
2910
|
+
|
|
2911
|
+
// src/datasource/skills/spotlight/index.ts
|
|
2912
|
+
var init_spotlight = __esm(() => {
|
|
2913
|
+
init_connector11();
|
|
2914
|
+
init_skill10();
|
|
2915
|
+
});
|
|
2916
|
+
|
|
2917
|
+
// src/datasource/skills/factory.ts
|
|
2918
|
+
function common(config, workspaceRoot) {
|
|
2919
|
+
return {
|
|
2920
|
+
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
2921
|
+
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
2922
|
+
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
2923
|
+
...workspaceRoot !== undefined ? { workspaceRoot } : {}
|
|
2924
|
+
};
|
|
2925
|
+
}
|
|
2926
|
+
function buildDatasourceSkills(config, workspaceRoot) {
|
|
2927
|
+
const skills = [];
|
|
2928
|
+
const unknown = [];
|
|
2929
|
+
for (const [name, raw] of Object.entries(config ?? {})) {
|
|
2930
|
+
if (raw === false)
|
|
2931
|
+
continue;
|
|
2932
|
+
const entry = raw === true ? {} : raw;
|
|
2933
|
+
if (entry.enabled === false)
|
|
2934
|
+
continue;
|
|
2935
|
+
const builder = BUILDERS[name];
|
|
2936
|
+
if (builder === undefined) {
|
|
2937
|
+
unknown.push(name);
|
|
2938
|
+
continue;
|
|
2939
|
+
}
|
|
2940
|
+
skills.push(builder(entry, workspaceRoot));
|
|
2941
|
+
}
|
|
2942
|
+
return { skills, unknown };
|
|
2943
|
+
}
|
|
2944
|
+
var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
2945
|
+
var init_factory = __esm(() => {
|
|
2946
|
+
init_discord();
|
|
2947
|
+
init_gdrive();
|
|
2948
|
+
init_rclone_connector();
|
|
2949
|
+
init_github();
|
|
2950
|
+
init_himalaya_connector();
|
|
2951
|
+
init_gmail();
|
|
2952
|
+
init_mail_export();
|
|
2953
|
+
init_notion();
|
|
2954
|
+
init_obsidian();
|
|
2955
|
+
init_rss();
|
|
2956
|
+
init_slack();
|
|
2957
|
+
init_spotlight();
|
|
2958
|
+
BUILDERS = {
|
|
2959
|
+
slack: (config, workspaceRoot) => new SlackSkill({ ...common(config, workspaceRoot), connectorOptions: config.connector }),
|
|
2960
|
+
discord: (config, workspaceRoot) => new DiscordSkill({
|
|
2961
|
+
...common(config, workspaceRoot),
|
|
2962
|
+
connectorOptions: config.connector
|
|
2963
|
+
}),
|
|
2964
|
+
notion: (config, workspaceRoot) => new NotionSkill({
|
|
2965
|
+
...common(config, workspaceRoot),
|
|
2966
|
+
connectorOptions: config.connector
|
|
2967
|
+
}),
|
|
2968
|
+
github: (config, workspaceRoot) => new GitHubSkill({
|
|
2969
|
+
...common(config, workspaceRoot),
|
|
2970
|
+
connectorOptions: config.connector
|
|
2971
|
+
}),
|
|
2972
|
+
gdrive: (config, workspaceRoot) => {
|
|
2973
|
+
const connector = config.connector;
|
|
2974
|
+
if (connector?.backend === "rclone") {
|
|
2975
|
+
const { backend: _backend, ...rcloneOptions } = connector;
|
|
2976
|
+
return new GDriveSkill({
|
|
2977
|
+
...common(config, workspaceRoot),
|
|
2978
|
+
connector: new RcloneConnector(rcloneOptions)
|
|
2979
|
+
});
|
|
2980
|
+
}
|
|
2981
|
+
return new GDriveSkill({
|
|
2982
|
+
...common(config, workspaceRoot),
|
|
2983
|
+
connectorOptions: connector
|
|
2984
|
+
});
|
|
2985
|
+
},
|
|
2986
|
+
gmail: (config, workspaceRoot) => {
|
|
2987
|
+
const connector = config.connector;
|
|
2988
|
+
if (connector?.backend === "himalaya") {
|
|
2989
|
+
const { backend: _backend, ...himalayaOptions } = connector;
|
|
2990
|
+
return new GmailSkill({
|
|
2991
|
+
...common(config, workspaceRoot),
|
|
2992
|
+
connector: new HimalayaConnector(himalayaOptions)
|
|
2993
|
+
});
|
|
2994
|
+
}
|
|
2995
|
+
return new GmailSkill({ ...common(config, workspaceRoot), connectorOptions: connector });
|
|
2996
|
+
},
|
|
2997
|
+
"mail-export": (config, workspaceRoot) => new MailExportSkill({
|
|
2998
|
+
...common(config, workspaceRoot),
|
|
2999
|
+
connectorOptions: config.connector
|
|
3000
|
+
}),
|
|
3001
|
+
obsidian: (config, workspaceRoot) => new ObsidianSkill({
|
|
3002
|
+
...common(config, workspaceRoot),
|
|
3003
|
+
connectorOptions: config.connector
|
|
3004
|
+
}),
|
|
3005
|
+
rss: (config, workspaceRoot) => new RssSkill({ ...common(config, workspaceRoot), connectorOptions: config.connector }),
|
|
3006
|
+
spotlight: (config, workspaceRoot) => new SpotlightSkill({
|
|
3007
|
+
...common(config, workspaceRoot),
|
|
3008
|
+
connectorOptions: config.connector
|
|
3009
|
+
})
|
|
3010
|
+
};
|
|
3011
|
+
BUILTIN_DATASOURCE_SKILL_NAMES = Object.keys(BUILDERS);
|
|
3012
|
+
});
|
|
3013
|
+
|
|
359
3014
|
// src/filesystem/file-lock.ts
|
|
360
3015
|
import { randomUUID } from "node:crypto";
|
|
361
3016
|
import {
|
|
362
3017
|
closeSync,
|
|
363
3018
|
lstatSync,
|
|
364
|
-
mkdirSync,
|
|
3019
|
+
mkdirSync as mkdirSync2,
|
|
365
3020
|
openSync,
|
|
366
3021
|
readdirSync,
|
|
367
|
-
readFileSync,
|
|
3022
|
+
readFileSync as readFileSync2,
|
|
368
3023
|
rmdirSync,
|
|
369
3024
|
unlinkSync,
|
|
370
3025
|
writeSync
|
|
371
3026
|
} from "node:fs";
|
|
372
|
-
import { join as
|
|
3027
|
+
import { join as join5 } from "node:path";
|
|
373
3028
|
function hasErrorCode(error, code) {
|
|
374
3029
|
return error instanceof Error && "code" in error && error.code === code;
|
|
375
3030
|
}
|
|
@@ -413,7 +3068,7 @@ function inspectRegularFile(path) {
|
|
|
413
3068
|
return { kind: "other" };
|
|
414
3069
|
let contents;
|
|
415
3070
|
try {
|
|
416
|
-
contents =
|
|
3071
|
+
contents = readFileSync2(path, "utf8");
|
|
417
3072
|
} catch (error) {
|
|
418
3073
|
if (hasErrorCode(error, "ENOENT"))
|
|
419
3074
|
return { kind: "missing" };
|
|
@@ -504,7 +3159,7 @@ function reapStaleLockDirectory(lockPath, staleMs) {
|
|
|
504
3159
|
return removeEmptyLockDirectory(lockPath);
|
|
505
3160
|
const staleSnapshots = [];
|
|
506
3161
|
for (const name of names) {
|
|
507
|
-
const inspection = inspectRegularFile(
|
|
3162
|
+
const inspection = inspectRegularFile(join5(lockPath, name));
|
|
508
3163
|
if (inspection.kind === "missing")
|
|
509
3164
|
continue;
|
|
510
3165
|
if (inspection.kind !== "file" || !isStaleSnapshot(inspection, staleMs))
|
|
@@ -559,14 +3214,14 @@ function createLockFile(lockPath, contents) {
|
|
|
559
3214
|
}
|
|
560
3215
|
function tryCreateOwnerMarker(lockPath, owner, contents) {
|
|
561
3216
|
try {
|
|
562
|
-
|
|
3217
|
+
mkdirSync2(lockPath, { mode: 448 });
|
|
563
3218
|
} catch (error) {
|
|
564
3219
|
if (hasErrorCode(error, "EEXIST"))
|
|
565
3220
|
return;
|
|
566
3221
|
throw error;
|
|
567
3222
|
}
|
|
568
3223
|
const markerName = `owner-${owner.token}.json`;
|
|
569
|
-
const markerPath =
|
|
3224
|
+
const markerPath = join5(lockPath, markerName);
|
|
570
3225
|
let acquired = false;
|
|
571
3226
|
try {
|
|
572
3227
|
try {
|
|
@@ -651,9 +3306,9 @@ var init_file_lock = __esm(() => {
|
|
|
651
3306
|
});
|
|
652
3307
|
|
|
653
3308
|
// src/subagents/local-models.ts
|
|
654
|
-
import { readFileSync as
|
|
3309
|
+
import { readFileSync as readFileSync3 } from "node:fs";
|
|
655
3310
|
import { homedir as homedir2 } from "node:os";
|
|
656
|
-
import { join as
|
|
3311
|
+
import { join as join6 } from "node:path";
|
|
657
3312
|
import { parse } from "smol-toml";
|
|
658
3313
|
function createModel(provider, baseUrl, id, defaultId, defaultName) {
|
|
659
3314
|
return {
|
|
@@ -670,8 +3325,8 @@ function createModel(provider, baseUrl, id, defaultId, defaultName) {
|
|
|
670
3325
|
};
|
|
671
3326
|
}
|
|
672
3327
|
function loadLocalAutoRAGModels(options = {}) {
|
|
673
|
-
const configPath = options.configPath ??
|
|
674
|
-
const config = parse(
|
|
3328
|
+
const configPath = options.configPath ?? join6(homedir2(), ".codex", "config.toml");
|
|
3329
|
+
const config = parse(readFileSync3(configPath, "utf8"));
|
|
675
3330
|
const provider = config.model_provider;
|
|
676
3331
|
if (typeof provider !== "string" || provider.length === 0) {
|
|
677
3332
|
throw new Error(`AutoRAG requires model_provider in ${configPath}`);
|
|
@@ -702,8 +3357,8 @@ var init_local_models = () => {};
|
|
|
702
3357
|
|
|
703
3358
|
// src/cli/config.ts
|
|
704
3359
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
705
|
-
import { existsSync, mkdirSync as
|
|
706
|
-
import { basename, dirname, isAbsolute as isAbsolute2, join as
|
|
3360
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync, unlinkSync as unlinkSync2, writeFileSync as writeFileSync2 } from "node:fs";
|
|
3361
|
+
import { basename as basename4, dirname as dirname3, isAbsolute as isAbsolute2, join as join7, resolve as resolve2 } from "node:path";
|
|
707
3362
|
import { findEnvKeys, getEnvApiKey, getModel, getProviders } from "@earendil-works/pi-ai/compat";
|
|
708
3363
|
function resolveConfigPath(input) {
|
|
709
3364
|
const flags = input.flags;
|
|
@@ -718,15 +3373,15 @@ function resolveConfigPath(input) {
|
|
|
718
3373
|
return { configPath: envConfig, explicit: true };
|
|
719
3374
|
}
|
|
720
3375
|
return {
|
|
721
|
-
configPath:
|
|
3376
|
+
configPath: join7(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
|
|
722
3377
|
explicit: false,
|
|
723
|
-
legacyPath:
|
|
3378
|
+
legacyPath: join7(cwd, LEGACY_CONFIG_FILENAME)
|
|
724
3379
|
};
|
|
725
3380
|
}
|
|
726
3381
|
function readConfigFile(configPath, explicit) {
|
|
727
3382
|
let exists;
|
|
728
3383
|
try {
|
|
729
|
-
exists =
|
|
3384
|
+
exists = existsSync2(configPath);
|
|
730
3385
|
} catch {
|
|
731
3386
|
exists = false;
|
|
732
3387
|
}
|
|
@@ -738,7 +3393,7 @@ function readConfigFile(configPath, explicit) {
|
|
|
738
3393
|
}
|
|
739
3394
|
let text;
|
|
740
3395
|
try {
|
|
741
|
-
text =
|
|
3396
|
+
text = readFileSync4(configPath, "utf8");
|
|
742
3397
|
} catch (err) {
|
|
743
3398
|
throw new ConfigError(`Failed to read config file: ${err.message}`);
|
|
744
3399
|
}
|
|
@@ -776,9 +3431,9 @@ function acquireConfigWriteLock(configPath) {
|
|
|
776
3431
|
});
|
|
777
3432
|
}
|
|
778
3433
|
function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
779
|
-
const temporaryPath =
|
|
3434
|
+
const temporaryPath = join7(dirname3(path), `.${basename4(path)}.${process.pid}.${randomUUID2()}.tmp`);
|
|
780
3435
|
try {
|
|
781
|
-
|
|
3436
|
+
writeFileSync2(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
782
3437
|
assertCommitAllowed?.();
|
|
783
3438
|
renameSync(temporaryPath, path);
|
|
784
3439
|
} finally {
|
|
@@ -786,14 +3441,14 @@ function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
|
786
3441
|
}
|
|
787
3442
|
}
|
|
788
3443
|
function migrateLegacyConfig(configPath, legacyPath) {
|
|
789
|
-
if (
|
|
3444
|
+
if (existsSync2(configPath) || !existsSync2(legacyPath))
|
|
790
3445
|
return;
|
|
791
3446
|
const legacy = readConfigFile(legacyPath, true);
|
|
792
|
-
const legacyBytes =
|
|
793
|
-
const migrated = normalizeLegacyConfigPaths(legacy ?? {},
|
|
3447
|
+
const legacyBytes = readFileSync4(legacyPath);
|
|
3448
|
+
const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname3(legacyPath));
|
|
794
3449
|
const migratedBytes = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths) ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
|
|
795
3450
|
`;
|
|
796
|
-
|
|
3451
|
+
mkdirSync3(dirname3(configPath), { recursive: true });
|
|
797
3452
|
const lock = acquireConfigWriteLock(configPath);
|
|
798
3453
|
try {
|
|
799
3454
|
const winner = readConfigFile(configPath, false);
|
|
@@ -823,7 +3478,7 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
|
|
|
823
3478
|
const legacy = readConfigFile(legacyPath, false);
|
|
824
3479
|
if (legacy === undefined)
|
|
825
3480
|
return {};
|
|
826
|
-
return normalizeLegacyConfigPaths(legacy,
|
|
3481
|
+
return normalizeLegacyConfigPaths(legacy, dirname3(legacyPath));
|
|
827
3482
|
}
|
|
828
3483
|
function resolveSearchPaths(searchPaths, origin) {
|
|
829
3484
|
return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
|
|
@@ -1092,10 +3747,10 @@ function resolveConfig(input) {
|
|
|
1092
3747
|
const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
|
|
1093
3748
|
const defaultSearchPaths = ["."];
|
|
1094
3749
|
const defaultWorkspacePath = cwd;
|
|
1095
|
-
const defaultMemoryPath =
|
|
3750
|
+
const defaultMemoryPath = join7(resolveAutoRAGHome(env), "memory.json");
|
|
1096
3751
|
const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
|
|
1097
3752
|
const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
|
|
1098
|
-
const configOrigin =
|
|
3753
|
+
const configOrigin = dirname3(resolve2(configPath));
|
|
1099
3754
|
const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
|
|
1100
3755
|
const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
|
|
1101
3756
|
const searchPaths = flagSearchPaths ?? envSearchPaths ?? fileSearchPaths ?? defaultSearchPaths;
|
|
@@ -1138,6 +3793,18 @@ function resolveConfig(input) {
|
|
|
1138
3793
|
config.jikji = file.jikji;
|
|
1139
3794
|
if (file.parserOptions)
|
|
1140
3795
|
config.parserOptions = file.parserOptions;
|
|
3796
|
+
if (file.datasources !== undefined) {
|
|
3797
|
+
if (typeof file.datasources !== "object" || file.datasources === null || Array.isArray(file.datasources)) {
|
|
3798
|
+
throw new ConfigError("Config field 'datasources' must be an object mapping skill names to their config");
|
|
3799
|
+
}
|
|
3800
|
+
config.datasources = file.datasources;
|
|
3801
|
+
}
|
|
3802
|
+
if (file.datasourceAccess !== undefined) {
|
|
3803
|
+
if (typeof file.datasourceAccess !== "object" || file.datasourceAccess === null || Array.isArray(file.datasourceAccess)) {
|
|
3804
|
+
throw new ConfigError("Config field 'datasourceAccess' must be an object with allowedTags/allowedScopes");
|
|
3805
|
+
}
|
|
3806
|
+
config.datasourceAccess = file.datasourceAccess;
|
|
3807
|
+
}
|
|
1141
3808
|
return config;
|
|
1142
3809
|
}
|
|
1143
3810
|
function resolveConfigReadOnly(input) {
|
|
@@ -1167,6 +3834,16 @@ function buildAgentOptions(config) {
|
|
|
1167
3834
|
opts.jikji = config.jikji;
|
|
1168
3835
|
if (config.parserOptions)
|
|
1169
3836
|
opts.parserOptions = config.parserOptions;
|
|
3837
|
+
if (config.datasources !== undefined) {
|
|
3838
|
+
const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath);
|
|
3839
|
+
if (unknown.length > 0) {
|
|
3840
|
+
throw new ConfigError(`Unknown datasource skill(s) in config: ${unknown.join(", ")}`);
|
|
3841
|
+
}
|
|
3842
|
+
if (skills.length > 0)
|
|
3843
|
+
opts.datasourceSkills = skills;
|
|
3844
|
+
}
|
|
3845
|
+
if (config.datasourceAccess !== undefined)
|
|
3846
|
+
opts.datasourceAccess = config.datasourceAccess;
|
|
1170
3847
|
return opts;
|
|
1171
3848
|
}
|
|
1172
3849
|
function isConfiguredEndpoint(reference) {
|
|
@@ -1385,7 +4062,7 @@ function resolveAgentModelDetailed(config, localOptions = {}) {
|
|
|
1385
4062
|
function writeDefaultConfig(path, partial, opts = {}) {
|
|
1386
4063
|
const cwd = resolve2(opts.cwd ?? process.cwd());
|
|
1387
4064
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
|
|
1388
|
-
const memoryPath = partial.memoryPath === undefined ?
|
|
4065
|
+
const memoryPath = partial.memoryPath === undefined ? join7(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
|
|
1389
4066
|
const full = {
|
|
1390
4067
|
searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
|
|
1391
4068
|
workspacePath,
|
|
@@ -1410,19 +4087,19 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
1410
4087
|
full.jikji = partial.jikji ?? {};
|
|
1411
4088
|
if (partial.parserOptions)
|
|
1412
4089
|
full.parserOptions = partial.parserOptions;
|
|
1413
|
-
|
|
4090
|
+
mkdirSync3(dirname3(path), { recursive: true });
|
|
1414
4091
|
const contents = `${JSON.stringify(full, null, 2)}
|
|
1415
4092
|
`;
|
|
1416
4093
|
const lock = acquireConfigWriteLock(path);
|
|
1417
4094
|
try {
|
|
1418
|
-
if (!opts.force &&
|
|
4095
|
+
if (!opts.force && existsSync2(path)) {
|
|
1419
4096
|
throw new ConfigError(`Config file already exists: ${path}`);
|
|
1420
4097
|
}
|
|
1421
4098
|
if (opts.force || opts.atomicCreate)
|
|
1422
4099
|
replaceFileAtomically(path, contents, lock.assertOwned);
|
|
1423
4100
|
else {
|
|
1424
4101
|
lock.assertOwned();
|
|
1425
|
-
|
|
4102
|
+
writeFileSync2(path, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
1426
4103
|
}
|
|
1427
4104
|
} catch (error) {
|
|
1428
4105
|
if (!opts.force && isEexistError(error)) {
|
|
@@ -1436,6 +4113,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
1436
4113
|
var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, BM25_ALLOWLIST, MINSYNC_ALLOWLIST;
|
|
1437
4114
|
var init_config = __esm(() => {
|
|
1438
4115
|
init_home();
|
|
4116
|
+
init_factory();
|
|
1439
4117
|
init_file_lock();
|
|
1440
4118
|
init_local_models();
|
|
1441
4119
|
init_home();
|
|
@@ -1483,8 +4161,8 @@ var exports_init = {};
|
|
|
1483
4161
|
__export(exports_init, {
|
|
1484
4162
|
runInit: () => runInit
|
|
1485
4163
|
});
|
|
1486
|
-
import { existsSync as
|
|
1487
|
-
import { dirname as
|
|
4164
|
+
import { existsSync as existsSync3, readFileSync as readFileSync5 } from "node:fs";
|
|
4165
|
+
import { dirname as dirname4 } from "node:path";
|
|
1488
4166
|
async function runInit(ctx) {
|
|
1489
4167
|
const flags = ctx.flags;
|
|
1490
4168
|
const partial = {};
|
|
@@ -1532,12 +4210,12 @@ async function runInit(ctx) {
|
|
|
1532
4210
|
throw error;
|
|
1533
4211
|
}
|
|
1534
4212
|
const resolvedPath = resolveConfigPath({ flags, cwd: ctx.cwd });
|
|
1535
|
-
const migratingLegacy = !resolvedPath.explicit && resolvedPath.legacyPath !== undefined &&
|
|
4213
|
+
const migratingLegacy = !resolvedPath.explicit && resolvedPath.legacyPath !== undefined && existsSync3(resolvedPath.legacyPath) && !existsSync3(resolvedPath.configPath);
|
|
1536
4214
|
if (migratingLegacy) {
|
|
1537
4215
|
try {
|
|
1538
4216
|
const legacyPath = resolvedPath.legacyPath;
|
|
1539
|
-
const legacy = JSON.parse(
|
|
1540
|
-
const normalizedLegacy = normalizeLegacyConfigPaths(legacy,
|
|
4217
|
+
const legacy = JSON.parse(readFileSync5(legacyPath, "utf8"));
|
|
4218
|
+
const normalizedLegacy = normalizeLegacyConfigPaths(legacy, dirname4(legacyPath));
|
|
1541
4219
|
partial.searchPaths ??= normalizedLegacy.searchPaths;
|
|
1542
4220
|
partial.workspacePath ??= normalizedLegacy.workspacePath;
|
|
1543
4221
|
partial.memoryPath ??= normalizedLegacy.memoryPath;
|
|
@@ -1623,49 +4301,6 @@ var init_init = __esm(() => {
|
|
|
1623
4301
|
init_config();
|
|
1624
4302
|
});
|
|
1625
4303
|
|
|
1626
|
-
// src/retrieval/scope.ts
|
|
1627
|
-
function normalizeVirtualPath(value) {
|
|
1628
|
-
const normalized = value.replace(/\\/g, "/").replace(/\/+/g, "/").trim();
|
|
1629
|
-
if (normalized.length === 0 || normalized === "/")
|
|
1630
|
-
return "/";
|
|
1631
|
-
const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`;
|
|
1632
|
-
return withLeadingSlash.length > 1 ? withLeadingSlash.replace(/\/+$/g, "") : withLeadingSlash;
|
|
1633
|
-
}
|
|
1634
|
-
function normalizeVirtualPathScope(scope) {
|
|
1635
|
-
if (scope === undefined)
|
|
1636
|
-
return;
|
|
1637
|
-
const normalized = normalizeVirtualPath(scope);
|
|
1638
|
-
return normalized === "/" ? undefined : normalized;
|
|
1639
|
-
}
|
|
1640
|
-
function matchesVirtualPathScope(virtualPath, scope) {
|
|
1641
|
-
const normalizedScope = normalizeVirtualPathScope(scope);
|
|
1642
|
-
if (normalizedScope === undefined)
|
|
1643
|
-
return true;
|
|
1644
|
-
return virtualPathScopeToRegExp(normalizedScope).test(normalizeVirtualPath(virtualPath));
|
|
1645
|
-
}
|
|
1646
|
-
function virtualPathScopeToRegExp(scope) {
|
|
1647
|
-
const normalized = normalizeVirtualPath(scope);
|
|
1648
|
-
const scoped = hasGlob(normalized) || looksLikeFileScope(normalized) ? normalized : `${normalized}/**`;
|
|
1649
|
-
const pattern = scoped.split("/").map((segment, index) => {
|
|
1650
|
-
if (index === 0)
|
|
1651
|
-
return "";
|
|
1652
|
-
if (segment === "**")
|
|
1653
|
-
return "(?:.*)";
|
|
1654
|
-
return escapeRegExp(segment).replace(/\\\*/g, "[^/]*");
|
|
1655
|
-
}).join("/").replace(/\/\(\?:\.\*\)$/u, "(?:/.*)?");
|
|
1656
|
-
return new RegExp(`^${pattern}$`);
|
|
1657
|
-
}
|
|
1658
|
-
function hasGlob(scope) {
|
|
1659
|
-
return scope.includes("*");
|
|
1660
|
-
}
|
|
1661
|
-
function looksLikeFileScope(scope) {
|
|
1662
|
-
const finalSegment = scope.split("/").at(-1) ?? "";
|
|
1663
|
-
return /\.[^/.]+$/u.test(finalSegment);
|
|
1664
|
-
}
|
|
1665
|
-
function escapeRegExp(literal) {
|
|
1666
|
-
return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1667
|
-
}
|
|
1668
|
-
|
|
1669
4304
|
// src/datasource/access-context.ts
|
|
1670
4305
|
class DatasourceAccessContext {
|
|
1671
4306
|
allowedTags;
|
|
@@ -1817,7 +4452,7 @@ var init_diagnostics = __esm(() => {
|
|
|
1817
4452
|
|
|
1818
4453
|
// src/jikji/answer-pack.ts
|
|
1819
4454
|
import { realpathSync } from "node:fs";
|
|
1820
|
-
import { isAbsolute as isAbsolute3, join as
|
|
4455
|
+
import { isAbsolute as isAbsolute3, join as join8, relative as relative2, resolve as resolve3 } from "node:path";
|
|
1821
4456
|
function isRecord(value) {
|
|
1822
4457
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1823
4458
|
}
|
|
@@ -1942,10 +4577,10 @@ function normalizeJikjiAnswerPath(rawPath, sourceRoots) {
|
|
|
1942
4577
|
} catch {
|
|
1943
4578
|
continue;
|
|
1944
4579
|
}
|
|
1945
|
-
const rel =
|
|
4580
|
+
const rel = relative2(realRoot, realCandidate);
|
|
1946
4581
|
if (rel.startsWith("..") || isAbsolute3(rel))
|
|
1947
4582
|
continue;
|
|
1948
|
-
return
|
|
4583
|
+
return join8(root.rootPath, rel);
|
|
1949
4584
|
}
|
|
1950
4585
|
return;
|
|
1951
4586
|
}
|
|
@@ -1959,9 +4594,9 @@ var init_answer_pack = __esm(() => {
|
|
|
1959
4594
|
});
|
|
1960
4595
|
|
|
1961
4596
|
// src/jikji/installer.ts
|
|
1962
|
-
import { spawn } from "node:child_process";
|
|
1963
|
-
import { existsSync as
|
|
1964
|
-
import { delimiter, join as
|
|
4597
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
4598
|
+
import { existsSync as existsSync4 } from "node:fs";
|
|
4599
|
+
import { delimiter, join as join9 } from "node:path";
|
|
1965
4600
|
function jikjiExecutableName(platform = process.platform) {
|
|
1966
4601
|
return platform === "win32" ? "jikji.exe" : "jikji";
|
|
1967
4602
|
}
|
|
@@ -1969,7 +4604,7 @@ function cargoExecutableName(platform = process.platform) {
|
|
|
1969
4604
|
return platform === "win32" ? "cargo.exe" : "cargo";
|
|
1970
4605
|
}
|
|
1971
4606
|
function cachedJikjiBinaryPath(root, platform = process.platform) {
|
|
1972
|
-
return
|
|
4607
|
+
return join9(root, ".autorag", "bin", jikjiExecutableName(platform));
|
|
1973
4608
|
}
|
|
1974
4609
|
function lookupExecutableInPath(executable, env) {
|
|
1975
4610
|
const pathEnv = env.PATH;
|
|
@@ -1978,15 +4613,15 @@ function lookupExecutableInPath(executable, env) {
|
|
|
1978
4613
|
for (const dir of pathEnv.split(delimiter)) {
|
|
1979
4614
|
if (dir.length === 0)
|
|
1980
4615
|
continue;
|
|
1981
|
-
const candidate =
|
|
1982
|
-
if (
|
|
4616
|
+
const candidate = join9(dir, executable);
|
|
4617
|
+
if (existsSync4(candidate))
|
|
1983
4618
|
return candidate;
|
|
1984
4619
|
}
|
|
1985
4620
|
return;
|
|
1986
4621
|
}
|
|
1987
4622
|
function defaultRunner(cargo, env) {
|
|
1988
4623
|
return (args, timeoutMs) => new Promise((resolvePromise) => {
|
|
1989
|
-
const child =
|
|
4624
|
+
const child = spawn4(cargo, args, { env, stdio: ["ignore", "ignore", "pipe"] });
|
|
1990
4625
|
let stderr = "";
|
|
1991
4626
|
const timeout = setTimeout(() => {
|
|
1992
4627
|
if (!child.killed)
|
|
@@ -2009,7 +4644,7 @@ function defaultRunner(cargo, env) {
|
|
|
2009
4644
|
async function ensureJikjiBinary(options) {
|
|
2010
4645
|
const env = options.env ?? process.env;
|
|
2011
4646
|
const cached = cachedJikjiBinaryPath(options.root);
|
|
2012
|
-
if (
|
|
4647
|
+
if (existsSync4(cached))
|
|
2013
4648
|
return { ok: true, binaryPath: cached, source: "cached" };
|
|
2014
4649
|
const cargoLocator = options.cargoLocator ?? ((e) => lookupExecutableInPath(cargoExecutableName(), e));
|
|
2015
4650
|
const cargo = cargoLocator(env);
|
|
@@ -2020,12 +4655,12 @@ async function ensureJikjiBinary(options) {
|
|
|
2020
4655
|
message: "Jikji auto-install needs the Rust toolchain (cargo) on PATH — install via https://rustup.rs, or install jikji manually (`cargo install jikji-cli`)."
|
|
2021
4656
|
};
|
|
2022
4657
|
}
|
|
2023
|
-
const args = ["install", JIKJI_CRATE_NAME, "--locked", "--root",
|
|
4658
|
+
const args = ["install", JIKJI_CRATE_NAME, "--locked", "--root", join9(options.root, ".autorag")];
|
|
2024
4659
|
if (options.version !== undefined)
|
|
2025
4660
|
args.push("--version", options.version);
|
|
2026
4661
|
const runner = options.runner ?? defaultRunner(cargo, env);
|
|
2027
4662
|
const result = await runner(args, options.timeoutMs ?? JIKJI_INSTALL_TIMEOUT_MS);
|
|
2028
|
-
if (result.code === 0 &&
|
|
4663
|
+
if (result.code === 0 && existsSync4(cached)) {
|
|
2029
4664
|
return { ok: true, binaryPath: cached, source: "installed" };
|
|
2030
4665
|
}
|
|
2031
4666
|
return {
|
|
@@ -2040,8 +4675,8 @@ var init_installer = __esm(() => {
|
|
|
2040
4675
|
});
|
|
2041
4676
|
|
|
2042
4677
|
// src/jikji/client.ts
|
|
2043
|
-
import { spawn as
|
|
2044
|
-
import { existsSync as
|
|
4678
|
+
import { spawn as spawn5 } from "node:child_process";
|
|
4679
|
+
import { existsSync as existsSync5 } from "node:fs";
|
|
2045
4680
|
|
|
2046
4681
|
class JikjiClient {
|
|
2047
4682
|
options;
|
|
@@ -2054,13 +4689,13 @@ class JikjiClient {
|
|
|
2054
4689
|
return commandFor(this.options.binaryPath);
|
|
2055
4690
|
if (this.resolvedCommand !== undefined)
|
|
2056
4691
|
return this.resolvedCommand;
|
|
2057
|
-
if (lookupExecutableInPath(
|
|
2058
|
-
this.resolvedCommand =
|
|
4692
|
+
if (lookupExecutableInPath(DEFAULT_BINARY3, process.env) !== undefined) {
|
|
4693
|
+
this.resolvedCommand = DEFAULT_BINARY3;
|
|
2059
4694
|
return this.resolvedCommand;
|
|
2060
4695
|
}
|
|
2061
4696
|
if (this.options.root !== undefined) {
|
|
2062
4697
|
const cached = cachedJikjiBinaryPath(this.options.root);
|
|
2063
|
-
if (
|
|
4698
|
+
if (existsSync5(cached)) {
|
|
2064
4699
|
this.resolvedCommand = cached;
|
|
2065
4700
|
return this.resolvedCommand;
|
|
2066
4701
|
}
|
|
@@ -2072,7 +4707,7 @@ class JikjiClient {
|
|
|
2072
4707
|
}
|
|
2073
4708
|
}
|
|
2074
4709
|
}
|
|
2075
|
-
this.resolvedCommand =
|
|
4710
|
+
this.resolvedCommand = DEFAULT_BINARY3;
|
|
2076
4711
|
return this.resolvedCommand;
|
|
2077
4712
|
}
|
|
2078
4713
|
async prepare(root, options = {}) {
|
|
@@ -2136,7 +4771,7 @@ class JikjiClient {
|
|
|
2136
4771
|
function spawnJikji(request) {
|
|
2137
4772
|
return new Promise((resolve4) => {
|
|
2138
4773
|
const options = request.options;
|
|
2139
|
-
const child =
|
|
4774
|
+
const child = spawn5(request.command, request.args, {
|
|
2140
4775
|
env: controlledEnv(options.env),
|
|
2141
4776
|
stdio: ["ignore", "pipe", "pipe"]
|
|
2142
4777
|
});
|
|
@@ -2147,7 +4782,7 @@ function spawnJikji(request) {
|
|
|
2147
4782
|
const timeout = setTimeout(() => {
|
|
2148
4783
|
finalReason = "timeout";
|
|
2149
4784
|
terminate(child);
|
|
2150
|
-
}, options.timeoutMs ??
|
|
4785
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS5);
|
|
2151
4786
|
const abortHandler = () => {
|
|
2152
4787
|
finalReason = "aborted";
|
|
2153
4788
|
terminate(child);
|
|
@@ -2194,7 +4829,7 @@ function spawnJikji(request) {
|
|
|
2194
4829
|
});
|
|
2195
4830
|
}
|
|
2196
4831
|
function commandFor(binaryPath) {
|
|
2197
|
-
return binaryPath === undefined || binaryPath ===
|
|
4832
|
+
return binaryPath === undefined || binaryPath === DEFAULT_BINARY3 ? DEFAULT_BINARY3 : binaryPath;
|
|
2198
4833
|
}
|
|
2199
4834
|
function buildPrepareArgs(options, root) {
|
|
2200
4835
|
const args = ["prepare", root, "--json"];
|
|
@@ -2261,19 +4896,19 @@ function appendBounded(state, chunk, maxBytes) {
|
|
|
2261
4896
|
const remainingBytes = Math.max(maxBytes - state.bytes, 0);
|
|
2262
4897
|
return { text: state.text + chunk.slice(0, remainingBytes), bytes: maxBytes, capped: true };
|
|
2263
4898
|
}
|
|
2264
|
-
var
|
|
4899
|
+
var DEFAULT_BINARY3 = "jikji", DEFAULT_TIMEOUT_MS5 = 1e4, DEFAULT_MAX_BUFFER_BYTES = 1048576;
|
|
2265
4900
|
var init_client = __esm(() => {
|
|
2266
4901
|
init_answer_pack();
|
|
2267
4902
|
init_installer();
|
|
2268
4903
|
});
|
|
2269
4904
|
|
|
2270
4905
|
// src/filesystem/source-paths.ts
|
|
2271
|
-
import { basename as
|
|
4906
|
+
import { basename as basename5, isAbsolute as isAbsolute4, relative as relative3, resolve as resolve4, sep as sep2 } from "node:path";
|
|
2272
4907
|
function planSourceRoots(searchPaths) {
|
|
2273
4908
|
const sorted = searchPaths.map((path) => resolve4(path)).sort((a, b) => a.localeCompare(b));
|
|
2274
4909
|
const used = new Set;
|
|
2275
4910
|
return sorted.map((rootPath) => {
|
|
2276
|
-
const base =
|
|
4911
|
+
const base = basename5(rootPath.replace(/[/\\]+$/, "")) || "root";
|
|
2277
4912
|
let prefix = `/${base}`;
|
|
2278
4913
|
let suffix = 2;
|
|
2279
4914
|
while (used.has(prefix)) {
|
|
@@ -2285,14 +4920,14 @@ function planSourceRoots(searchPaths) {
|
|
|
2285
4920
|
});
|
|
2286
4921
|
}
|
|
2287
4922
|
function sourceIdentifier(sourceRoot, sourcePath) {
|
|
2288
|
-
const rel =
|
|
2289
|
-
const suffix = rel === "" ? "" : `/${rel.split(
|
|
4923
|
+
const rel = relative3(sourceRoot.rootPath, resolve4(sourcePath));
|
|
4924
|
+
const suffix = rel === "" ? "" : `/${rel.split(sep2).join("/")}`;
|
|
2290
4925
|
return `${sourceRoot.prefix}${suffix}`;
|
|
2291
4926
|
}
|
|
2292
4927
|
var init_source_paths = () => {};
|
|
2293
4928
|
|
|
2294
4929
|
// src/jikji/path-map.ts
|
|
2295
|
-
import { isAbsolute as isAbsolute5, relative as
|
|
4930
|
+
import { isAbsolute as isAbsolute5, relative as relative4, resolve as resolve5, sep as sep3 } from "node:path";
|
|
2296
4931
|
function planJikjiSourceRoots(searchPaths) {
|
|
2297
4932
|
return planSourceRoots(searchPaths).map((sourceRoot) => ({
|
|
2298
4933
|
...sourceRoot,
|
|
@@ -2319,8 +4954,8 @@ function isUnsafeReturnedPath(normalizedPath) {
|
|
|
2319
4954
|
return normalizedPath.startsWith("/") || /^[A-Za-z]:\//.test(normalizedPath) || normalizedPath.startsWith("//") || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(normalizedPath);
|
|
2320
4955
|
}
|
|
2321
4956
|
function isWithinRoot(rootPath, sourcePath) {
|
|
2322
|
-
const rel =
|
|
2323
|
-
return rel === "" || !rel.startsWith("..") && !rel.includes(`..${
|
|
4957
|
+
const rel = relative4(resolve5(rootPath), resolve5(sourcePath));
|
|
4958
|
+
return rel === "" || !rel.startsWith("..") && !rel.includes(`..${sep3}`) && !isAbsolute5(rel);
|
|
2324
4959
|
}
|
|
2325
4960
|
var init_path_map = __esm(() => {
|
|
2326
4961
|
init_source_paths();
|
|
@@ -2353,8 +4988,8 @@ var init_jikji = __esm(() => {
|
|
|
2353
4988
|
});
|
|
2354
4989
|
|
|
2355
4990
|
// src/manifest/loader.ts
|
|
2356
|
-
import { existsSync as
|
|
2357
|
-
import { extname, join as
|
|
4991
|
+
import { existsSync as existsSync6, readdirSync as readdirSync2, readFileSync as readFileSync6 } from "node:fs";
|
|
4992
|
+
import { extname as extname4, join as join10 } from "node:path";
|
|
2358
4993
|
import { parse as parseYaml } from "yaml";
|
|
2359
4994
|
function isValidManifest(data) {
|
|
2360
4995
|
if (typeof data !== "object" || data === null)
|
|
@@ -2363,8 +4998,8 @@ function isValidManifest(data) {
|
|
|
2363
4998
|
return typeof obj.name === "string" && obj.name.length > 0 && typeof obj.description === "string" && obj.description.length > 0 && typeof obj.type === "string" && ["vector", "bm25", "hybrid", "visual"].includes(obj.type);
|
|
2364
4999
|
}
|
|
2365
5000
|
function loadManifest(filePath) {
|
|
2366
|
-
const content =
|
|
2367
|
-
const ext =
|
|
5001
|
+
const content = readFileSync6(filePath, "utf-8");
|
|
5002
|
+
const ext = extname4(filePath).toLowerCase();
|
|
2368
5003
|
let data;
|
|
2369
5004
|
if (ext === ".yaml" || ext === ".yml") {
|
|
2370
5005
|
data = parseYaml(content);
|
|
@@ -2386,7 +5021,7 @@ function loadManifest(filePath) {
|
|
|
2386
5021
|
};
|
|
2387
5022
|
}
|
|
2388
5023
|
function loadManifests(dirPath) {
|
|
2389
|
-
if (!
|
|
5024
|
+
if (!existsSync6(dirPath)) {
|
|
2390
5025
|
return [];
|
|
2391
5026
|
}
|
|
2392
5027
|
let entries;
|
|
@@ -2397,10 +5032,10 @@ function loadManifests(dirPath) {
|
|
|
2397
5032
|
}
|
|
2398
5033
|
const manifests = [];
|
|
2399
5034
|
for (const entry of entries) {
|
|
2400
|
-
const ext =
|
|
5035
|
+
const ext = extname4(entry).toLowerCase();
|
|
2401
5036
|
if (ext !== ".yaml" && ext !== ".yml" && ext !== ".json")
|
|
2402
5037
|
continue;
|
|
2403
|
-
const filePath =
|
|
5038
|
+
const filePath = join10(dirPath, entry);
|
|
2404
5039
|
try {
|
|
2405
5040
|
const manifest = loadManifest(filePath);
|
|
2406
5041
|
manifests.push(manifest);
|
|
@@ -2486,7 +5121,7 @@ var init_check_memory_tool = __esm(() => {
|
|
|
2486
5121
|
});
|
|
2487
5122
|
|
|
2488
5123
|
// src/retrieval/evidence-id.ts
|
|
2489
|
-
import { createHash } from "node:crypto";
|
|
5124
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
2490
5125
|
function normalizeEvidenceText(text) {
|
|
2491
5126
|
return text.replace(/\r\n?/gu, `
|
|
2492
5127
|
`).trim().normalize("NFC");
|
|
@@ -2516,7 +5151,7 @@ function stableEvidenceId(input) {
|
|
|
2516
5151
|
input.lineNumber ?? "",
|
|
2517
5152
|
normalizeEvidenceText(evidenceText)
|
|
2518
5153
|
].join("\x00");
|
|
2519
|
-
const digest =
|
|
5154
|
+
const digest = createHash3("sha256").update(payload).digest("hex").slice(0, 24);
|
|
2520
5155
|
return `${method}:${digest}`;
|
|
2521
5156
|
}
|
|
2522
5157
|
function normalizeEvidenceRef(input) {
|
|
@@ -2528,9 +5163,9 @@ function normalizeEvidenceRef(input) {
|
|
|
2528
5163
|
var init_evidence_id = () => {};
|
|
2529
5164
|
|
|
2530
5165
|
// src/memory/memory.ts
|
|
2531
|
-
import { createHash as
|
|
2532
|
-
import { existsSync as
|
|
2533
|
-
import { dirname as
|
|
5166
|
+
import { createHash as createHash4, randomUUID as randomUUID3 } from "node:crypto";
|
|
5167
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync4, readFileSync as readFileSync7, renameSync as renameSync2, unlinkSync as unlinkSync3, writeFileSync as writeFileSync3 } from "node:fs";
|
|
5168
|
+
import { dirname as dirname5 } from "node:path";
|
|
2534
5169
|
function emptyMemoryV4() {
|
|
2535
5170
|
return {
|
|
2536
5171
|
version: 4,
|
|
@@ -2587,7 +5222,7 @@ function normalizeV4(data) {
|
|
|
2587
5222
|
};
|
|
2588
5223
|
}
|
|
2589
5224
|
function hashText(value) {
|
|
2590
|
-
return
|
|
5225
|
+
return createHash4("sha256").update(value).digest("hex");
|
|
2591
5226
|
}
|
|
2592
5227
|
function resultId(sessionId, number) {
|
|
2593
5228
|
return `${sessionId}:${number}`;
|
|
@@ -2687,9 +5322,9 @@ class RetrievalMemory {
|
|
|
2687
5322
|
this.persistedData = cloneMemory(this.data);
|
|
2688
5323
|
}
|
|
2689
5324
|
save() {
|
|
2690
|
-
const dir =
|
|
2691
|
-
if (!
|
|
2692
|
-
|
|
5325
|
+
const dir = dirname5(this.storagePath);
|
|
5326
|
+
if (!existsSync7(dir))
|
|
5327
|
+
mkdirSync4(dir, { recursive: true });
|
|
2693
5328
|
const lock = this.acquireLock();
|
|
2694
5329
|
const localData = this.data;
|
|
2695
5330
|
let tmpPath;
|
|
@@ -2697,7 +5332,7 @@ class RetrievalMemory {
|
|
|
2697
5332
|
this.data = this.mergeWithPersisted(this.readPersistedData());
|
|
2698
5333
|
this.capData();
|
|
2699
5334
|
tmpPath = `${this.storagePath}.${randomUUID3()}.tmp`;
|
|
2700
|
-
|
|
5335
|
+
writeFileSync3(tmpPath, `${JSON.stringify(this.data, null, 2)}
|
|
2701
5336
|
`, "utf-8");
|
|
2702
5337
|
lock.assertOwned();
|
|
2703
5338
|
renameSync2(tmpPath, this.storagePath);
|
|
@@ -2707,7 +5342,7 @@ class RetrievalMemory {
|
|
|
2707
5342
|
throw error;
|
|
2708
5343
|
} finally {
|
|
2709
5344
|
try {
|
|
2710
|
-
if (tmpPath &&
|
|
5345
|
+
if (tmpPath && existsSync7(tmpPath))
|
|
2711
5346
|
unlinkSync3(tmpPath);
|
|
2712
5347
|
} finally {
|
|
2713
5348
|
lock.release();
|
|
@@ -2911,10 +5546,10 @@ class RetrievalMemory {
|
|
|
2911
5546
|
});
|
|
2912
5547
|
}
|
|
2913
5548
|
readPersistedData() {
|
|
2914
|
-
if (!
|
|
5549
|
+
if (!existsSync7(this.storagePath))
|
|
2915
5550
|
return emptyMemoryV4();
|
|
2916
5551
|
try {
|
|
2917
|
-
const parsed = JSON.parse(
|
|
5552
|
+
const parsed = JSON.parse(readFileSync7(this.storagePath, "utf-8"));
|
|
2918
5553
|
if (isV4(parsed))
|
|
2919
5554
|
return normalizeV4(parsed);
|
|
2920
5555
|
} catch (error) {
|
|
@@ -3165,17 +5800,17 @@ var init_memory = __esm(() => {
|
|
|
3165
5800
|
});
|
|
3166
5801
|
|
|
3167
5802
|
// src/minsync/embedder-config.ts
|
|
3168
|
-
import { readFileSync as
|
|
3169
|
-
import { join as
|
|
5803
|
+
import { readFileSync as readFileSync8, writeFileSync as writeFileSync4 } from "node:fs";
|
|
5804
|
+
import { join as join11 } from "node:path";
|
|
3170
5805
|
import { parse as parse2, stringify } from "smol-toml";
|
|
3171
5806
|
function minSyncConfigPath(workspacePath) {
|
|
3172
|
-
return
|
|
5807
|
+
return join11(workspacePath, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE);
|
|
3173
5808
|
}
|
|
3174
5809
|
function rewriteEmbedderConfig(workspacePath, embedder) {
|
|
3175
5810
|
const configPath = minSyncConfigPath(workspacePath);
|
|
3176
5811
|
let raw;
|
|
3177
5812
|
try {
|
|
3178
|
-
raw =
|
|
5813
|
+
raw = readFileSync8(configPath, "utf8");
|
|
3179
5814
|
} catch {
|
|
3180
5815
|
return false;
|
|
3181
5816
|
}
|
|
@@ -3205,17 +5840,17 @@ function rewriteEmbedderConfig(workspacePath, embedder) {
|
|
|
3205
5840
|
}
|
|
3206
5841
|
if (embedder.dimension !== undefined)
|
|
3207
5842
|
options.dimension = embedder.dimension;
|
|
3208
|
-
|
|
5843
|
+
writeFileSync4(configPath, stringify(parsed));
|
|
3209
5844
|
return true;
|
|
3210
5845
|
}
|
|
3211
5846
|
var MINSYNC_CONFIG_DIR = ".minsync", MINSYNC_CONFIG_FILE = "config.toml";
|
|
3212
5847
|
var init_embedder_config = () => {};
|
|
3213
5848
|
|
|
3214
5849
|
// src/minsync/process.ts
|
|
3215
|
-
import { spawn as
|
|
5850
|
+
import { spawn as spawn6 } from "node:child_process";
|
|
3216
5851
|
function spawnProcess(command, args, cwd, options = {}) {
|
|
3217
5852
|
return new Promise((resolve6) => {
|
|
3218
|
-
const child =
|
|
5853
|
+
const child = spawn6(command, [...args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
3219
5854
|
let stdout = "";
|
|
3220
5855
|
let stderr = "";
|
|
3221
5856
|
let timedOut = false;
|
|
@@ -3250,7 +5885,7 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
3250
5885
|
var init_process = () => {};
|
|
3251
5886
|
|
|
3252
5887
|
// src/minsync/client.ts
|
|
3253
|
-
import { existsSync as
|
|
5888
|
+
import { existsSync as existsSync8 } from "node:fs";
|
|
3254
5889
|
|
|
3255
5890
|
class MinSyncClient {
|
|
3256
5891
|
binaryPath;
|
|
@@ -3262,7 +5897,7 @@ class MinSyncClient {
|
|
|
3262
5897
|
this.embedder = options.embedder;
|
|
3263
5898
|
}
|
|
3264
5899
|
async sync() {
|
|
3265
|
-
if (!
|
|
5900
|
+
if (!existsSync8(this.binaryPath)) {
|
|
3266
5901
|
return { ok: false, synced: 0, workspacePath: this.workspacePath, reason: "missing-binary" };
|
|
3267
5902
|
}
|
|
3268
5903
|
if (this.embedder?.apiKeyEnv) {
|
|
@@ -3309,7 +5944,7 @@ class MinSyncClient {
|
|
|
3309
5944
|
return { ok: true, synced: readSyncedCount(result.stdout), workspacePath: this.workspacePath };
|
|
3310
5945
|
}
|
|
3311
5946
|
async query(text, topK) {
|
|
3312
|
-
if (!
|
|
5947
|
+
if (!existsSync8(this.binaryPath))
|
|
3313
5948
|
return [];
|
|
3314
5949
|
const result = await spawnProcess(this.binaryPath, ["query", "--format", "json", "-k", String(topK), text], this.workspacePath);
|
|
3315
5950
|
if (!result.ok)
|
|
@@ -3364,23 +5999,23 @@ var init_client2 = __esm(() => {
|
|
|
3364
5999
|
});
|
|
3365
6000
|
|
|
3366
6001
|
// src/minsync/installer.ts
|
|
3367
|
-
import { createHash as
|
|
3368
|
-
import { createWriteStream, existsSync as
|
|
6002
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
6003
|
+
import { createWriteStream, existsSync as existsSync9, mkdirSync as mkdirSync5, renameSync as renameSync3, rmSync } from "node:fs";
|
|
3369
6004
|
import { chmod, mkdtemp } from "node:fs/promises";
|
|
3370
6005
|
import { get } from "node:https";
|
|
3371
6006
|
import { tmpdir } from "node:os";
|
|
3372
|
-
import { basename as
|
|
6007
|
+
import { basename as basename6, dirname as dirname6, join as join12 } from "node:path";
|
|
3373
6008
|
import { pipeline } from "node:stream/promises";
|
|
3374
6009
|
async function ensureMinSyncBinary(options) {
|
|
3375
|
-
const binaryPath =
|
|
3376
|
-
if (
|
|
6010
|
+
const binaryPath = join12(options.root, ".autorag", "bin", executableName(options.platform ?? process.platform));
|
|
6011
|
+
if (existsSync9(binaryPath))
|
|
3377
6012
|
return { binaryPath, version: "cached" };
|
|
3378
6013
|
const releaseProvider = options.releaseProvider ?? fetchLatestMinSyncRelease;
|
|
3379
6014
|
const release = await releaseProvider();
|
|
3380
6015
|
const asset = selectReleaseAsset(release, options.platform ?? process.platform, options.arch ?? process.arch);
|
|
3381
6016
|
requireSha256(asset);
|
|
3382
6017
|
const assetInstaller = options.assetInstaller ?? installReleaseAsset;
|
|
3383
|
-
|
|
6018
|
+
mkdirSync5(dirname6(binaryPath), { recursive: true });
|
|
3384
6019
|
await assetInstaller(asset, binaryPath);
|
|
3385
6020
|
await chmod(binaryPath, 493);
|
|
3386
6021
|
return { binaryPath, version: release.tagName };
|
|
@@ -3409,9 +6044,9 @@ function selectReleaseAsset(release, platform, arch) {
|
|
|
3409
6044
|
}
|
|
3410
6045
|
async function installReleaseAsset(asset, destination) {
|
|
3411
6046
|
const expectedSha256 = requireSha256(asset);
|
|
3412
|
-
const tempDir = await mkdtemp(
|
|
6047
|
+
const tempDir = await mkdtemp(join12(tmpdir(), "autorag-minsync-install-"));
|
|
3413
6048
|
try {
|
|
3414
|
-
const archive =
|
|
6049
|
+
const archive = join12(tempDir, asset.name);
|
|
3415
6050
|
await downloadFile(asset.downloadUrl, archive);
|
|
3416
6051
|
await verifySha256(archive, expectedSha256);
|
|
3417
6052
|
if (!asset.name.endsWith(".tar.gz") && !asset.name.endsWith(".zip")) {
|
|
@@ -3420,7 +6055,7 @@ async function installReleaseAsset(asset, destination) {
|
|
|
3420
6055
|
const extracted = await spawnProcess("tar", ["-xf", archive, "-C", tempDir], tempDir);
|
|
3421
6056
|
if (!extracted.ok)
|
|
3422
6057
|
throw new MinSyncReleaseError(extracted.stderr || `Could not extract ${asset.name}`);
|
|
3423
|
-
renameSync3(
|
|
6058
|
+
renameSync3(join12(tempDir, basename6(destination)), destination);
|
|
3424
6059
|
} finally {
|
|
3425
6060
|
rmSync(tempDir, { recursive: true, force: true });
|
|
3426
6061
|
}
|
|
@@ -3429,7 +6064,7 @@ async function downloadFile(url, destination) {
|
|
|
3429
6064
|
await pipeline(await openHttps(url), createWriteStream(destination));
|
|
3430
6065
|
}
|
|
3431
6066
|
async function verifySha256(path, expected) {
|
|
3432
|
-
const hash =
|
|
6067
|
+
const hash = createHash5("sha256");
|
|
3433
6068
|
const stream = await import("node:fs").then((fs) => fs.createReadStream(path));
|
|
3434
6069
|
await pipeline(stream, hash);
|
|
3435
6070
|
const actual = hash.digest("hex");
|
|
@@ -3531,56 +6166,56 @@ var init_installer2 = __esm(() => {
|
|
|
3531
6166
|
});
|
|
3532
6167
|
|
|
3533
6168
|
// src/minsync/paths.ts
|
|
3534
|
-
import { join as
|
|
6169
|
+
import { join as join13 } from "node:path";
|
|
3535
6170
|
function minSyncWorkspaceRoot(root) {
|
|
3536
|
-
return
|
|
6171
|
+
return join13(root, MINSYNC_SUBDIR);
|
|
3537
6172
|
}
|
|
3538
6173
|
function minSyncDocumentPath(workspacePath, virtualPath) {
|
|
3539
6174
|
const safePath = virtualPath.replace(/^\/+/, "");
|
|
3540
|
-
return
|
|
6175
|
+
return join13(workspacePath, MINSYNC_FILES_SUBDIR, `${safePath}.md`);
|
|
3541
6176
|
}
|
|
3542
6177
|
var MINSYNC_SUBDIR, MINSYNC_FILES_SUBDIR = "files";
|
|
3543
6178
|
var init_paths = __esm(() => {
|
|
3544
|
-
MINSYNC_SUBDIR =
|
|
6179
|
+
MINSYNC_SUBDIR = join13(".autorag", "minsync");
|
|
3545
6180
|
});
|
|
3546
6181
|
|
|
3547
6182
|
// src/mirror/paths.ts
|
|
3548
|
-
import { createHash as
|
|
3549
|
-
import { join as
|
|
6183
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
6184
|
+
import { join as join14 } from "node:path";
|
|
3550
6185
|
function parsedMirrorRoot(root) {
|
|
3551
|
-
return
|
|
6186
|
+
return join14(root, PARSED_MIRROR_SUBDIR);
|
|
3552
6187
|
}
|
|
3553
6188
|
function parsedMirrorIndexPath(root) {
|
|
3554
|
-
return
|
|
6189
|
+
return join14(parsedMirrorRoot(root), PARSED_INDEX_FILE);
|
|
3555
6190
|
}
|
|
3556
6191
|
function parsedOutputPath(root, virtualPath) {
|
|
3557
|
-
const digest =
|
|
3558
|
-
return
|
|
6192
|
+
const digest = createHash6("sha256").update(virtualPath).digest("hex");
|
|
6193
|
+
return join14(parsedMirrorRoot(root), PARSED_FILES_SUBDIR, `${digest}.md`);
|
|
3559
6194
|
}
|
|
3560
6195
|
var PARSED_MIRROR_SUBDIR, PARSED_FILES_SUBDIR = "files", PARSED_INDEX_FILE = "index.json";
|
|
3561
6196
|
var init_paths2 = __esm(() => {
|
|
3562
|
-
PARSED_MIRROR_SUBDIR =
|
|
6197
|
+
PARSED_MIRROR_SUBDIR = join14(".autorag", "parsed");
|
|
3563
6198
|
});
|
|
3564
6199
|
|
|
3565
6200
|
// src/mirror/index-store.ts
|
|
3566
6201
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
3567
|
-
import { existsSync as
|
|
3568
|
-
import { dirname as
|
|
6202
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync6, readFileSync as readFileSync9, renameSync as renameSync4, writeFileSync as writeFileSync5 } from "node:fs";
|
|
6203
|
+
import { dirname as dirname7 } from "node:path";
|
|
3569
6204
|
function emptyMirrorIndex() {
|
|
3570
6205
|
return { version: 1, entries: {} };
|
|
3571
6206
|
}
|
|
3572
6207
|
function loadMirrorIndex(root) {
|
|
3573
6208
|
const path = parsedMirrorIndexPath(root);
|
|
3574
|
-
if (!
|
|
6209
|
+
if (!existsSync10(path))
|
|
3575
6210
|
return emptyMirrorIndex();
|
|
3576
|
-
const parsed = JSON.parse(
|
|
6211
|
+
const parsed = JSON.parse(readFileSync9(path, "utf8"));
|
|
3577
6212
|
return isParsedMirrorIndex(parsed) ? parsed : emptyMirrorIndex();
|
|
3578
6213
|
}
|
|
3579
6214
|
function saveMirrorIndex(root, index) {
|
|
3580
6215
|
const path = parsedMirrorIndexPath(root);
|
|
3581
|
-
|
|
6216
|
+
mkdirSync6(dirname7(path), { recursive: true });
|
|
3582
6217
|
const tmp = `${path}.${randomUUID4()}.tmp`;
|
|
3583
|
-
|
|
6218
|
+
writeFileSync5(tmp, `${JSON.stringify(index, null, 2)}
|
|
3584
6219
|
`);
|
|
3585
6220
|
renameSync4(tmp, path);
|
|
3586
6221
|
}
|
|
@@ -3600,17 +6235,17 @@ var init_index_store = __esm(() => {
|
|
|
3600
6235
|
});
|
|
3601
6236
|
|
|
3602
6237
|
// src/minsync/workspace.ts
|
|
3603
|
-
import { copyFileSync, existsSync as
|
|
3604
|
-
import { dirname as
|
|
6238
|
+
import { copyFileSync, existsSync as existsSync11, mkdirSync as mkdirSync7, rmSync as rmSync2 } from "node:fs";
|
|
6239
|
+
import { dirname as dirname8, join as join15 } from "node:path";
|
|
3605
6240
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
3606
6241
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
3607
|
-
const filesRoot =
|
|
6242
|
+
const filesRoot = join15(workspacePath, "files");
|
|
3608
6243
|
rmSync2(filesRoot, { recursive: true, force: true });
|
|
3609
|
-
|
|
6244
|
+
mkdirSync7(filesRoot, { recursive: true });
|
|
3610
6245
|
const index = loadMirrorIndex(root);
|
|
3611
|
-
const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) =>
|
|
6246
|
+
const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => existsSync11(entry.outputPath)).map((entry) => {
|
|
3612
6247
|
const minSyncPath = minSyncDocumentPath(workspacePath, entry.virtualPath);
|
|
3613
|
-
|
|
6248
|
+
mkdirSync7(dirname8(minSyncPath), { recursive: true });
|
|
3614
6249
|
copyFileSync(entry.outputPath, minSyncPath);
|
|
3615
6250
|
return {
|
|
3616
6251
|
virtualPath: entry.virtualPath,
|
|
@@ -3640,7 +6275,7 @@ function buildMinSyncPathMap(root, workspacePath) {
|
|
|
3640
6275
|
return byPath;
|
|
3641
6276
|
}
|
|
3642
6277
|
function legacyParsedVirtualPath(root, virtualPath) {
|
|
3643
|
-
return
|
|
6278
|
+
return join15(root, ".autorag", "parsed", "files", `${virtualPath.replace(/^\/+/, "")}.md`);
|
|
3644
6279
|
}
|
|
3645
6280
|
var init_workspace = __esm(() => {
|
|
3646
6281
|
init_index_store();
|
|
@@ -3648,8 +6283,8 @@ var init_workspace = __esm(() => {
|
|
|
3648
6283
|
});
|
|
3649
6284
|
|
|
3650
6285
|
// src/minsync/method.ts
|
|
3651
|
-
import { accessSync, constants, existsSync as
|
|
3652
|
-
import { basename as
|
|
6286
|
+
import { accessSync, constants, existsSync as existsSync12 } from "node:fs";
|
|
6287
|
+
import { basename as basename7, delimiter as delimiter2, join as join16 } from "node:path";
|
|
3653
6288
|
function degrade(workspacePath, reason) {
|
|
3654
6289
|
return { ok: false, synced: 0, workspacePath, reason };
|
|
3655
6290
|
}
|
|
@@ -3661,7 +6296,7 @@ function lookupInPath(env) {
|
|
|
3661
6296
|
for (const dir of pathEnv.split(delimiter2)) {
|
|
3662
6297
|
if (dir.length === 0)
|
|
3663
6298
|
continue;
|
|
3664
|
-
const candidate =
|
|
6299
|
+
const candidate = join16(dir, execName);
|
|
3665
6300
|
try {
|
|
3666
6301
|
accessSync(candidate, constants.X_OK);
|
|
3667
6302
|
return candidate;
|
|
@@ -3711,7 +6346,7 @@ class MinSyncVectorMethod {
|
|
|
3711
6346
|
return binaryResult;
|
|
3712
6347
|
}
|
|
3713
6348
|
isBinaryMissing() {
|
|
3714
|
-
return this.binaryPath !== undefined && !
|
|
6349
|
+
return this.binaryPath !== undefined && !existsSync12(this.binaryPath);
|
|
3715
6350
|
}
|
|
3716
6351
|
async retrieve(query, options) {
|
|
3717
6352
|
const topK = options.topK ?? 20;
|
|
@@ -3732,7 +6367,7 @@ class MinSyncVectorMethod {
|
|
|
3732
6367
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
3733
6368
|
continue;
|
|
3734
6369
|
results.push({
|
|
3735
|
-
id: `minsync:${entry.virtualPath}:${
|
|
6370
|
+
id: `minsync:${entry.virtualPath}:${basename7(hit.path)}`,
|
|
3736
6371
|
content: hit.text,
|
|
3737
6372
|
source: entry.virtualPath,
|
|
3738
6373
|
score: hit.score,
|
|
@@ -3744,14 +6379,14 @@ class MinSyncVectorMethod {
|
|
|
3744
6379
|
return results;
|
|
3745
6380
|
}
|
|
3746
6381
|
async resolveBinary() {
|
|
3747
|
-
if (this.binaryPath &&
|
|
6382
|
+
if (this.binaryPath && existsSync12(this.binaryPath)) {
|
|
3748
6383
|
return this.binaryPath;
|
|
3749
6384
|
}
|
|
3750
6385
|
const pathBinary = lookupInPath(process.env);
|
|
3751
6386
|
if (pathBinary)
|
|
3752
6387
|
return pathBinary;
|
|
3753
|
-
const cachedBinary =
|
|
3754
|
-
if (
|
|
6388
|
+
const cachedBinary = join16(this.root, ".autorag", "bin", executableName(process.platform));
|
|
6389
|
+
if (existsSync12(cachedBinary))
|
|
3755
6390
|
return cachedBinary;
|
|
3756
6391
|
if (this.autoInstall) {
|
|
3757
6392
|
try {
|
|
@@ -3835,7 +6470,7 @@ class Parser {
|
|
|
3835
6470
|
}
|
|
3836
6471
|
|
|
3837
6472
|
// src/parser/eml.ts
|
|
3838
|
-
import { simpleParser } from "mailparser";
|
|
6473
|
+
import { simpleParser as simpleParser2 } from "mailparser";
|
|
3839
6474
|
function addressText(address) {
|
|
3840
6475
|
if (address === undefined)
|
|
3841
6476
|
return;
|
|
@@ -3853,7 +6488,7 @@ var init_eml = __esm(() => {
|
|
|
3853
6488
|
extensions = [".eml"];
|
|
3854
6489
|
async parse(input) {
|
|
3855
6490
|
try {
|
|
3856
|
-
const parsed = await
|
|
6491
|
+
const parsed = await simpleParser2(Buffer.from(input.bytes));
|
|
3857
6492
|
const from = addressText(parsed.from);
|
|
3858
6493
|
const to = addressText(parsed.to);
|
|
3859
6494
|
const lines = [
|
|
@@ -3882,9 +6517,480 @@ var init_eml = __esm(() => {
|
|
|
3882
6517
|
};
|
|
3883
6518
|
});
|
|
3884
6519
|
|
|
6520
|
+
// src/parser/hwp-markdown.ts
|
|
6521
|
+
function renderHwpMarkdown(document) {
|
|
6522
|
+
const tables = new Map;
|
|
6523
|
+
for (const table of document.tables) {
|
|
6524
|
+
const key = `${table.sectionIndex}:${table.parentParagraphIndex}`;
|
|
6525
|
+
const group = tables.get(key) ?? [];
|
|
6526
|
+
group.push(table);
|
|
6527
|
+
tables.set(key, group);
|
|
6528
|
+
}
|
|
6529
|
+
for (const group of tables.values()) {
|
|
6530
|
+
group.sort((left, right) => left.controlIndex - right.controlIndex);
|
|
6531
|
+
}
|
|
6532
|
+
const blocks = [];
|
|
6533
|
+
for (const paragraph of document.paragraphs) {
|
|
6534
|
+
const text = cleanText(paragraph.text);
|
|
6535
|
+
if (hasRenderableText(text))
|
|
6536
|
+
blocks.push(text);
|
|
6537
|
+
for (const table of tables.get(`${paragraph.sectionIndex}:${paragraph.paragraphIndex}`) ?? []) {
|
|
6538
|
+
const rendered = renderTable(table);
|
|
6539
|
+
if (rendered.length > 0)
|
|
6540
|
+
blocks.push(rendered);
|
|
6541
|
+
}
|
|
6542
|
+
}
|
|
6543
|
+
return blocks.join(`
|
|
6544
|
+
|
|
6545
|
+
`);
|
|
6546
|
+
}
|
|
6547
|
+
function cleanText(text) {
|
|
6548
|
+
return text.replace(/[\u0000\uFFFC]/gu, "");
|
|
6549
|
+
}
|
|
6550
|
+
function hasRenderableText(text) {
|
|
6551
|
+
return text.trim().length > 0;
|
|
6552
|
+
}
|
|
6553
|
+
function renderTable(table) {
|
|
6554
|
+
const cells = table.cells.map((cell) => ({ ...cell, text: renderCell(cell) }));
|
|
6555
|
+
if (!cells.some((cell) => hasRenderableText(cell.text)))
|
|
6556
|
+
return "";
|
|
6557
|
+
if (cells.some((cell) => cell.rowSpan > 1 || cell.columnSpan > 1))
|
|
6558
|
+
return renderMergedTable(cells);
|
|
6559
|
+
const cellsByPosition = new Map(cells.map((cell) => [`${cell.row}:${cell.column}`, cell.text]));
|
|
6560
|
+
const header = Array.from({ length: table.columnCount }, (_, column) => `Column ${column + 1}`);
|
|
6561
|
+
const rows = Array.from({ length: table.rowCount }, (_, row) => Array.from({ length: table.columnCount }, (_2, column) => cellsByPosition.get(`${row}:${column}`) ?? ""));
|
|
6562
|
+
return [
|
|
6563
|
+
`| ${header.join(" | ")} |`,
|
|
6564
|
+
`| ${header.map(() => "---").join(" | ")} |`,
|
|
6565
|
+
...rows.map((row) => `| ${row.join(" | ")} |`)
|
|
6566
|
+
].join(`
|
|
6567
|
+
`);
|
|
6568
|
+
}
|
|
6569
|
+
function renderMergedTable(cells) {
|
|
6570
|
+
const rows = new Map;
|
|
6571
|
+
const coveredPositions = new Set;
|
|
6572
|
+
for (const cell of [...cells].sort((left, right) => left.row - right.row || left.column - right.column)) {
|
|
6573
|
+
const position = `${cell.row}:${cell.column}`;
|
|
6574
|
+
if (coveredPositions.has(position))
|
|
6575
|
+
continue;
|
|
6576
|
+
const row = rows.get(cell.row) ?? [];
|
|
6577
|
+
row.push(cell);
|
|
6578
|
+
rows.set(cell.row, row);
|
|
6579
|
+
for (let rowIndex = cell.row;rowIndex < cell.row + Math.max(1, cell.rowSpan); rowIndex += 1) {
|
|
6580
|
+
for (let columnIndex = cell.column;columnIndex < cell.column + Math.max(1, cell.columnSpan); columnIndex += 1) {
|
|
6581
|
+
if (rowIndex !== cell.row || columnIndex !== cell.column)
|
|
6582
|
+
coveredPositions.add(`${rowIndex}:${columnIndex}`);
|
|
6583
|
+
}
|
|
6584
|
+
}
|
|
6585
|
+
}
|
|
6586
|
+
const renderedRows = [...rows.entries()].sort(([left], [right]) => left - right).map(([row, cellsInRow]) => {
|
|
6587
|
+
const text = cellsInRow.sort((left, right) => left.column - right.column).map((cell) => cell.text).filter(hasRenderableText).join(" | ");
|
|
6588
|
+
return hasRenderableText(text) ? `Row ${row + 1}: ${text}` : "";
|
|
6589
|
+
}).filter((row) => row.length > 0);
|
|
6590
|
+
return ["[Table]", ...renderedRows].join(`
|
|
6591
|
+
`);
|
|
6592
|
+
}
|
|
6593
|
+
function renderCell(cell) {
|
|
6594
|
+
return cell.paragraphs.map(cleanText).filter(hasRenderableText).join("<br>").replaceAll("\\", "\\\\").replace(/\r\n|\r|\n/gu, "<br>").replaceAll("|", "\\|");
|
|
6595
|
+
}
|
|
6596
|
+
|
|
6597
|
+
// src/parser/rhwp-adapter.ts
|
|
6598
|
+
import { readFile as readFile3 } from "node:fs/promises";
|
|
6599
|
+
import initRhwp, { HwpDocument } from "@rhwp/core";
|
|
6600
|
+
function createRhwpExtractor(runtime = defaultRhwpRuntime, overrides = {}) {
|
|
6601
|
+
const limits = resolveLimits(overrides);
|
|
6602
|
+
const initialize = runtime === defaultRhwpRuntime ? initializeDefaultRhwpRuntime : createRuntimeInitializer(runtime);
|
|
6603
|
+
return async (bytes) => {
|
|
6604
|
+
await initialize();
|
|
6605
|
+
const document = runtime.open(bytes);
|
|
6606
|
+
try {
|
|
6607
|
+
return extractDocument(document, limits);
|
|
6608
|
+
} finally {
|
|
6609
|
+
document.free();
|
|
6610
|
+
}
|
|
6611
|
+
};
|
|
6612
|
+
}
|
|
6613
|
+
function createRuntimeInitializer(runtime) {
|
|
6614
|
+
let initialization;
|
|
6615
|
+
return () => {
|
|
6616
|
+
if (initialization === undefined) {
|
|
6617
|
+
const attempt = runtime.initialize();
|
|
6618
|
+
initialization = attempt.catch((error) => {
|
|
6619
|
+
initialization = undefined;
|
|
6620
|
+
throw error;
|
|
6621
|
+
});
|
|
6622
|
+
}
|
|
6623
|
+
return initialization;
|
|
6624
|
+
};
|
|
6625
|
+
}
|
|
6626
|
+
function extractDocument(document, limits) {
|
|
6627
|
+
const paragraphs = [];
|
|
6628
|
+
const paragraphCounts = [];
|
|
6629
|
+
const counters = { characters: 0, tables: 0, cells: 0, cellParagraphs: 0 };
|
|
6630
|
+
const sectionCount = readCount(document.getSectionCount(), "section count");
|
|
6631
|
+
checkBudget("maxSections", sectionCount, limits);
|
|
6632
|
+
for (let sectionIndex = 0;sectionIndex < sectionCount; sectionIndex += 1) {
|
|
6633
|
+
const paragraphCount = readCount(document.getParagraphCount(sectionIndex), `paragraph count for section ${sectionIndex}`);
|
|
6634
|
+
paragraphCounts.push(paragraphCount);
|
|
6635
|
+
checkBudget("maxBodyParagraphs", paragraphs.length + paragraphCount, limits);
|
|
6636
|
+
for (let paragraphIndex = 0;paragraphIndex < paragraphCount; paragraphIndex += 1) {
|
|
6637
|
+
const length = readCount(document.getParagraphLength(sectionIndex, paragraphIndex), `paragraph length at ${sectionIndex}:${paragraphIndex}`);
|
|
6638
|
+
const text = readText(document.getTextRange(sectionIndex, paragraphIndex, 0, length), `paragraph text at ${sectionIndex}:${paragraphIndex}`);
|
|
6639
|
+
counters.characters += [...text].length;
|
|
6640
|
+
checkBudget("maxCharacters", counters.characters, limits);
|
|
6641
|
+
paragraphs.push({
|
|
6642
|
+
sectionIndex,
|
|
6643
|
+
paragraphIndex,
|
|
6644
|
+
text
|
|
6645
|
+
});
|
|
6646
|
+
}
|
|
6647
|
+
}
|
|
6648
|
+
return { paragraphs, tables: extractTables(document, limits, counters, paragraphCounts) };
|
|
6649
|
+
}
|
|
6650
|
+
function extractTables(document, limits, counters, paragraphCounts) {
|
|
6651
|
+
const tables = [];
|
|
6652
|
+
const seen = new Set;
|
|
6653
|
+
const pageCount = readCount(document.pageCount(), "page count");
|
|
6654
|
+
for (let pageIndex = 0;pageIndex < pageCount; pageIndex += 1) {
|
|
6655
|
+
const layout = readJsonRecord(document.getPageControlLayout(pageIndex), `page ${pageIndex} control layout`);
|
|
6656
|
+
const controls = readArray(layout.controls, `controls for page ${pageIndex}`);
|
|
6657
|
+
for (let layoutIndex = 0;layoutIndex < controls.length; layoutIndex += 1) {
|
|
6658
|
+
const control = readRecord(controls[layoutIndex], `control ${layoutIndex} on page ${pageIndex}`);
|
|
6659
|
+
if (typeof control.type !== "string") {
|
|
6660
|
+
throw new TypeError(`rhwp returned an invalid control type at page ${pageIndex}:${layoutIndex}`);
|
|
6661
|
+
}
|
|
6662
|
+
if (control.type !== "table")
|
|
6663
|
+
continue;
|
|
6664
|
+
const label = `table at page ${pageIndex}:${layoutIndex}`;
|
|
6665
|
+
const coordinates = readTableCoordinates(control, label, paragraphCounts, limits);
|
|
6666
|
+
if (coordinates === undefined)
|
|
6667
|
+
continue;
|
|
6668
|
+
const key = `${coordinates.sectionIndex}:${coordinates.parentParagraphIndex}:${coordinates.controlIndex}`;
|
|
6669
|
+
if (seen.has(key))
|
|
6670
|
+
continue;
|
|
6671
|
+
seen.add(key);
|
|
6672
|
+
counters.tables += 1;
|
|
6673
|
+
checkBudget("maxTables", counters.tables, limits);
|
|
6674
|
+
tables.push(extractTable(document, coordinates, limits, counters));
|
|
6675
|
+
}
|
|
6676
|
+
}
|
|
6677
|
+
return tables;
|
|
6678
|
+
}
|
|
6679
|
+
function readTableCoordinates(control, label, paragraphCounts, limits) {
|
|
6680
|
+
const hasSectionIndex = Object.hasOwn(control, "secIdx");
|
|
6681
|
+
const hasParagraphIndex = Object.hasOwn(control, "paraIdx");
|
|
6682
|
+
const hasControlIndex = Object.hasOwn(control, "controlIdx");
|
|
6683
|
+
const coordinateCount = Number(hasSectionIndex) + Number(hasParagraphIndex) + Number(hasControlIndex);
|
|
6684
|
+
if (coordinateCount === 0) {
|
|
6685
|
+
validateDeferredNestedTable(control, label, limits);
|
|
6686
|
+
return;
|
|
6687
|
+
}
|
|
6688
|
+
if (coordinateCount !== 3) {
|
|
6689
|
+
throw new TypeError(`rhwp returned a partial coordinate set for ${label}`);
|
|
6690
|
+
}
|
|
6691
|
+
if (Object.hasOwn(control, "cellPath") || Object.hasOwn(control, "parentParaIdx")) {
|
|
6692
|
+
throw new TypeError(`rhwp returned conflicting top-level and nested context for ${label}`);
|
|
6693
|
+
}
|
|
6694
|
+
const coordinates = {
|
|
6695
|
+
sectionIndex: readCount(control.secIdx, `secIdx for ${label}`),
|
|
6696
|
+
parentParagraphIndex: readCount(control.paraIdx, `paraIdx for ${label}`),
|
|
6697
|
+
controlIndex: readCount(control.controlIdx, `controlIdx for ${label}`)
|
|
6698
|
+
};
|
|
6699
|
+
const paragraphCount = paragraphCounts[coordinates.sectionIndex];
|
|
6700
|
+
if (paragraphCount === undefined) {
|
|
6701
|
+
throw new TypeError(`rhwp returned a section coordinate outside the announced document range for ${label}`);
|
|
6702
|
+
}
|
|
6703
|
+
if (coordinates.parentParagraphIndex >= paragraphCount) {
|
|
6704
|
+
throw new TypeError(`rhwp returned a parent paragraph coordinate outside the announced section range for ${label}`);
|
|
6705
|
+
}
|
|
6706
|
+
return coordinates;
|
|
6707
|
+
}
|
|
6708
|
+
function validateDeferredNestedTable(control, label, limits) {
|
|
6709
|
+
if (!Object.hasOwn(control, "cellPath") && !Object.hasOwn(control, "parentParaIdx")) {
|
|
6710
|
+
validateDeferredNestedLayout(control, label, limits);
|
|
6711
|
+
return;
|
|
6712
|
+
}
|
|
6713
|
+
readCount(control.parentParaIdx, `parentParaIdx for deferred nested ${label}`);
|
|
6714
|
+
const cellPath = readArray(control.cellPath, `cellPath for deferred nested ${label}`);
|
|
6715
|
+
if (cellPath.length === 0) {
|
|
6716
|
+
throw new TypeError(`rhwp returned an empty cellPath for deferred nested ${label}`);
|
|
6717
|
+
}
|
|
6718
|
+
checkBudget("maxCells", cellPath.length, limits);
|
|
6719
|
+
for (let pathIndex = 0;pathIndex < cellPath.length; pathIndex += 1) {
|
|
6720
|
+
const entry = readRecord(cellPath[pathIndex], `cellPath entry ${pathIndex} for deferred nested ${label}`);
|
|
6721
|
+
readCount(entry.controlIndex, `controlIndex in cellPath entry ${pathIndex} for deferred nested ${label}`);
|
|
6722
|
+
readCount(entry.cellIndex, `cellIndex in cellPath entry ${pathIndex} for deferred nested ${label}`);
|
|
6723
|
+
readCount(entry.cellParaIndex, `cellParaIndex in cellPath entry ${pathIndex} for deferred nested ${label}`);
|
|
6724
|
+
}
|
|
6725
|
+
}
|
|
6726
|
+
function validateDeferredNestedLayout(control, label, limits) {
|
|
6727
|
+
readFiniteNumber(control.x, `x for deferred nested ${label}`);
|
|
6728
|
+
readFiniteNumber(control.y, `y for deferred nested ${label}`);
|
|
6729
|
+
readPositiveFiniteNumber(control.w, `w for deferred nested ${label}`);
|
|
6730
|
+
readPositiveFiniteNumber(control.h, `h for deferred nested ${label}`);
|
|
6731
|
+
const rowCount = readPositiveCount(control.rowCount, `rowCount for deferred nested ${label}`);
|
|
6732
|
+
const columnCount = readPositiveCount(control.colCount, `colCount for deferred nested ${label}`);
|
|
6733
|
+
const logicalCellCount = readLogicalCellCount(rowCount, columnCount, limits);
|
|
6734
|
+
readCount(control.plane, `plane for deferred nested ${label}`);
|
|
6735
|
+
readSafeInteger(control.zOrder, `zOrder for deferred nested ${label}`);
|
|
6736
|
+
readCount(control.stableIndex, `stableIndex for deferred nested ${label}`);
|
|
6737
|
+
const cells = readArray(control.cells, `cells for deferred nested ${label}`);
|
|
6738
|
+
if (cells.length === 0) {
|
|
6739
|
+
throw new TypeError(`rhwp returned empty cells for deferred nested ${label}`);
|
|
6740
|
+
}
|
|
6741
|
+
checkBudget("maxCells", cells.length, limits);
|
|
6742
|
+
if (cells.length > logicalCellCount) {
|
|
6743
|
+
throw new TypeError(`rhwp returned too many cells for deferred nested ${label}`);
|
|
6744
|
+
}
|
|
6745
|
+
const anchors = new Set;
|
|
6746
|
+
const cellIndexes = new Set;
|
|
6747
|
+
const coveredCoordinates = new Set;
|
|
6748
|
+
for (let cellIndex = 0;cellIndex < cells.length; cellIndex += 1) {
|
|
6749
|
+
const cell = readRecord(cells[cellIndex], `cell ${cellIndex} for deferred nested ${label}`);
|
|
6750
|
+
readFiniteNumber(cell.x, `x for cell ${cellIndex} in deferred nested ${label}`);
|
|
6751
|
+
readFiniteNumber(cell.y, `y for cell ${cellIndex} in deferred nested ${label}`);
|
|
6752
|
+
readPositiveFiniteNumber(cell.w, `w for cell ${cellIndex} in deferred nested ${label}`);
|
|
6753
|
+
readPositiveFiniteNumber(cell.h, `h for cell ${cellIndex} in deferred nested ${label}`);
|
|
6754
|
+
const row = readCount(cell.row, `row for cell ${cellIndex} in deferred nested ${label}`);
|
|
6755
|
+
const column = readCount(cell.col, `col for cell ${cellIndex} in deferred nested ${label}`);
|
|
6756
|
+
const rowSpan = readPositiveCount(cell.rowSpan, `rowSpan for cell ${cellIndex} in deferred nested ${label}`);
|
|
6757
|
+
const columnSpan = readPositiveCount(cell.colSpan, `colSpan for cell ${cellIndex} in deferred nested ${label}`);
|
|
6758
|
+
const runtimeCellIndex = readCount(cell.cellIdx, `cellIdx for cell ${cellIndex} in deferred nested ${label}`);
|
|
6759
|
+
if (row >= rowCount || rowSpan > rowCount - row) {
|
|
6760
|
+
throw new TypeError(`rhwp returned an out-of-range row span for cell ${cellIndex} in deferred nested ${label}`);
|
|
6761
|
+
}
|
|
6762
|
+
if (column >= columnCount || columnSpan > columnCount - column) {
|
|
6763
|
+
throw new TypeError(`rhwp returned an out-of-range col span for cell ${cellIndex} in deferred nested ${label}`);
|
|
6764
|
+
}
|
|
6765
|
+
if (cellIndexes.has(runtimeCellIndex)) {
|
|
6766
|
+
throw new TypeError(`rhwp returned a duplicate cellIdx for deferred nested ${label}`);
|
|
6767
|
+
}
|
|
6768
|
+
cellIndexes.add(runtimeCellIndex);
|
|
6769
|
+
const anchor = `${row}:${column}`;
|
|
6770
|
+
if (anchors.has(anchor)) {
|
|
6771
|
+
throw new TypeError(`rhwp returned a duplicate cell anchor for deferred nested ${label}`);
|
|
6772
|
+
}
|
|
6773
|
+
anchors.add(anchor);
|
|
6774
|
+
for (let coveredRow = row;coveredRow < row + rowSpan; coveredRow += 1) {
|
|
6775
|
+
for (let coveredColumn = column;coveredColumn < column + columnSpan; coveredColumn += 1) {
|
|
6776
|
+
const coordinate = `${coveredRow}:${coveredColumn}`;
|
|
6777
|
+
if (coveredCoordinates.has(coordinate)) {
|
|
6778
|
+
throw new TypeError(`rhwp returned overlapping cell spans for deferred nested ${label}`);
|
|
6779
|
+
}
|
|
6780
|
+
coveredCoordinates.add(coordinate);
|
|
6781
|
+
}
|
|
6782
|
+
}
|
|
6783
|
+
}
|
|
6784
|
+
}
|
|
6785
|
+
function extractTable(document, coordinates, limits, counters) {
|
|
6786
|
+
const { sectionIndex, parentParagraphIndex, controlIndex } = coordinates;
|
|
6787
|
+
const key = `${sectionIndex}:${parentParagraphIndex}:${controlIndex}`;
|
|
6788
|
+
const dimensions = readJsonRecord(document.getTableDimensions(sectionIndex, parentParagraphIndex, controlIndex), `table dimensions at ${key}`);
|
|
6789
|
+
const rowCount = readPositiveCount(dimensions.rowCount, `rowCount for table ${key}`);
|
|
6790
|
+
const columnCount = readPositiveCount(dimensions.colCount, `colCount for table ${key}`);
|
|
6791
|
+
const cellCount = readCount(dimensions.cellCount, `cellCount for table ${key}`);
|
|
6792
|
+
const logicalCellCount = chargeLogicalCells(rowCount, columnCount, counters, limits);
|
|
6793
|
+
if (cellCount === 0 || cellCount > logicalCellCount) {
|
|
6794
|
+
throw new TypeError(`rhwp returned an invalid cellCount for table ${key}; expected 1..${logicalCellCount}, received ${cellCount}`);
|
|
6795
|
+
}
|
|
6796
|
+
const cellGeometry = [];
|
|
6797
|
+
const anchors = new Set;
|
|
6798
|
+
const coveredCoordinates = new Set;
|
|
6799
|
+
for (let cellIndex = 0;cellIndex < cellCount; cellIndex += 1) {
|
|
6800
|
+
const cellInfo = readJsonRecord(document.getCellInfo(sectionIndex, parentParagraphIndex, controlIndex, cellIndex), `cell metadata at ${key}:${cellIndex}`);
|
|
6801
|
+
const row = readCount(cellInfo.row, `row for cell ${key}:${cellIndex}`);
|
|
6802
|
+
const column = readCount(cellInfo.col, `col for cell ${key}:${cellIndex}`);
|
|
6803
|
+
const rowSpan = readPositiveCount(cellInfo.rowSpan, `rowSpan for cell ${key}:${cellIndex}`);
|
|
6804
|
+
const columnSpan = readPositiveCount(cellInfo.colSpan, `colSpan for cell ${key}:${cellIndex}`);
|
|
6805
|
+
if (row >= rowCount) {
|
|
6806
|
+
throw new TypeError(`rhwp returned a row outside the table range for cell ${key}:${cellIndex}`);
|
|
6807
|
+
}
|
|
6808
|
+
if (column >= columnCount) {
|
|
6809
|
+
throw new TypeError(`rhwp returned a col outside the table range for cell ${key}:${cellIndex}`);
|
|
6810
|
+
}
|
|
6811
|
+
if (rowSpan > rowCount - row) {
|
|
6812
|
+
throw new TypeError(`rhwp returned a rowSpan outside the table range for cell ${key}:${cellIndex}`);
|
|
6813
|
+
}
|
|
6814
|
+
if (columnSpan > columnCount - column) {
|
|
6815
|
+
throw new TypeError(`rhwp returned a colSpan outside the table range for cell ${key}:${cellIndex}`);
|
|
6816
|
+
}
|
|
6817
|
+
const anchor = row * columnCount + column;
|
|
6818
|
+
if (anchors.has(anchor)) {
|
|
6819
|
+
throw new TypeError(`rhwp returned a duplicate cell anchor for table ${key} at ${row}:${column}`);
|
|
6820
|
+
}
|
|
6821
|
+
anchors.add(anchor);
|
|
6822
|
+
for (let coveredRow = row;coveredRow < row + rowSpan; coveredRow += 1) {
|
|
6823
|
+
for (let coveredColumn = column;coveredColumn < column + columnSpan; coveredColumn += 1) {
|
|
6824
|
+
const coordinate = coveredRow * columnCount + coveredColumn;
|
|
6825
|
+
if (coveredCoordinates.has(coordinate)) {
|
|
6826
|
+
throw new TypeError(`rhwp returned overlapping cell spans for table ${key} at ${coveredRow}:${coveredColumn}`);
|
|
6827
|
+
}
|
|
6828
|
+
coveredCoordinates.add(coordinate);
|
|
6829
|
+
}
|
|
6830
|
+
}
|
|
6831
|
+
cellGeometry.push({ row, column, rowSpan, columnSpan });
|
|
6832
|
+
}
|
|
6833
|
+
if (coveredCoordinates.size !== logicalCellCount) {
|
|
6834
|
+
throw new TypeError(`rhwp returned cellCount ${cellCount} that covers ${coveredCoordinates.size} of ${logicalCellCount} logical grid coordinates for table ${key}`);
|
|
6835
|
+
}
|
|
6836
|
+
const cells = [];
|
|
6837
|
+
for (let cellIndex = 0;cellIndex < cellGeometry.length; cellIndex += 1) {
|
|
6838
|
+
const geometry = cellGeometry[cellIndex];
|
|
6839
|
+
if (geometry === undefined)
|
|
6840
|
+
throw new TypeError(`rhwp omitted cell geometry at ${key}:${cellIndex}`);
|
|
6841
|
+
const paragraphCount = readCount(document.getCellParagraphCount(sectionIndex, parentParagraphIndex, controlIndex, cellIndex), `cell paragraph count at ${key}:${cellIndex}`);
|
|
6842
|
+
counters.cellParagraphs += paragraphCount;
|
|
6843
|
+
checkBudget("maxCellParagraphs", counters.cellParagraphs, limits);
|
|
6844
|
+
const paragraphs = [];
|
|
6845
|
+
for (let cellParagraphIndex = 0;cellParagraphIndex < paragraphCount; cellParagraphIndex += 1) {
|
|
6846
|
+
const length = readCount(document.getCellParagraphLength(sectionIndex, parentParagraphIndex, controlIndex, cellIndex, cellParagraphIndex), `cell paragraph length at ${key}:${cellIndex}:${cellParagraphIndex}`);
|
|
6847
|
+
const text = readText(document.getTextInCell(sectionIndex, parentParagraphIndex, controlIndex, cellIndex, cellParagraphIndex, 0, length), `cell paragraph text at ${key}:${cellIndex}:${cellParagraphIndex}`);
|
|
6848
|
+
counters.characters += [...text].length;
|
|
6849
|
+
checkBudget("maxCharacters", counters.characters, limits);
|
|
6850
|
+
paragraphs.push(text);
|
|
6851
|
+
}
|
|
6852
|
+
cells.push({
|
|
6853
|
+
...geometry,
|
|
6854
|
+
paragraphs
|
|
6855
|
+
});
|
|
6856
|
+
}
|
|
6857
|
+
return {
|
|
6858
|
+
sectionIndex,
|
|
6859
|
+
parentParagraphIndex,
|
|
6860
|
+
controlIndex,
|
|
6861
|
+
rowCount,
|
|
6862
|
+
columnCount,
|
|
6863
|
+
cells
|
|
6864
|
+
};
|
|
6865
|
+
}
|
|
6866
|
+
function chargeLogicalCells(rowCount, columnCount, counters, limits) {
|
|
6867
|
+
const logicalCellCount = readLogicalCellCount(rowCount, columnCount, limits);
|
|
6868
|
+
if (logicalCellCount > limits.maxCells - counters.cells) {
|
|
6869
|
+
throw new HwpExtractionBudgetError("maxCells", counters.cells + logicalCellCount, limits.maxCells);
|
|
6870
|
+
}
|
|
6871
|
+
counters.cells += logicalCellCount;
|
|
6872
|
+
return logicalCellCount;
|
|
6873
|
+
}
|
|
6874
|
+
function readLogicalCellCount(rowCount, columnCount, limits) {
|
|
6875
|
+
if (rowCount > Math.floor(limits.maxCells / columnCount)) {
|
|
6876
|
+
throw new HwpExtractionBudgetError("maxCells", rowCount * columnCount, limits.maxCells);
|
|
6877
|
+
}
|
|
6878
|
+
return rowCount * columnCount;
|
|
6879
|
+
}
|
|
6880
|
+
function resolveLimits(overrides) {
|
|
6881
|
+
const limits = { ...DEFAULT_LIMITS };
|
|
6882
|
+
for (const name of Object.keys(DEFAULT_LIMITS)) {
|
|
6883
|
+
const value = overrides[name];
|
|
6884
|
+
if (value === undefined)
|
|
6885
|
+
continue;
|
|
6886
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
6887
|
+
throw new TypeError(`${name} must be a positive finite safe integer`);
|
|
6888
|
+
}
|
|
6889
|
+
limits[name] = value;
|
|
6890
|
+
}
|
|
6891
|
+
return limits;
|
|
6892
|
+
}
|
|
6893
|
+
function checkBudget(name, actual, limits) {
|
|
6894
|
+
if (actual > limits[name])
|
|
6895
|
+
throw new HwpExtractionBudgetError(name, actual, limits[name]);
|
|
6896
|
+
}
|
|
6897
|
+
function readCount(value, label) {
|
|
6898
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
6899
|
+
throw new TypeError(`rhwp returned an invalid ${label}; expected a non-negative safe integer`);
|
|
6900
|
+
}
|
|
6901
|
+
return value;
|
|
6902
|
+
}
|
|
6903
|
+
function readPositiveCount(value, label) {
|
|
6904
|
+
const count = readCount(value, label);
|
|
6905
|
+
if (count === 0) {
|
|
6906
|
+
throw new TypeError(`rhwp returned an invalid ${label}; expected a positive safe integer`);
|
|
6907
|
+
}
|
|
6908
|
+
return count;
|
|
6909
|
+
}
|
|
6910
|
+
function readSafeInteger(value, label) {
|
|
6911
|
+
if (!Number.isSafeInteger(value)) {
|
|
6912
|
+
throw new TypeError(`rhwp returned an invalid ${label}; expected a safe integer`);
|
|
6913
|
+
}
|
|
6914
|
+
return value;
|
|
6915
|
+
}
|
|
6916
|
+
function readFiniteNumber(value, label) {
|
|
6917
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
6918
|
+
throw new TypeError(`rhwp returned an invalid ${label}; expected a finite number`);
|
|
6919
|
+
}
|
|
6920
|
+
return value;
|
|
6921
|
+
}
|
|
6922
|
+
function readPositiveFiniteNumber(value, label) {
|
|
6923
|
+
const number = readFiniteNumber(value, label);
|
|
6924
|
+
if (number <= 0) {
|
|
6925
|
+
throw new TypeError(`rhwp returned an invalid ${label}; expected a positive finite number`);
|
|
6926
|
+
}
|
|
6927
|
+
return number;
|
|
6928
|
+
}
|
|
6929
|
+
function readText(value, label) {
|
|
6930
|
+
if (typeof value !== "string")
|
|
6931
|
+
throw new TypeError(`rhwp returned invalid ${label}; expected a string`);
|
|
6932
|
+
return value;
|
|
6933
|
+
}
|
|
6934
|
+
function readJsonRecord(value, label) {
|
|
6935
|
+
const text = readText(value, label);
|
|
6936
|
+
let parsed;
|
|
6937
|
+
try {
|
|
6938
|
+
parsed = JSON.parse(text);
|
|
6939
|
+
} catch (cause) {
|
|
6940
|
+
throw new TypeError(`rhwp returned malformed JSON for ${label}`, { cause });
|
|
6941
|
+
}
|
|
6942
|
+
return readRecord(parsed, label);
|
|
6943
|
+
}
|
|
6944
|
+
function readRecord(value, label) {
|
|
6945
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
6946
|
+
throw new TypeError(`rhwp returned invalid ${label}; expected an object`);
|
|
6947
|
+
}
|
|
6948
|
+
return value;
|
|
6949
|
+
}
|
|
6950
|
+
function readArray(value, label) {
|
|
6951
|
+
if (!Array.isArray(value))
|
|
6952
|
+
throw new TypeError(`rhwp returned invalid ${label}; expected an array`);
|
|
6953
|
+
return value;
|
|
6954
|
+
}
|
|
6955
|
+
var DEFAULT_LIMITS, HwpExtractionBudgetError, defaultRhwpRuntime, initializeDefaultRhwpRuntime;
|
|
6956
|
+
var init_rhwp_adapter = __esm(() => {
|
|
6957
|
+
DEFAULT_LIMITS = {
|
|
6958
|
+
maxSections: 256,
|
|
6959
|
+
maxBodyParagraphs: 200000,
|
|
6960
|
+
maxTables: 20000,
|
|
6961
|
+
maxCells: 500000,
|
|
6962
|
+
maxCellParagraphs: 1e6,
|
|
6963
|
+
maxCharacters: 20000000
|
|
6964
|
+
};
|
|
6965
|
+
HwpExtractionBudgetError = class HwpExtractionBudgetError extends Error {
|
|
6966
|
+
code = "HWP_EXTRACTION_BUDGET_EXCEEDED";
|
|
6967
|
+
limit;
|
|
6968
|
+
actual;
|
|
6969
|
+
maximum;
|
|
6970
|
+
constructor(limit, actual, maximum) {
|
|
6971
|
+
super(`HWP extraction exceeded ${limit} budget of ${maximum} (received ${actual})`);
|
|
6972
|
+
this.name = "HwpExtractionBudgetError";
|
|
6973
|
+
this.limit = limit;
|
|
6974
|
+
this.actual = actual;
|
|
6975
|
+
this.maximum = maximum;
|
|
6976
|
+
}
|
|
6977
|
+
};
|
|
6978
|
+
defaultRhwpRuntime = {
|
|
6979
|
+
async initialize() {
|
|
6980
|
+
const rhwpModuleUrl = import.meta.resolve("@rhwp/core");
|
|
6981
|
+
const wasmBytes = await readFile3(new URL("./rhwp_bg.wasm", rhwpModuleUrl));
|
|
6982
|
+
await initRhwp({ module_or_path: wasmBytes });
|
|
6983
|
+
},
|
|
6984
|
+
open(bytes) {
|
|
6985
|
+
return new HwpDocument(bytes);
|
|
6986
|
+
}
|
|
6987
|
+
};
|
|
6988
|
+
initializeDefaultRhwpRuntime = createRuntimeInitializer(defaultRhwpRuntime);
|
|
6989
|
+
});
|
|
6990
|
+
|
|
3885
6991
|
// src/parser/xml-text.ts
|
|
3886
6992
|
import { Buffer as Buffer2 } from "node:buffer";
|
|
3887
|
-
import { XMLParser } from "fast-xml-parser";
|
|
6993
|
+
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
3888
6994
|
import JSZip from "jszip";
|
|
3889
6995
|
async function readZipXmlText(bytes, pathPattern) {
|
|
3890
6996
|
const zip = await JSZip.loadAsync(bytes);
|
|
@@ -3974,7 +7080,7 @@ function isRecord6(value) {
|
|
|
3974
7080
|
}
|
|
3975
7081
|
var parser, MAX_XML_FILES = 64, MAX_XML_BYTES = 5000000, MAX_TEXT_CHUNKS = 20000;
|
|
3976
7082
|
var init_xml_text = __esm(() => {
|
|
3977
|
-
parser = new
|
|
7083
|
+
parser = new XMLParser2({
|
|
3978
7084
|
ignoreAttributes: false,
|
|
3979
7085
|
parseTagValue: false,
|
|
3980
7086
|
processEntities: true,
|
|
@@ -3983,19 +7089,29 @@ var init_xml_text = __esm(() => {
|
|
|
3983
7089
|
});
|
|
3984
7090
|
|
|
3985
7091
|
// src/parser/hwp.ts
|
|
3986
|
-
import { extname as
|
|
7092
|
+
import { extname as extname5 } from "node:path";
|
|
3987
7093
|
var HwpParser;
|
|
3988
7094
|
var init_hwp = __esm(() => {
|
|
3989
7095
|
init_errors();
|
|
7096
|
+
init_rhwp_adapter();
|
|
3990
7097
|
init_text();
|
|
3991
7098
|
init_xml_text();
|
|
3992
7099
|
HwpParser = class HwpParser extends Parser {
|
|
3993
7100
|
name = "hwp";
|
|
3994
7101
|
extensions = [".hwpx", ".hwp"];
|
|
7102
|
+
extractor;
|
|
7103
|
+
constructor(options = {}) {
|
|
7104
|
+
super();
|
|
7105
|
+
this.extractor = options.extractor ?? createRhwpExtractor(undefined, options.limits);
|
|
7106
|
+
}
|
|
3995
7107
|
async parse(input) {
|
|
3996
7108
|
try {
|
|
3997
|
-
if (
|
|
3998
|
-
|
|
7109
|
+
if (extname5(input.virtualPath).toLowerCase() === ".hwp") {
|
|
7110
|
+
const document = await this.extractor(input.bytes);
|
|
7111
|
+
return {
|
|
7112
|
+
markdown: normalizeMarkdown(renderHwpMarkdown(document)),
|
|
7113
|
+
metadata: { parser: this.name, format: "hwp5" }
|
|
7114
|
+
};
|
|
3999
7115
|
}
|
|
4000
7116
|
const chunks = await readZipXmlText(input.bytes, /^(?:Contents|content)\/.*\.xml$/i);
|
|
4001
7117
|
return {
|
|
@@ -4119,7 +7235,7 @@ var init_ocr = __esm(() => {
|
|
|
4119
7235
|
});
|
|
4120
7236
|
|
|
4121
7237
|
// src/parser/office.ts
|
|
4122
|
-
import { extname as
|
|
7238
|
+
import { extname as extname6 } from "node:path";
|
|
4123
7239
|
function formatMarkdown(parserName, chunks) {
|
|
4124
7240
|
const markdown = normalizeMarkdown(chunks.filter((chunk) => chunk.trim().length > 0).join(`
|
|
4125
7241
|
|
|
@@ -4160,7 +7276,7 @@ var init_office = __esm(() => {
|
|
|
4160
7276
|
extensions = [".xlsx", ".xls"];
|
|
4161
7277
|
async parse(input) {
|
|
4162
7278
|
try {
|
|
4163
|
-
if (
|
|
7279
|
+
if (extname6(input.virtualPath).toLowerCase() === ".xls") {
|
|
4164
7280
|
throw new Error("legacy XLS binary parsing is not supported by the pure JavaScript parser");
|
|
4165
7281
|
}
|
|
4166
7282
|
const sharedStrings = await readZipXmlText(input.bytes, /^xl\/sharedStrings\.xml$/);
|
|
@@ -4174,12 +7290,12 @@ var init_office = __esm(() => {
|
|
|
4174
7290
|
});
|
|
4175
7291
|
|
|
4176
7292
|
// src/parser/opendataloader-pdf.ts
|
|
4177
|
-
import { mkdtemp as mkdtemp2, readFile, rm, writeFile } from "node:fs/promises";
|
|
7293
|
+
import { mkdtemp as mkdtemp2, readFile as readFile4, rm, writeFile } from "node:fs/promises";
|
|
4178
7294
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
4179
|
-
import { basename as
|
|
7295
|
+
import { basename as basename8, join as join17, parse as parse3 } from "node:path";
|
|
4180
7296
|
import { convert } from "@opendataloader/pdf";
|
|
4181
7297
|
function markdownOutputPath(outputDir, inputPath) {
|
|
4182
|
-
return
|
|
7298
|
+
return join17(outputDir, `${parse3(inputPath).name}.md`);
|
|
4183
7299
|
}
|
|
4184
7300
|
var OpendataloaderPdfParser;
|
|
4185
7301
|
var init_opendataloader_pdf = __esm(() => {
|
|
@@ -4196,9 +7312,9 @@ var init_opendataloader_pdf = __esm(() => {
|
|
|
4196
7312
|
this.converter = options.converter ?? convert;
|
|
4197
7313
|
}
|
|
4198
7314
|
async parse(input) {
|
|
4199
|
-
const tempRoot = await mkdtemp2(
|
|
7315
|
+
const tempRoot = await mkdtemp2(join17(tmpdir2(), "autorag-pdf-"));
|
|
4200
7316
|
try {
|
|
4201
|
-
const inputPath = input.sourcePath ??
|
|
7317
|
+
const inputPath = input.sourcePath ?? join17(tempRoot, basename8(input.virtualPath));
|
|
4202
7318
|
if (!input.sourcePath) {
|
|
4203
7319
|
await writeFile(inputPath, input.bytes);
|
|
4204
7320
|
}
|
|
@@ -4217,7 +7333,7 @@ var init_opendataloader_pdf = __esm(() => {
|
|
|
4217
7333
|
} : {}
|
|
4218
7334
|
});
|
|
4219
7335
|
return {
|
|
4220
|
-
markdown: normalizeMarkdown(await
|
|
7336
|
+
markdown: normalizeMarkdown(await readFile4(markdownOutputPath(tempRoot, inputPath), "utf8")),
|
|
4221
7337
|
metadata: { parser: this.name }
|
|
4222
7338
|
};
|
|
4223
7339
|
} catch (cause) {
|
|
@@ -4243,7 +7359,7 @@ var init_plain_text = __esm(() => {
|
|
|
4243
7359
|
});
|
|
4244
7360
|
|
|
4245
7361
|
// src/parser/registry.ts
|
|
4246
|
-
import { extname as
|
|
7362
|
+
import { extname as extname7 } from "node:path";
|
|
4247
7363
|
|
|
4248
7364
|
class ParserRegistry {
|
|
4249
7365
|
parsersByExtension = new Map;
|
|
@@ -4262,7 +7378,7 @@ class ParserRegistry {
|
|
|
4262
7378
|
}
|
|
4263
7379
|
}
|
|
4264
7380
|
getForVirtualPath(virtualPath) {
|
|
4265
|
-
return this.parsersByExtension.get(normalizeExtension(
|
|
7381
|
+
return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
|
|
4266
7382
|
}
|
|
4267
7383
|
list() {
|
|
4268
7384
|
return Array.from(new Set(this.parsersByExtension.values()));
|
|
@@ -4282,7 +7398,7 @@ function createDefaultParserRegistry(options = {}) {
|
|
|
4282
7398
|
new DocxParser,
|
|
4283
7399
|
new PptxParser,
|
|
4284
7400
|
new XlsxParser,
|
|
4285
|
-
new HwpParser,
|
|
7401
|
+
new HwpParser(options.hwp),
|
|
4286
7402
|
new EmlParser
|
|
4287
7403
|
];
|
|
4288
7404
|
if (options.ocr?.enabled)
|
|
@@ -4301,9 +7417,9 @@ var init_defaults = __esm(() => {
|
|
|
4301
7417
|
|
|
4302
7418
|
// src/mirror/sync.ts
|
|
4303
7419
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
4304
|
-
import { existsSync as
|
|
4305
|
-
import { opendir, readFile as
|
|
4306
|
-
import { dirname as
|
|
7420
|
+
import { existsSync as existsSync13, mkdirSync as mkdirSync8, renameSync as renameSync5, rmSync as rmSync3, writeFileSync as writeFileSync6 } from "node:fs";
|
|
7421
|
+
import { opendir, readFile as readFile5, stat as stat4 } from "node:fs/promises";
|
|
7422
|
+
import { dirname as dirname9, extname as extname8, resolve as resolve6 } from "node:path";
|
|
4307
7423
|
async function syncParsedMirrors(options) {
|
|
4308
7424
|
const registry = options.registry ?? createDefaultParserRegistry(options.parserOptions);
|
|
4309
7425
|
const maxSourceBytes = options.maxSourceBytes ?? DEFAULT_MAX_SOURCE_BYTES;
|
|
@@ -4351,11 +7467,11 @@ async function syncParsedMirrors(options) {
|
|
|
4351
7467
|
}
|
|
4352
7468
|
const previousEntry = previous.entries[entry.virtualPath];
|
|
4353
7469
|
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
4354
|
-
const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName === parser2.name &&
|
|
7470
|
+
const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName === parser2.name && existsSync13(outputPath);
|
|
4355
7471
|
if (!unchanged) {
|
|
4356
7472
|
let parsed;
|
|
4357
7473
|
try {
|
|
4358
|
-
const bytes = await
|
|
7474
|
+
const bytes = await readFile5(entry.sourcePath);
|
|
4359
7475
|
parsed = await parser2.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
|
|
4360
7476
|
} catch (error) {
|
|
4361
7477
|
if (!(error instanceof ParseError))
|
|
@@ -4475,12 +7591,12 @@ async function collectFiles(sourceRoot, directory, entries, supportedExtensions)
|
|
|
4475
7591
|
}
|
|
4476
7592
|
if (!entry.isFile())
|
|
4477
7593
|
continue;
|
|
4478
|
-
const extension = normalizeExtension2(
|
|
7594
|
+
const extension = normalizeExtension2(extname8(entry.name));
|
|
4479
7595
|
if (!supportedExtensions.has(extension))
|
|
4480
7596
|
continue;
|
|
4481
7597
|
let fileStat;
|
|
4482
7598
|
try {
|
|
4483
|
-
fileStat = await
|
|
7599
|
+
fileStat = await stat4(sourcePath, { bigint: true });
|
|
4484
7600
|
} catch {
|
|
4485
7601
|
continue;
|
|
4486
7602
|
}
|
|
@@ -4500,13 +7616,13 @@ function removePrevious(root, index, virtualPath) {
|
|
|
4500
7616
|
return 1;
|
|
4501
7617
|
}
|
|
4502
7618
|
function writeAtomic(path, content) {
|
|
4503
|
-
|
|
7619
|
+
mkdirSync8(dirname9(path), { recursive: true });
|
|
4504
7620
|
const tmp = `${path}.${randomUUID5()}.tmp`;
|
|
4505
|
-
|
|
7621
|
+
writeFileSync6(tmp, content);
|
|
4506
7622
|
renameSync5(tmp, path);
|
|
4507
7623
|
}
|
|
4508
7624
|
function removeFile(path) {
|
|
4509
|
-
if (
|
|
7625
|
+
if (existsSync13(path))
|
|
4510
7626
|
rmSync3(path, { force: true });
|
|
4511
7627
|
}
|
|
4512
7628
|
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, SKIP_DIR_NAMES;
|
|
@@ -4543,8 +7659,8 @@ var init_sync = __esm(() => {
|
|
|
4543
7659
|
});
|
|
4544
7660
|
|
|
4545
7661
|
// src/observability/run-log.ts
|
|
4546
|
-
import { appendFileSync, mkdirSync as
|
|
4547
|
-
import { dirname as
|
|
7662
|
+
import { appendFileSync, mkdirSync as mkdirSync9 } from "node:fs";
|
|
7663
|
+
import { dirname as dirname10 } from "node:path";
|
|
4548
7664
|
|
|
4549
7665
|
class AutoRAGRunLogger {
|
|
4550
7666
|
path;
|
|
@@ -4553,7 +7669,7 @@ class AutoRAGRunLogger {
|
|
|
4553
7669
|
}
|
|
4554
7670
|
write(event) {
|
|
4555
7671
|
try {
|
|
4556
|
-
|
|
7672
|
+
mkdirSync9(dirname10(this.path), { recursive: true });
|
|
4557
7673
|
appendFileSync(this.path, `${JSON.stringify(event)}
|
|
4558
7674
|
`, "utf8");
|
|
4559
7675
|
} catch (error) {}
|
|
@@ -4644,9 +7760,9 @@ function methodFailureCode(name) {
|
|
|
4644
7760
|
}
|
|
4645
7761
|
|
|
4646
7762
|
// src/retrieval/methods/bm25.ts
|
|
4647
|
-
import { createHash as
|
|
4648
|
-
import { existsSync as
|
|
4649
|
-
import { dirname as
|
|
7763
|
+
import { createHash as createHash7 } from "node:crypto";
|
|
7764
|
+
import { existsSync as existsSync14, mkdirSync as mkdirSync10, readFileSync as readFileSync10, rmSync as rmSync4, writeFileSync as writeFileSync7 } from "node:fs";
|
|
7765
|
+
import { dirname as dirname11, join as join18 } from "node:path";
|
|
4650
7766
|
|
|
4651
7767
|
class BM25Method {
|
|
4652
7768
|
root;
|
|
@@ -4657,16 +7773,16 @@ class BM25Method {
|
|
|
4657
7773
|
status;
|
|
4658
7774
|
constructor(options) {
|
|
4659
7775
|
this.root = options.root;
|
|
4660
|
-
this.indexPath = options.indexPath ??
|
|
7776
|
+
this.indexPath = options.indexPath ?? join18(options.root, BM25_SUBDIR);
|
|
4661
7777
|
this.fallback = options.fallback ?? "typescript";
|
|
4662
7778
|
this.forceEngine = options.forceEngine;
|
|
4663
7779
|
this.importBinding = options.importBinding ?? (() => import("@pngwasi/node-tantivy-binding"));
|
|
4664
7780
|
if (options.enabled === false) {
|
|
4665
7781
|
this.status = { readiness: "disabled", engine: "none", message: "BM25 is disabled" };
|
|
4666
7782
|
} else {
|
|
4667
|
-
const tantivyDir =
|
|
4668
|
-
const hasTantivy =
|
|
4669
|
-
const hasFallback =
|
|
7783
|
+
const tantivyDir = join18(this.indexPath, TANTIVY_SUBDIR);
|
|
7784
|
+
const hasTantivy = existsSync14(join18(tantivyDir, "meta.json"));
|
|
7785
|
+
const hasFallback = existsSync14(join18(this.indexPath, FALLBACK_INDEX_FILE));
|
|
4670
7786
|
if (hasTantivy) {
|
|
4671
7787
|
this.status = { readiness: "ready", engine: "tantivy" };
|
|
4672
7788
|
} else if (hasFallback && this.fallback !== "disabled") {
|
|
@@ -4732,7 +7848,7 @@ class BM25Method {
|
|
|
4732
7848
|
const trimmedQuery = query.trim();
|
|
4733
7849
|
if (trimmedQuery.length === 0)
|
|
4734
7850
|
return [];
|
|
4735
|
-
const topK = options.topK ??
|
|
7851
|
+
const topK = options.topK ?? DEFAULT_TOP_K2;
|
|
4736
7852
|
if (this.status.readiness === "disabled" || this.status.readiness === "dependency_unavailable") {
|
|
4737
7853
|
throw new BM25UnavailableError(this.status.readiness, this.status.message ?? "BM25 is unavailable");
|
|
4738
7854
|
}
|
|
@@ -4759,7 +7875,7 @@ class BM25Method {
|
|
|
4759
7875
|
}
|
|
4760
7876
|
const indexDir = this.tantivyIndexPath();
|
|
4761
7877
|
rmSync4(indexDir, { recursive: true, force: true });
|
|
4762
|
-
|
|
7878
|
+
mkdirSync10(indexDir, { recursive: true });
|
|
4763
7879
|
const schema = new binding.SchemaBuilder().addTextField("virtualPath", { stored: true, indexOption: "basic", tokenizerName: "raw" }).addTextField("chunkId", { stored: true, indexOption: "basic", tokenizerName: "raw" }).addTextField("content", { stored: true, indexOption: "position" }).build();
|
|
4764
7880
|
const index = new binding.Index(schema, indexDir, false);
|
|
4765
7881
|
const writer = index.writer(30000000, 1);
|
|
@@ -4775,7 +7891,7 @@ class BM25Method {
|
|
|
4775
7891
|
async retrieveTantivy(queryText, topK, scope) {
|
|
4776
7892
|
const binding = await this.importBinding();
|
|
4777
7893
|
const indexDir = this.tantivyIndexPath();
|
|
4778
|
-
if (!
|
|
7894
|
+
if (!existsSync14(indexDir) || !binding.Index.exists(indexDir)) {
|
|
4779
7895
|
this.status = { readiness: "index_missing", engine: "none", message: "Tantivy BM25 index is missing" };
|
|
4780
7896
|
throw new BM25UnavailableError("index_missing", "Tantivy BM25 index is missing; call refresh() first");
|
|
4781
7897
|
}
|
|
@@ -4820,18 +7936,18 @@ class BM25Method {
|
|
|
4820
7936
|
writeFallbackIndex(chunks) {
|
|
4821
7937
|
const index = { version: 1, chunks };
|
|
4822
7938
|
const path = this.fallbackIndexPath();
|
|
4823
|
-
|
|
4824
|
-
|
|
7939
|
+
mkdirSync10(dirname11(path), { recursive: true });
|
|
7940
|
+
writeFileSync7(path, `${JSON.stringify(index, null, 2)}
|
|
4825
7941
|
`);
|
|
4826
7942
|
}
|
|
4827
7943
|
retrieveFallback(query, topK, scope) {
|
|
4828
7944
|
const index = this.readFallbackIndex();
|
|
4829
|
-
const queryTerms =
|
|
7945
|
+
const queryTerms = tokenize2(query);
|
|
4830
7946
|
if (queryTerms.length === 0)
|
|
4831
7947
|
return [];
|
|
4832
7948
|
const scopedChunks = index.chunks.filter((chunk) => matchesVirtualPathScope(chunk.virtualPath, scope));
|
|
4833
7949
|
const documentFrequencies = new Map;
|
|
4834
|
-
const tokenized = scopedChunks.map((chunk) => ({ chunk, terms:
|
|
7950
|
+
const tokenized = scopedChunks.map((chunk) => ({ chunk, terms: tokenize2(chunk.content) }));
|
|
4835
7951
|
for (const term of new Set(queryTerms)) {
|
|
4836
7952
|
documentFrequencies.set(term, tokenized.filter(({ terms }) => terms.includes(term)).length);
|
|
4837
7953
|
}
|
|
@@ -4854,19 +7970,19 @@ class BM25Method {
|
|
|
4854
7970
|
}
|
|
4855
7971
|
readFallbackIndex() {
|
|
4856
7972
|
const path = this.fallbackIndexPath();
|
|
4857
|
-
if (!
|
|
7973
|
+
if (!existsSync14(path))
|
|
4858
7974
|
throw new BM25UnavailableError("index_missing", "BM25 fallback index is missing");
|
|
4859
|
-
const parsed = JSON.parse(
|
|
7975
|
+
const parsed = JSON.parse(readFileSync10(path, "utf8"));
|
|
4860
7976
|
return parsed.version === 1 && Array.isArray(parsed.chunks) ? parsed : { version: 1, chunks: [] };
|
|
4861
7977
|
}
|
|
4862
7978
|
hasFallbackIndex() {
|
|
4863
|
-
return
|
|
7979
|
+
return existsSync14(this.fallbackIndexPath());
|
|
4864
7980
|
}
|
|
4865
7981
|
tantivyIndexPath() {
|
|
4866
|
-
return
|
|
7982
|
+
return join18(this.indexPath, TANTIVY_SUBDIR);
|
|
4867
7983
|
}
|
|
4868
7984
|
fallbackIndexPath() {
|
|
4869
|
-
return
|
|
7985
|
+
return join18(this.indexPath, FALLBACK_INDEX_FILE);
|
|
4870
7986
|
}
|
|
4871
7987
|
syncResult(indexedChunks) {
|
|
4872
7988
|
return {
|
|
@@ -4881,9 +7997,9 @@ function loadChunks(root) {
|
|
|
4881
7997
|
const index = loadMirrorIndex(root);
|
|
4882
7998
|
const chunks = [];
|
|
4883
7999
|
for (const entry of Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath))) {
|
|
4884
|
-
if (!
|
|
8000
|
+
if (!existsSync14(entry.outputPath))
|
|
4885
8001
|
continue;
|
|
4886
|
-
const content = normalizeMarkdown(
|
|
8002
|
+
const content = normalizeMarkdown(readFileSync10(entry.outputPath, "utf8"));
|
|
4887
8003
|
for (const [chunkIndex, chunkContent] of chunkMarkdown(content).entries()) {
|
|
4888
8004
|
chunks.push({
|
|
4889
8005
|
id: `bm25:${entry.virtualPath}:${chunkIndex}:${hash(chunkContent)}`,
|
|
@@ -4913,7 +8029,7 @@ ${paragraph}`;
|
|
|
4913
8029
|
chunks.push(current);
|
|
4914
8030
|
return chunks;
|
|
4915
8031
|
}
|
|
4916
|
-
function
|
|
8032
|
+
function tokenize2(value) {
|
|
4917
8033
|
return value.toLowerCase().match(/[\p{Letter}\p{Number}_]+/gu) ?? [];
|
|
4918
8034
|
}
|
|
4919
8035
|
function bm25Score(queryTerms, documentTerms, documentFrequencies, documentCount, avgDocumentLength) {
|
|
@@ -4939,13 +8055,13 @@ function firstString(values) {
|
|
|
4939
8055
|
return typeof first === "string" ? first : undefined;
|
|
4940
8056
|
}
|
|
4941
8057
|
function hash(value) {
|
|
4942
|
-
return
|
|
8058
|
+
return createHash7("sha256").update(value).digest("hex").slice(0, 12);
|
|
4943
8059
|
}
|
|
4944
|
-
var BM25_SUBDIR, TANTIVY_SUBDIR = "tantivy", FALLBACK_INDEX_FILE = "fallback-index.json",
|
|
8060
|
+
var BM25_SUBDIR, TANTIVY_SUBDIR = "tantivy", FALLBACK_INDEX_FILE = "fallback-index.json", DEFAULT_TOP_K2 = 20, MAX_CHUNK_CHARS = 2400, BM25UnavailableError;
|
|
4945
8061
|
var init_bm25 = __esm(() => {
|
|
4946
8062
|
init_index_store();
|
|
4947
8063
|
init_text();
|
|
4948
|
-
BM25_SUBDIR =
|
|
8064
|
+
BM25_SUBDIR = join18(".autorag", "bm25");
|
|
4949
8065
|
BM25UnavailableError = class BM25UnavailableError extends Error {
|
|
4950
8066
|
readiness;
|
|
4951
8067
|
constructor(readiness, message) {
|
|
@@ -4979,7 +8095,7 @@ class RetrievalMethodRegistry {
|
|
|
4979
8095
|
|
|
4980
8096
|
// src/subagents/contracts.ts
|
|
4981
8097
|
function parseExplorerAssignment(value) {
|
|
4982
|
-
const record =
|
|
8098
|
+
const record = asRecord2(value, "explorer assignment");
|
|
4983
8099
|
const originalQuery = requiredText(record, "originalQuery", "MISSING_ORIGINAL_QUERY", "INVALID_ORIGINAL_QUERY");
|
|
4984
8100
|
const method = requiredMethod(record, "method", "MISSING_RETRIEVAL_METHOD", "INVALID_RETRIEVAL_METHOD");
|
|
4985
8101
|
const queryVariant = requiredText(record, "queryVariant", "MISSING_QUERY_VARIANT", "INVALID_QUERY_VARIANT");
|
|
@@ -4995,7 +8111,7 @@ function parseExplorerAssignment(value) {
|
|
|
4995
8111
|
return { originalQuery, method, queryVariant, queryVariants: [...variantsRaw] };
|
|
4996
8112
|
}
|
|
4997
8113
|
function parseEvidenceCandidate(value) {
|
|
4998
|
-
const record =
|
|
8114
|
+
const record = asRecord2(value, "evidence candidate");
|
|
4999
8115
|
const source = requiredText(record, "source", "MISSING_SOURCE", "INVALID_SOURCE");
|
|
5000
8116
|
const method = requiredMethod(record, "method", "MISSING_EVIDENCE_METHOD", "INVALID_EVIDENCE_METHOD");
|
|
5001
8117
|
const evidence = requiredText(record, "evidence", "MISSING_EVIDENCE", "INVALID_EVIDENCE");
|
|
@@ -5019,7 +8135,7 @@ function parseEvidenceCandidate(value) {
|
|
|
5019
8135
|
};
|
|
5020
8136
|
}
|
|
5021
8137
|
function parseExplorerReport(value) {
|
|
5022
|
-
const record =
|
|
8138
|
+
const record = asRecord2(value, "explorer report");
|
|
5023
8139
|
const assignmentRaw = record.assignment;
|
|
5024
8140
|
if (assignmentRaw === undefined) {
|
|
5025
8141
|
throw new ContractValidationError("MISSING_ASSIGNMENT", "assignment is required.", "assignment");
|
|
@@ -5048,7 +8164,7 @@ function safeParseExplorerReport(value) {
|
|
|
5048
8164
|
return safeParse(() => parseExplorerReport(value));
|
|
5049
8165
|
}
|
|
5050
8166
|
function parseSourceTemporal(value) {
|
|
5051
|
-
const record =
|
|
8167
|
+
const record = asRecord2(value, "sourceTemporal");
|
|
5052
8168
|
const status = record.status;
|
|
5053
8169
|
if (status === undefined) {
|
|
5054
8170
|
throw new ContractValidationError("INVALID_SOURCE_TEMPORAL", "sourceTemporal.status is required.", "sourceTemporal.status");
|
|
@@ -5068,7 +8184,7 @@ function parseSourceTemporal(value) {
|
|
|
5068
8184
|
}
|
|
5069
8185
|
return status === "asOf" ? { status, asOf: date } : status === "updatedAt" ? { status, updatedAt: date } : { status, createdAt: date };
|
|
5070
8186
|
}
|
|
5071
|
-
function
|
|
8187
|
+
function asRecord2(value, label) {
|
|
5072
8188
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
5073
8189
|
throw new ContractValidationError("INVALID_OBJECT", `${label} must be an object.`);
|
|
5074
8190
|
}
|
|
@@ -6079,21 +9195,21 @@ var init_model_policy = __esm(() => {
|
|
|
6079
9195
|
|
|
6080
9196
|
// src/subagents/runtime.ts
|
|
6081
9197
|
import childProcess from "node:child_process";
|
|
6082
|
-
import { createHash as
|
|
9198
|
+
import { createHash as createHash8, randomUUID as randomUUID6 } from "node:crypto";
|
|
6083
9199
|
import {
|
|
6084
9200
|
chmodSync,
|
|
6085
|
-
existsSync as
|
|
9201
|
+
existsSync as existsSync15,
|
|
6086
9202
|
linkSync,
|
|
6087
|
-
mkdirSync as
|
|
6088
|
-
readFileSync as
|
|
9203
|
+
mkdirSync as mkdirSync11,
|
|
9204
|
+
readFileSync as readFileSync11,
|
|
6089
9205
|
realpathSync as realpathSync3,
|
|
6090
9206
|
renameSync as renameSync6,
|
|
6091
9207
|
statSync as statSync2,
|
|
6092
9208
|
unlinkSync as unlinkSync4,
|
|
6093
|
-
writeFileSync as
|
|
9209
|
+
writeFileSync as writeFileSync8
|
|
6094
9210
|
} from "node:fs";
|
|
6095
9211
|
import { createRequire as createRequire2 } from "node:module";
|
|
6096
|
-
import { basename as
|
|
9212
|
+
import { basename as basename9, dirname as dirname12, extname as extname9, isAbsolute as isAbsolute6, join as join19, relative as relative5, resolve as resolve8, sep as sep4 } from "node:path";
|
|
6097
9213
|
import { fileURLToPath } from "node:url";
|
|
6098
9214
|
import { InMemoryCredentialStore, InMemoryModelsStore } from "@earendil-works/pi-ai";
|
|
6099
9215
|
import {
|
|
@@ -6105,24 +9221,24 @@ import {
|
|
|
6105
9221
|
SettingsManager
|
|
6106
9222
|
} from "@earendil-works/pi-coding-agent";
|
|
6107
9223
|
function resolveExplorerToolsExtensionPath() {
|
|
6108
|
-
const dir =
|
|
9224
|
+
const dir = dirname12(runtimePath);
|
|
6109
9225
|
const candidates = [
|
|
6110
|
-
|
|
6111
|
-
|
|
6112
|
-
|
|
9226
|
+
join19(dir, `explorer-tools-extension${extname9(runtimePath)}`),
|
|
9227
|
+
join19(dir, "subagents", "explorer-tools-extension.js"),
|
|
9228
|
+
join19(dir, "..", "subagents", "explorer-tools-extension.js")
|
|
6113
9229
|
];
|
|
6114
9230
|
for (const candidate of candidates) {
|
|
6115
|
-
if (
|
|
9231
|
+
if (existsSync15(candidate))
|
|
6116
9232
|
return realpathSync3(candidate);
|
|
6117
9233
|
}
|
|
6118
9234
|
throw new Error(`AutoRAG explorer tools extension not found; looked in: ${candidates.join(", ")}`);
|
|
6119
9235
|
}
|
|
6120
9236
|
function resolvePiSubagentsPackageJson() {
|
|
6121
9237
|
const require2 = createRequire2(import.meta.url);
|
|
6122
|
-
return
|
|
9238
|
+
return join19(dirname12(require2.resolve("pi-subagents")), "package.json");
|
|
6123
9239
|
}
|
|
6124
9240
|
function resolvePiSubagentsExtension() {
|
|
6125
|
-
return
|
|
9241
|
+
return join19(dirname12(resolvePiSubagentsPackageJson()), "src", "extension", "index.ts");
|
|
6126
9242
|
}
|
|
6127
9243
|
function resolveSubagentPiBinary() {
|
|
6128
9244
|
const configured = process.env.PI_SUBAGENT_PI_BINARY?.trim();
|
|
@@ -6130,12 +9246,12 @@ function resolveSubagentPiBinary() {
|
|
|
6130
9246
|
return configured;
|
|
6131
9247
|
const require2 = createRequire2(import.meta.url);
|
|
6132
9248
|
const subagentsPackage = resolvePiSubagentsPackageJson();
|
|
6133
|
-
const packageJsonPath =
|
|
9249
|
+
const packageJsonPath = join19(dirname12(dirname12(subagentsPackage)), "@earendil-works", "pi-coding-agent", "package.json");
|
|
6134
9250
|
const packageJson = require2(packageJsonPath);
|
|
6135
9251
|
const bin = typeof packageJson.bin === "string" ? packageJson.bin : packageJson.bin?.pi;
|
|
6136
9252
|
if (!bin)
|
|
6137
9253
|
throw new Error("Mandatory pi-subagents runtime could not resolve the Pi CLI binary");
|
|
6138
|
-
return
|
|
9254
|
+
return join19(dirname12(packageJsonPath), bin);
|
|
6139
9255
|
}
|
|
6140
9256
|
function asToolDefinition(tool) {
|
|
6141
9257
|
return {
|
|
@@ -6144,12 +9260,12 @@ function asToolDefinition(tool) {
|
|
|
6144
9260
|
};
|
|
6145
9261
|
}
|
|
6146
9262
|
function defaultAgentDir() {
|
|
6147
|
-
return
|
|
9263
|
+
return join19(resolveAutoRAGHome(), "pi-agent");
|
|
6148
9264
|
}
|
|
6149
9265
|
function ensurePiSettingsFile(agentDir) {
|
|
6150
|
-
const settingsPath =
|
|
9266
|
+
const settingsPath = join19(agentDir, "settings.json");
|
|
6151
9267
|
try {
|
|
6152
|
-
|
|
9268
|
+
writeFileSync8(settingsPath, "{}", { encoding: "utf8", flag: "wx" });
|
|
6153
9269
|
} catch (error) {
|
|
6154
9270
|
if (typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST")
|
|
6155
9271
|
return;
|
|
@@ -6172,7 +9288,7 @@ function isRecognizedManagedExplorerDefinition(definition) {
|
|
|
6172
9288
|
function replaceExplorerDefinitionAtomically(explorerPath) {
|
|
6173
9289
|
const temporaryPath = `${explorerPath}.${randomUUID6()}.tmp`;
|
|
6174
9290
|
try {
|
|
6175
|
-
|
|
9291
|
+
writeFileSync8(temporaryPath, AUTORAG_EXPLORER_AGENT_DEFINITION, {
|
|
6176
9292
|
encoding: "utf8",
|
|
6177
9293
|
flag: "wx",
|
|
6178
9294
|
mode: 384
|
|
@@ -6180,30 +9296,30 @@ function replaceExplorerDefinitionAtomically(explorerPath) {
|
|
|
6180
9296
|
chmodSync(temporaryPath, 384);
|
|
6181
9297
|
renameSync6(temporaryPath, explorerPath);
|
|
6182
9298
|
} finally {
|
|
6183
|
-
if (
|
|
9299
|
+
if (existsSync15(temporaryPath))
|
|
6184
9300
|
unlinkSync4(temporaryPath);
|
|
6185
9301
|
}
|
|
6186
9302
|
}
|
|
6187
9303
|
function ensureMandatoryExplorerAgent(agentDir) {
|
|
6188
|
-
const agentsDir =
|
|
6189
|
-
const explorerPath =
|
|
6190
|
-
if (
|
|
6191
|
-
const existingDefinition =
|
|
9304
|
+
const agentsDir = join19(agentDir, "agents");
|
|
9305
|
+
const explorerPath = join19(agentsDir, "autorag-explorer.md");
|
|
9306
|
+
if (existsSync15(explorerPath)) {
|
|
9307
|
+
const existingDefinition = readFileSync11(explorerPath, "utf8");
|
|
6192
9308
|
if (existingDefinition === AUTORAG_EXPLORER_AGENT_DEFINITION)
|
|
6193
9309
|
return;
|
|
6194
9310
|
if (!isRecognizedManagedExplorerDefinition(existingDefinition)) {
|
|
6195
9311
|
throw new Error(`AutoRAG persistent explorer definition at ${explorerPath} is not canonical`);
|
|
6196
9312
|
}
|
|
6197
9313
|
replaceExplorerDefinitionAtomically(explorerPath);
|
|
6198
|
-
if (
|
|
9314
|
+
if (readFileSync11(explorerPath, "utf8") !== AUTORAG_EXPLORER_AGENT_DEFINITION || (statSync2(explorerPath).mode & 511) !== 384) {
|
|
6199
9315
|
throw new Error(`AutoRAG persistent explorer definition at ${explorerPath} is not canonical`);
|
|
6200
9316
|
}
|
|
6201
9317
|
return;
|
|
6202
9318
|
}
|
|
6203
|
-
|
|
6204
|
-
const temporaryPath =
|
|
9319
|
+
mkdirSync11(agentsDir, { recursive: true });
|
|
9320
|
+
const temporaryPath = join19(agentsDir, `.autorag-explorer.${randomUUID6()}.tmp`);
|
|
6205
9321
|
try {
|
|
6206
|
-
|
|
9322
|
+
writeFileSync8(temporaryPath, AUTORAG_EXPLORER_AGENT_DEFINITION, {
|
|
6207
9323
|
encoding: "utf8",
|
|
6208
9324
|
flag: "wx",
|
|
6209
9325
|
mode: 384
|
|
@@ -6216,18 +9332,18 @@ function ensureMandatoryExplorerAgent(agentDir) {
|
|
|
6216
9332
|
}
|
|
6217
9333
|
}
|
|
6218
9334
|
} finally {
|
|
6219
|
-
if (
|
|
9335
|
+
if (existsSync15(temporaryPath))
|
|
6220
9336
|
unlinkSync4(temporaryPath);
|
|
6221
9337
|
}
|
|
6222
|
-
if (
|
|
9338
|
+
if (readFileSync11(explorerPath, "utf8") !== AUTORAG_EXPLORER_AGENT_DEFINITION) {
|
|
6223
9339
|
throw new Error(`AutoRAG persistent explorer definition at ${explorerPath} is not canonical`);
|
|
6224
9340
|
}
|
|
6225
9341
|
}
|
|
6226
9342
|
function defaultSessionDir(cwd, agentDir) {
|
|
6227
9343
|
const canonicalCwd = resolve8(cwd);
|
|
6228
9344
|
const readablePath = canonicalCwd.replace(/^[/\\]/, "").replace(/[/\\:]/g, "-");
|
|
6229
|
-
const cwdHash =
|
|
6230
|
-
return
|
|
9345
|
+
const cwdHash = createHash8("sha256").update(canonicalCwd).digest("hex").slice(0, 12);
|
|
9346
|
+
return join19(agentDir, "sessions", `--${readablePath}-${cwdHash}--`);
|
|
6231
9347
|
}
|
|
6232
9348
|
function childEnvironmentFromSpawnOptions(args) {
|
|
6233
9349
|
const optionsIndex = Array.isArray(args[1]) ? 2 : 1;
|
|
@@ -6265,7 +9381,7 @@ function isPiChildSpawn(args, context) {
|
|
|
6265
9381
|
return childArgsFromSpawnArguments(args)[0] === context.piBinary;
|
|
6266
9382
|
}
|
|
6267
9383
|
function isAsyncSubagentRunnerSpawn(args) {
|
|
6268
|
-
return childArgsFromSpawnArguments(args).some((argument) => typeof argument === "string" &&
|
|
9384
|
+
return childArgsFromSpawnArguments(args).some((argument) => typeof argument === "string" && basename9(argument) === "subagent-runner.ts");
|
|
6269
9385
|
}
|
|
6270
9386
|
function createScopedChildProcessModule(context) {
|
|
6271
9387
|
const originalSpawn = childProcess.spawn.bind(childProcess);
|
|
@@ -6287,12 +9403,12 @@ function createScopedChildProcessModule(context) {
|
|
|
6287
9403
|
return { ...scopedModule, default: scopedModule };
|
|
6288
9404
|
}
|
|
6289
9405
|
function isWithinDirectory(root, candidate) {
|
|
6290
|
-
const relativePath =
|
|
6291
|
-
return relativePath === "" || relativePath !== ".." && !relativePath.startsWith(`..${
|
|
9406
|
+
const relativePath = relative5(root, candidate);
|
|
9407
|
+
return relativePath === "" || relativePath !== ".." && !relativePath.startsWith(`..${sep4}`) && !isAbsolute6(relativePath);
|
|
6292
9408
|
}
|
|
6293
9409
|
async function loadScopedPiSubagentsExtension(extensionPath, environment) {
|
|
6294
9410
|
const packageJsonPath = resolvePiSubagentsPackageJson();
|
|
6295
|
-
const packageRoot =
|
|
9411
|
+
const packageRoot = dirname12(packageJsonPath);
|
|
6296
9412
|
const packageRequire = createRequire2(packageJsonPath);
|
|
6297
9413
|
const jitiModule = packageRequire("jiti");
|
|
6298
9414
|
if (typeof jitiModule.createJiti !== "function") {
|
|
@@ -6300,7 +9416,7 @@ async function loadScopedPiSubagentsExtension(extensionPath, environment) {
|
|
|
6300
9416
|
}
|
|
6301
9417
|
const createJiti = jitiModule.createJiti;
|
|
6302
9418
|
const baseJiti = createJiti(packageJsonPath, { moduleCache: false, fsCache: false });
|
|
6303
|
-
const utilsPath =
|
|
9419
|
+
const utilsPath = join19(packageRoot, "src", "shared", "utils.ts");
|
|
6304
9420
|
const upstreamUtils = await baseJiti.import(utilsPath);
|
|
6305
9421
|
if (typeof upstreamUtils.getAgentDir !== "function") {
|
|
6306
9422
|
throw new Error("pi-subagents did not expose its expected getAgentDir utility");
|
|
@@ -6330,11 +9446,11 @@ function isJsonObject(value) {
|
|
|
6330
9446
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6331
9447
|
}
|
|
6332
9448
|
function readPiModelsConfig(modelsPath) {
|
|
6333
|
-
if (!
|
|
9449
|
+
if (!existsSync15(modelsPath))
|
|
6334
9450
|
return { providers: {} };
|
|
6335
9451
|
let parsed;
|
|
6336
9452
|
try {
|
|
6337
|
-
parsed = JSON.parse(
|
|
9453
|
+
parsed = JSON.parse(readFileSync11(modelsPath, "utf8"));
|
|
6338
9454
|
} catch (error) {
|
|
6339
9455
|
throw new Error(`AutoRAG could not parse Pi models.json at ${modelsPath}: ${error.message}`, {
|
|
6340
9456
|
cause: error
|
|
@@ -6508,13 +9624,13 @@ async function persistPiModels(modelsPath, roleModels, providerCredentials) {
|
|
|
6508
9624
|
}
|
|
6509
9625
|
}
|
|
6510
9626
|
temporaryPath = `${modelsPath}.${randomUUID6()}.tmp`;
|
|
6511
|
-
|
|
9627
|
+
writeFileSync8(temporaryPath, content, { encoding: "utf8", flag: "wx", mode: 384 });
|
|
6512
9628
|
await validatePiModelsFile(temporaryPath, roleModels, "generated");
|
|
6513
9629
|
lock.assertOwned();
|
|
6514
9630
|
renameSync6(temporaryPath, modelsPath);
|
|
6515
9631
|
} finally {
|
|
6516
9632
|
try {
|
|
6517
|
-
if (temporaryPath &&
|
|
9633
|
+
if (temporaryPath && existsSync15(temporaryPath))
|
|
6518
9634
|
unlinkSync4(temporaryPath);
|
|
6519
9635
|
} finally {
|
|
6520
9636
|
lock.release();
|
|
@@ -6579,7 +9695,7 @@ function explorerEnvironmentIdentity(environment) {
|
|
|
6579
9695
|
agentDir: environment.agentDir,
|
|
6580
9696
|
piBinary: environment.piBinary,
|
|
6581
9697
|
apiKeyName: environment.apiKeyName,
|
|
6582
|
-
apiKeyHash: environment.apiKey === undefined ? undefined :
|
|
9698
|
+
apiKeyHash: environment.apiKey === undefined ? undefined : createHash8("sha256").update(environment.apiKey).digest("hex")
|
|
6583
9699
|
});
|
|
6584
9700
|
}
|
|
6585
9701
|
function defaultExplorerModel(model) {
|
|
@@ -6587,7 +9703,7 @@ function defaultExplorerModel(model) {
|
|
|
6587
9703
|
}
|
|
6588
9704
|
async function createMandatorySubagentSession(options) {
|
|
6589
9705
|
const agentDir = resolve8(options.agentDir ?? defaultAgentDir());
|
|
6590
|
-
|
|
9706
|
+
mkdirSync11(agentDir, { recursive: true });
|
|
6591
9707
|
ensureMandatoryExplorerAgent(agentDir);
|
|
6592
9708
|
const piBinary = resolveSubagentPiBinary();
|
|
6593
9709
|
const extensionPath = options.extensionPath ?? resolvePiSubagentsExtension();
|
|
@@ -6609,7 +9725,7 @@ async function createMandatorySubagentSession(options) {
|
|
|
6609
9725
|
const explorerRoleModel = activeRoleModel(explorerModel, "explorer");
|
|
6610
9726
|
const roleModels = [orchestratorRoleModel, explorerRoleModel];
|
|
6611
9727
|
validateProviderApiKeyEnvNames(roleModels);
|
|
6612
|
-
const authPath =
|
|
9728
|
+
const authPath = join19(agentDir, "auth.json");
|
|
6613
9729
|
for (const { reference } of roleModels) {
|
|
6614
9730
|
if (parentProviderCredentials.has(reference.provider))
|
|
6615
9731
|
continue;
|
|
@@ -6618,7 +9734,7 @@ async function createMandatorySubagentSession(options) {
|
|
|
6618
9734
|
continue;
|
|
6619
9735
|
parentProviderCredentials.set(reference.provider, stored.key);
|
|
6620
9736
|
}
|
|
6621
|
-
const modelsPath =
|
|
9737
|
+
const modelsPath = join19(agentDir, "models.json");
|
|
6622
9738
|
const explorerCredential = parentProviderCredentials.get(explorerRoleModel.reference.provider);
|
|
6623
9739
|
const explorerEnvironment = {
|
|
6624
9740
|
agentDir,
|
|
@@ -6927,12 +10043,12 @@ Required handoff: include temporal metadata.
|
|
|
6927
10043
|
});
|
|
6928
10044
|
|
|
6929
10045
|
// src/agent/bash-tool.ts
|
|
6930
|
-
import { spawn as
|
|
10046
|
+
import { spawn as spawn7 } from "node:child_process";
|
|
6931
10047
|
import { Type as Type2 } from "typebox";
|
|
6932
10048
|
function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
6933
10049
|
return new Promise((resolve9) => {
|
|
6934
10050
|
const shell = process.platform === "win32" ? "bash.exe" : "/bin/bash";
|
|
6935
|
-
const child =
|
|
10051
|
+
const child = spawn7(shell, ["-c", command], { cwd });
|
|
6936
10052
|
const chunks = [];
|
|
6937
10053
|
let total = 0;
|
|
6938
10054
|
let truncated = false;
|
|
@@ -6976,7 +10092,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
6976
10092
|
});
|
|
6977
10093
|
}
|
|
6978
10094
|
function createBashTool(options) {
|
|
6979
|
-
const defaultTimeout = options.timeoutMs ??
|
|
10095
|
+
const defaultTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
6980
10096
|
const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
6981
10097
|
return {
|
|
6982
10098
|
name: BASH_TOOL_NAME,
|
|
@@ -7035,7 +10151,7 @@ function createBashTool(options) {
|
|
|
7035
10151
|
}
|
|
7036
10152
|
};
|
|
7037
10153
|
}
|
|
7038
|
-
var BASH_TOOL_NAME = "bash",
|
|
10154
|
+
var BASH_TOOL_NAME = "bash", DEFAULT_TIMEOUT_MS6 = 120000, DEFAULT_MAX_OUTPUT_BYTES = 131072, bashSchema;
|
|
7039
10155
|
var init_bash_tool = __esm(() => {
|
|
7040
10156
|
bashSchema = Type2.Object({
|
|
7041
10157
|
command: Type2.String({
|
|
@@ -7585,7 +10701,8 @@ function recordStructuredResultsSession(sessionId, query, details, sessions, mem
|
|
|
7585
10701
|
summary: result.summary,
|
|
7586
10702
|
evidence: result.evidence.map((evidence) => evidence.lineNumber !== undefined ? { excerpt: evidence.excerpt, lineNumber: evidence.lineNumber } : { excerpt: evidence.excerpt }),
|
|
7587
10703
|
confidence: confidenceFrom(result.confidence),
|
|
7588
|
-
feedbackId: `${sessionId}:${result.number}
|
|
10704
|
+
feedbackId: `${sessionId}:${result.number}`,
|
|
10705
|
+
source: registry.get(result.number)?.source
|
|
7589
10706
|
}));
|
|
7590
10707
|
const answer = details.answer;
|
|
7591
10708
|
const diagnostics = [];
|
|
@@ -7992,7 +11109,7 @@ var init_watch_refresh = __esm(() => {
|
|
|
7992
11109
|
// src/agent/agent.ts
|
|
7993
11110
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
7994
11111
|
import { watch as fsWatch, realpathSync as realpathSync4, statSync as statSync3 } from "node:fs";
|
|
7995
|
-
import { dirname as
|
|
11112
|
+
import { dirname as dirname13, join as join20, resolve as resolve9 } from "node:path";
|
|
7996
11113
|
import { Agent } from "@earendil-works/pi-agent-core";
|
|
7997
11114
|
|
|
7998
11115
|
class AutoRAGAgent {
|
|
@@ -8079,10 +11196,10 @@ class AutoRAGAgent {
|
|
|
8079
11196
|
if (options.jikji) {
|
|
8080
11197
|
this.jikjiClient = new JikjiClient({ ...options.jikji, root: this.workspaceProjectRoot });
|
|
8081
11198
|
}
|
|
8082
|
-
const memPath = memoryPath ??
|
|
11199
|
+
const memPath = memoryPath ?? join20(resolveAutoRAGHome(), "memory.json");
|
|
8083
11200
|
this.memory = new RetrievalMemory({ storagePath: memPath });
|
|
8084
11201
|
this.memory.load();
|
|
8085
|
-
this.runLogger = new AutoRAGRunLogger(
|
|
11202
|
+
this.runLogger = new AutoRAGRunLogger(join20(dirname13(memPath), "logs", "runs.jsonl"));
|
|
8086
11203
|
const checkMemoryTool = createCheckMemoryTool(this.memory);
|
|
8087
11204
|
const searchBM25Tool = createSearchBM25DocumentsTool(() => this.bm25Method);
|
|
8088
11205
|
const searchDatasourceTool = createSearchDatasourceDocumentsTool(this);
|
|
@@ -8835,7 +11952,7 @@ ${allowedRoots}
|
|
|
8835
11952
|
written: 0,
|
|
8836
11953
|
deleted: 0,
|
|
8837
11954
|
skipped: 0,
|
|
8838
|
-
indexPath:
|
|
11955
|
+
indexPath: join20(this.workspaceProjectRoot, PARSED_MIRROR_SUBDIR),
|
|
8839
11956
|
diagnostics
|
|
8840
11957
|
};
|
|
8841
11958
|
}
|
|
@@ -9946,8 +13063,8 @@ var exports_commands = {};
|
|
|
9946
13063
|
__export(exports_commands, {
|
|
9947
13064
|
runIndex: () => runIndex
|
|
9948
13065
|
});
|
|
9949
|
-
import { existsSync as
|
|
9950
|
-
import { join as
|
|
13066
|
+
import { existsSync as existsSync16, lstatSync as lstatSync2, rmSync as rmSync5 } from "node:fs";
|
|
13067
|
+
import { join as join21, resolve as resolve10, sep as sep5 } from "node:path";
|
|
9951
13068
|
async function runIndex(ctx) {
|
|
9952
13069
|
const sub = ctx.positionals[0];
|
|
9953
13070
|
if (sub !== "reset" && sub !== "rebuild") {
|
|
@@ -9966,7 +13083,7 @@ async function runIndex(ctx) {
|
|
|
9966
13083
|
const autoragDir = resolve10(config.workspacePath, ".autorag");
|
|
9967
13084
|
const targets = targetSubdirs.map((subdir) => resolve10(config.workspacePath, subdir));
|
|
9968
13085
|
for (const dir of [autoragDir, ...targets]) {
|
|
9969
|
-
if (
|
|
13086
|
+
if (existsSync16(dir) && lstatSync2(dir).isSymbolicLink()) {
|
|
9970
13087
|
ctx.stderr(renderError(new Error("Refusing to reset: an index path is a symlink, not a real directory."), {
|
|
9971
13088
|
json: ctx.json
|
|
9972
13089
|
}));
|
|
@@ -9981,7 +13098,7 @@ async function runIndex(ctx) {
|
|
|
9981
13098
|
}
|
|
9982
13099
|
if (!ctx.flags.yes) {
|
|
9983
13100
|
if (ctx.promptYesNo) {
|
|
9984
|
-
const ok = await ctx.promptYesNo(`Reset the ${targetNames.join(", ")} indexes under ${
|
|
13101
|
+
const ok = await ctx.promptYesNo(`Reset the ${targetNames.join(", ")} indexes under ${join21(config.workspacePath, ".autorag")}?`);
|
|
9985
13102
|
if (!ok) {
|
|
9986
13103
|
ctx.stderr(renderError(new Error("Reset declined."), { json: ctx.json }));
|
|
9987
13104
|
return 2;
|
|
@@ -10040,7 +13157,7 @@ function resolveResetScope(methods) {
|
|
|
10040
13157
|
};
|
|
10041
13158
|
}
|
|
10042
13159
|
function isWithin(target, base) {
|
|
10043
|
-
const prefix = base.endsWith(
|
|
13160
|
+
const prefix = base.endsWith(sep5) ? base : `${base}${sep5}`;
|
|
10044
13161
|
return target === base || target.startsWith(prefix);
|
|
10045
13162
|
}
|
|
10046
13163
|
var ALL_RESET_TARGETS, ALL_RESET_TARGET_NAMES;
|
|
@@ -10131,7 +13248,7 @@ __export(exports_health, {
|
|
|
10131
13248
|
});
|
|
10132
13249
|
import { mkdtempSync, rmSync as rmSync6 } from "node:fs";
|
|
10133
13250
|
import { tmpdir as tmpdir3 } from "node:os";
|
|
10134
|
-
import { join as
|
|
13251
|
+
import { join as join22 } from "node:path";
|
|
10135
13252
|
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
10136
13253
|
function defaultResolveConfig(input) {
|
|
10137
13254
|
return resolveConfigReadOnly({ flags: input.flags, cwd: input.cwd, env: input.env });
|
|
@@ -10224,7 +13341,7 @@ function sanitizeMessage(raw) {
|
|
|
10224
13341
|
function parseTimeoutMs(flags) {
|
|
10225
13342
|
const raw = flags["timeout-ms"];
|
|
10226
13343
|
if (raw === undefined || raw === true)
|
|
10227
|
-
return
|
|
13344
|
+
return DEFAULT_TIMEOUT_MS7;
|
|
10228
13345
|
const text = typeof raw === "string" ? raw.trim() : "";
|
|
10229
13346
|
if (text.length === 0)
|
|
10230
13347
|
return { error: "--timeout-ms requires a value" };
|
|
@@ -10332,10 +13449,10 @@ async function defaultOrchestratorProbe(input, signal) {
|
|
|
10332
13449
|
}
|
|
10333
13450
|
}
|
|
10334
13451
|
async function defaultExplorerProbe(input, signal) {
|
|
10335
|
-
const tempRoot = mkdtempSync(
|
|
13452
|
+
const tempRoot = mkdtempSync(join22(tmpdir3(), "autorag-health-"));
|
|
10336
13453
|
try {
|
|
10337
|
-
const agentDir =
|
|
10338
|
-
const sessionDir =
|
|
13454
|
+
const agentDir = join22(tempRoot, "agent");
|
|
13455
|
+
const sessionDir = join22(tempRoot, "sessions");
|
|
10339
13456
|
const providerApiKeys = input.providerApiKeys;
|
|
10340
13457
|
const modelRef = `${input.model.provider}/${input.model.id}`;
|
|
10341
13458
|
let probeSession;
|
|
@@ -10581,7 +13698,7 @@ async function runHealth(ctx, deps = {}) {
|
|
|
10581
13698
|
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
10582
13699
|
return exitCodeFor(topCategory);
|
|
10583
13700
|
}
|
|
10584
|
-
var CATEGORY_PRECEDENCE, SECRET_PATTERNS,
|
|
13701
|
+
var CATEGORY_PRECEDENCE, SECRET_PATTERNS, DEFAULT_TIMEOUT_MS7 = 1e4, AUTH_ERROR_PATTERNS, NETWORK_ERROR_CODES, NETWORK_ERROR_PATTERNS, HEALTH_EXPLORER_SYSTEM_PROMPT = "You are a health probe. Reply with HEALTH_OK. Do not read any files.";
|
|
10585
13702
|
var init_health = __esm(() => {
|
|
10586
13703
|
init_runtime();
|
|
10587
13704
|
init_config();
|