@autorag/librarian 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +45 -1
  2. package/dist/agent/search-documents.d.ts +1 -0
  3. package/dist/cli/config.d.ts +6 -0
  4. package/dist/cli/index.js +3423 -306
  5. package/dist/datasource/chunk-store.d.ts +67 -0
  6. package/dist/datasource/connector-skill.d.ts +107 -0
  7. package/dist/datasource/connector.d.ts +72 -0
  8. package/dist/datasource/http.d.ts +49 -0
  9. package/dist/datasource/index.d.ts +14 -0
  10. package/dist/datasource/skills/discord/connector.d.ts +30 -0
  11. package/dist/datasource/skills/discord/index.d.ts +2 -0
  12. package/dist/datasource/skills/discord/skill.d.ts +20 -0
  13. package/dist/datasource/skills/factory.d.ts +35 -0
  14. package/dist/datasource/skills/gdrive/connector.d.ts +30 -0
  15. package/dist/datasource/skills/gdrive/index.d.ts +3 -0
  16. package/dist/datasource/skills/gdrive/rclone-connector.d.ts +51 -0
  17. package/dist/datasource/skills/gdrive/skill.d.ts +24 -0
  18. package/dist/datasource/skills/github/connector.d.ts +28 -0
  19. package/dist/datasource/skills/github/index.d.ts +2 -0
  20. package/dist/datasource/skills/github/skill.d.ts +20 -0
  21. package/dist/datasource/skills/gmail/connector.d.ts +30 -0
  22. package/dist/datasource/skills/gmail/himalaya-connector.d.ts +40 -0
  23. package/dist/datasource/skills/gmail/index.d.ts +3 -0
  24. package/dist/datasource/skills/gmail/skill.d.ts +24 -0
  25. package/dist/datasource/skills/mail-export/connector.d.ts +21 -0
  26. package/dist/datasource/skills/mail-export/index.d.ts +2 -0
  27. package/dist/datasource/skills/mail-export/skill.d.ts +20 -0
  28. package/dist/datasource/skills/notion/connector.d.ts +26 -0
  29. package/dist/datasource/skills/notion/index.d.ts +2 -0
  30. package/dist/datasource/skills/notion/skill.d.ts +20 -0
  31. package/dist/datasource/skills/obsidian/connector.d.ts +22 -0
  32. package/dist/datasource/skills/obsidian/index.d.ts +2 -0
  33. package/dist/datasource/skills/obsidian/skill.d.ts +20 -0
  34. package/dist/datasource/skills/rss/connector.d.ts +28 -0
  35. package/dist/datasource/skills/rss/index.d.ts +2 -0
  36. package/dist/datasource/skills/rss/skill.d.ts +22 -0
  37. package/dist/datasource/skills/slack/connector.d.ts +28 -0
  38. package/dist/datasource/skills/slack/index.d.ts +2 -0
  39. package/dist/datasource/skills/slack/skill.d.ts +20 -0
  40. package/dist/datasource/skills/spotlight/connector.d.ts +59 -0
  41. package/dist/datasource/skills/spotlight/index.d.ts +2 -0
  42. package/dist/datasource/skills/spotlight/skill.d.ts +20 -0
  43. package/dist/index.js +3267 -152
  44. package/dist/parser/defaults.d.ts +2 -0
  45. package/dist/parser/hwp-markdown.d.ts +25 -0
  46. package/dist/parser/hwp.d.ts +7 -0
  47. package/dist/parser/index.d.ts +2 -1
  48. package/dist/parser/rhwp-adapter.d.ts +30 -0
  49. package/package.json +3 -2
  50. package/skills/autorag-setup/SKILL.md +52 -0
package/dist/cli/index.js CHANGED
@@ -135,6 +135,8 @@ function searchEnvelope(resp, debug) {
135
135
  title: r.title,
136
136
  summary: r.summary
137
137
  };
138
+ if (r.source !== undefined)
139
+ base.source = r.source;
138
140
  if (debug) {
139
141
  base.confidence = r.confidence;
140
142
  base.evidence = r.evidence.map((e) => {
@@ -169,6 +171,8 @@ function renderSearchHuman(resp, debug) {
169
171
  for (const r of resp.results) {
170
172
  const header = `${r.number}. ${r.title}`;
171
173
  lines.push(header);
174
+ if (r.source !== undefined)
175
+ lines.push(` source: ${r.source}`);
172
176
  if (r.summary)
173
177
  lines.push(` ${r.summary}`);
174
178
  if (debug) {
@@ -356,20 +360,2671 @@ function resolveAutoRAGHome(env = process.env) {
356
360
  var AUTORAG_HOME_ENV = "AUTORAG_HOME";
357
361
  var init_home = () => {};
358
362
 
363
+ // src/datasource/http.ts
364
+ function combineSignals(timeoutMs, outer) {
365
+ const timeout = AbortSignal.timeout(timeoutMs);
366
+ return outer === undefined ? timeout : AbortSignal.any([timeout, outer]);
367
+ }
368
+ function statusToReason(status) {
369
+ if (status === 401)
370
+ return "auth";
371
+ if (status === 403)
372
+ return "permission";
373
+ if (status === 429)
374
+ return "rate-limited";
375
+ return "api-error";
376
+ }
377
+ async function requestText(url, options) {
378
+ const fetchImpl = options.fetchImpl ?? globalThis.fetch;
379
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
380
+ let response;
381
+ try {
382
+ response = await fetchImpl(url, {
383
+ method: options.method ?? "GET",
384
+ headers: options.headers,
385
+ body: options.body,
386
+ signal: combineSignals(timeoutMs, options.signal)
387
+ });
388
+ } catch {
389
+ return { ok: false, reason: "unavailable", message: "request failed or timed out" };
390
+ }
391
+ if (!response.ok) {
392
+ return { ok: false, reason: statusToReason(response.status), message: `http-${response.status}` };
393
+ }
394
+ let text;
395
+ try {
396
+ text = await response.text();
397
+ } catch {
398
+ return { ok: false, reason: "invalid-data", message: "response body unreadable" };
399
+ }
400
+ return { ok: true, status: response.status, text };
401
+ }
402
+ function httpText(url, options = {}) {
403
+ return requestText(url, options);
404
+ }
405
+ async function httpJson(url, options = {}) {
406
+ const result = await requestText(url, options);
407
+ if (!result.ok)
408
+ return result;
409
+ try {
410
+ return { ok: true, status: result.status, json: JSON.parse(result.text) };
411
+ } catch {
412
+ return { ok: false, reason: "invalid-data", message: "response was not valid JSON" };
413
+ }
414
+ }
415
+ function asRecord(value) {
416
+ return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
417
+ }
418
+ function asArray(value) {
419
+ return Array.isArray(value) ? value : [];
420
+ }
421
+ function asString(value) {
422
+ return typeof value === "string" ? value : undefined;
423
+ }
424
+ function asNumber(value) {
425
+ return typeof value === "number" && Number.isFinite(value) ? value : undefined;
426
+ }
427
+ function parseEpochMs(value) {
428
+ if (typeof value !== "string" || value.length === 0)
429
+ return;
430
+ const parsed = Date.parse(value);
431
+ return Number.isNaN(parsed) ? undefined : parsed;
432
+ }
433
+ function resolveToken(token, tokenEnv) {
434
+ if (token !== undefined && token.length > 0)
435
+ return token;
436
+ if (tokenEnv !== undefined && tokenEnv.length > 0) {
437
+ const fromEnv = process.env[tokenEnv];
438
+ if (fromEnv !== undefined && fromEnv.length > 0)
439
+ return fromEnv;
440
+ }
441
+ return;
442
+ }
443
+ var DEFAULT_TIMEOUT_MS = 15000;
444
+
445
+ // src/datasource/skills/discord/connector.ts
446
+ class DiscordConnector {
447
+ options;
448
+ constructor(options = {}) {
449
+ this.options = options;
450
+ }
451
+ async fetch(signal) {
452
+ const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV);
453
+ if (token === undefined)
454
+ return { ok: false, reason: "not-configured", message: "auth token not configured" };
455
+ const guildId = this.options.guildId;
456
+ if (guildId === undefined || guildId.length === 0) {
457
+ return { ok: false, reason: "not-configured", message: "guild id not configured" };
458
+ }
459
+ const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL;
460
+ const request = {
461
+ headers: { Authorization: `Bot ${token}` },
462
+ timeoutMs: this.options.timeoutMs,
463
+ fetchImpl: this.options.fetchImpl,
464
+ signal
465
+ };
466
+ const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES;
467
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS;
468
+ const listResult = await httpJson(`${baseUrl}/guilds/${guildId}/channels`, request);
469
+ if (!listResult.ok) {
470
+ return { ok: false, reason: listResult.reason, message: `channel list failed: ${listResult.message}` };
471
+ }
472
+ const channels = [];
473
+ for (const raw of asArray(listResult.json)) {
474
+ const channel = asRecord(raw);
475
+ const id = asString(channel?.id);
476
+ const name = asString(channel?.name);
477
+ const type = asNumber(channel?.type);
478
+ if (id === undefined || name === undefined || type === undefined)
479
+ continue;
480
+ if (!TEXT_CHANNEL_TYPES.has(type))
481
+ continue;
482
+ channels.push({ id, name });
483
+ }
484
+ const filter = this.options.channels;
485
+ const selected = filter === undefined || filter.length === 0 ? channels : channels.filter((channel) => filter.includes(channel.id) || filter.includes(channel.name));
486
+ const documents = [];
487
+ const warnings = [];
488
+ for (const channel of selected) {
489
+ if (documents.length >= maxDocuments)
490
+ break;
491
+ let before;
492
+ for (let page = 0;page < maxPages && documents.length < maxDocuments; page += 1) {
493
+ const url = new URL(`${baseUrl}/channels/${channel.id}/messages`);
494
+ url.searchParams.set("limit", "100");
495
+ if (before !== undefined)
496
+ url.searchParams.set("before", before);
497
+ const result = await httpJson(url.toString(), request);
498
+ if (!result.ok) {
499
+ if (result.reason === "rate-limited") {
500
+ return { ok: false, reason: "rate-limited", message: "discord: rate limited" };
501
+ }
502
+ if (result.reason === "permission") {
503
+ warnings.push(`channel ${channel.name} skipped: missing permission`);
504
+ } else {
505
+ warnings.push(`channel ${channel.name} history failed: ${result.message}`);
506
+ }
507
+ break;
508
+ }
509
+ const messages = asArray(result.json);
510
+ if (messages.length === 0)
511
+ break;
512
+ let lastId;
513
+ for (const raw of messages) {
514
+ const message = asRecord(raw);
515
+ const id = asString(message?.id);
516
+ if (id !== undefined)
517
+ lastId = id;
518
+ if (documents.length >= maxDocuments)
519
+ continue;
520
+ const content = asString(message?.content);
521
+ if (message === undefined || id === undefined || content === undefined || content.length === 0)
522
+ continue;
523
+ const author = asString(asRecord(message.author)?.username) ?? "unknown";
524
+ documents.push({
525
+ docId: `${channel.id}-${id}`,
526
+ hierarchy: ["channels", channel.name],
527
+ title: `#${channel.name}`,
528
+ content: `${author}: ${content}`,
529
+ publishedAt: parseEpochMs(message.timestamp),
530
+ metadata: { channelId: channel.id, messageId: id, author }
531
+ });
532
+ }
533
+ if (lastId === undefined || messages.length < 100)
534
+ break;
535
+ before = lastId;
536
+ }
537
+ }
538
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
539
+ }
540
+ }
541
+ var DEFAULT_BASE_URL = "https://discord.com/api/v10", DEFAULT_TOKEN_ENV = "DISCORD_BOT_TOKEN", DEFAULT_MAX_PAGES = 10, DEFAULT_MAX_DOCUMENTS = 500, TEXT_CHANNEL_TYPES;
542
+ var init_connector = __esm(() => {
543
+ TEXT_CHANNEL_TYPES = new Set([0, 5]);
544
+ });
545
+
546
+ // src/datasource/connector.ts
547
+ import { createHash } from "node:crypto";
548
+ function connectorFailureToDiagnosticCode(reason) {
549
+ switch (reason) {
550
+ case "auth":
551
+ return "datasource-auth-error";
552
+ case "permission":
553
+ return "datasource-permission-denied";
554
+ case "rate-limited":
555
+ return "datasource-rate-limited";
556
+ case "unavailable":
557
+ case "not-configured":
558
+ return "datasource-unavailable";
559
+ case "empty":
560
+ return "datasource-empty";
561
+ case "api-error":
562
+ case "invalid-data":
563
+ return "datasource-index-failed";
564
+ }
565
+ }
566
+ function boundDiagnosticText(value) {
567
+ const trimmed = value.trim();
568
+ return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
569
+ }
570
+ function sanitizeIdSegment(value) {
571
+ const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
572
+ let start = 0;
573
+ while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
574
+ start += 1;
575
+ let end = normalized.length;
576
+ while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
577
+ end -= 1;
578
+ const cleaned = normalized.slice(start, end);
579
+ if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
580
+ return cleaned;
581
+ const hash = createHash("sha256").update(value).digest("hex").slice(0, 8);
582
+ const stem = cleaned.slice(0, 60);
583
+ return stem.length > 0 ? `${stem}-${hash}` : hash;
584
+ }
585
+ var init_connector2 = () => {};
586
+
587
+ // src/datasource/chunk-store.ts
588
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
589
+ import { dirname, join as join2 } from "node:path";
590
+ function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
591
+ return join2(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
592
+ }
593
+
594
+ class DatasourceChunkStore {
595
+ skillName;
596
+ instanceId;
597
+ storePath;
598
+ maxChunkChars;
599
+ chunkList = [];
600
+ loaded = false;
601
+ constructor(options) {
602
+ this.skillName = options.skillName;
603
+ this.instanceId = options.instanceId;
604
+ this.maxChunkChars = options.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS;
605
+ this.storePath = options.workspaceRoot === undefined ? undefined : datasourceChunkStorePath(options.workspaceRoot, options.skillName, options.instanceId);
606
+ }
607
+ replaceDocuments(documents) {
608
+ const chunks = [];
609
+ const seenIds = new Set;
610
+ for (const document of documents) {
611
+ const docSegment = sanitizeIdSegment(document.docId);
612
+ const hierarchy = (document.hierarchy ?? []).map(sanitizeIdSegment).filter((segment) => segment.length > 0);
613
+ const parts = splitContent(document.content, this.maxChunkChars);
614
+ for (const [index, part] of parts.entries()) {
615
+ let chunkId = parts.length === 1 ? docSegment : `${docSegment}-p${index + 1}`;
616
+ while (seenIds.has(chunkId))
617
+ chunkId = `${chunkId}x`;
618
+ seenIds.add(chunkId);
619
+ chunks.push({
620
+ chunkId,
621
+ docId: document.docId,
622
+ hierarchy,
623
+ ...document.title !== undefined ? { title: document.title } : {},
624
+ content: part,
625
+ ...document.publishedAt !== undefined ? { publishedAt: document.publishedAt } : {},
626
+ metadata: { ...document.metadata ?? {} }
627
+ });
628
+ }
629
+ }
630
+ this.chunkList = chunks;
631
+ this.loaded = true;
632
+ this.persist();
633
+ return chunks.length;
634
+ }
635
+ load() {
636
+ if (this.storePath === undefined)
637
+ return false;
638
+ try {
639
+ if (!existsSync(this.storePath))
640
+ return false;
641
+ const parsed = JSON.parse(readFileSync(this.storePath, "utf8"));
642
+ if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
643
+ return false;
644
+ this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
645
+ this.loaded = true;
646
+ return true;
647
+ } catch {
648
+ return false;
649
+ }
650
+ }
651
+ ensureLoaded() {
652
+ if (!this.loaded) {
653
+ this.load();
654
+ this.loaded = true;
655
+ }
656
+ }
657
+ chunks() {
658
+ this.ensureLoaded();
659
+ return this.chunkList;
660
+ }
661
+ hierarchies() {
662
+ this.ensureLoaded();
663
+ const seen = new Map;
664
+ for (const chunk of this.chunkList) {
665
+ seen.set(chunk.hierarchy.join("/"), chunk.hierarchy);
666
+ }
667
+ return [...seen.values()];
668
+ }
669
+ search(query, topK) {
670
+ this.ensureLoaded();
671
+ const queryTerms = [...new Set(tokenize(query))];
672
+ if (queryTerms.length === 0 || this.chunkList.length === 0 || topK <= 0)
673
+ return [];
674
+ const docCount = this.chunkList.length;
675
+ const tokenized = this.chunkList.map((chunk) => tokenize(`${chunk.title ?? ""} ${chunk.content}`));
676
+ const avgLength = tokenized.reduce((sum, tokens) => sum + tokens.length, 0) / docCount || 1;
677
+ const termFrequency = (term, tokens) => {
678
+ let tf = 0;
679
+ for (const token of tokens) {
680
+ if (token === term)
681
+ tf += 1;
682
+ else if (term.length >= 2 && token.startsWith(term))
683
+ tf += 0.75;
684
+ }
685
+ return tf;
686
+ };
687
+ const frequencies = queryTerms.map((term) => tokenized.map((tokens) => termFrequency(term, tokens)));
688
+ const documentFrequency = frequencies.map((perDoc) => perDoc.reduce((count, tf) => count + (tf > 0 ? 1 : 0), 0));
689
+ const k1 = 1.2;
690
+ const b = 0.75;
691
+ const scored = [];
692
+ for (const [index, chunk] of this.chunkList.entries()) {
693
+ const tokens = tokenized[index] ?? [];
694
+ let score = 0;
695
+ for (const [termIndex] of queryTerms.entries()) {
696
+ const df = documentFrequency[termIndex] ?? 0;
697
+ if (df === 0)
698
+ continue;
699
+ const tf = frequencies[termIndex]?.[index] ?? 0;
700
+ if (tf === 0)
701
+ continue;
702
+ const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
703
+ score += idf * tf * (k1 + 1) / (tf + k1 * (1 - b + b * tokens.length / avgLength));
704
+ }
705
+ if (score > 0)
706
+ scored.push({ chunk, score });
707
+ }
708
+ scored.sort((a, b2) => b2.score - a.score || a.chunk.chunkId.localeCompare(b2.chunk.chunkId));
709
+ return scored.slice(0, topK);
710
+ }
711
+ persist() {
712
+ if (this.storePath === undefined)
713
+ return;
714
+ try {
715
+ mkdirSync(dirname(this.storePath), { recursive: true });
716
+ const payload = {
717
+ version: STORE_VERSION,
718
+ skill: this.skillName,
719
+ instance: this.instanceId,
720
+ updatedAt: Date.now(),
721
+ chunks: this.chunkList
722
+ };
723
+ writeFileSync(this.storePath, `${JSON.stringify(payload)}
724
+ `, "utf8");
725
+ } catch {}
726
+ }
727
+ }
728
+ function splitContent(content, maxChars) {
729
+ const trimmed = content.trim();
730
+ if (trimmed.length === 0)
731
+ return [];
732
+ if (trimmed.length <= maxChars)
733
+ return [trimmed];
734
+ const parts = [];
735
+ const paragraphs = trimmed.split(/\n{2,}/u);
736
+ let current = "";
737
+ for (const paragraph of paragraphs) {
738
+ if (current.length > 0 && current.length + paragraph.length + 2 > maxChars) {
739
+ parts.push(current);
740
+ current = "";
741
+ }
742
+ if (paragraph.length > maxChars) {
743
+ if (current.length > 0) {
744
+ parts.push(current);
745
+ current = "";
746
+ }
747
+ for (let offset = 0;offset < paragraph.length; offset += maxChars) {
748
+ parts.push(paragraph.slice(offset, offset + maxChars));
749
+ }
750
+ continue;
751
+ }
752
+ current = current.length === 0 ? paragraph : `${current}
753
+
754
+ ${paragraph}`;
755
+ }
756
+ if (current.length > 0)
757
+ parts.push(current);
758
+ return parts.filter((part) => part.trim().length > 0);
759
+ }
760
+ function tokenize(text) {
761
+ return text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length > 0);
762
+ }
763
+ var STORE_VERSION = 1, DEFAULT_MAX_CHUNK_CHARS = 2000;
764
+ var init_chunk_store = __esm(() => {
765
+ init_connector2();
766
+ });
767
+
768
+ // src/retrieval/scope.ts
769
+ function normalizeVirtualPath(value) {
770
+ const normalized = value.replace(/\\/g, "/").replace(/\/+/g, "/").trim();
771
+ if (normalized.length === 0 || normalized === "/")
772
+ return "/";
773
+ const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`;
774
+ return withLeadingSlash.length > 1 ? withLeadingSlash.replace(/\/+$/g, "") : withLeadingSlash;
775
+ }
776
+ function normalizeVirtualPathScope(scope) {
777
+ if (scope === undefined)
778
+ return;
779
+ const normalized = normalizeVirtualPath(scope);
780
+ return normalized === "/" ? undefined : normalized;
781
+ }
782
+ function matchesVirtualPathScope(virtualPath, scope) {
783
+ const normalizedScope = normalizeVirtualPathScope(scope);
784
+ if (normalizedScope === undefined)
785
+ return true;
786
+ return virtualPathScopeToRegExp(normalizedScope).test(normalizeVirtualPath(virtualPath));
787
+ }
788
+ function virtualPathScopeToRegExp(scope) {
789
+ const normalized = normalizeVirtualPath(scope);
790
+ const scoped = hasGlob(normalized) || looksLikeFileScope(normalized) ? normalized : `${normalized}/**`;
791
+ const pattern = scoped.split("/").map((segment, index) => {
792
+ if (index === 0)
793
+ return "";
794
+ if (segment === "**")
795
+ return "(?:.*)";
796
+ return escapeRegExp(segment).replace(/\\\*/g, "[^/]*");
797
+ }).join("/").replace(/\/\(\?:\.\*\)$/u, "(?:/.*)?");
798
+ return new RegExp(`^${pattern}$`);
799
+ }
800
+ function hasGlob(scope) {
801
+ return scope.includes("*");
802
+ }
803
+ function looksLikeFileScope(scope) {
804
+ const finalSegment = scope.split("/").at(-1) ?? "";
805
+ return /\.[^/.]+$/u.test(finalSegment);
806
+ }
807
+ function escapeRegExp(literal) {
808
+ return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
809
+ }
810
+
811
+ // src/datasource/scope.ts
812
+ function buildDatasourceInstanceSource(skillName, instanceId) {
813
+ return normalizeVirtualPath(`/${skillName}/${instanceId}`);
814
+ }
815
+ function buildDatasourceChunkSource(skillName, instanceId, chunkId) {
816
+ return normalizeVirtualPath(`/${skillName}/${instanceId}/${DATASOURCE_CHUNKS_SEGMENT}/${chunkId}`);
817
+ }
818
+ function datasourceSourcePath(skillName, instanceId, chunkId) {
819
+ return chunkId === undefined ? buildDatasourceInstanceSource(skillName, instanceId) : buildDatasourceChunkSource(skillName, instanceId, chunkId);
820
+ }
821
+ function datasourceSourceHasFragment(source) {
822
+ return typeof source === "string" && source.includes("#");
823
+ }
824
+ function isDatasourceSource(source) {
825
+ if (typeof source !== "string" || source.length === 0)
826
+ return false;
827
+ if (datasourceSourceHasFragment(source))
828
+ return false;
829
+ const normalized = normalizeVirtualPath(source);
830
+ return normalized !== "/";
831
+ }
832
+ function matchesDatasourceScope(source, scope) {
833
+ if (datasourceSourceHasFragment(source))
834
+ return false;
835
+ return matchesVirtualPathScope(source, scope);
836
+ }
837
+ var DATASOURCE_CHUNKS_SEGMENT = "chunks";
838
+ var init_scope = () => {};
839
+
840
+ // src/datasource/connector-skill.ts
841
+ class ConnectorDatasourceSkill {
842
+ definition;
843
+ connector;
844
+ instanceId;
845
+ instances;
846
+ pollingIntervalMs;
847
+ tags;
848
+ store;
849
+ dedupeWindowMs;
850
+ lastIndexedAt;
851
+ lastPolledAt;
852
+ lastError;
853
+ seenDocs = new Map;
854
+ constructor(definition, options) {
855
+ this.definition = definition;
856
+ this.connector = options.connector;
857
+ this.instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID;
858
+ this.instances = options.instances !== undefined && options.instances.length > 0 ? options.instances : [this.instanceId];
859
+ this.pollingIntervalMs = options.pollingIntervalMs ?? DEFAULT_POLLING_INTERVAL_MS;
860
+ this.tags = options.tags ?? definition.defaultTags;
861
+ this.lastIndexedAt = options.lastIndexedAt;
862
+ this.dedupeWindowMs = options.dedupeWindowMs ?? 0;
863
+ this.store = new DatasourceChunkStore({
864
+ skillName: definition.skillName,
865
+ instanceId: this.instanceId,
866
+ workspaceRoot: options.workspaceRoot,
867
+ maxChunkChars: options.maxChunkChars
868
+ });
869
+ }
870
+ describe() {
871
+ return {
872
+ name: this.definition.skillName,
873
+ id: this.definition.skillName,
874
+ type: this.definition.skillType,
875
+ description: this.definition.description,
876
+ capabilities: this.definition.capabilities,
877
+ tags: this.tags,
878
+ status: "active",
879
+ datasourceId: this.definition.skillName,
880
+ instanceId: this.instanceId,
881
+ instances: this.instances
882
+ };
883
+ }
884
+ polling() {
885
+ return {
886
+ mode: this.pollingIntervalMs > 0 ? "poll" : "none",
887
+ ...this.pollingIntervalMs > 0 ? { intervalMs: this.pollingIntervalMs } : {},
888
+ lastIndexedAt: this.lastIndexedAt,
889
+ ...this.lastPolledAt !== undefined ? { lastPolledAt: this.lastPolledAt } : {},
890
+ ...this.lastError !== undefined ? { lastError: this.lastError } : {}
891
+ };
892
+ }
893
+ async index() {
894
+ this.lastPolledAt = Date.now();
895
+ let fetched;
896
+ try {
897
+ fetched = await this.connector.fetch();
898
+ } catch {
899
+ return this.fail("datasource-unavailable", "connector failed unexpectedly");
900
+ }
901
+ if (!fetched.ok) {
902
+ const code = connectorFailureToDiagnosticCode(fetched.reason);
903
+ const message = boundDiagnosticText(fetched.message ?? fetched.reason);
904
+ return this.fail(code, `${fetched.reason}: ${message}`);
905
+ }
906
+ const documents = this.applyDedupeWindow(fetched.documents);
907
+ const chunkCount = this.store.replaceDocuments(documents);
908
+ this.lastIndexedAt = Date.now();
909
+ this.lastError = undefined;
910
+ const diagnostics = (fetched.warnings ?? []).map((warning) => ({
911
+ code: "datasource-index-failed",
912
+ severity: "warning",
913
+ message: boundDiagnosticText(warning),
914
+ instanceId: this.instanceId,
915
+ source: this.definition.skillName
916
+ }));
917
+ if (chunkCount === 0) {
918
+ diagnostics.push({
919
+ code: "datasource-empty",
920
+ severity: "info",
921
+ message: "datasource fetch returned no indexable documents",
922
+ instanceId: this.instanceId,
923
+ source: this.definition.skillName
924
+ });
925
+ }
926
+ return {
927
+ ok: true,
928
+ instanceId: this.instanceId,
929
+ skill: this.definition.skillName,
930
+ chunkCount,
931
+ indexedAt: this.lastIndexedAt,
932
+ diagnostics
933
+ };
934
+ }
935
+ retrievalMethods() {
936
+ return [
937
+ new ConnectorLexicalMethod({
938
+ skillName: this.definition.skillName,
939
+ skillType: this.definition.skillType,
940
+ instanceId: this.instanceId,
941
+ tags: this.tags,
942
+ store: this.store
943
+ })
944
+ ];
945
+ }
946
+ describeSources() {
947
+ const out = this.instances.map((instanceId) => ({
948
+ source: datasourceSourcePath(this.definition.skillName, instanceId),
949
+ datasourceId: this.definition.skillName,
950
+ skill: this.definition.skillName,
951
+ instanceId,
952
+ contentType: this.definition.contentType,
953
+ metadata: {
954
+ datasourceId: this.definition.skillName,
955
+ instanceId,
956
+ tags: this.tags
957
+ }
958
+ }));
959
+ for (const hierarchy of this.store.hierarchies()) {
960
+ if (hierarchy.length === 0)
961
+ continue;
962
+ out.push({
963
+ source: `${datasourceSourcePath(this.definition.skillName, this.instanceId)}/${hierarchy.join("/")}`,
964
+ datasourceId: this.definition.skillName,
965
+ skill: this.definition.skillName,
966
+ instanceId: this.instanceId,
967
+ contentType: this.definition.contentType,
968
+ metadata: {
969
+ datasourceId: this.definition.skillName,
970
+ instanceId: this.instanceId,
971
+ hierarchy
972
+ }
973
+ });
974
+ }
975
+ return out;
976
+ }
977
+ skillManifest() {
978
+ const skillName = this.definition.skillName;
979
+ const instanceScopes = this.instances.map((instanceId) => `- \`${datasourceSourcePath(skillName, instanceId)}\``).join(`
980
+ `);
981
+ const cadence = this.pollingIntervalMs > 0 ? `roughly every ${Math.round(this.pollingIntervalMs / 60000)} minute(s) when auto-refresh runs` : "on manual refresh only";
982
+ return {
983
+ name: `datasource-${skillName}`,
984
+ description: this.definition.manifestDescription,
985
+ content: [
986
+ `# ${this.definition.description} (${this.definition.skillType})`,
987
+ "",
988
+ "## When to use",
989
+ this.definition.manifestDescription,
990
+ "",
991
+ "## Indexing",
992
+ `Indexing is server-managed and refreshed ${cadence}. You do not trigger indexing; just search.`,
993
+ "",
994
+ "## How to search",
995
+ "Call `search_datasource_documents` with a natural-language `query`. Optionally pass `topK` and a narrowing `scope`. Available authorized scopes:",
996
+ instanceScopes.length > 0 ? instanceScopes : "- (no authorized instances)",
997
+ "",
998
+ "`scope` can only narrow within already-authorized scopes; it can never widen access.",
999
+ "",
1000
+ "## Output rules",
1001
+ `Datasource source identifiers such as \`/${skillName}/<instance>/chunks/<id>\` are stable and traceable. Result metadata may carry real file paths or account identifiers; you may cite them in the visible answer when they help the user locate the underlying item. Privacy is the operator's responsibility: run AutoRAG with a local LLM if results must not leave this machine.`,
1002
+ ...this.definition.manifestNotes !== undefined && this.definition.manifestNotes.length > 0 ? ["", ...this.definition.manifestNotes] : []
1003
+ ].join(`
1004
+ `)
1005
+ };
1006
+ }
1007
+ applyDedupeWindow(documents) {
1008
+ if (this.dedupeWindowMs <= 0)
1009
+ return documents;
1010
+ const now = Date.now();
1011
+ for (const [docId, seenAt] of this.seenDocs) {
1012
+ if (now - seenAt > this.dedupeWindowMs)
1013
+ this.seenDocs.delete(docId);
1014
+ }
1015
+ const out = [];
1016
+ const inBatch = new Set;
1017
+ for (const document of documents) {
1018
+ if (inBatch.has(document.docId))
1019
+ continue;
1020
+ inBatch.add(document.docId);
1021
+ this.seenDocs.set(document.docId, now);
1022
+ out.push(document);
1023
+ }
1024
+ return out;
1025
+ }
1026
+ fail(code, message) {
1027
+ const sanitized = boundDiagnosticText(message);
1028
+ this.lastError = sanitized;
1029
+ const diagnostic = {
1030
+ code,
1031
+ severity: code === "datasource-unavailable" || code === "datasource-empty" ? "warning" : "error",
1032
+ message: sanitized,
1033
+ instanceId: this.instanceId,
1034
+ source: this.definition.skillName
1035
+ };
1036
+ return {
1037
+ ok: false,
1038
+ instanceId: this.instanceId,
1039
+ skill: this.definition.skillName,
1040
+ indexedAt: Date.now(),
1041
+ diagnostics: [diagnostic],
1042
+ error: sanitized,
1043
+ code,
1044
+ message: sanitized
1045
+ };
1046
+ }
1047
+ }
1048
+
1049
+ class ConnectorLexicalMethod {
1050
+ options;
1051
+ constructor(options) {
1052
+ this.options = options;
1053
+ }
1054
+ describe() {
1055
+ const { skillName, skillType, tags } = this.options;
1056
+ return {
1057
+ name: `${skillName}-lexical`,
1058
+ type: "bm25",
1059
+ description: `Lexical retrieval over indexed ${skillType} chunks`,
1060
+ status: "active",
1061
+ capabilities: ["lexical", "scoped", "path-opaque-sources"],
1062
+ datasourceId: skillName,
1063
+ tags: [...tags]
1064
+ };
1065
+ }
1066
+ async retrieve(query, options) {
1067
+ const trimmed = query.trim();
1068
+ if (trimmed.length === 0)
1069
+ return [];
1070
+ const topK = options.topK ?? DEFAULT_TOP_K;
1071
+ let hits;
1072
+ try {
1073
+ hits = this.options.store.search(trimmed, Math.max(topK * 3, topK));
1074
+ } catch {
1075
+ return [];
1076
+ }
1077
+ const { skillName, instanceId } = this.options;
1078
+ const mapped = [];
1079
+ for (const { chunk, score } of hits) {
1080
+ const source = datasourceSourcePath(skillName, instanceId, chunk.chunkId);
1081
+ if (!this.matchesScope(source, options.scope, options.allowedScopes))
1082
+ continue;
1083
+ mapped.push({
1084
+ id: `${skillName}:${instanceId}:${chunk.chunkId}`,
1085
+ content: chunk.content,
1086
+ source,
1087
+ score,
1088
+ metadata: {
1089
+ ...chunk.metadata,
1090
+ method: `${skillName}-lexical`,
1091
+ datasourceId: skillName,
1092
+ instanceId,
1093
+ chunkId: chunk.chunkId,
1094
+ ...chunk.title !== undefined ? { title: chunk.title } : {},
1095
+ ...chunk.hierarchy.length > 0 ? { hierarchy: chunk.hierarchy.join("/") } : {},
1096
+ ...chunk.publishedAt !== undefined ? { publishedAt: chunk.publishedAt } : {}
1097
+ }
1098
+ });
1099
+ if (mapped.length >= topK)
1100
+ break;
1101
+ }
1102
+ return mapped;
1103
+ }
1104
+ matchesScope(source, scope, allowedScopes) {
1105
+ if (!matchesDatasourceScope(source, scope))
1106
+ return false;
1107
+ if (allowedScopes === undefined || allowedScopes.length === 0)
1108
+ return true;
1109
+ return allowedScopes.some((entry) => matchesDatasourceScope(source, entry));
1110
+ }
1111
+ }
1112
+ var DEFAULT_POLLING_INTERVAL_MS, DEFAULT_INSTANCE_ID = "default", DEFAULT_TOP_K = 20;
1113
+ var init_connector_skill = __esm(() => {
1114
+ init_chunk_store();
1115
+ init_connector2();
1116
+ init_scope();
1117
+ DEFAULT_POLLING_INTERVAL_MS = 15 * 60 * 1000;
1118
+ });
1119
+
1120
+ // src/datasource/skills/discord/skill.ts
1121
+ var DISCORD_SKILL_DEFINITION, DiscordSkill;
1122
+ var init_skill = __esm(() => {
1123
+ init_connector_skill();
1124
+ init_connector();
1125
+ DISCORD_SKILL_DEFINITION = {
1126
+ skillName: "discord",
1127
+ skillType: "discord-guild",
1128
+ description: "Discord guild datasource",
1129
+ capabilities: ["chat", "api", "polling"],
1130
+ defaultTags: ["discord", "chat", "pii"],
1131
+ contentType: "chat",
1132
+ manifestDescription: "Search indexed Discord messages across authorized guild channels and threads. Use for questions about Discord conversations or community discussions.",
1133
+ manifestNotes: ["Channel visibility and history windows are bounded by the server-configured bot permissions."]
1134
+ };
1135
+ DiscordSkill = class DiscordSkill extends ConnectorDatasourceSkill {
1136
+ constructor(options = {}) {
1137
+ const { connector, connectorOptions, ...rest } = options;
1138
+ super(DISCORD_SKILL_DEFINITION, {
1139
+ ...rest,
1140
+ connector: connector ?? new DiscordConnector(connectorOptions ?? {})
1141
+ });
1142
+ }
1143
+ };
1144
+ });
1145
+
1146
+ // src/datasource/skills/discord/index.ts
1147
+ var init_discord = __esm(() => {
1148
+ init_connector();
1149
+ init_skill();
1150
+ });
1151
+
1152
+ // src/datasource/skills/gdrive/connector.ts
1153
+ class GDriveConnector {
1154
+ options;
1155
+ constructor(options = {}) {
1156
+ this.options = options;
1157
+ }
1158
+ async fetch(signal) {
1159
+ const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV2);
1160
+ if (token === undefined)
1161
+ return { ok: false, reason: "not-configured", message: "auth token not configured" };
1162
+ const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL2;
1163
+ const request = {
1164
+ headers: { Authorization: `Bearer ${token}` },
1165
+ timeoutMs: this.options.timeoutMs,
1166
+ fetchImpl: this.options.fetchImpl,
1167
+ signal
1168
+ };
1169
+ const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES2;
1170
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS2;
1171
+ const files = [];
1172
+ let pageToken;
1173
+ for (let page = 0;page < maxPages; page += 1) {
1174
+ const url = new URL(`${baseUrl}/files`);
1175
+ url.searchParams.set("pageSize", "100");
1176
+ url.searchParams.set("fields", "nextPageToken,files(id,name,mimeType,modifiedTime,parents)");
1177
+ const query = this.options.folderId !== undefined ? `trashed=false and '${this.options.folderId}' in parents` : "trashed=false";
1178
+ url.searchParams.set("q", query);
1179
+ if (this.options.includeSharedDrives === true) {
1180
+ url.searchParams.set("supportsAllDrives", "true");
1181
+ url.searchParams.set("includeItemsFromAllDrives", "true");
1182
+ url.searchParams.set("corpora", "allDrives");
1183
+ }
1184
+ if (pageToken !== undefined)
1185
+ url.searchParams.set("pageToken", pageToken);
1186
+ const result = await httpJson(url.toString(), request);
1187
+ if (!result.ok)
1188
+ return { ok: false, reason: result.reason, message: `file list failed: ${result.message}` };
1189
+ const envelope = asRecord(result.json);
1190
+ if (envelope === undefined)
1191
+ return { ok: false, reason: "invalid-data", message: "file list malformed" };
1192
+ for (const raw of asArray(envelope.files)) {
1193
+ const file = asRecord(raw);
1194
+ const id = asString(file?.id);
1195
+ const name = asString(file?.name);
1196
+ const mimeType = asString(file?.mimeType);
1197
+ if (id === undefined || name === undefined || mimeType === undefined)
1198
+ continue;
1199
+ files.push({ id, name, mimeType, modifiedTime: asString(file?.modifiedTime) });
1200
+ }
1201
+ pageToken = asString(envelope.nextPageToken);
1202
+ if (pageToken === undefined || pageToken.length === 0)
1203
+ break;
1204
+ }
1205
+ const documents = [];
1206
+ const warnings = [];
1207
+ for (const file of files) {
1208
+ if (documents.length >= maxDocuments)
1209
+ break;
1210
+ const exportMime = EXPORT_MIME[file.mimeType];
1211
+ let contentUrl;
1212
+ if (exportMime !== undefined) {
1213
+ contentUrl = `${baseUrl}/files/${file.id}/export?mimeType=${encodeURIComponent(exportMime)}`;
1214
+ } else if (DOWNLOAD_MIME.has(file.mimeType)) {
1215
+ contentUrl = `${baseUrl}/files/${file.id}?alt=media`;
1216
+ } else {
1217
+ continue;
1218
+ }
1219
+ const contentResult = await httpText(contentUrl, request);
1220
+ if (!contentResult.ok) {
1221
+ warnings.push(`file export failed: ${contentResult.message}`);
1222
+ continue;
1223
+ }
1224
+ const content = contentResult.text.trim().slice(0, MAX_CONTENT_CHARS);
1225
+ if (content.length === 0)
1226
+ continue;
1227
+ documents.push({
1228
+ docId: file.id,
1229
+ hierarchy: ["files"],
1230
+ title: file.name,
1231
+ content,
1232
+ publishedAt: parseEpochMs(file.modifiedTime),
1233
+ metadata: {
1234
+ mimeType: file.mimeType,
1235
+ ...file.modifiedTime !== undefined ? { modifiedTime: file.modifiedTime } : {}
1236
+ }
1237
+ });
1238
+ }
1239
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
1240
+ }
1241
+ }
1242
+ var DEFAULT_BASE_URL2 = "https://www.googleapis.com/drive/v3", DEFAULT_TOKEN_ENV2 = "GDRIVE_ACCESS_TOKEN", DEFAULT_MAX_PAGES2 = 10, DEFAULT_MAX_DOCUMENTS2 = 500, MAX_CONTENT_CHARS = 1e5, EXPORT_MIME, DOWNLOAD_MIME;
1243
+ var init_connector3 = __esm(() => {
1244
+ EXPORT_MIME = {
1245
+ "application/vnd.google-apps.document": "text/plain",
1246
+ "application/vnd.google-apps.spreadsheet": "text/csv"
1247
+ };
1248
+ DOWNLOAD_MIME = new Set(["text/plain", "text/markdown", "text/csv", "application/json"]);
1249
+ });
1250
+
1251
+ // src/datasource/skills/gdrive/rclone-connector.ts
1252
+ import { spawn } from "node:child_process";
1253
+
1254
+ class RcloneConnector {
1255
+ options;
1256
+ runner;
1257
+ constructor(options = {}) {
1258
+ this.options = options;
1259
+ this.runner = options.runner ?? ((args, timeoutMs) => runBinary(options.binaryPath ?? DEFAULT_BINARY, args, timeoutMs));
1260
+ }
1261
+ async fetch() {
1262
+ const remote = this.options.remote;
1263
+ if (remote === undefined || remote.length === 0) {
1264
+ return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
1265
+ }
1266
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
1267
+ const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
1268
+ const extensions = this.options.extensions ?? DEFAULT_EXTENSIONS;
1269
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS3;
1270
+ const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE;
1271
+ let listResult;
1272
+ try {
1273
+ listResult = await this.runner(["lsjson", remote, "--recursive", "--files-only", "--drive-export-formats", exportFormats], timeoutMs);
1274
+ } catch {
1275
+ return { ok: false, reason: "unavailable", message: "rclone binary not found or failed to spawn" };
1276
+ }
1277
+ if (!listResult.ok) {
1278
+ return { ok: false, reason: classifyFailure(listResult.stderr), message: shortFailure(listResult) };
1279
+ }
1280
+ let entries;
1281
+ try {
1282
+ entries = asArray(JSON.parse(listResult.stdout));
1283
+ } catch {
1284
+ return { ok: false, reason: "invalid-data", message: "rclone listing was not valid JSON" };
1285
+ }
1286
+ const documents = [];
1287
+ let readFailures = 0;
1288
+ let skipped = 0;
1289
+ for (const raw of entries) {
1290
+ if (documents.length >= maxDocuments)
1291
+ break;
1292
+ const entry = asRecord(raw);
1293
+ const path = asString(entry?.Path);
1294
+ const name = asString(entry?.Name);
1295
+ if (entry === undefined || path === undefined || name === undefined)
1296
+ continue;
1297
+ const extension = name.includes(".") ? name.slice(name.lastIndexOf(".")).toLowerCase() : "";
1298
+ if (!extensions.includes(extension)) {
1299
+ skipped += 1;
1300
+ continue;
1301
+ }
1302
+ const size = typeof entry.Size === "number" ? entry.Size : 0;
1303
+ if (size > maxBytes) {
1304
+ skipped += 1;
1305
+ continue;
1306
+ }
1307
+ const target = remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
1308
+ let content = "";
1309
+ try {
1310
+ const catResult = await this.runner(["cat", target, "--drive-export-formats", exportFormats, "--count", String(maxBytes)], timeoutMs);
1311
+ if (!catResult.ok) {
1312
+ readFailures += 1;
1313
+ continue;
1314
+ }
1315
+ content = catResult.stdout.trim().slice(0, MAX_CONTENT_CHARS2);
1316
+ } catch {
1317
+ readFailures += 1;
1318
+ continue;
1319
+ }
1320
+ if (content.length === 0)
1321
+ continue;
1322
+ const segments = path.split("/");
1323
+ documents.push({
1324
+ docId: path,
1325
+ hierarchy: ["files", ...segments.slice(0, -1)],
1326
+ title: name,
1327
+ content,
1328
+ publishedAt: parseEpochMs(asString(entry.ModTime)),
1329
+ metadata: {
1330
+ ...asString(entry.MimeType) !== undefined ? { mimeType: asString(entry.MimeType) } : {},
1331
+ size
1332
+ }
1333
+ });
1334
+ }
1335
+ const warnings = [];
1336
+ if (readFailures > 0)
1337
+ warnings.push(`${readFailures} file(s) failed to read`);
1338
+ if (skipped > 0)
1339
+ warnings.push(`${skipped} file(s) skipped (non-text or oversized)`);
1340
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
1341
+ }
1342
+ }
1343
+ function classifyFailure(stderr) {
1344
+ const lower = stderr.toLowerCase();
1345
+ if (lower.includes("didn't find section") || lower.includes("config file") || lower.includes("no remotes")) {
1346
+ return "not-configured";
1347
+ }
1348
+ if (lower.includes("oauth") || lower.includes("token") || lower.includes("unauthorized") || lower.includes("401")) {
1349
+ return "auth";
1350
+ }
1351
+ if (lower.includes("permission") || lower.includes("403") || lower.includes("forbidden"))
1352
+ return "permission";
1353
+ return "api-error";
1354
+ }
1355
+ function shortFailure(result) {
1356
+ return `rclone exited with code ${result.code ?? "unknown"}`;
1357
+ }
1358
+ function runBinary(binary, args, timeoutMs) {
1359
+ return new Promise((resolvePromise) => {
1360
+ const child = spawn(binary, args, { stdio: ["ignore", "pipe", "pipe"] });
1361
+ let stdout = "";
1362
+ let stderr = "";
1363
+ const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
1364
+ child.stdout.on("data", (chunk) => {
1365
+ stdout += chunk.toString("utf8");
1366
+ });
1367
+ child.stderr.on("data", (chunk) => {
1368
+ stderr += chunk.toString("utf8");
1369
+ });
1370
+ child.on("error", () => {
1371
+ clearTimeout(timer);
1372
+ resolvePromise({ ok: false, stdout, stderr, code: null });
1373
+ });
1374
+ child.on("close", (code) => {
1375
+ clearTimeout(timer);
1376
+ resolvePromise({ ok: code === 0, stdout, stderr, code });
1377
+ });
1378
+ });
1379
+ }
1380
+ var DEFAULT_BINARY = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS3 = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS2 = 60000, MAX_CONTENT_CHARS2 = 1e5;
1381
+ var init_rclone_connector = __esm(() => {
1382
+ DEFAULT_EXTENSIONS = [".txt", ".md", ".csv", ".json", ".html", ".rst", ".org"];
1383
+ DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
1384
+ });
1385
+
1386
+ // src/datasource/skills/gdrive/skill.ts
1387
+ var GDRIVE_SKILL_DEFINITION, GDriveSkill;
1388
+ var init_skill2 = __esm(() => {
1389
+ init_connector_skill();
1390
+ init_connector3();
1391
+ GDRIVE_SKILL_DEFINITION = {
1392
+ skillName: "gdrive",
1393
+ skillType: "google-drive",
1394
+ description: "Google Drive datasource",
1395
+ capabilities: ["documents", "api", "polling"],
1396
+ defaultTags: ["gdrive", "documents", "pii"],
1397
+ contentType: "document",
1398
+ manifestDescription: "Search indexed Google Drive documents (Docs, Sheets, plain text). Use for questions about files stored in the authorized Drive account or folder.",
1399
+ manifestNotes: [
1400
+ "Shared-drive permissions are enforced by the server-configured OAuth token; unexported binary formats are not indexed."
1401
+ ]
1402
+ };
1403
+ GDriveSkill = class GDriveSkill extends ConnectorDatasourceSkill {
1404
+ constructor(options = {}) {
1405
+ const { connector, connectorOptions, ...rest } = options;
1406
+ super(GDRIVE_SKILL_DEFINITION, {
1407
+ ...rest,
1408
+ connector: connector ?? new GDriveConnector(connectorOptions ?? {})
1409
+ });
1410
+ }
1411
+ };
1412
+ });
1413
+
1414
+ // src/datasource/skills/gdrive/index.ts
1415
+ var init_gdrive = __esm(() => {
1416
+ init_connector3();
1417
+ init_rclone_connector();
1418
+ init_skill2();
1419
+ });
1420
+
1421
+ // src/datasource/skills/github/connector.ts
1422
+ class GitHubConnector {
1423
+ options;
1424
+ constructor(options = {}) {
1425
+ this.options = options;
1426
+ }
1427
+ async fetch(signal) {
1428
+ const repos = this.options.repos ?? [];
1429
+ if (repos.length === 0)
1430
+ return { ok: false, reason: "not-configured", message: "no repositories configured" };
1431
+ const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV3);
1432
+ const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL3;
1433
+ const request = {
1434
+ headers: {
1435
+ Accept: "application/vnd.github+json",
1436
+ "X-GitHub-Api-Version": "2022-11-28",
1437
+ "User-Agent": "autorag-datasource",
1438
+ ...token !== undefined ? { Authorization: `Bearer ${token}` } : {}
1439
+ },
1440
+ timeoutMs: this.options.timeoutMs,
1441
+ fetchImpl: this.options.fetchImpl,
1442
+ signal
1443
+ };
1444
+ const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES3;
1445
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4;
1446
+ const documents = [];
1447
+ const warnings = [];
1448
+ for (const repo of repos) {
1449
+ if (documents.length >= maxDocuments)
1450
+ break;
1451
+ const [owner, name] = repo.split("/", 2);
1452
+ if (owner === undefined || name === undefined || owner.length === 0 || name.length === 0) {
1453
+ warnings.push(`repository entry ${repos.indexOf(repo) + 1} is not owner/repo`);
1454
+ continue;
1455
+ }
1456
+ for (let page = 1;page <= maxPages && documents.length < maxDocuments; page += 1) {
1457
+ const url = `${baseUrl}/repos/${owner}/${name}/issues?state=all&per_page=100&page=${page}`;
1458
+ const result = await httpJson(url, request);
1459
+ if (!result.ok) {
1460
+ if (result.reason === "auth")
1461
+ return { ok: false, reason: "auth", message: "github: unauthorized" };
1462
+ if (result.reason === "rate-limited" || result.reason === "permission") {
1463
+ return { ok: false, reason: "rate-limited", message: "github: rate limited or forbidden" };
1464
+ }
1465
+ warnings.push(`repo ${owner}-${name} fetch failed: ${result.message}`);
1466
+ break;
1467
+ }
1468
+ const items = asArray(result.json);
1469
+ if (items.length === 0)
1470
+ break;
1471
+ for (const raw of items) {
1472
+ if (documents.length >= maxDocuments)
1473
+ break;
1474
+ const item = asRecord(raw);
1475
+ const number = asNumber(item?.number);
1476
+ const title = asString(item?.title);
1477
+ if (item === undefined || number === undefined || title === undefined)
1478
+ continue;
1479
+ const isPull = asRecord(item.pull_request) !== undefined;
1480
+ const body = asString(item.body) ?? "";
1481
+ const labels = asArray(item.labels).map((label) => asString(asRecord(label)?.name)).filter((label) => label !== undefined);
1482
+ documents.push({
1483
+ docId: `${owner}-${name}-${number}`,
1484
+ hierarchy: [owner, name, isPull ? "pulls" : "issues"],
1485
+ title: `#${number} ${title}`,
1486
+ content: body.length > 0 ? `${title}
1487
+
1488
+ ${body}` : title,
1489
+ publishedAt: parseEpochMs(item.updated_at),
1490
+ metadata: {
1491
+ state: asString(item.state) ?? "unknown",
1492
+ number,
1493
+ kind: isPull ? "pull" : "issue",
1494
+ labels
1495
+ }
1496
+ });
1497
+ }
1498
+ if (items.length < 100)
1499
+ break;
1500
+ }
1501
+ }
1502
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
1503
+ }
1504
+ }
1505
+ var DEFAULT_BASE_URL3 = "https://api.github.com", DEFAULT_TOKEN_ENV3 = "GITHUB_TOKEN", DEFAULT_MAX_PAGES3 = 10, DEFAULT_MAX_DOCUMENTS4 = 500;
1506
+ var init_connector4 = () => {};
1507
+
1508
+ // src/datasource/skills/github/skill.ts
1509
+ var GITHUB_SKILL_DEFINITION, GitHubSkill;
1510
+ var init_skill3 = __esm(() => {
1511
+ init_connector_skill();
1512
+ init_connector4();
1513
+ GITHUB_SKILL_DEFINITION = {
1514
+ skillName: "github",
1515
+ skillType: "github-issues",
1516
+ description: "GitHub Issues/PRs datasource",
1517
+ capabilities: ["issues", "api", "polling"],
1518
+ defaultTags: ["github", "issues"],
1519
+ contentType: "issue",
1520
+ manifestDescription: "Search indexed GitHub issues and pull requests for the configured repositories. Use for questions about bug reports, feature requests, and review discussions.",
1521
+ manifestNotes: ["Repository visibility is bounded by the server-configured token scopes."]
1522
+ };
1523
+ GitHubSkill = class GitHubSkill extends ConnectorDatasourceSkill {
1524
+ constructor(options = {}) {
1525
+ const { connector, connectorOptions, ...rest } = options;
1526
+ super(GITHUB_SKILL_DEFINITION, {
1527
+ ...rest,
1528
+ connector: connector ?? new GitHubConnector(connectorOptions ?? {})
1529
+ });
1530
+ }
1531
+ };
1532
+ });
1533
+
1534
+ // src/datasource/skills/github/index.ts
1535
+ var init_github = __esm(() => {
1536
+ init_connector4();
1537
+ init_skill3();
1538
+ });
1539
+
1540
+ // src/datasource/skills/gmail/himalaya-connector.ts
1541
+ import { spawn as spawn2 } from "node:child_process";
1542
+
1543
+ class HimalayaConnector {
1544
+ options;
1545
+ runner;
1546
+ constructor(options = {}) {
1547
+ this.options = options;
1548
+ this.runner = options.runner ?? ((args, timeoutMs) => runBinary2(options.binaryPath ?? DEFAULT_BINARY2, args, timeoutMs));
1549
+ }
1550
+ async fetch() {
1551
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS3;
1552
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS5;
1553
+ const accountArgs = this.options.account !== undefined ? ["--account", this.options.account] : [];
1554
+ const folderArgs = this.options.folder !== undefined ? ["--folder", this.options.folder] : [];
1555
+ const listArgs = [
1556
+ "envelope",
1557
+ "list",
1558
+ ...accountArgs,
1559
+ ...folderArgs,
1560
+ "--page-size",
1561
+ String(this.options.pageSize ?? DEFAULT_PAGE_SIZE),
1562
+ "--output",
1563
+ "json"
1564
+ ];
1565
+ let listResult;
1566
+ try {
1567
+ listResult = await this.runner(listArgs, timeoutMs);
1568
+ } catch {
1569
+ return { ok: false, reason: "unavailable", message: "himalaya binary not found or failed to spawn" };
1570
+ }
1571
+ if (!listResult.ok) {
1572
+ return { ok: false, reason: classifyFailure2(listResult.stderr), message: shortFailure2(listResult) };
1573
+ }
1574
+ let envelopes;
1575
+ try {
1576
+ envelopes = asArray(JSON.parse(listResult.stdout));
1577
+ } catch {
1578
+ return { ok: false, reason: "invalid-data", message: "envelope listing was not valid JSON" };
1579
+ }
1580
+ const account = this.options.account ?? "default";
1581
+ const folder = this.options.folder ?? "INBOX";
1582
+ const documents = [];
1583
+ let readFailures = 0;
1584
+ for (const raw of envelopes.slice(0, maxDocuments)) {
1585
+ const envelope = asRecord(raw);
1586
+ const id = asString(envelope?.id);
1587
+ if (envelope === undefined || id === undefined)
1588
+ continue;
1589
+ const subject = asString(envelope.subject) ?? "(no subject)";
1590
+ const from = asRecord(envelope.from);
1591
+ const fromText = [asString(from?.name), asString(from?.addr)].filter(Boolean).join(" ").trim();
1592
+ let body = "";
1593
+ try {
1594
+ const readResult = await this.runner(["message", "read", ...accountArgs, ...folderArgs, id], timeoutMs);
1595
+ if (readResult.ok)
1596
+ body = readResult.stdout.trim().slice(0, MAX_BODY_CHARS);
1597
+ else
1598
+ readFailures += 1;
1599
+ } catch {
1600
+ readFailures += 1;
1601
+ }
1602
+ const headerBlock = [
1603
+ `Subject: ${subject}`,
1604
+ ...fromText.length > 0 ? [`From: ${fromText}`] : [],
1605
+ ...asString(envelope.date) !== undefined ? [`Date: ${asString(envelope.date)}`] : []
1606
+ ].join(`
1607
+ `);
1608
+ documents.push({
1609
+ docId: `${account}-${folder}-${id}`,
1610
+ hierarchy: ["accounts", account, folder],
1611
+ title: subject,
1612
+ content: body.length > 0 ? `${headerBlock}
1613
+
1614
+ ${body}` : headerBlock,
1615
+ publishedAt: parseHimalayaDate(asString(envelope.date)),
1616
+ metadata: {
1617
+ account,
1618
+ folder,
1619
+ messageId: id,
1620
+ ...asArray(envelope.flags).length > 0 ? {
1621
+ flags: asArray(envelope.flags).map((flag) => asString(flag)).filter(Boolean)
1622
+ } : {}
1623
+ }
1624
+ });
1625
+ }
1626
+ const warnings = readFailures > 0 ? [`${readFailures} message(s) failed to read`] : undefined;
1627
+ return { ok: true, documents, ...warnings !== undefined ? { warnings } : {} };
1628
+ }
1629
+ }
1630
+ function parseHimalayaDate(value) {
1631
+ if (value === undefined || value.length === 0)
1632
+ return;
1633
+ const parsed = Date.parse(value.replace(" ", "T"));
1634
+ return Number.isNaN(parsed) ? undefined : parsed;
1635
+ }
1636
+ function classifyFailure2(stderr) {
1637
+ const lower = stderr.toLowerCase();
1638
+ if (lower.includes("cannot find") && lower.includes("account"))
1639
+ return "not-configured";
1640
+ if (lower.includes("auth") || lower.includes("login") || lower.includes("credential"))
1641
+ return "auth";
1642
+ return "api-error";
1643
+ }
1644
+ function shortFailure2(result) {
1645
+ return `himalaya exited with code ${result.code ?? "unknown"}`;
1646
+ }
1647
+ function runBinary2(binary, args, timeoutMs) {
1648
+ return new Promise((resolvePromise) => {
1649
+ const child = spawn2(binary, args, { stdio: ["ignore", "pipe", "pipe"] });
1650
+ let stdout = "";
1651
+ let stderr = "";
1652
+ const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
1653
+ child.stdout.on("data", (chunk) => {
1654
+ stdout += chunk.toString("utf8");
1655
+ });
1656
+ child.stderr.on("data", (chunk) => {
1657
+ stderr += chunk.toString("utf8");
1658
+ });
1659
+ child.on("error", () => {
1660
+ clearTimeout(timer);
1661
+ resolvePromise({ ok: false, stdout, stderr, code: null });
1662
+ });
1663
+ child.on("close", (code) => {
1664
+ clearTimeout(timer);
1665
+ resolvePromise({ ok: code === 0, stdout, stderr, code });
1666
+ });
1667
+ });
1668
+ }
1669
+ var DEFAULT_BINARY2 = "himalaya", DEFAULT_PAGE_SIZE = 100, DEFAULT_MAX_DOCUMENTS5 = 50, DEFAULT_TIMEOUT_MS3 = 30000, MAX_BODY_CHARS = 50000;
1670
+ var init_himalaya_connector = () => {};
1671
+
1672
+ // src/datasource/skills/gmail/connector.ts
1673
+ class GmailConnector {
1674
+ options;
1675
+ constructor(options = {}) {
1676
+ this.options = options;
1677
+ }
1678
+ async fetch(signal) {
1679
+ const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV4);
1680
+ if (token === undefined)
1681
+ return { ok: false, reason: "not-configured", message: "auth token not configured" };
1682
+ const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL4;
1683
+ const request = {
1684
+ headers: { Authorization: `Bearer ${token}` },
1685
+ timeoutMs: this.options.timeoutMs,
1686
+ fetchImpl: this.options.fetchImpl,
1687
+ signal
1688
+ };
1689
+ const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES4;
1690
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS6;
1691
+ const messageIds = [];
1692
+ let pageToken;
1693
+ for (let page = 0;page < maxPages && messageIds.length < maxDocuments; page += 1) {
1694
+ const url = new URL(`${baseUrl}/users/me/messages`);
1695
+ url.searchParams.set("maxResults", "100");
1696
+ for (const labelId of this.options.labelIds ?? [])
1697
+ url.searchParams.append("labelIds", labelId);
1698
+ if (this.options.query !== undefined)
1699
+ url.searchParams.set("q", this.options.query);
1700
+ if (pageToken !== undefined)
1701
+ url.searchParams.set("pageToken", pageToken);
1702
+ const result = await httpJson(url.toString(), request);
1703
+ if (!result.ok)
1704
+ return { ok: false, reason: result.reason, message: `message list failed: ${result.message}` };
1705
+ const envelope = asRecord(result.json);
1706
+ if (envelope === undefined)
1707
+ return { ok: false, reason: "invalid-data", message: "message list malformed" };
1708
+ for (const raw of asArray(envelope.messages)) {
1709
+ const id = asString(asRecord(raw)?.id);
1710
+ if (id !== undefined)
1711
+ messageIds.push(id);
1712
+ }
1713
+ pageToken = asString(envelope.nextPageToken);
1714
+ if (pageToken === undefined || pageToken.length === 0)
1715
+ break;
1716
+ }
1717
+ const documents = [];
1718
+ let failedFetches = 0;
1719
+ for (const id of messageIds.slice(0, maxDocuments)) {
1720
+ const result = await httpJson(`${baseUrl}/users/me/messages/${id}?format=full`, request);
1721
+ if (!result.ok) {
1722
+ if (result.reason === "auth")
1723
+ return { ok: false, reason: "auth", message: "gmail: unauthorized" };
1724
+ if (result.reason === "rate-limited") {
1725
+ return { ok: false, reason: "rate-limited", message: "gmail: rate limited" };
1726
+ }
1727
+ failedFetches += 1;
1728
+ continue;
1729
+ }
1730
+ const message = asRecord(result.json);
1731
+ if (message === undefined) {
1732
+ failedFetches += 1;
1733
+ continue;
1734
+ }
1735
+ const payload = asRecord(message.payload);
1736
+ const headers = headerMap(payload);
1737
+ const subject = headers.get("subject") ?? "(no subject)";
1738
+ const body = (extractBody(payload) ?? asString(message.snippet) ?? "").slice(0, MAX_BODY_CHARS2);
1739
+ const labelIds = asArray(message.labelIds).map((label) => asString(label)).filter((label) => label !== undefined);
1740
+ const internalDate = asString(message.internalDate);
1741
+ const publishedAt = internalDate !== undefined && /^\d+$/u.test(internalDate) ? Number(internalDate) : undefined;
1742
+ const headerBlock = [
1743
+ `Subject: ${subject}`,
1744
+ ...headers.has("from") ? [`From: ${headers.get("from")}`] : [],
1745
+ ...headers.has("to") ? [`To: ${headers.get("to")}`] : [],
1746
+ ...headers.has("date") ? [`Date: ${headers.get("date")}`] : []
1747
+ ].join(`
1748
+ `);
1749
+ documents.push({
1750
+ docId: id,
1751
+ hierarchy: ["labels", this.options.labelIds?.[0] ?? "all"],
1752
+ title: subject,
1753
+ content: `${headerBlock}
1754
+
1755
+ ${body}`.trim(),
1756
+ ...publishedAt !== undefined ? { publishedAt } : {},
1757
+ metadata: {
1758
+ ...asString(message.threadId) !== undefined ? { threadId: asString(message.threadId) } : {},
1759
+ labelIds
1760
+ }
1761
+ });
1762
+ }
1763
+ const warnings = failedFetches > 0 ? [`${failedFetches} message(s) failed to fetch`] : undefined;
1764
+ return { ok: true, documents, ...warnings !== undefined ? { warnings } : {} };
1765
+ }
1766
+ }
1767
+ function headerMap(payload) {
1768
+ const map = new Map;
1769
+ for (const raw of asArray(payload?.headers)) {
1770
+ const header = asRecord(raw);
1771
+ const name = asString(header?.name)?.toLowerCase();
1772
+ const value = asString(header?.value);
1773
+ if (name !== undefined && value !== undefined && !map.has(name))
1774
+ map.set(name, value);
1775
+ }
1776
+ return map;
1777
+ }
1778
+ function extractBody(payload) {
1779
+ if (payload === undefined)
1780
+ return;
1781
+ const mimeType = asString(payload.mimeType);
1782
+ const data = asString(asRecord(payload.body)?.data);
1783
+ if (mimeType === "text/plain" && data !== undefined)
1784
+ return decodeBase64Url(data);
1785
+ const collected = [];
1786
+ for (const raw of asArray(payload.parts)) {
1787
+ const part = asRecord(raw);
1788
+ const text = extractBody(part);
1789
+ if (text !== undefined && text.length > 0)
1790
+ collected.push(text);
1791
+ }
1792
+ if (collected.length > 0)
1793
+ return collected.join(`
1794
+ `);
1795
+ return data !== undefined ? decodeBase64Url(data) : undefined;
1796
+ }
1797
+ function decodeBase64Url(data) {
1798
+ try {
1799
+ return Buffer.from(data, "base64url").toString("utf8");
1800
+ } catch {
1801
+ return "";
1802
+ }
1803
+ }
1804
+ var DEFAULT_BASE_URL4 = "https://gmail.googleapis.com/gmail/v1", DEFAULT_TOKEN_ENV4 = "GMAIL_ACCESS_TOKEN", DEFAULT_MAX_PAGES4 = 10, DEFAULT_MAX_DOCUMENTS6 = 500, MAX_BODY_CHARS2 = 50000;
1805
+ var init_connector5 = () => {};
1806
+
1807
+ // src/datasource/skills/gmail/skill.ts
1808
+ var GMAIL_SKILL_DEFINITION, GmailSkill;
1809
+ var init_skill4 = __esm(() => {
1810
+ init_connector_skill();
1811
+ init_connector5();
1812
+ GMAIL_SKILL_DEFINITION = {
1813
+ skillName: "gmail",
1814
+ skillType: "gmail-account",
1815
+ description: "Gmail datasource",
1816
+ capabilities: ["email", "api", "polling"],
1817
+ defaultTags: ["gmail", "email", "pii"],
1818
+ contentType: "email",
1819
+ manifestDescription: "Search indexed Gmail messages from the authorized account and labels. Use for questions about email conversations, decisions, or attachments discussed over email.",
1820
+ manifestNotes: ["Label and folder visibility is bounded by the server-configured OAuth token."]
1821
+ };
1822
+ GmailSkill = class GmailSkill extends ConnectorDatasourceSkill {
1823
+ constructor(options = {}) {
1824
+ const { connector, connectorOptions, ...rest } = options;
1825
+ super(GMAIL_SKILL_DEFINITION, {
1826
+ ...rest,
1827
+ connector: connector ?? new GmailConnector(connectorOptions ?? {})
1828
+ });
1829
+ }
1830
+ };
1831
+ });
1832
+
1833
+ // src/datasource/skills/gmail/index.ts
1834
+ var init_gmail = __esm(() => {
1835
+ init_connector5();
1836
+ init_himalaya_connector();
1837
+ init_skill4();
1838
+ });
1839
+
1840
+ // src/datasource/skills/mail-export/connector.ts
1841
+ import { readdir, readFile, stat } from "node:fs/promises";
1842
+ import { basename, dirname as dirname2, extname, join as join3 } from "node:path";
1843
+ import { simpleParser } from "mailparser";
1844
+
1845
+ class MailExportConnector {
1846
+ options;
1847
+ constructor(options = {}) {
1848
+ this.options = options;
1849
+ }
1850
+ async fetch() {
1851
+ const paths = this.options.paths ?? [];
1852
+ if (paths.length === 0)
1853
+ return { ok: false, reason: "not-configured", message: "no mail export paths configured" };
1854
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS7;
1855
+ const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
1856
+ const files = [];
1857
+ let missingPaths = 0;
1858
+ for (const path of paths) {
1859
+ try {
1860
+ const info = await stat(path);
1861
+ if (info.isDirectory()) {
1862
+ const entries = await readdir(path, { recursive: true, withFileTypes: true });
1863
+ for (const entry of entries) {
1864
+ if (!entry.isFile())
1865
+ continue;
1866
+ const extension = extname(entry.name).toLowerCase();
1867
+ if (extension === ".eml" || extension === ".mbox") {
1868
+ files.push(join3(entry.parentPath, entry.name));
1869
+ }
1870
+ }
1871
+ } else if (info.isFile()) {
1872
+ files.push(path);
1873
+ }
1874
+ } catch {
1875
+ missingPaths += 1;
1876
+ }
1877
+ }
1878
+ if (missingPaths === paths.length) {
1879
+ return { ok: false, reason: "unavailable", message: "no configured mail export paths were readable" };
1880
+ }
1881
+ const documents = [];
1882
+ const warnings = [];
1883
+ if (missingPaths > 0)
1884
+ warnings.push(`${missingPaths} configured path(s) were unreadable`);
1885
+ let parseFailures = 0;
1886
+ let skippedLarge = 0;
1887
+ for (const file of files.sort()) {
1888
+ if (documents.length >= maxDocuments)
1889
+ break;
1890
+ let bytes;
1891
+ try {
1892
+ const info = await stat(file);
1893
+ if (info.size > maxBytes) {
1894
+ skippedLarge += 1;
1895
+ continue;
1896
+ }
1897
+ bytes = await readFile(file);
1898
+ } catch {
1899
+ parseFailures += 1;
1900
+ continue;
1901
+ }
1902
+ const stem = basename(file, extname(file));
1903
+ const mailboxDir = basename(dirname2(file));
1904
+ const hierarchy = ["mailboxes", mailboxDir, stem];
1905
+ const extension = extname(file).toLowerCase();
1906
+ const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
1907
+ for (const [index, raw] of rawMessages.entries()) {
1908
+ if (documents.length >= maxDocuments)
1909
+ break;
1910
+ try {
1911
+ const parsed = await simpleParser(raw);
1912
+ const subject = parsed.subject ?? "(no subject)";
1913
+ const from = Array.isArray(parsed.from) ? parsed.from[0]?.text : parsed.from?.text;
1914
+ const to = Array.isArray(parsed.to) ? parsed.to[0]?.text : parsed.to?.text;
1915
+ const body = (parsed.text ?? "").trim().slice(0, MAX_BODY_CHARS3);
1916
+ const headerBlock = [
1917
+ `Subject: ${subject}`,
1918
+ ...from !== undefined ? [`From: ${from}`] : [],
1919
+ ...to !== undefined ? [`To: ${to}`] : [],
1920
+ ...parsed.date !== undefined ? [`Date: ${parsed.date.toISOString()}`] : []
1921
+ ].join(`
1922
+ `);
1923
+ const content = `${headerBlock}
1924
+
1925
+ ${body}`.trim();
1926
+ if (content.length === 0)
1927
+ continue;
1928
+ documents.push({
1929
+ docId: extension === ".mbox" ? `${stem}-${index + 1}` : stem,
1930
+ hierarchy,
1931
+ title: subject,
1932
+ content,
1933
+ ...parsed.date !== undefined ? { publishedAt: parsed.date.getTime() } : {},
1934
+ metadata: {
1935
+ path: file,
1936
+ subject,
1937
+ ...from !== undefined ? { from } : {},
1938
+ messageIndex: index
1939
+ }
1940
+ });
1941
+ } catch {
1942
+ parseFailures += 1;
1943
+ }
1944
+ }
1945
+ }
1946
+ if (skippedLarge > 0)
1947
+ warnings.push(`${skippedLarge} file(s) exceeded the size limit and were skipped`);
1948
+ if (parseFailures > 0)
1949
+ warnings.push(`${parseFailures} message(s) failed to parse`);
1950
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
1951
+ }
1952
+ }
1953
+ function splitMbox(text) {
1954
+ const lines = text.split(/\r?\n/u);
1955
+ const messages = [];
1956
+ let current = [];
1957
+ for (const line of lines) {
1958
+ if (MBOX_SEPARATOR.test(line) && line.startsWith("From ")) {
1959
+ if (current.length > 0)
1960
+ messages.push(current.join(`
1961
+ `));
1962
+ current = [];
1963
+ continue;
1964
+ }
1965
+ current.push(line);
1966
+ }
1967
+ if (current.length > 0)
1968
+ messages.push(current.join(`
1969
+ `));
1970
+ return messages.filter((message) => message.trim().length > 0);
1971
+ }
1972
+ var DEFAULT_MAX_DOCUMENTS7 = 500, DEFAULT_MAX_BYTES_PER_FILE2, MAX_BODY_CHARS3 = 50000, MBOX_SEPARATOR;
1973
+ var init_connector6 = __esm(() => {
1974
+ DEFAULT_MAX_BYTES_PER_FILE2 = 50 * 1024 * 1024;
1975
+ MBOX_SEPARATOR = /^From .*\d{4}$/mu;
1976
+ });
1977
+
1978
+ // src/datasource/skills/mail-export/skill.ts
1979
+ var MAIL_EXPORT_SKILL_DEFINITION, MailExportSkill;
1980
+ var init_skill5 = __esm(() => {
1981
+ init_connector_skill();
1982
+ init_connector6();
1983
+ MAIL_EXPORT_SKILL_DEFINITION = {
1984
+ skillName: "mail-export",
1985
+ skillType: "mail-export",
1986
+ description: "Local mail export datasource",
1987
+ capabilities: ["email", "filesystem"],
1988
+ defaultTags: ["mail-export", "email", "pii"],
1989
+ contentType: "email",
1990
+ manifestDescription: "Search indexed local mail exports (mbox/eml archives). Use for questions about archived email conversations.",
1991
+ manifestNotes: [
1992
+ "Only the server-configured export paths are indexed; archives are read locally and never uploaded."
1993
+ ]
1994
+ };
1995
+ MailExportSkill = class MailExportSkill extends ConnectorDatasourceSkill {
1996
+ constructor(options = {}) {
1997
+ const { connector, connectorOptions, ...rest } = options;
1998
+ super(MAIL_EXPORT_SKILL_DEFINITION, {
1999
+ ...rest,
2000
+ connector: connector ?? new MailExportConnector(connectorOptions ?? {})
2001
+ });
2002
+ }
2003
+ };
2004
+ });
2005
+
2006
+ // src/datasource/skills/mail-export/index.ts
2007
+ var init_mail_export = __esm(() => {
2008
+ init_connector6();
2009
+ init_skill5();
2010
+ });
2011
+
2012
+ // src/datasource/skills/notion/connector.ts
2013
+ class NotionConnector {
2014
+ options;
2015
+ constructor(options = {}) {
2016
+ this.options = options;
2017
+ }
2018
+ async fetch(signal) {
2019
+ const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV5);
2020
+ if (token === undefined)
2021
+ return { ok: false, reason: "not-configured", message: "auth token not configured" };
2022
+ const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL5;
2023
+ const request = {
2024
+ headers: {
2025
+ Authorization: `Bearer ${token}`,
2026
+ "Notion-Version": NOTION_VERSION,
2027
+ "Content-Type": "application/json"
2028
+ },
2029
+ timeoutMs: this.options.timeoutMs,
2030
+ fetchImpl: this.options.fetchImpl,
2031
+ signal
2032
+ };
2033
+ const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES5;
2034
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS8;
2035
+ const entries = [];
2036
+ let cursor;
2037
+ for (let page = 0;page < maxPages; page += 1) {
2038
+ const body = JSON.stringify({ page_size: 100, ...cursor !== undefined ? { start_cursor: cursor } : {} });
2039
+ const result = await httpJson(`${baseUrl}/search`, { ...request, method: "POST", body });
2040
+ if (!result.ok)
2041
+ return { ok: false, reason: result.reason, message: `search failed: ${result.message}` };
2042
+ const envelope = asRecord(result.json);
2043
+ if (envelope === undefined)
2044
+ return { ok: false, reason: "invalid-data", message: "search response malformed" };
2045
+ for (const raw of asArray(envelope.results)) {
2046
+ const entry = asRecord(raw);
2047
+ if (entry !== undefined)
2048
+ entries.push(entry);
2049
+ }
2050
+ cursor = envelope.has_more === true ? asString(envelope.next_cursor) : undefined;
2051
+ if (cursor === undefined)
2052
+ break;
2053
+ }
2054
+ const documents = [];
2055
+ const warnings = [];
2056
+ for (const entry of entries) {
2057
+ if (documents.length >= maxDocuments)
2058
+ break;
2059
+ const id = asString(entry.id);
2060
+ if (id === undefined)
2061
+ continue;
2062
+ const objectType = asString(entry.object) ?? "page";
2063
+ const title = extractTitle(entry);
2064
+ let blockText = "";
2065
+ if (objectType === "page") {
2066
+ const blocksResult = await httpJson(`${baseUrl}/blocks/${id}/children?page_size=100`, request);
2067
+ if (blocksResult.ok) {
2068
+ blockText = extractBlockText(blocksResult.json);
2069
+ } else {
2070
+ warnings.push(`page content fetch failed: ${blocksResult.message}`);
2071
+ }
2072
+ }
2073
+ const content = [title, blockText].filter((part) => part.length > 0).join(`
2074
+
2075
+ `);
2076
+ if (content.length === 0)
2077
+ continue;
2078
+ documents.push({
2079
+ docId: id,
2080
+ hierarchy: hierarchyFor(entry, objectType),
2081
+ ...title.length > 0 ? { title } : {},
2082
+ content,
2083
+ publishedAt: parseEpochMs(entry.last_edited_time),
2084
+ metadata: {
2085
+ objectType,
2086
+ ...asString(entry.last_edited_time) !== undefined ? { lastEditedTime: asString(entry.last_edited_time) } : {}
2087
+ }
2088
+ });
2089
+ }
2090
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
2091
+ }
2092
+ }
2093
+ function hierarchyFor(entry, objectType) {
2094
+ if (objectType === "database")
2095
+ return ["databases", asString(entry.id) ?? "unknown"];
2096
+ const parent = asRecord(entry.parent);
2097
+ const parentType = asString(parent?.type);
2098
+ if (parentType === "database_id") {
2099
+ return ["databases", asString(parent?.database_id) ?? "unknown"];
2100
+ }
2101
+ if (parentType === "page_id") {
2102
+ return ["pages", asString(parent?.page_id) ?? "unknown"];
2103
+ }
2104
+ return ["pages"];
2105
+ }
2106
+ function extractTitle(entry) {
2107
+ const directTitle = plainText(entry.title);
2108
+ if (directTitle.length > 0)
2109
+ return directTitle;
2110
+ const properties = asRecord(entry.properties);
2111
+ if (properties === undefined)
2112
+ return "";
2113
+ for (const value of Object.values(properties)) {
2114
+ const property = asRecord(value);
2115
+ if (property === undefined || property.type !== "title")
2116
+ continue;
2117
+ const text = plainText(property.title);
2118
+ if (text.length > 0)
2119
+ return text;
2120
+ }
2121
+ return "";
2122
+ }
2123
+ function extractBlockText(json) {
2124
+ const envelope = asRecord(json);
2125
+ const lines = [];
2126
+ for (const raw of asArray(envelope?.results)) {
2127
+ const block = asRecord(raw);
2128
+ const type = asString(block?.type);
2129
+ if (block === undefined || type === undefined)
2130
+ continue;
2131
+ if (!TEXT_BLOCK_TYPES.includes(type))
2132
+ continue;
2133
+ const payload = asRecord(block[type]);
2134
+ const text = plainText(payload?.rich_text);
2135
+ if (text.length > 0)
2136
+ lines.push(text);
2137
+ }
2138
+ return lines.join(`
2139
+ `);
2140
+ }
2141
+ function plainText(richText) {
2142
+ return asArray(richText).map((raw) => asString(asRecord(raw)?.plain_text) ?? "").filter((text) => text.length > 0).join("").trim();
2143
+ }
2144
+ var DEFAULT_BASE_URL5 = "https://api.notion.com/v1", DEFAULT_TOKEN_ENV5 = "NOTION_TOKEN", NOTION_VERSION = "2022-06-28", DEFAULT_MAX_PAGES5 = 10, DEFAULT_MAX_DOCUMENTS8 = 500, TEXT_BLOCK_TYPES;
2145
+ var init_connector7 = __esm(() => {
2146
+ TEXT_BLOCK_TYPES = [
2147
+ "paragraph",
2148
+ "heading_1",
2149
+ "heading_2",
2150
+ "heading_3",
2151
+ "bulleted_list_item",
2152
+ "numbered_list_item",
2153
+ "to_do",
2154
+ "quote",
2155
+ "callout",
2156
+ "code"
2157
+ ];
2158
+ });
2159
+
2160
+ // src/datasource/skills/notion/skill.ts
2161
+ var NOTION_SKILL_DEFINITION, NotionSkill;
2162
+ var init_skill6 = __esm(() => {
2163
+ init_connector_skill();
2164
+ init_connector7();
2165
+ NOTION_SKILL_DEFINITION = {
2166
+ skillName: "notion",
2167
+ skillType: "notion-workspace",
2168
+ description: "Notion workspace datasource",
2169
+ capabilities: ["documents", "api", "polling"],
2170
+ defaultTags: ["notion", "documents"],
2171
+ contentType: "document",
2172
+ manifestDescription: "Search indexed Notion pages and databases shared with the integration. Use for questions about Notion docs, wikis, and database entries.",
2173
+ manifestNotes: ["Only pages shared with the server-configured integration token are indexed."]
2174
+ };
2175
+ NotionSkill = class NotionSkill extends ConnectorDatasourceSkill {
2176
+ constructor(options = {}) {
2177
+ const { connector, connectorOptions, ...rest } = options;
2178
+ super(NOTION_SKILL_DEFINITION, {
2179
+ ...rest,
2180
+ connector: connector ?? new NotionConnector(connectorOptions ?? {})
2181
+ });
2182
+ }
2183
+ };
2184
+ });
2185
+
2186
+ // src/datasource/skills/notion/index.ts
2187
+ var init_notion = __esm(() => {
2188
+ init_connector7();
2189
+ init_skill6();
2190
+ });
2191
+
2192
+ // src/datasource/skills/obsidian/connector.ts
2193
+ import { readdir as readdir2, readFile as readFile2, stat as stat2 } from "node:fs/promises";
2194
+ import { basename as basename2, extname as extname2, join as join4, relative, sep } from "node:path";
2195
+
2196
+ class ObsidianConnector {
2197
+ options;
2198
+ constructor(options = {}) {
2199
+ this.options = options;
2200
+ }
2201
+ async fetch() {
2202
+ const vaultPath = this.options.vaultPath;
2203
+ if (vaultPath === undefined || vaultPath.length === 0) {
2204
+ return { ok: false, reason: "not-configured", message: "vault path not configured" };
2205
+ }
2206
+ try {
2207
+ const info = await stat2(vaultPath);
2208
+ if (!info.isDirectory())
2209
+ return { ok: false, reason: "unavailable", message: "vault path is not a directory" };
2210
+ } catch {
2211
+ return { ok: false, reason: "unavailable", message: "vault path is not readable" };
2212
+ }
2213
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS9;
2214
+ const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE3;
2215
+ const files = [];
2216
+ const walk = async (dir) => {
2217
+ let entries;
2218
+ try {
2219
+ entries = await readdir2(dir, { withFileTypes: true });
2220
+ } catch {
2221
+ return;
2222
+ }
2223
+ for (const entry of entries) {
2224
+ if (entry.isDirectory()) {
2225
+ if (!SKIPPED_DIRS.has(entry.name))
2226
+ await walk(join4(dir, entry.name));
2227
+ } else if (entry.isFile() && extname2(entry.name).toLowerCase() === ".md") {
2228
+ files.push(join4(dir, entry.name));
2229
+ }
2230
+ }
2231
+ };
2232
+ await walk(vaultPath);
2233
+ const documents = [];
2234
+ let readFailures = 0;
2235
+ let skippedLarge = 0;
2236
+ for (const file of files.sort()) {
2237
+ if (documents.length >= maxDocuments)
2238
+ break;
2239
+ let text;
2240
+ try {
2241
+ const info = await stat2(file);
2242
+ if (info.size > maxBytes) {
2243
+ skippedLarge += 1;
2244
+ continue;
2245
+ }
2246
+ text = await readFile2(file, "utf8");
2247
+ const relativePath = relative(vaultPath, file);
2248
+ const segments = relativePath.split(sep);
2249
+ const directorySegments = segments.slice(0, -1);
2250
+ const stem = basename2(file, ".md");
2251
+ const { body, frontmatterTags } = stripFrontmatter(text);
2252
+ const inlineTags = matchAll(body, INLINE_TAG_PATTERN, 1);
2253
+ const links = [];
2254
+ const embeds = [];
2255
+ for (const match of body.matchAll(WIKI_LINK_PATTERN)) {
2256
+ const target = match[2]?.trim();
2257
+ if (target === undefined || target.length === 0)
2258
+ continue;
2259
+ (match[1] === "!" ? embeds : links).push(target);
2260
+ }
2261
+ const heading = /^#\s+(.+)$/mu.exec(body)?.[1]?.trim();
2262
+ const content = body.trim().slice(0, MAX_CONTENT_CHARS3);
2263
+ if (content.length === 0)
2264
+ continue;
2265
+ documents.push({
2266
+ docId: relativePath.split(sep).join("__").replace(/\.md$/u, ""),
2267
+ hierarchy: ["folders", ...directorySegments],
2268
+ title: heading ?? stem,
2269
+ content,
2270
+ publishedAt: info.mtimeMs,
2271
+ metadata: {
2272
+ path: file,
2273
+ tags: [...new Set([...frontmatterTags, ...inlineTags])],
2274
+ links,
2275
+ embeds,
2276
+ folder: directorySegments.join("/")
2277
+ }
2278
+ });
2279
+ } catch {
2280
+ readFailures += 1;
2281
+ }
2282
+ }
2283
+ const warnings = [];
2284
+ if (skippedLarge > 0)
2285
+ warnings.push(`${skippedLarge} note(s) exceeded the size limit and were skipped`);
2286
+ if (readFailures > 0)
2287
+ warnings.push(`${readFailures} note(s) failed to read`);
2288
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
2289
+ }
2290
+ }
2291
+ function stripFrontmatter(text) {
2292
+ if (!text.startsWith("---"))
2293
+ return { body: text, frontmatterTags: [] };
2294
+ const end = text.indexOf(`
2295
+ ---`, 3);
2296
+ if (end === -1)
2297
+ return { body: text, frontmatterTags: [] };
2298
+ const frontmatter = text.slice(3, end);
2299
+ const body = text.slice(text.indexOf(`
2300
+ `, end + 1) + 1);
2301
+ return { body, frontmatterTags: parseFrontmatterTags(frontmatter) };
2302
+ }
2303
+ function parseFrontmatterTags(frontmatter) {
2304
+ const lines = frontmatter.split(`
2305
+ `);
2306
+ const tags = [];
2307
+ for (const [index, line] of lines.entries()) {
2308
+ const match = /^tags:\s*(.*)$/u.exec(line.trim());
2309
+ if (match === null)
2310
+ continue;
2311
+ const inline = match[1]?.trim() ?? "";
2312
+ if (inline.length > 0) {
2313
+ const cleaned = inline.replace(/^\[/u, "").replace(/\]$/u, "");
2314
+ for (const tag of cleaned.split(/[\s,]+/u)) {
2315
+ const trimmed = tag.trim().replace(/^["'#]+|["']+$/gu, "");
2316
+ if (trimmed.length > 0)
2317
+ tags.push(trimmed);
2318
+ }
2319
+ break;
2320
+ }
2321
+ for (let cursor = index + 1;cursor < lines.length; cursor += 1) {
2322
+ const item = /^\s*-\s*(.+)$/u.exec(lines[cursor] ?? "");
2323
+ if (item === null)
2324
+ break;
2325
+ const trimmed = item[1]?.trim().replace(/^["'#]+|["']+$/gu, "") ?? "";
2326
+ if (trimmed.length > 0)
2327
+ tags.push(trimmed);
2328
+ }
2329
+ break;
2330
+ }
2331
+ return tags;
2332
+ }
2333
+ function matchAll(text, pattern, group) {
2334
+ const out = [];
2335
+ for (const match of text.matchAll(pattern)) {
2336
+ const value = match[group]?.trim();
2337
+ if (value !== undefined && value.length > 0)
2338
+ out.push(value);
2339
+ }
2340
+ return out;
2341
+ }
2342
+ var DEFAULT_MAX_DOCUMENTS9 = 1000, DEFAULT_MAX_BYTES_PER_FILE3, MAX_CONTENT_CHARS3 = 1e5, SKIPPED_DIRS, INLINE_TAG_PATTERN, WIKI_LINK_PATTERN;
2343
+ var init_connector8 = __esm(() => {
2344
+ DEFAULT_MAX_BYTES_PER_FILE3 = 2 * 1024 * 1024;
2345
+ SKIPPED_DIRS = new Set([".obsidian", ".trash", ".git", "node_modules"]);
2346
+ INLINE_TAG_PATTERN = /(?:^|\s)#([A-Za-z0-9_/-]+)/gu;
2347
+ WIKI_LINK_PATTERN = /(!?)\[\[([^\]|]+)(?:\|[^\]]*)?\]\]/gu;
2348
+ });
2349
+
2350
+ // src/datasource/skills/obsidian/skill.ts
2351
+ var OBSIDIAN_SKILL_DEFINITION, ObsidianSkill;
2352
+ var init_skill7 = __esm(() => {
2353
+ init_connector_skill();
2354
+ init_connector8();
2355
+ OBSIDIAN_SKILL_DEFINITION = {
2356
+ skillName: "obsidian",
2357
+ skillType: "obsidian-vault",
2358
+ description: "Obsidian vault datasource",
2359
+ capabilities: ["notes", "filesystem"],
2360
+ defaultTags: ["obsidian", "notes"],
2361
+ contentType: "note",
2362
+ manifestDescription: "Search indexed Obsidian vault notes, including tags, wiki links, and embeds. Use for questions about personal notes and knowledge-base content.",
2363
+ manifestNotes: ["Only the server-configured vault is indexed; note metadata includes tags, links, and embeds."]
2364
+ };
2365
+ ObsidianSkill = class ObsidianSkill extends ConnectorDatasourceSkill {
2366
+ constructor(options = {}) {
2367
+ const { connector, connectorOptions, ...rest } = options;
2368
+ super(OBSIDIAN_SKILL_DEFINITION, {
2369
+ ...rest,
2370
+ connector: connector ?? new ObsidianConnector(connectorOptions ?? {})
2371
+ });
2372
+ }
2373
+ };
2374
+ });
2375
+
2376
+ // src/datasource/skills/obsidian/index.ts
2377
+ var init_obsidian = __esm(() => {
2378
+ init_connector8();
2379
+ init_skill7();
2380
+ });
2381
+
2382
+ // src/datasource/skills/rss/connector.ts
2383
+ import { XMLParser } from "fast-xml-parser";
2384
+
2385
+ class RssConnector {
2386
+ options;
2387
+ parser = new XMLParser({ ignoreAttributes: false });
2388
+ constructor(options = {}) {
2389
+ this.options = options;
2390
+ }
2391
+ async fetch(signal) {
2392
+ const feeds = this.options.feeds ?? [];
2393
+ if (feeds.length === 0)
2394
+ return { ok: false, reason: "not-configured", message: "no feeds configured" };
2395
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS10;
2396
+ const maxItemsPerFeed = this.options.maxItemsPerFeed ?? DEFAULT_MAX_ITEMS_PER_FEED;
2397
+ const documents = [];
2398
+ const warnings = [];
2399
+ let failedFeeds = 0;
2400
+ for (const [index, feed] of feeds.entries()) {
2401
+ if (documents.length >= maxDocuments)
2402
+ break;
2403
+ const result = await httpText(feed.url, {
2404
+ headers: { Accept: ACCEPT_HEADER },
2405
+ timeoutMs: this.options.timeoutMs ?? DEFAULT_FEED_TIMEOUT_MS,
2406
+ fetchImpl: this.options.fetchImpl,
2407
+ signal
2408
+ });
2409
+ if (!result.ok) {
2410
+ failedFeeds += 1;
2411
+ warnings.push(`feed ${index + 1} failed: ${result.message}`);
2412
+ continue;
2413
+ }
2414
+ let parsed;
2415
+ try {
2416
+ parsed = this.parser.parse(result.text);
2417
+ } catch {
2418
+ failedFeeds += 1;
2419
+ warnings.push(`feed ${index + 1} failed: unparseable XML`);
2420
+ continue;
2421
+ }
2422
+ const normalized = normalizeFeed(parsed);
2423
+ if (normalized === undefined) {
2424
+ failedFeeds += 1;
2425
+ warnings.push(`feed ${index + 1} failed: unrecognized feed shape`);
2426
+ continue;
2427
+ }
2428
+ const feedTitle = normalized.title ?? `feed-${index + 1}`;
2429
+ const hierarchy = feed.category !== undefined ? ["feeds", feed.category, feedTitle] : ["feeds", feedTitle];
2430
+ for (const item of normalized.items.slice(0, maxItemsPerFeed)) {
2431
+ if (documents.length >= maxDocuments)
2432
+ break;
2433
+ const title = item.title ?? "(untitled)";
2434
+ const body = stripHtml(item.body ?? "");
2435
+ const content = body.length > 0 ? `${title}
2436
+
2437
+ ${body}` : title;
2438
+ documents.push({
2439
+ docId: item.id,
2440
+ hierarchy,
2441
+ title,
2442
+ content: content.slice(0, MAX_CONTENT_CHARS4),
2443
+ ...item.publishedAt !== undefined ? { publishedAt: item.publishedAt } : {},
2444
+ metadata: { feedIndex: index, feedTitle, categories: item.categories }
2445
+ });
2446
+ }
2447
+ }
2448
+ if (failedFeeds === feeds.length) {
2449
+ return { ok: false, reason: "unavailable", message: "all configured feeds failed" };
2450
+ }
2451
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
2452
+ }
2453
+ }
2454
+ function normalizeFeed(parsed) {
2455
+ const root = asRecord(parsed);
2456
+ if (root === undefined)
2457
+ return;
2458
+ const channel = asRecord(asRecord(root.rss)?.channel);
2459
+ if (channel !== undefined) {
2460
+ const items = asArray(channel.item ?? []).concat(asRecord(channel.item) !== undefined ? [channel.item] : []);
2461
+ return { title: textOf(channel.title), items: items.map(normalizeRssItem).filter(present) };
2462
+ }
2463
+ const atom = asRecord(root.feed);
2464
+ if (atom !== undefined) {
2465
+ const entries = asArray(atom.entry ?? []).concat(asRecord(atom.entry) !== undefined ? [atom.entry] : []);
2466
+ return { title: textOf(atom.title), items: entries.map(normalizeAtomEntry).filter(present) };
2467
+ }
2468
+ return;
2469
+ }
2470
+ function normalizeRssItem(raw) {
2471
+ const item = asRecord(raw);
2472
+ if (item === undefined)
2473
+ return;
2474
+ const title = textOf(item.title);
2475
+ const guid = textOf(item.guid) ?? textOf(item.link) ?? title;
2476
+ if (guid === undefined)
2477
+ return;
2478
+ return {
2479
+ id: guid,
2480
+ title,
2481
+ body: textOf(item.description),
2482
+ publishedAt: parseEpochMs(textOf(item.pubDate)),
2483
+ categories: categoriesOf(item.category)
2484
+ };
2485
+ }
2486
+ function normalizeAtomEntry(raw) {
2487
+ const entry = asRecord(raw);
2488
+ if (entry === undefined)
2489
+ return;
2490
+ const title = textOf(entry.title);
2491
+ const id = textOf(entry.id) ?? title;
2492
+ if (id === undefined)
2493
+ return;
2494
+ return {
2495
+ id,
2496
+ title,
2497
+ body: textOf(entry.summary) ?? textOf(entry.content),
2498
+ publishedAt: parseEpochMs(textOf(entry.published) ?? textOf(entry.updated)),
2499
+ categories: categoriesOf(entry.category)
2500
+ };
2501
+ }
2502
+ function textOf(value) {
2503
+ const direct = asString(value);
2504
+ if (direct !== undefined)
2505
+ return direct.trim();
2506
+ if (typeof value === "number")
2507
+ return String(value);
2508
+ const record = asRecord(value);
2509
+ if (record !== undefined) {
2510
+ const text = asString(record["#text"]) ?? asString(record["@_term"]);
2511
+ if (text !== undefined)
2512
+ return text.trim();
2513
+ if (typeof record["#text"] === "number")
2514
+ return String(record["#text"]);
2515
+ }
2516
+ return;
2517
+ }
2518
+ function categoriesOf(value) {
2519
+ const values = Array.isArray(value) ? value : value !== undefined ? [value] : [];
2520
+ return values.map(textOf).filter((category) => category !== undefined && category.length > 0);
2521
+ }
2522
+ function stripHtml(text) {
2523
+ return text.replace(/<[^>]+>/gu, " ").replace(/\s+/gu, " ").trim();
2524
+ }
2525
+ function present(item) {
2526
+ return item !== undefined;
2527
+ }
2528
+ var DEFAULT_MAX_DOCUMENTS10 = 500, DEFAULT_MAX_ITEMS_PER_FEED = 100, DEFAULT_FEED_TIMEOUT_MS = 30000, MAX_CONTENT_CHARS4 = 20000, ACCEPT_HEADER = "application/rss+xml, application/atom+xml, application/xml, text/xml";
2529
+ var init_connector9 = () => {};
2530
+
2531
+ // src/datasource/skills/rss/skill.ts
2532
+ var RSS_SKILL_DEFINITION, DEFAULT_DEDUPE_WINDOW_MS, RssSkill;
2533
+ var init_skill8 = __esm(() => {
2534
+ init_connector_skill();
2535
+ init_connector9();
2536
+ RSS_SKILL_DEFINITION = {
2537
+ skillName: "rss",
2538
+ skillType: "rss-feeds",
2539
+ description: "RSS/news datasource",
2540
+ capabilities: ["news", "api", "polling"],
2541
+ defaultTags: ["rss", "news", "public"],
2542
+ contentType: "article",
2543
+ manifestDescription: "Search indexed RSS/news feed articles from the configured feeds. Use for questions about recent news items or blog posts the server subscribes to.",
2544
+ manifestNotes: ["Feeds are polled with a dedupe window so re-delivered items are indexed once per window."]
2545
+ };
2546
+ DEFAULT_DEDUPE_WINDOW_MS = 24 * 60 * 60 * 1000;
2547
+ RssSkill = class RssSkill extends ConnectorDatasourceSkill {
2548
+ constructor(options = {}) {
2549
+ const { connector, connectorOptions, ...rest } = options;
2550
+ super(RSS_SKILL_DEFINITION, {
2551
+ ...rest,
2552
+ connector: connector ?? new RssConnector(connectorOptions ?? {}),
2553
+ dedupeWindowMs: options.dedupeWindowMs ?? DEFAULT_DEDUPE_WINDOW_MS
2554
+ });
2555
+ }
2556
+ };
2557
+ });
2558
+
2559
+ // src/datasource/skills/rss/index.ts
2560
+ var init_rss = __esm(() => {
2561
+ init_connector9();
2562
+ init_skill8();
2563
+ });
2564
+
2565
+ // src/datasource/skills/slack/connector.ts
2566
+ class SlackConnector {
2567
+ options;
2568
+ constructor(options = {}) {
2569
+ this.options = options;
2570
+ }
2571
+ async fetch(signal) {
2572
+ const token = resolveToken(this.options.token, this.options.tokenEnv ?? DEFAULT_TOKEN_ENV6);
2573
+ if (token === undefined)
2574
+ return { ok: false, reason: "not-configured", message: "auth token not configured" };
2575
+ const baseUrl = this.options.baseUrl ?? DEFAULT_BASE_URL6;
2576
+ const headers = { Authorization: `Bearer ${token}` };
2577
+ const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES6;
2578
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS11;
2579
+ const request = { headers, timeoutMs: this.options.timeoutMs, fetchImpl: this.options.fetchImpl, signal };
2580
+ const channels = [];
2581
+ let cursor;
2582
+ for (let page = 0;page < maxPages; page += 1) {
2583
+ const url = new URL(`${baseUrl}/conversations.list`);
2584
+ url.searchParams.set("limit", "200");
2585
+ url.searchParams.set("types", "public_channel,private_channel");
2586
+ if (cursor !== undefined)
2587
+ url.searchParams.set("cursor", cursor);
2588
+ const result = await httpJson(url.toString(), request);
2589
+ if (!result.ok)
2590
+ return { ok: false, reason: result.reason, message: `channel list failed: ${result.message}` };
2591
+ const envelope = asRecord(result.json);
2592
+ if (envelope === undefined)
2593
+ return { ok: false, reason: "invalid-data", message: "channel list malformed" };
2594
+ if (envelope.ok !== true) {
2595
+ const error = asString(envelope.error) ?? "unknown-error";
2596
+ if (AUTH_ERRORS.has(error))
2597
+ return { ok: false, reason: "auth", message: `slack: ${error}` };
2598
+ if (error === "ratelimited")
2599
+ return { ok: false, reason: "rate-limited", message: "slack: ratelimited" };
2600
+ return { ok: false, reason: "api-error", message: `slack: ${error}` };
2601
+ }
2602
+ for (const raw of asArray(envelope.channels)) {
2603
+ const channel = asRecord(raw);
2604
+ const id = asString(channel?.id);
2605
+ const name = asString(channel?.name);
2606
+ if (id !== undefined && name !== undefined)
2607
+ channels.push({ id, name });
2608
+ }
2609
+ cursor = asString(asRecord(envelope.response_metadata)?.next_cursor);
2610
+ if (cursor === undefined || cursor.length === 0)
2611
+ break;
2612
+ }
2613
+ const filter = this.options.channels;
2614
+ const selected = filter === undefined || filter.length === 0 ? channels : channels.filter((channel) => filter.includes(channel.id) || filter.includes(channel.name));
2615
+ const documents = [];
2616
+ const warnings = [];
2617
+ for (const channel of selected) {
2618
+ if (documents.length >= maxDocuments)
2619
+ break;
2620
+ let historyCursor;
2621
+ for (let page = 0;page < maxPages && documents.length < maxDocuments; page += 1) {
2622
+ const url = new URL(`${baseUrl}/conversations.history`);
2623
+ url.searchParams.set("channel", channel.id);
2624
+ url.searchParams.set("limit", "200");
2625
+ if (historyCursor !== undefined)
2626
+ url.searchParams.set("cursor", historyCursor);
2627
+ const result = await httpJson(url.toString(), request);
2628
+ if (!result.ok) {
2629
+ warnings.push(`channel ${channel.name} history failed: ${result.message}`);
2630
+ break;
2631
+ }
2632
+ const envelope = asRecord(result.json);
2633
+ if (envelope === undefined || envelope.ok !== true) {
2634
+ const error = asString(envelope?.error) ?? "malformed response";
2635
+ if (AUTH_ERRORS.has(error))
2636
+ return { ok: false, reason: "auth", message: `slack: ${error}` };
2637
+ if (error === "ratelimited")
2638
+ return { ok: false, reason: "rate-limited", message: "slack: ratelimited" };
2639
+ if (CHANNEL_SKIP_ERRORS.has(error))
2640
+ warnings.push(`channel ${channel.name} skipped: ${error}`);
2641
+ else
2642
+ warnings.push(`channel ${channel.name} history failed: ${error}`);
2643
+ break;
2644
+ }
2645
+ for (const raw of asArray(envelope.messages)) {
2646
+ if (documents.length >= maxDocuments)
2647
+ break;
2648
+ const message = asRecord(raw);
2649
+ if (message === undefined)
2650
+ continue;
2651
+ const text = asString(message.text);
2652
+ const ts = asString(message.ts);
2653
+ const subtype = asString(message.subtype);
2654
+ if (text === undefined || text.length === 0 || ts === undefined)
2655
+ continue;
2656
+ if (subtype === "channel_join")
2657
+ continue;
2658
+ const user = asString(message.user) ?? asString(message.username) ?? "unknown";
2659
+ documents.push({
2660
+ docId: `${channel.id}-${ts.replace(/\./gu, "-")}`,
2661
+ hierarchy: ["channels", channel.name],
2662
+ title: `#${channel.name}`,
2663
+ content: `${user}: ${text}`,
2664
+ publishedAt: Math.round(Number.parseFloat(ts) * 1000),
2665
+ metadata: { channelId: channel.id, ts, user }
2666
+ });
2667
+ }
2668
+ historyCursor = asString(asRecord(envelope.response_metadata)?.next_cursor);
2669
+ if (historyCursor === undefined || historyCursor.length === 0)
2670
+ break;
2671
+ }
2672
+ }
2673
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
2674
+ }
2675
+ }
2676
+ var DEFAULT_BASE_URL6 = "https://slack.com/api", DEFAULT_TOKEN_ENV6 = "SLACK_BOT_TOKEN", DEFAULT_MAX_PAGES6 = 10, DEFAULT_MAX_DOCUMENTS11 = 500, AUTH_ERRORS, CHANNEL_SKIP_ERRORS;
2677
+ var init_connector10 = __esm(() => {
2678
+ AUTH_ERRORS = new Set(["invalid_auth", "not_authed", "token_revoked", "account_inactive"]);
2679
+ CHANNEL_SKIP_ERRORS = new Set(["missing_scope", "access_denied", "channel_not_found", "not_in_channel"]);
2680
+ });
2681
+
2682
+ // src/datasource/skills/slack/skill.ts
2683
+ var SLACK_SKILL_DEFINITION, SlackSkill;
2684
+ var init_skill9 = __esm(() => {
2685
+ init_connector_skill();
2686
+ init_connector10();
2687
+ SLACK_SKILL_DEFINITION = {
2688
+ skillName: "slack",
2689
+ skillType: "slack-workspace",
2690
+ description: "Slack workspace datasource",
2691
+ capabilities: ["chat", "api", "polling"],
2692
+ defaultTags: ["slack", "chat", "pii"],
2693
+ contentType: "chat",
2694
+ manifestDescription: "Search indexed Slack messages across authorized workspace channels. Use for questions about Slack conversations, decisions made in channels, or who said what.",
2695
+ manifestNotes: [
2696
+ "Message history windows and channel visibility are bounded by the server-configured bot token scopes."
2697
+ ]
2698
+ };
2699
+ SlackSkill = class SlackSkill extends ConnectorDatasourceSkill {
2700
+ constructor(options = {}) {
2701
+ const { connector, connectorOptions, ...rest } = options;
2702
+ super(SLACK_SKILL_DEFINITION, {
2703
+ ...rest,
2704
+ connector: connector ?? new SlackConnector(connectorOptions ?? {})
2705
+ });
2706
+ }
2707
+ };
2708
+ });
2709
+
2710
+ // src/datasource/skills/slack/index.ts
2711
+ var init_slack = __esm(() => {
2712
+ init_connector10();
2713
+ init_skill9();
2714
+ });
2715
+
2716
+ // src/datasource/skills/spotlight/connector.ts
2717
+ import { spawn as spawn3 } from "node:child_process";
2718
+ import { createHash as createHash2 } from "node:crypto";
2719
+ import { readFile as fsReadFile, stat as stat3 } from "node:fs/promises";
2720
+ import { basename as basename3, extname as extname3 } from "node:path";
2721
+
2722
+ class SpotlightConnector {
2723
+ options;
2724
+ run;
2725
+ readFile;
2726
+ platform;
2727
+ constructor(options = {}) {
2728
+ this.options = options;
2729
+ this.run = options.run ?? spawnRunner;
2730
+ this.readFile = options.readFile ?? ((path) => fsReadFile(path, "utf8"));
2731
+ this.platform = options.platform ?? process.platform;
2732
+ }
2733
+ async fetch() {
2734
+ if (this.platform !== "darwin") {
2735
+ return { ok: false, reason: "unavailable", message: "spotlight datasource is supported on macOS only" };
2736
+ }
2737
+ const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
2738
+ if (queries.length === 0) {
2739
+ return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
2740
+ }
2741
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS4;
2742
+ const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
2743
+ if (status.ok && /indexing disabled/iu.test(status.stdout)) {
2744
+ return {
2745
+ ok: false,
2746
+ reason: "unavailable",
2747
+ message: "Spotlight indexing is disabled; enable it with `sudo mdutil -i on /`"
2748
+ };
2749
+ }
2750
+ const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
2751
+ const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS12;
2752
+ const seen = new Set;
2753
+ const paths = [];
2754
+ let mdfindMissing = false;
2755
+ let failedQueries = 0;
2756
+ for (const query of queries) {
2757
+ const args = this.options.onlyIn !== undefined ? ["-onlyin", this.options.onlyIn, query] : [query];
2758
+ const result = await this.run("mdfind", args, timeoutMs);
2759
+ if (!result.ok) {
2760
+ if (result.error === "ENOENT") {
2761
+ mdfindMissing = true;
2762
+ break;
2763
+ }
2764
+ failedQueries += 1;
2765
+ continue;
2766
+ }
2767
+ let kept = 0;
2768
+ for (const line of result.stdout.split(`
2769
+ `)) {
2770
+ const path = line.trim();
2771
+ if (path.length === 0 || seen.has(path))
2772
+ continue;
2773
+ if (kept >= maxPerQuery)
2774
+ break;
2775
+ seen.add(path);
2776
+ paths.push(path);
2777
+ kept += 1;
2778
+ }
2779
+ }
2780
+ if (mdfindMissing) {
2781
+ return { ok: false, reason: "unavailable", message: "mdfind binary not found; Spotlight requires macOS" };
2782
+ }
2783
+ if (failedQueries === queries.length) {
2784
+ return { ok: false, reason: "unavailable", message: "every configured mdfind query failed" };
2785
+ }
2786
+ const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE4;
2787
+ const documents = [];
2788
+ let permissionDenied = 0;
2789
+ let readFailures = 0;
2790
+ for (const path of paths) {
2791
+ if (documents.length >= maxDocuments)
2792
+ break;
2793
+ try {
2794
+ const info = await stat3(path);
2795
+ if (!info.isFile() || info.size > maxBytes)
2796
+ continue;
2797
+ const text = await this.readFile(path);
2798
+ const content = text.trim().slice(0, MAX_CONTENT_CHARS5);
2799
+ if (content.length === 0)
2800
+ continue;
2801
+ documents.push({
2802
+ docId: createHash2("sha256").update(path).digest("hex").slice(0, 16),
2803
+ hierarchy: ["files"],
2804
+ title: basename3(path),
2805
+ content,
2806
+ publishedAt: info.mtimeMs,
2807
+ metadata: { path, extension: extname3(path).toLowerCase() }
2808
+ });
2809
+ } catch (error) {
2810
+ const code = error.code;
2811
+ if (code === "EPERM" || code === "EACCES")
2812
+ permissionDenied += 1;
2813
+ else
2814
+ readFailures += 1;
2815
+ }
2816
+ }
2817
+ if (documents.length === 0 && permissionDenied > 0) {
2818
+ return {
2819
+ ok: false,
2820
+ reason: "permission",
2821
+ message: `macOS denied access to all Spotlight hits; ${FDA_GUIDANCE}`
2822
+ };
2823
+ }
2824
+ const warnings = [];
2825
+ if (failedQueries > 0)
2826
+ warnings.push(`${failedQueries} mdfind quer(y/ies) failed`);
2827
+ if (permissionDenied > 0) {
2828
+ warnings.push(`${permissionDenied} Spotlight hit(s) were denied by macOS privacy protections; ${FDA_GUIDANCE}`);
2829
+ }
2830
+ if (readFailures > 0)
2831
+ warnings.push(`${readFailures} Spotlight hit(s) failed to read`);
2832
+ return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
2833
+ }
2834
+ }
2835
+ var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS12 = 300, DEFAULT_MAX_BYTES_PER_FILE4, DEFAULT_TIMEOUT_MS4 = 15000, MAX_CONTENT_CHARS5 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve2) => {
2836
+ const child = spawn3(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
2837
+ let stdout = "";
2838
+ let stderr = "";
2839
+ let settled = false;
2840
+ const finish = (result) => {
2841
+ if (settled)
2842
+ return;
2843
+ settled = true;
2844
+ clearTimeout(timer);
2845
+ resolve2(result);
2846
+ };
2847
+ const timer = setTimeout(() => {
2848
+ child.kill("SIGKILL");
2849
+ finish({ ok: false, stdout, stderr: "process timed out", code: null });
2850
+ }, timeoutMs);
2851
+ child.stdout.setEncoding("utf8");
2852
+ child.stderr.setEncoding("utf8");
2853
+ child.stdout.on("data", (chunk) => {
2854
+ if (stdout.length < MAX_BUFFER_BYTES)
2855
+ stdout += chunk;
2856
+ });
2857
+ child.stderr.on("data", (chunk) => {
2858
+ if (stderr.length < MAX_BUFFER_BYTES)
2859
+ stderr += chunk;
2860
+ });
2861
+ child.on("error", (error) => {
2862
+ finish({ ok: false, stdout, stderr: error.message, code: null, error: error.code });
2863
+ });
2864
+ child.on("close", (code) => {
2865
+ finish({ ok: code === 0, stdout, stderr, code });
2866
+ });
2867
+ });
2868
+ var init_connector11 = __esm(() => {
2869
+ DEFAULT_MAX_BYTES_PER_FILE4 = 256 * 1024;
2870
+ MAX_BUFFER_BYTES = 4 * 1024 * 1024;
2871
+ });
2872
+
2873
+ // src/datasource/skills/spotlight/skill.ts
2874
+ var SPOTLIGHT_SKILL_DEFINITION, SpotlightSkill;
2875
+ var init_skill10 = __esm(() => {
2876
+ init_connector_skill();
2877
+ init_connector11();
2878
+ SPOTLIGHT_SKILL_DEFINITION = {
2879
+ skillName: "spotlight",
2880
+ skillType: "spotlight-search",
2881
+ description: "macOS Spotlight datasource",
2882
+ capabilities: ["files", "macos-only", "local-search"],
2883
+ defaultTags: ["spotlight", "files"],
2884
+ contentType: "document",
2885
+ manifestDescription: "Search files indexed by macOS Spotlight via the built-in mdfind CLI. macOS only; no external installs. Use for questions about local documents on the Mac running AutoRAG.",
2886
+ manifestNotes: [
2887
+ "## Platform requirements",
2888
+ "- macOS only: this datasource is unavailable on any other platform.",
2889
+ "- Spotlight indexing must be enabled (`sudo mdutil -i on /`; check with `mdutil -s /`).",
2890
+ "- **Full Disk Access**: Spotlight can find protected files (Mail, Messages, Safari data, parts of ~/Library), but reading their content fails unless the terminal or app running AutoRAG has Full Disk Access. Grant it under System Settings -> Privacy & Security -> Full Disk Access, add the host app (e.g. Terminal, iTerm, or the AutoRAG binary), then restart the app. Permission failures surface as `datasource-permission-denied`.",
2891
+ "",
2892
+ "## Operator cheat sheet",
2893
+ "- `mdfind <query>` — Spotlight search; `mdfind -onlyin <dir> <query>` scopes to a directory; `mdfind -name <text>` matches file names.",
2894
+ "- `mdls <file>` — show Spotlight metadata attributes (kMDItemDisplayName, kMDItemContentType, …) for one file.",
2895
+ "- `mdutil -s /` — index status; `sudo mdutil -i on /` / `-i off /` — enable/disable indexing; `sudo mdutil -E /` — erase and rebuild the index.",
2896
+ "",
2897
+ "Queries are trusted server configuration; indexing re-runs the configured mdfind queries and hydrates text content. Result metadata carries the real absolute file path (`path`) so hits are traceable back to disk; if paths must not leave this machine, run AutoRAG with a local LLM."
2898
+ ]
2899
+ };
2900
+ SpotlightSkill = class SpotlightSkill extends ConnectorDatasourceSkill {
2901
+ constructor(options = {}) {
2902
+ const { connector, connectorOptions, ...rest } = options;
2903
+ super(SPOTLIGHT_SKILL_DEFINITION, {
2904
+ ...rest,
2905
+ connector: connector ?? new SpotlightConnector(connectorOptions ?? {})
2906
+ });
2907
+ }
2908
+ };
2909
+ });
2910
+
2911
+ // src/datasource/skills/spotlight/index.ts
2912
+ var init_spotlight = __esm(() => {
2913
+ init_connector11();
2914
+ init_skill10();
2915
+ });
2916
+
2917
+ // src/datasource/skills/factory.ts
2918
+ function common(config, workspaceRoot) {
2919
+ return {
2920
+ ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
2921
+ ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
2922
+ ...config.tags !== undefined ? { tags: config.tags } : {},
2923
+ ...workspaceRoot !== undefined ? { workspaceRoot } : {}
2924
+ };
2925
+ }
2926
+ function buildDatasourceSkills(config, workspaceRoot) {
2927
+ const skills = [];
2928
+ const unknown = [];
2929
+ for (const [name, raw] of Object.entries(config ?? {})) {
2930
+ if (raw === false)
2931
+ continue;
2932
+ const entry = raw === true ? {} : raw;
2933
+ if (entry.enabled === false)
2934
+ continue;
2935
+ const builder = BUILDERS[name];
2936
+ if (builder === undefined) {
2937
+ unknown.push(name);
2938
+ continue;
2939
+ }
2940
+ skills.push(builder(entry, workspaceRoot));
2941
+ }
2942
+ return { skills, unknown };
2943
+ }
2944
+ var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
2945
+ var init_factory = __esm(() => {
2946
+ init_discord();
2947
+ init_gdrive();
2948
+ init_rclone_connector();
2949
+ init_github();
2950
+ init_himalaya_connector();
2951
+ init_gmail();
2952
+ init_mail_export();
2953
+ init_notion();
2954
+ init_obsidian();
2955
+ init_rss();
2956
+ init_slack();
2957
+ init_spotlight();
2958
+ BUILDERS = {
2959
+ slack: (config, workspaceRoot) => new SlackSkill({ ...common(config, workspaceRoot), connectorOptions: config.connector }),
2960
+ discord: (config, workspaceRoot) => new DiscordSkill({
2961
+ ...common(config, workspaceRoot),
2962
+ connectorOptions: config.connector
2963
+ }),
2964
+ notion: (config, workspaceRoot) => new NotionSkill({
2965
+ ...common(config, workspaceRoot),
2966
+ connectorOptions: config.connector
2967
+ }),
2968
+ github: (config, workspaceRoot) => new GitHubSkill({
2969
+ ...common(config, workspaceRoot),
2970
+ connectorOptions: config.connector
2971
+ }),
2972
+ gdrive: (config, workspaceRoot) => {
2973
+ const connector = config.connector;
2974
+ if (connector?.backend === "rclone") {
2975
+ const { backend: _backend, ...rcloneOptions } = connector;
2976
+ return new GDriveSkill({
2977
+ ...common(config, workspaceRoot),
2978
+ connector: new RcloneConnector(rcloneOptions)
2979
+ });
2980
+ }
2981
+ return new GDriveSkill({
2982
+ ...common(config, workspaceRoot),
2983
+ connectorOptions: connector
2984
+ });
2985
+ },
2986
+ gmail: (config, workspaceRoot) => {
2987
+ const connector = config.connector;
2988
+ if (connector?.backend === "himalaya") {
2989
+ const { backend: _backend, ...himalayaOptions } = connector;
2990
+ return new GmailSkill({
2991
+ ...common(config, workspaceRoot),
2992
+ connector: new HimalayaConnector(himalayaOptions)
2993
+ });
2994
+ }
2995
+ return new GmailSkill({ ...common(config, workspaceRoot), connectorOptions: connector });
2996
+ },
2997
+ "mail-export": (config, workspaceRoot) => new MailExportSkill({
2998
+ ...common(config, workspaceRoot),
2999
+ connectorOptions: config.connector
3000
+ }),
3001
+ obsidian: (config, workspaceRoot) => new ObsidianSkill({
3002
+ ...common(config, workspaceRoot),
3003
+ connectorOptions: config.connector
3004
+ }),
3005
+ rss: (config, workspaceRoot) => new RssSkill({ ...common(config, workspaceRoot), connectorOptions: config.connector }),
3006
+ spotlight: (config, workspaceRoot) => new SpotlightSkill({
3007
+ ...common(config, workspaceRoot),
3008
+ connectorOptions: config.connector
3009
+ })
3010
+ };
3011
+ BUILTIN_DATASOURCE_SKILL_NAMES = Object.keys(BUILDERS);
3012
+ });
3013
+
359
3014
  // src/filesystem/file-lock.ts
360
3015
  import { randomUUID } from "node:crypto";
361
3016
  import {
362
3017
  closeSync,
363
3018
  lstatSync,
364
- mkdirSync,
3019
+ mkdirSync as mkdirSync2,
365
3020
  openSync,
366
3021
  readdirSync,
367
- readFileSync,
3022
+ readFileSync as readFileSync2,
368
3023
  rmdirSync,
369
3024
  unlinkSync,
370
3025
  writeSync
371
3026
  } from "node:fs";
372
- import { join as join2 } from "node:path";
3027
+ import { join as join5 } from "node:path";
373
3028
  function hasErrorCode(error, code) {
374
3029
  return error instanceof Error && "code" in error && error.code === code;
375
3030
  }
@@ -413,7 +3068,7 @@ function inspectRegularFile(path) {
413
3068
  return { kind: "other" };
414
3069
  let contents;
415
3070
  try {
416
- contents = readFileSync(path, "utf8");
3071
+ contents = readFileSync2(path, "utf8");
417
3072
  } catch (error) {
418
3073
  if (hasErrorCode(error, "ENOENT"))
419
3074
  return { kind: "missing" };
@@ -504,7 +3159,7 @@ function reapStaleLockDirectory(lockPath, staleMs) {
504
3159
  return removeEmptyLockDirectory(lockPath);
505
3160
  const staleSnapshots = [];
506
3161
  for (const name of names) {
507
- const inspection = inspectRegularFile(join2(lockPath, name));
3162
+ const inspection = inspectRegularFile(join5(lockPath, name));
508
3163
  if (inspection.kind === "missing")
509
3164
  continue;
510
3165
  if (inspection.kind !== "file" || !isStaleSnapshot(inspection, staleMs))
@@ -559,14 +3214,14 @@ function createLockFile(lockPath, contents) {
559
3214
  }
560
3215
  function tryCreateOwnerMarker(lockPath, owner, contents) {
561
3216
  try {
562
- mkdirSync(lockPath, { mode: 448 });
3217
+ mkdirSync2(lockPath, { mode: 448 });
563
3218
  } catch (error) {
564
3219
  if (hasErrorCode(error, "EEXIST"))
565
3220
  return;
566
3221
  throw error;
567
3222
  }
568
3223
  const markerName = `owner-${owner.token}.json`;
569
- const markerPath = join2(lockPath, markerName);
3224
+ const markerPath = join5(lockPath, markerName);
570
3225
  let acquired = false;
571
3226
  try {
572
3227
  try {
@@ -651,9 +3306,9 @@ var init_file_lock = __esm(() => {
651
3306
  });
652
3307
 
653
3308
  // src/subagents/local-models.ts
654
- import { readFileSync as readFileSync2 } from "node:fs";
3309
+ import { readFileSync as readFileSync3 } from "node:fs";
655
3310
  import { homedir as homedir2 } from "node:os";
656
- import { join as join3 } from "node:path";
3311
+ import { join as join6 } from "node:path";
657
3312
  import { parse } from "smol-toml";
658
3313
  function createModel(provider, baseUrl, id, defaultId, defaultName) {
659
3314
  return {
@@ -670,8 +3325,8 @@ function createModel(provider, baseUrl, id, defaultId, defaultName) {
670
3325
  };
671
3326
  }
672
3327
  function loadLocalAutoRAGModels(options = {}) {
673
- const configPath = options.configPath ?? join3(homedir2(), ".codex", "config.toml");
674
- const config = parse(readFileSync2(configPath, "utf8"));
3328
+ const configPath = options.configPath ?? join6(homedir2(), ".codex", "config.toml");
3329
+ const config = parse(readFileSync3(configPath, "utf8"));
675
3330
  const provider = config.model_provider;
676
3331
  if (typeof provider !== "string" || provider.length === 0) {
677
3332
  throw new Error(`AutoRAG requires model_provider in ${configPath}`);
@@ -702,8 +3357,8 @@ var init_local_models = () => {};
702
3357
 
703
3358
  // src/cli/config.ts
704
3359
  import { randomUUID as randomUUID2 } from "node:crypto";
705
- import { existsSync, mkdirSync as mkdirSync2, readFileSync as readFileSync3, renameSync, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
706
- import { basename, dirname, isAbsolute as isAbsolute2, join as join4, resolve as resolve2 } from "node:path";
3360
+ import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync, unlinkSync as unlinkSync2, writeFileSync as writeFileSync2 } from "node:fs";
3361
+ import { basename as basename4, dirname as dirname3, isAbsolute as isAbsolute2, join as join7, resolve as resolve2 } from "node:path";
707
3362
  import { findEnvKeys, getEnvApiKey, getModel, getProviders } from "@earendil-works/pi-ai/compat";
708
3363
  function resolveConfigPath(input) {
709
3364
  const flags = input.flags;
@@ -718,15 +3373,15 @@ function resolveConfigPath(input) {
718
3373
  return { configPath: envConfig, explicit: true };
719
3374
  }
720
3375
  return {
721
- configPath: join4(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
3376
+ configPath: join7(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
722
3377
  explicit: false,
723
- legacyPath: join4(cwd, LEGACY_CONFIG_FILENAME)
3378
+ legacyPath: join7(cwd, LEGACY_CONFIG_FILENAME)
724
3379
  };
725
3380
  }
726
3381
  function readConfigFile(configPath, explicit) {
727
3382
  let exists;
728
3383
  try {
729
- exists = existsSync(configPath);
3384
+ exists = existsSync2(configPath);
730
3385
  } catch {
731
3386
  exists = false;
732
3387
  }
@@ -738,7 +3393,7 @@ function readConfigFile(configPath, explicit) {
738
3393
  }
739
3394
  let text;
740
3395
  try {
741
- text = readFileSync3(configPath, "utf8");
3396
+ text = readFileSync4(configPath, "utf8");
742
3397
  } catch (err) {
743
3398
  throw new ConfigError(`Failed to read config file: ${err.message}`);
744
3399
  }
@@ -776,9 +3431,9 @@ function acquireConfigWriteLock(configPath) {
776
3431
  });
777
3432
  }
778
3433
  function replaceFileAtomically(path, contents, assertCommitAllowed) {
779
- const temporaryPath = join4(dirname(path), `.${basename(path)}.${process.pid}.${randomUUID2()}.tmp`);
3434
+ const temporaryPath = join7(dirname3(path), `.${basename4(path)}.${process.pid}.${randomUUID2()}.tmp`);
780
3435
  try {
781
- writeFileSync(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
3436
+ writeFileSync2(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
782
3437
  assertCommitAllowed?.();
783
3438
  renameSync(temporaryPath, path);
784
3439
  } finally {
@@ -786,14 +3441,14 @@ function replaceFileAtomically(path, contents, assertCommitAllowed) {
786
3441
  }
787
3442
  }
788
3443
  function migrateLegacyConfig(configPath, legacyPath) {
789
- if (existsSync(configPath) || !existsSync(legacyPath))
3444
+ if (existsSync2(configPath) || !existsSync2(legacyPath))
790
3445
  return;
791
3446
  const legacy = readConfigFile(legacyPath, true);
792
- const legacyBytes = readFileSync3(legacyPath);
793
- const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname(legacyPath));
3447
+ const legacyBytes = readFileSync4(legacyPath);
3448
+ const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname3(legacyPath));
794
3449
  const migratedBytes = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths) ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
795
3450
  `;
796
- mkdirSync2(dirname(configPath), { recursive: true });
3451
+ mkdirSync3(dirname3(configPath), { recursive: true });
797
3452
  const lock = acquireConfigWriteLock(configPath);
798
3453
  try {
799
3454
  const winner = readConfigFile(configPath, false);
@@ -823,7 +3478,7 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
823
3478
  const legacy = readConfigFile(legacyPath, false);
824
3479
  if (legacy === undefined)
825
3480
  return {};
826
- return normalizeLegacyConfigPaths(legacy, dirname(legacyPath));
3481
+ return normalizeLegacyConfigPaths(legacy, dirname3(legacyPath));
827
3482
  }
828
3483
  function resolveSearchPaths(searchPaths, origin) {
829
3484
  return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
@@ -1092,10 +3747,10 @@ function resolveConfig(input) {
1092
3747
  const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
1093
3748
  const defaultSearchPaths = ["."];
1094
3749
  const defaultWorkspacePath = cwd;
1095
- const defaultMemoryPath = join4(resolveAutoRAGHome(env), "memory.json");
3750
+ const defaultMemoryPath = join7(resolveAutoRAGHome(env), "memory.json");
1096
3751
  const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
1097
3752
  const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
1098
- const configOrigin = dirname(resolve2(configPath));
3753
+ const configOrigin = dirname3(resolve2(configPath));
1099
3754
  const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
1100
3755
  const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
1101
3756
  const searchPaths = flagSearchPaths ?? envSearchPaths ?? fileSearchPaths ?? defaultSearchPaths;
@@ -1138,6 +3793,18 @@ function resolveConfig(input) {
1138
3793
  config.jikji = file.jikji;
1139
3794
  if (file.parserOptions)
1140
3795
  config.parserOptions = file.parserOptions;
3796
+ if (file.datasources !== undefined) {
3797
+ if (typeof file.datasources !== "object" || file.datasources === null || Array.isArray(file.datasources)) {
3798
+ throw new ConfigError("Config field 'datasources' must be an object mapping skill names to their config");
3799
+ }
3800
+ config.datasources = file.datasources;
3801
+ }
3802
+ if (file.datasourceAccess !== undefined) {
3803
+ if (typeof file.datasourceAccess !== "object" || file.datasourceAccess === null || Array.isArray(file.datasourceAccess)) {
3804
+ throw new ConfigError("Config field 'datasourceAccess' must be an object with allowedTags/allowedScopes");
3805
+ }
3806
+ config.datasourceAccess = file.datasourceAccess;
3807
+ }
1141
3808
  return config;
1142
3809
  }
1143
3810
  function resolveConfigReadOnly(input) {
@@ -1167,6 +3834,16 @@ function buildAgentOptions(config) {
1167
3834
  opts.jikji = config.jikji;
1168
3835
  if (config.parserOptions)
1169
3836
  opts.parserOptions = config.parserOptions;
3837
+ if (config.datasources !== undefined) {
3838
+ const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath);
3839
+ if (unknown.length > 0) {
3840
+ throw new ConfigError(`Unknown datasource skill(s) in config: ${unknown.join(", ")}`);
3841
+ }
3842
+ if (skills.length > 0)
3843
+ opts.datasourceSkills = skills;
3844
+ }
3845
+ if (config.datasourceAccess !== undefined)
3846
+ opts.datasourceAccess = config.datasourceAccess;
1170
3847
  return opts;
1171
3848
  }
1172
3849
  function isConfiguredEndpoint(reference) {
@@ -1385,7 +4062,7 @@ function resolveAgentModelDetailed(config, localOptions = {}) {
1385
4062
  function writeDefaultConfig(path, partial, opts = {}) {
1386
4063
  const cwd = resolve2(opts.cwd ?? process.cwd());
1387
4064
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
1388
- const memoryPath = partial.memoryPath === undefined ? join4(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
4065
+ const memoryPath = partial.memoryPath === undefined ? join7(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
1389
4066
  const full = {
1390
4067
  searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
1391
4068
  workspacePath,
@@ -1410,19 +4087,19 @@ function writeDefaultConfig(path, partial, opts = {}) {
1410
4087
  full.jikji = partial.jikji ?? {};
1411
4088
  if (partial.parserOptions)
1412
4089
  full.parserOptions = partial.parserOptions;
1413
- mkdirSync2(dirname(path), { recursive: true });
4090
+ mkdirSync3(dirname3(path), { recursive: true });
1414
4091
  const contents = `${JSON.stringify(full, null, 2)}
1415
4092
  `;
1416
4093
  const lock = acquireConfigWriteLock(path);
1417
4094
  try {
1418
- if (!opts.force && existsSync(path)) {
4095
+ if (!opts.force && existsSync2(path)) {
1419
4096
  throw new ConfigError(`Config file already exists: ${path}`);
1420
4097
  }
1421
4098
  if (opts.force || opts.atomicCreate)
1422
4099
  replaceFileAtomically(path, contents, lock.assertOwned);
1423
4100
  else {
1424
4101
  lock.assertOwned();
1425
- writeFileSync(path, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
4102
+ writeFileSync2(path, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
1426
4103
  }
1427
4104
  } catch (error) {
1428
4105
  if (!opts.force && isEexistError(error)) {
@@ -1436,6 +4113,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
1436
4113
  var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, BM25_ALLOWLIST, MINSYNC_ALLOWLIST;
1437
4114
  var init_config = __esm(() => {
1438
4115
  init_home();
4116
+ init_factory();
1439
4117
  init_file_lock();
1440
4118
  init_local_models();
1441
4119
  init_home();
@@ -1483,8 +4161,8 @@ var exports_init = {};
1483
4161
  __export(exports_init, {
1484
4162
  runInit: () => runInit
1485
4163
  });
1486
- import { existsSync as existsSync2, readFileSync as readFileSync4 } from "node:fs";
1487
- import { dirname as dirname2 } from "node:path";
4164
+ import { existsSync as existsSync3, readFileSync as readFileSync5 } from "node:fs";
4165
+ import { dirname as dirname4 } from "node:path";
1488
4166
  async function runInit(ctx) {
1489
4167
  const flags = ctx.flags;
1490
4168
  const partial = {};
@@ -1532,12 +4210,12 @@ async function runInit(ctx) {
1532
4210
  throw error;
1533
4211
  }
1534
4212
  const resolvedPath = resolveConfigPath({ flags, cwd: ctx.cwd });
1535
- const migratingLegacy = !resolvedPath.explicit && resolvedPath.legacyPath !== undefined && existsSync2(resolvedPath.legacyPath) && !existsSync2(resolvedPath.configPath);
4213
+ const migratingLegacy = !resolvedPath.explicit && resolvedPath.legacyPath !== undefined && existsSync3(resolvedPath.legacyPath) && !existsSync3(resolvedPath.configPath);
1536
4214
  if (migratingLegacy) {
1537
4215
  try {
1538
4216
  const legacyPath = resolvedPath.legacyPath;
1539
- const legacy = JSON.parse(readFileSync4(legacyPath, "utf8"));
1540
- const normalizedLegacy = normalizeLegacyConfigPaths(legacy, dirname2(legacyPath));
4217
+ const legacy = JSON.parse(readFileSync5(legacyPath, "utf8"));
4218
+ const normalizedLegacy = normalizeLegacyConfigPaths(legacy, dirname4(legacyPath));
1541
4219
  partial.searchPaths ??= normalizedLegacy.searchPaths;
1542
4220
  partial.workspacePath ??= normalizedLegacy.workspacePath;
1543
4221
  partial.memoryPath ??= normalizedLegacy.memoryPath;
@@ -1623,49 +4301,6 @@ var init_init = __esm(() => {
1623
4301
  init_config();
1624
4302
  });
1625
4303
 
1626
- // src/retrieval/scope.ts
1627
- function normalizeVirtualPath(value) {
1628
- const normalized = value.replace(/\\/g, "/").replace(/\/+/g, "/").trim();
1629
- if (normalized.length === 0 || normalized === "/")
1630
- return "/";
1631
- const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`;
1632
- return withLeadingSlash.length > 1 ? withLeadingSlash.replace(/\/+$/g, "") : withLeadingSlash;
1633
- }
1634
- function normalizeVirtualPathScope(scope) {
1635
- if (scope === undefined)
1636
- return;
1637
- const normalized = normalizeVirtualPath(scope);
1638
- return normalized === "/" ? undefined : normalized;
1639
- }
1640
- function matchesVirtualPathScope(virtualPath, scope) {
1641
- const normalizedScope = normalizeVirtualPathScope(scope);
1642
- if (normalizedScope === undefined)
1643
- return true;
1644
- return virtualPathScopeToRegExp(normalizedScope).test(normalizeVirtualPath(virtualPath));
1645
- }
1646
- function virtualPathScopeToRegExp(scope) {
1647
- const normalized = normalizeVirtualPath(scope);
1648
- const scoped = hasGlob(normalized) || looksLikeFileScope(normalized) ? normalized : `${normalized}/**`;
1649
- const pattern = scoped.split("/").map((segment, index) => {
1650
- if (index === 0)
1651
- return "";
1652
- if (segment === "**")
1653
- return "(?:.*)";
1654
- return escapeRegExp(segment).replace(/\\\*/g, "[^/]*");
1655
- }).join("/").replace(/\/\(\?:\.\*\)$/u, "(?:/.*)?");
1656
- return new RegExp(`^${pattern}$`);
1657
- }
1658
- function hasGlob(scope) {
1659
- return scope.includes("*");
1660
- }
1661
- function looksLikeFileScope(scope) {
1662
- const finalSegment = scope.split("/").at(-1) ?? "";
1663
- return /\.[^/.]+$/u.test(finalSegment);
1664
- }
1665
- function escapeRegExp(literal) {
1666
- return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1667
- }
1668
-
1669
4304
  // src/datasource/access-context.ts
1670
4305
  class DatasourceAccessContext {
1671
4306
  allowedTags;
@@ -1817,7 +4452,7 @@ var init_diagnostics = __esm(() => {
1817
4452
 
1818
4453
  // src/jikji/answer-pack.ts
1819
4454
  import { realpathSync } from "node:fs";
1820
- import { isAbsolute as isAbsolute3, join as join5, relative, resolve as resolve3 } from "node:path";
4455
+ import { isAbsolute as isAbsolute3, join as join8, relative as relative2, resolve as resolve3 } from "node:path";
1821
4456
  function isRecord(value) {
1822
4457
  return typeof value === "object" && value !== null && !Array.isArray(value);
1823
4458
  }
@@ -1942,10 +4577,10 @@ function normalizeJikjiAnswerPath(rawPath, sourceRoots) {
1942
4577
  } catch {
1943
4578
  continue;
1944
4579
  }
1945
- const rel = relative(realRoot, realCandidate);
4580
+ const rel = relative2(realRoot, realCandidate);
1946
4581
  if (rel.startsWith("..") || isAbsolute3(rel))
1947
4582
  continue;
1948
- return join5(root.rootPath, rel);
4583
+ return join8(root.rootPath, rel);
1949
4584
  }
1950
4585
  return;
1951
4586
  }
@@ -1959,9 +4594,9 @@ var init_answer_pack = __esm(() => {
1959
4594
  });
1960
4595
 
1961
4596
  // src/jikji/installer.ts
1962
- import { spawn } from "node:child_process";
1963
- import { existsSync as existsSync3 } from "node:fs";
1964
- import { delimiter, join as join6 } from "node:path";
4597
+ import { spawn as spawn4 } from "node:child_process";
4598
+ import { existsSync as existsSync4 } from "node:fs";
4599
+ import { delimiter, join as join9 } from "node:path";
1965
4600
  function jikjiExecutableName(platform = process.platform) {
1966
4601
  return platform === "win32" ? "jikji.exe" : "jikji";
1967
4602
  }
@@ -1969,7 +4604,7 @@ function cargoExecutableName(platform = process.platform) {
1969
4604
  return platform === "win32" ? "cargo.exe" : "cargo";
1970
4605
  }
1971
4606
  function cachedJikjiBinaryPath(root, platform = process.platform) {
1972
- return join6(root, ".autorag", "bin", jikjiExecutableName(platform));
4607
+ return join9(root, ".autorag", "bin", jikjiExecutableName(platform));
1973
4608
  }
1974
4609
  function lookupExecutableInPath(executable, env) {
1975
4610
  const pathEnv = env.PATH;
@@ -1978,15 +4613,15 @@ function lookupExecutableInPath(executable, env) {
1978
4613
  for (const dir of pathEnv.split(delimiter)) {
1979
4614
  if (dir.length === 0)
1980
4615
  continue;
1981
- const candidate = join6(dir, executable);
1982
- if (existsSync3(candidate))
4616
+ const candidate = join9(dir, executable);
4617
+ if (existsSync4(candidate))
1983
4618
  return candidate;
1984
4619
  }
1985
4620
  return;
1986
4621
  }
1987
4622
  function defaultRunner(cargo, env) {
1988
4623
  return (args, timeoutMs) => new Promise((resolvePromise) => {
1989
- const child = spawn(cargo, args, { env, stdio: ["ignore", "ignore", "pipe"] });
4624
+ const child = spawn4(cargo, args, { env, stdio: ["ignore", "ignore", "pipe"] });
1990
4625
  let stderr = "";
1991
4626
  const timeout = setTimeout(() => {
1992
4627
  if (!child.killed)
@@ -2009,7 +4644,7 @@ function defaultRunner(cargo, env) {
2009
4644
  async function ensureJikjiBinary(options) {
2010
4645
  const env = options.env ?? process.env;
2011
4646
  const cached = cachedJikjiBinaryPath(options.root);
2012
- if (existsSync3(cached))
4647
+ if (existsSync4(cached))
2013
4648
  return { ok: true, binaryPath: cached, source: "cached" };
2014
4649
  const cargoLocator = options.cargoLocator ?? ((e) => lookupExecutableInPath(cargoExecutableName(), e));
2015
4650
  const cargo = cargoLocator(env);
@@ -2020,12 +4655,12 @@ async function ensureJikjiBinary(options) {
2020
4655
  message: "Jikji auto-install needs the Rust toolchain (cargo) on PATH — install via https://rustup.rs, or install jikji manually (`cargo install jikji-cli`)."
2021
4656
  };
2022
4657
  }
2023
- const args = ["install", JIKJI_CRATE_NAME, "--locked", "--root", join6(options.root, ".autorag")];
4658
+ const args = ["install", JIKJI_CRATE_NAME, "--locked", "--root", join9(options.root, ".autorag")];
2024
4659
  if (options.version !== undefined)
2025
4660
  args.push("--version", options.version);
2026
4661
  const runner = options.runner ?? defaultRunner(cargo, env);
2027
4662
  const result = await runner(args, options.timeoutMs ?? JIKJI_INSTALL_TIMEOUT_MS);
2028
- if (result.code === 0 && existsSync3(cached)) {
4663
+ if (result.code === 0 && existsSync4(cached)) {
2029
4664
  return { ok: true, binaryPath: cached, source: "installed" };
2030
4665
  }
2031
4666
  return {
@@ -2040,8 +4675,8 @@ var init_installer = __esm(() => {
2040
4675
  });
2041
4676
 
2042
4677
  // src/jikji/client.ts
2043
- import { spawn as spawn2 } from "node:child_process";
2044
- import { existsSync as existsSync4 } from "node:fs";
4678
+ import { spawn as spawn5 } from "node:child_process";
4679
+ import { existsSync as existsSync5 } from "node:fs";
2045
4680
 
2046
4681
  class JikjiClient {
2047
4682
  options;
@@ -2054,13 +4689,13 @@ class JikjiClient {
2054
4689
  return commandFor(this.options.binaryPath);
2055
4690
  if (this.resolvedCommand !== undefined)
2056
4691
  return this.resolvedCommand;
2057
- if (lookupExecutableInPath(DEFAULT_BINARY, process.env) !== undefined) {
2058
- this.resolvedCommand = DEFAULT_BINARY;
4692
+ if (lookupExecutableInPath(DEFAULT_BINARY3, process.env) !== undefined) {
4693
+ this.resolvedCommand = DEFAULT_BINARY3;
2059
4694
  return this.resolvedCommand;
2060
4695
  }
2061
4696
  if (this.options.root !== undefined) {
2062
4697
  const cached = cachedJikjiBinaryPath(this.options.root);
2063
- if (existsSync4(cached)) {
4698
+ if (existsSync5(cached)) {
2064
4699
  this.resolvedCommand = cached;
2065
4700
  return this.resolvedCommand;
2066
4701
  }
@@ -2072,7 +4707,7 @@ class JikjiClient {
2072
4707
  }
2073
4708
  }
2074
4709
  }
2075
- this.resolvedCommand = DEFAULT_BINARY;
4710
+ this.resolvedCommand = DEFAULT_BINARY3;
2076
4711
  return this.resolvedCommand;
2077
4712
  }
2078
4713
  async prepare(root, options = {}) {
@@ -2136,7 +4771,7 @@ class JikjiClient {
2136
4771
  function spawnJikji(request) {
2137
4772
  return new Promise((resolve4) => {
2138
4773
  const options = request.options;
2139
- const child = spawn2(request.command, request.args, {
4774
+ const child = spawn5(request.command, request.args, {
2140
4775
  env: controlledEnv(options.env),
2141
4776
  stdio: ["ignore", "pipe", "pipe"]
2142
4777
  });
@@ -2147,7 +4782,7 @@ function spawnJikji(request) {
2147
4782
  const timeout = setTimeout(() => {
2148
4783
  finalReason = "timeout";
2149
4784
  terminate(child);
2150
- }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
4785
+ }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS5);
2151
4786
  const abortHandler = () => {
2152
4787
  finalReason = "aborted";
2153
4788
  terminate(child);
@@ -2194,7 +4829,7 @@ function spawnJikji(request) {
2194
4829
  });
2195
4830
  }
2196
4831
  function commandFor(binaryPath) {
2197
- return binaryPath === undefined || binaryPath === DEFAULT_BINARY ? DEFAULT_BINARY : binaryPath;
4832
+ return binaryPath === undefined || binaryPath === DEFAULT_BINARY3 ? DEFAULT_BINARY3 : binaryPath;
2198
4833
  }
2199
4834
  function buildPrepareArgs(options, root) {
2200
4835
  const args = ["prepare", root, "--json"];
@@ -2261,19 +4896,19 @@ function appendBounded(state, chunk, maxBytes) {
2261
4896
  const remainingBytes = Math.max(maxBytes - state.bytes, 0);
2262
4897
  return { text: state.text + chunk.slice(0, remainingBytes), bytes: maxBytes, capped: true };
2263
4898
  }
2264
- var DEFAULT_BINARY = "jikji", DEFAULT_TIMEOUT_MS = 1e4, DEFAULT_MAX_BUFFER_BYTES = 1048576;
4899
+ var DEFAULT_BINARY3 = "jikji", DEFAULT_TIMEOUT_MS5 = 1e4, DEFAULT_MAX_BUFFER_BYTES = 1048576;
2265
4900
  var init_client = __esm(() => {
2266
4901
  init_answer_pack();
2267
4902
  init_installer();
2268
4903
  });
2269
4904
 
2270
4905
  // src/filesystem/source-paths.ts
2271
- import { basename as basename2, isAbsolute as isAbsolute4, relative as relative2, resolve as resolve4, sep } from "node:path";
4906
+ import { basename as basename5, isAbsolute as isAbsolute4, relative as relative3, resolve as resolve4, sep as sep2 } from "node:path";
2272
4907
  function planSourceRoots(searchPaths) {
2273
4908
  const sorted = searchPaths.map((path) => resolve4(path)).sort((a, b) => a.localeCompare(b));
2274
4909
  const used = new Set;
2275
4910
  return sorted.map((rootPath) => {
2276
- const base = basename2(rootPath.replace(/[/\\]+$/, "")) || "root";
4911
+ const base = basename5(rootPath.replace(/[/\\]+$/, "")) || "root";
2277
4912
  let prefix = `/${base}`;
2278
4913
  let suffix = 2;
2279
4914
  while (used.has(prefix)) {
@@ -2285,14 +4920,14 @@ function planSourceRoots(searchPaths) {
2285
4920
  });
2286
4921
  }
2287
4922
  function sourceIdentifier(sourceRoot, sourcePath) {
2288
- const rel = relative2(sourceRoot.rootPath, resolve4(sourcePath));
2289
- const suffix = rel === "" ? "" : `/${rel.split(sep).join("/")}`;
4923
+ const rel = relative3(sourceRoot.rootPath, resolve4(sourcePath));
4924
+ const suffix = rel === "" ? "" : `/${rel.split(sep2).join("/")}`;
2290
4925
  return `${sourceRoot.prefix}${suffix}`;
2291
4926
  }
2292
4927
  var init_source_paths = () => {};
2293
4928
 
2294
4929
  // src/jikji/path-map.ts
2295
- import { isAbsolute as isAbsolute5, relative as relative3, resolve as resolve5, sep as sep2 } from "node:path";
4930
+ import { isAbsolute as isAbsolute5, relative as relative4, resolve as resolve5, sep as sep3 } from "node:path";
2296
4931
  function planJikjiSourceRoots(searchPaths) {
2297
4932
  return planSourceRoots(searchPaths).map((sourceRoot) => ({
2298
4933
  ...sourceRoot,
@@ -2319,8 +4954,8 @@ function isUnsafeReturnedPath(normalizedPath) {
2319
4954
  return normalizedPath.startsWith("/") || /^[A-Za-z]:\//.test(normalizedPath) || normalizedPath.startsWith("//") || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(normalizedPath);
2320
4955
  }
2321
4956
  function isWithinRoot(rootPath, sourcePath) {
2322
- const rel = relative3(resolve5(rootPath), resolve5(sourcePath));
2323
- return rel === "" || !rel.startsWith("..") && !rel.includes(`..${sep2}`) && !isAbsolute5(rel);
4957
+ const rel = relative4(resolve5(rootPath), resolve5(sourcePath));
4958
+ return rel === "" || !rel.startsWith("..") && !rel.includes(`..${sep3}`) && !isAbsolute5(rel);
2324
4959
  }
2325
4960
  var init_path_map = __esm(() => {
2326
4961
  init_source_paths();
@@ -2353,8 +4988,8 @@ var init_jikji = __esm(() => {
2353
4988
  });
2354
4989
 
2355
4990
  // src/manifest/loader.ts
2356
- import { existsSync as existsSync5, readdirSync as readdirSync2, readFileSync as readFileSync5 } from "node:fs";
2357
- import { extname, join as join7 } from "node:path";
4991
+ import { existsSync as existsSync6, readdirSync as readdirSync2, readFileSync as readFileSync6 } from "node:fs";
4992
+ import { extname as extname4, join as join10 } from "node:path";
2358
4993
  import { parse as parseYaml } from "yaml";
2359
4994
  function isValidManifest(data) {
2360
4995
  if (typeof data !== "object" || data === null)
@@ -2363,8 +4998,8 @@ function isValidManifest(data) {
2363
4998
  return typeof obj.name === "string" && obj.name.length > 0 && typeof obj.description === "string" && obj.description.length > 0 && typeof obj.type === "string" && ["vector", "bm25", "hybrid", "visual"].includes(obj.type);
2364
4999
  }
2365
5000
  function loadManifest(filePath) {
2366
- const content = readFileSync5(filePath, "utf-8");
2367
- const ext = extname(filePath).toLowerCase();
5001
+ const content = readFileSync6(filePath, "utf-8");
5002
+ const ext = extname4(filePath).toLowerCase();
2368
5003
  let data;
2369
5004
  if (ext === ".yaml" || ext === ".yml") {
2370
5005
  data = parseYaml(content);
@@ -2386,7 +5021,7 @@ function loadManifest(filePath) {
2386
5021
  };
2387
5022
  }
2388
5023
  function loadManifests(dirPath) {
2389
- if (!existsSync5(dirPath)) {
5024
+ if (!existsSync6(dirPath)) {
2390
5025
  return [];
2391
5026
  }
2392
5027
  let entries;
@@ -2397,10 +5032,10 @@ function loadManifests(dirPath) {
2397
5032
  }
2398
5033
  const manifests = [];
2399
5034
  for (const entry of entries) {
2400
- const ext = extname(entry).toLowerCase();
5035
+ const ext = extname4(entry).toLowerCase();
2401
5036
  if (ext !== ".yaml" && ext !== ".yml" && ext !== ".json")
2402
5037
  continue;
2403
- const filePath = join7(dirPath, entry);
5038
+ const filePath = join10(dirPath, entry);
2404
5039
  try {
2405
5040
  const manifest = loadManifest(filePath);
2406
5041
  manifests.push(manifest);
@@ -2486,7 +5121,7 @@ var init_check_memory_tool = __esm(() => {
2486
5121
  });
2487
5122
 
2488
5123
  // src/retrieval/evidence-id.ts
2489
- import { createHash } from "node:crypto";
5124
+ import { createHash as createHash3 } from "node:crypto";
2490
5125
  function normalizeEvidenceText(text) {
2491
5126
  return text.replace(/\r\n?/gu, `
2492
5127
  `).trim().normalize("NFC");
@@ -2516,7 +5151,7 @@ function stableEvidenceId(input) {
2516
5151
  input.lineNumber ?? "",
2517
5152
  normalizeEvidenceText(evidenceText)
2518
5153
  ].join("\x00");
2519
- const digest = createHash("sha256").update(payload).digest("hex").slice(0, 24);
5154
+ const digest = createHash3("sha256").update(payload).digest("hex").slice(0, 24);
2520
5155
  return `${method}:${digest}`;
2521
5156
  }
2522
5157
  function normalizeEvidenceRef(input) {
@@ -2528,9 +5163,9 @@ function normalizeEvidenceRef(input) {
2528
5163
  var init_evidence_id = () => {};
2529
5164
 
2530
5165
  // src/memory/memory.ts
2531
- import { createHash as createHash2, randomUUID as randomUUID3 } from "node:crypto";
2532
- import { existsSync as existsSync6, mkdirSync as mkdirSync3, readFileSync as readFileSync6, renameSync as renameSync2, unlinkSync as unlinkSync3, writeFileSync as writeFileSync2 } from "node:fs";
2533
- import { dirname as dirname3 } from "node:path";
5166
+ import { createHash as createHash4, randomUUID as randomUUID3 } from "node:crypto";
5167
+ import { existsSync as existsSync7, mkdirSync as mkdirSync4, readFileSync as readFileSync7, renameSync as renameSync2, unlinkSync as unlinkSync3, writeFileSync as writeFileSync3 } from "node:fs";
5168
+ import { dirname as dirname5 } from "node:path";
2534
5169
  function emptyMemoryV4() {
2535
5170
  return {
2536
5171
  version: 4,
@@ -2587,7 +5222,7 @@ function normalizeV4(data) {
2587
5222
  };
2588
5223
  }
2589
5224
  function hashText(value) {
2590
- return createHash2("sha256").update(value).digest("hex");
5225
+ return createHash4("sha256").update(value).digest("hex");
2591
5226
  }
2592
5227
  function resultId(sessionId, number) {
2593
5228
  return `${sessionId}:${number}`;
@@ -2687,9 +5322,9 @@ class RetrievalMemory {
2687
5322
  this.persistedData = cloneMemory(this.data);
2688
5323
  }
2689
5324
  save() {
2690
- const dir = dirname3(this.storagePath);
2691
- if (!existsSync6(dir))
2692
- mkdirSync3(dir, { recursive: true });
5325
+ const dir = dirname5(this.storagePath);
5326
+ if (!existsSync7(dir))
5327
+ mkdirSync4(dir, { recursive: true });
2693
5328
  const lock = this.acquireLock();
2694
5329
  const localData = this.data;
2695
5330
  let tmpPath;
@@ -2697,7 +5332,7 @@ class RetrievalMemory {
2697
5332
  this.data = this.mergeWithPersisted(this.readPersistedData());
2698
5333
  this.capData();
2699
5334
  tmpPath = `${this.storagePath}.${randomUUID3()}.tmp`;
2700
- writeFileSync2(tmpPath, `${JSON.stringify(this.data, null, 2)}
5335
+ writeFileSync3(tmpPath, `${JSON.stringify(this.data, null, 2)}
2701
5336
  `, "utf-8");
2702
5337
  lock.assertOwned();
2703
5338
  renameSync2(tmpPath, this.storagePath);
@@ -2707,7 +5342,7 @@ class RetrievalMemory {
2707
5342
  throw error;
2708
5343
  } finally {
2709
5344
  try {
2710
- if (tmpPath && existsSync6(tmpPath))
5345
+ if (tmpPath && existsSync7(tmpPath))
2711
5346
  unlinkSync3(tmpPath);
2712
5347
  } finally {
2713
5348
  lock.release();
@@ -2911,10 +5546,10 @@ class RetrievalMemory {
2911
5546
  });
2912
5547
  }
2913
5548
  readPersistedData() {
2914
- if (!existsSync6(this.storagePath))
5549
+ if (!existsSync7(this.storagePath))
2915
5550
  return emptyMemoryV4();
2916
5551
  try {
2917
- const parsed = JSON.parse(readFileSync6(this.storagePath, "utf-8"));
5552
+ const parsed = JSON.parse(readFileSync7(this.storagePath, "utf-8"));
2918
5553
  if (isV4(parsed))
2919
5554
  return normalizeV4(parsed);
2920
5555
  } catch (error) {
@@ -3165,17 +5800,17 @@ var init_memory = __esm(() => {
3165
5800
  });
3166
5801
 
3167
5802
  // src/minsync/embedder-config.ts
3168
- import { readFileSync as readFileSync7, writeFileSync as writeFileSync3 } from "node:fs";
3169
- import { join as join8 } from "node:path";
5803
+ import { readFileSync as readFileSync8, writeFileSync as writeFileSync4 } from "node:fs";
5804
+ import { join as join11 } from "node:path";
3170
5805
  import { parse as parse2, stringify } from "smol-toml";
3171
5806
  function minSyncConfigPath(workspacePath) {
3172
- return join8(workspacePath, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE);
5807
+ return join11(workspacePath, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE);
3173
5808
  }
3174
5809
  function rewriteEmbedderConfig(workspacePath, embedder) {
3175
5810
  const configPath = minSyncConfigPath(workspacePath);
3176
5811
  let raw;
3177
5812
  try {
3178
- raw = readFileSync7(configPath, "utf8");
5813
+ raw = readFileSync8(configPath, "utf8");
3179
5814
  } catch {
3180
5815
  return false;
3181
5816
  }
@@ -3205,17 +5840,17 @@ function rewriteEmbedderConfig(workspacePath, embedder) {
3205
5840
  }
3206
5841
  if (embedder.dimension !== undefined)
3207
5842
  options.dimension = embedder.dimension;
3208
- writeFileSync3(configPath, stringify(parsed));
5843
+ writeFileSync4(configPath, stringify(parsed));
3209
5844
  return true;
3210
5845
  }
3211
5846
  var MINSYNC_CONFIG_DIR = ".minsync", MINSYNC_CONFIG_FILE = "config.toml";
3212
5847
  var init_embedder_config = () => {};
3213
5848
 
3214
5849
  // src/minsync/process.ts
3215
- import { spawn as spawn3 } from "node:child_process";
5850
+ import { spawn as spawn6 } from "node:child_process";
3216
5851
  function spawnProcess(command, args, cwd, options = {}) {
3217
5852
  return new Promise((resolve6) => {
3218
- const child = spawn3(command, [...args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
5853
+ const child = spawn6(command, [...args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
3219
5854
  let stdout = "";
3220
5855
  let stderr = "";
3221
5856
  let timedOut = false;
@@ -3250,7 +5885,7 @@ function spawnProcess(command, args, cwd, options = {}) {
3250
5885
  var init_process = () => {};
3251
5886
 
3252
5887
  // src/minsync/client.ts
3253
- import { existsSync as existsSync7 } from "node:fs";
5888
+ import { existsSync as existsSync8 } from "node:fs";
3254
5889
 
3255
5890
  class MinSyncClient {
3256
5891
  binaryPath;
@@ -3262,7 +5897,7 @@ class MinSyncClient {
3262
5897
  this.embedder = options.embedder;
3263
5898
  }
3264
5899
  async sync() {
3265
- if (!existsSync7(this.binaryPath)) {
5900
+ if (!existsSync8(this.binaryPath)) {
3266
5901
  return { ok: false, synced: 0, workspacePath: this.workspacePath, reason: "missing-binary" };
3267
5902
  }
3268
5903
  if (this.embedder?.apiKeyEnv) {
@@ -3309,7 +5944,7 @@ class MinSyncClient {
3309
5944
  return { ok: true, synced: readSyncedCount(result.stdout), workspacePath: this.workspacePath };
3310
5945
  }
3311
5946
  async query(text, topK) {
3312
- if (!existsSync7(this.binaryPath))
5947
+ if (!existsSync8(this.binaryPath))
3313
5948
  return [];
3314
5949
  const result = await spawnProcess(this.binaryPath, ["query", "--format", "json", "-k", String(topK), text], this.workspacePath);
3315
5950
  if (!result.ok)
@@ -3364,23 +5999,23 @@ var init_client2 = __esm(() => {
3364
5999
  });
3365
6000
 
3366
6001
  // src/minsync/installer.ts
3367
- import { createHash as createHash3 } from "node:crypto";
3368
- import { createWriteStream, existsSync as existsSync8, mkdirSync as mkdirSync4, renameSync as renameSync3, rmSync } from "node:fs";
6002
+ import { createHash as createHash5 } from "node:crypto";
6003
+ import { createWriteStream, existsSync as existsSync9, mkdirSync as mkdirSync5, renameSync as renameSync3, rmSync } from "node:fs";
3369
6004
  import { chmod, mkdtemp } from "node:fs/promises";
3370
6005
  import { get } from "node:https";
3371
6006
  import { tmpdir } from "node:os";
3372
- import { basename as basename3, dirname as dirname4, join as join9 } from "node:path";
6007
+ import { basename as basename6, dirname as dirname6, join as join12 } from "node:path";
3373
6008
  import { pipeline } from "node:stream/promises";
3374
6009
  async function ensureMinSyncBinary(options) {
3375
- const binaryPath = join9(options.root, ".autorag", "bin", executableName(options.platform ?? process.platform));
3376
- if (existsSync8(binaryPath))
6010
+ const binaryPath = join12(options.root, ".autorag", "bin", executableName(options.platform ?? process.platform));
6011
+ if (existsSync9(binaryPath))
3377
6012
  return { binaryPath, version: "cached" };
3378
6013
  const releaseProvider = options.releaseProvider ?? fetchLatestMinSyncRelease;
3379
6014
  const release = await releaseProvider();
3380
6015
  const asset = selectReleaseAsset(release, options.platform ?? process.platform, options.arch ?? process.arch);
3381
6016
  requireSha256(asset);
3382
6017
  const assetInstaller = options.assetInstaller ?? installReleaseAsset;
3383
- mkdirSync4(dirname4(binaryPath), { recursive: true });
6018
+ mkdirSync5(dirname6(binaryPath), { recursive: true });
3384
6019
  await assetInstaller(asset, binaryPath);
3385
6020
  await chmod(binaryPath, 493);
3386
6021
  return { binaryPath, version: release.tagName };
@@ -3409,9 +6044,9 @@ function selectReleaseAsset(release, platform, arch) {
3409
6044
  }
3410
6045
  async function installReleaseAsset(asset, destination) {
3411
6046
  const expectedSha256 = requireSha256(asset);
3412
- const tempDir = await mkdtemp(join9(tmpdir(), "autorag-minsync-install-"));
6047
+ const tempDir = await mkdtemp(join12(tmpdir(), "autorag-minsync-install-"));
3413
6048
  try {
3414
- const archive = join9(tempDir, asset.name);
6049
+ const archive = join12(tempDir, asset.name);
3415
6050
  await downloadFile(asset.downloadUrl, archive);
3416
6051
  await verifySha256(archive, expectedSha256);
3417
6052
  if (!asset.name.endsWith(".tar.gz") && !asset.name.endsWith(".zip")) {
@@ -3420,7 +6055,7 @@ async function installReleaseAsset(asset, destination) {
3420
6055
  const extracted = await spawnProcess("tar", ["-xf", archive, "-C", tempDir], tempDir);
3421
6056
  if (!extracted.ok)
3422
6057
  throw new MinSyncReleaseError(extracted.stderr || `Could not extract ${asset.name}`);
3423
- renameSync3(join9(tempDir, basename3(destination)), destination);
6058
+ renameSync3(join12(tempDir, basename6(destination)), destination);
3424
6059
  } finally {
3425
6060
  rmSync(tempDir, { recursive: true, force: true });
3426
6061
  }
@@ -3429,7 +6064,7 @@ async function downloadFile(url, destination) {
3429
6064
  await pipeline(await openHttps(url), createWriteStream(destination));
3430
6065
  }
3431
6066
  async function verifySha256(path, expected) {
3432
- const hash = createHash3("sha256");
6067
+ const hash = createHash5("sha256");
3433
6068
  const stream = await import("node:fs").then((fs) => fs.createReadStream(path));
3434
6069
  await pipeline(stream, hash);
3435
6070
  const actual = hash.digest("hex");
@@ -3531,56 +6166,56 @@ var init_installer2 = __esm(() => {
3531
6166
  });
3532
6167
 
3533
6168
  // src/minsync/paths.ts
3534
- import { join as join10 } from "node:path";
6169
+ import { join as join13 } from "node:path";
3535
6170
  function minSyncWorkspaceRoot(root) {
3536
- return join10(root, MINSYNC_SUBDIR);
6171
+ return join13(root, MINSYNC_SUBDIR);
3537
6172
  }
3538
6173
  function minSyncDocumentPath(workspacePath, virtualPath) {
3539
6174
  const safePath = virtualPath.replace(/^\/+/, "");
3540
- return join10(workspacePath, MINSYNC_FILES_SUBDIR, `${safePath}.md`);
6175
+ return join13(workspacePath, MINSYNC_FILES_SUBDIR, `${safePath}.md`);
3541
6176
  }
3542
6177
  var MINSYNC_SUBDIR, MINSYNC_FILES_SUBDIR = "files";
3543
6178
  var init_paths = __esm(() => {
3544
- MINSYNC_SUBDIR = join10(".autorag", "minsync");
6179
+ MINSYNC_SUBDIR = join13(".autorag", "minsync");
3545
6180
  });
3546
6181
 
3547
6182
  // src/mirror/paths.ts
3548
- import { createHash as createHash4 } from "node:crypto";
3549
- import { join as join11 } from "node:path";
6183
+ import { createHash as createHash6 } from "node:crypto";
6184
+ import { join as join14 } from "node:path";
3550
6185
  function parsedMirrorRoot(root) {
3551
- return join11(root, PARSED_MIRROR_SUBDIR);
6186
+ return join14(root, PARSED_MIRROR_SUBDIR);
3552
6187
  }
3553
6188
  function parsedMirrorIndexPath(root) {
3554
- return join11(parsedMirrorRoot(root), PARSED_INDEX_FILE);
6189
+ return join14(parsedMirrorRoot(root), PARSED_INDEX_FILE);
3555
6190
  }
3556
6191
  function parsedOutputPath(root, virtualPath) {
3557
- const digest = createHash4("sha256").update(virtualPath).digest("hex");
3558
- return join11(parsedMirrorRoot(root), PARSED_FILES_SUBDIR, `${digest}.md`);
6192
+ const digest = createHash6("sha256").update(virtualPath).digest("hex");
6193
+ return join14(parsedMirrorRoot(root), PARSED_FILES_SUBDIR, `${digest}.md`);
3559
6194
  }
3560
6195
  var PARSED_MIRROR_SUBDIR, PARSED_FILES_SUBDIR = "files", PARSED_INDEX_FILE = "index.json";
3561
6196
  var init_paths2 = __esm(() => {
3562
- PARSED_MIRROR_SUBDIR = join11(".autorag", "parsed");
6197
+ PARSED_MIRROR_SUBDIR = join14(".autorag", "parsed");
3563
6198
  });
3564
6199
 
3565
6200
  // src/mirror/index-store.ts
3566
6201
  import { randomUUID as randomUUID4 } from "node:crypto";
3567
- import { existsSync as existsSync9, mkdirSync as mkdirSync5, readFileSync as readFileSync8, renameSync as renameSync4, writeFileSync as writeFileSync4 } from "node:fs";
3568
- import { dirname as dirname5 } from "node:path";
6202
+ import { existsSync as existsSync10, mkdirSync as mkdirSync6, readFileSync as readFileSync9, renameSync as renameSync4, writeFileSync as writeFileSync5 } from "node:fs";
6203
+ import { dirname as dirname7 } from "node:path";
3569
6204
  function emptyMirrorIndex() {
3570
6205
  return { version: 1, entries: {} };
3571
6206
  }
3572
6207
  function loadMirrorIndex(root) {
3573
6208
  const path = parsedMirrorIndexPath(root);
3574
- if (!existsSync9(path))
6209
+ if (!existsSync10(path))
3575
6210
  return emptyMirrorIndex();
3576
- const parsed = JSON.parse(readFileSync8(path, "utf8"));
6211
+ const parsed = JSON.parse(readFileSync9(path, "utf8"));
3577
6212
  return isParsedMirrorIndex(parsed) ? parsed : emptyMirrorIndex();
3578
6213
  }
3579
6214
  function saveMirrorIndex(root, index) {
3580
6215
  const path = parsedMirrorIndexPath(root);
3581
- mkdirSync5(dirname5(path), { recursive: true });
6216
+ mkdirSync6(dirname7(path), { recursive: true });
3582
6217
  const tmp = `${path}.${randomUUID4()}.tmp`;
3583
- writeFileSync4(tmp, `${JSON.stringify(index, null, 2)}
6218
+ writeFileSync5(tmp, `${JSON.stringify(index, null, 2)}
3584
6219
  `);
3585
6220
  renameSync4(tmp, path);
3586
6221
  }
@@ -3600,17 +6235,17 @@ var init_index_store = __esm(() => {
3600
6235
  });
3601
6236
 
3602
6237
  // src/minsync/workspace.ts
3603
- import { copyFileSync, existsSync as existsSync10, mkdirSync as mkdirSync6, rmSync as rmSync2 } from "node:fs";
3604
- import { dirname as dirname6, join as join12 } from "node:path";
6238
+ import { copyFileSync, existsSync as existsSync11, mkdirSync as mkdirSync7, rmSync as rmSync2 } from "node:fs";
6239
+ import { dirname as dirname8, join as join15 } from "node:path";
3605
6240
  function syncMinSyncWorkspace(root, options = {}) {
3606
6241
  const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
3607
- const filesRoot = join12(workspacePath, "files");
6242
+ const filesRoot = join15(workspacePath, "files");
3608
6243
  rmSync2(filesRoot, { recursive: true, force: true });
3609
- mkdirSync6(filesRoot, { recursive: true });
6244
+ mkdirSync7(filesRoot, { recursive: true });
3610
6245
  const index = loadMirrorIndex(root);
3611
- const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => existsSync10(entry.outputPath)).map((entry) => {
6246
+ const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => existsSync11(entry.outputPath)).map((entry) => {
3612
6247
  const minSyncPath = minSyncDocumentPath(workspacePath, entry.virtualPath);
3613
- mkdirSync6(dirname6(minSyncPath), { recursive: true });
6248
+ mkdirSync7(dirname8(minSyncPath), { recursive: true });
3614
6249
  copyFileSync(entry.outputPath, minSyncPath);
3615
6250
  return {
3616
6251
  virtualPath: entry.virtualPath,
@@ -3640,7 +6275,7 @@ function buildMinSyncPathMap(root, workspacePath) {
3640
6275
  return byPath;
3641
6276
  }
3642
6277
  function legacyParsedVirtualPath(root, virtualPath) {
3643
- return join12(root, ".autorag", "parsed", "files", `${virtualPath.replace(/^\/+/, "")}.md`);
6278
+ return join15(root, ".autorag", "parsed", "files", `${virtualPath.replace(/^\/+/, "")}.md`);
3644
6279
  }
3645
6280
  var init_workspace = __esm(() => {
3646
6281
  init_index_store();
@@ -3648,8 +6283,8 @@ var init_workspace = __esm(() => {
3648
6283
  });
3649
6284
 
3650
6285
  // src/minsync/method.ts
3651
- import { accessSync, constants, existsSync as existsSync11 } from "node:fs";
3652
- import { basename as basename4, delimiter as delimiter2, join as join13 } from "node:path";
6286
+ import { accessSync, constants, existsSync as existsSync12 } from "node:fs";
6287
+ import { basename as basename7, delimiter as delimiter2, join as join16 } from "node:path";
3653
6288
  function degrade(workspacePath, reason) {
3654
6289
  return { ok: false, synced: 0, workspacePath, reason };
3655
6290
  }
@@ -3661,7 +6296,7 @@ function lookupInPath(env) {
3661
6296
  for (const dir of pathEnv.split(delimiter2)) {
3662
6297
  if (dir.length === 0)
3663
6298
  continue;
3664
- const candidate = join13(dir, execName);
6299
+ const candidate = join16(dir, execName);
3665
6300
  try {
3666
6301
  accessSync(candidate, constants.X_OK);
3667
6302
  return candidate;
@@ -3711,7 +6346,7 @@ class MinSyncVectorMethod {
3711
6346
  return binaryResult;
3712
6347
  }
3713
6348
  isBinaryMissing() {
3714
- return this.binaryPath !== undefined && !existsSync11(this.binaryPath);
6349
+ return this.binaryPath !== undefined && !existsSync12(this.binaryPath);
3715
6350
  }
3716
6351
  async retrieve(query, options) {
3717
6352
  const topK = options.topK ?? 20;
@@ -3732,7 +6367,7 @@ class MinSyncVectorMethod {
3732
6367
  if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
3733
6368
  continue;
3734
6369
  results.push({
3735
- id: `minsync:${entry.virtualPath}:${basename4(hit.path)}`,
6370
+ id: `minsync:${entry.virtualPath}:${basename7(hit.path)}`,
3736
6371
  content: hit.text,
3737
6372
  source: entry.virtualPath,
3738
6373
  score: hit.score,
@@ -3744,14 +6379,14 @@ class MinSyncVectorMethod {
3744
6379
  return results;
3745
6380
  }
3746
6381
  async resolveBinary() {
3747
- if (this.binaryPath && existsSync11(this.binaryPath)) {
6382
+ if (this.binaryPath && existsSync12(this.binaryPath)) {
3748
6383
  return this.binaryPath;
3749
6384
  }
3750
6385
  const pathBinary = lookupInPath(process.env);
3751
6386
  if (pathBinary)
3752
6387
  return pathBinary;
3753
- const cachedBinary = join13(this.root, ".autorag", "bin", executableName(process.platform));
3754
- if (existsSync11(cachedBinary))
6388
+ const cachedBinary = join16(this.root, ".autorag", "bin", executableName(process.platform));
6389
+ if (existsSync12(cachedBinary))
3755
6390
  return cachedBinary;
3756
6391
  if (this.autoInstall) {
3757
6392
  try {
@@ -3835,7 +6470,7 @@ class Parser {
3835
6470
  }
3836
6471
 
3837
6472
  // src/parser/eml.ts
3838
- import { simpleParser } from "mailparser";
6473
+ import { simpleParser as simpleParser2 } from "mailparser";
3839
6474
  function addressText(address) {
3840
6475
  if (address === undefined)
3841
6476
  return;
@@ -3853,7 +6488,7 @@ var init_eml = __esm(() => {
3853
6488
  extensions = [".eml"];
3854
6489
  async parse(input) {
3855
6490
  try {
3856
- const parsed = await simpleParser(Buffer.from(input.bytes));
6491
+ const parsed = await simpleParser2(Buffer.from(input.bytes));
3857
6492
  const from = addressText(parsed.from);
3858
6493
  const to = addressText(parsed.to);
3859
6494
  const lines = [
@@ -3882,9 +6517,480 @@ var init_eml = __esm(() => {
3882
6517
  };
3883
6518
  });
3884
6519
 
6520
+ // src/parser/hwp-markdown.ts
6521
+ function renderHwpMarkdown(document) {
6522
+ const tables = new Map;
6523
+ for (const table of document.tables) {
6524
+ const key = `${table.sectionIndex}:${table.parentParagraphIndex}`;
6525
+ const group = tables.get(key) ?? [];
6526
+ group.push(table);
6527
+ tables.set(key, group);
6528
+ }
6529
+ for (const group of tables.values()) {
6530
+ group.sort((left, right) => left.controlIndex - right.controlIndex);
6531
+ }
6532
+ const blocks = [];
6533
+ for (const paragraph of document.paragraphs) {
6534
+ const text = cleanText(paragraph.text);
6535
+ if (hasRenderableText(text))
6536
+ blocks.push(text);
6537
+ for (const table of tables.get(`${paragraph.sectionIndex}:${paragraph.paragraphIndex}`) ?? []) {
6538
+ const rendered = renderTable(table);
6539
+ if (rendered.length > 0)
6540
+ blocks.push(rendered);
6541
+ }
6542
+ }
6543
+ return blocks.join(`
6544
+
6545
+ `);
6546
+ }
6547
+ function cleanText(text) {
6548
+ return text.replace(/[\u0000\uFFFC]/gu, "");
6549
+ }
6550
+ function hasRenderableText(text) {
6551
+ return text.trim().length > 0;
6552
+ }
6553
+ function renderTable(table) {
6554
+ const cells = table.cells.map((cell) => ({ ...cell, text: renderCell(cell) }));
6555
+ if (!cells.some((cell) => hasRenderableText(cell.text)))
6556
+ return "";
6557
+ if (cells.some((cell) => cell.rowSpan > 1 || cell.columnSpan > 1))
6558
+ return renderMergedTable(cells);
6559
+ const cellsByPosition = new Map(cells.map((cell) => [`${cell.row}:${cell.column}`, cell.text]));
6560
+ const header = Array.from({ length: table.columnCount }, (_, column) => `Column ${column + 1}`);
6561
+ const rows = Array.from({ length: table.rowCount }, (_, row) => Array.from({ length: table.columnCount }, (_2, column) => cellsByPosition.get(`${row}:${column}`) ?? ""));
6562
+ return [
6563
+ `| ${header.join(" | ")} |`,
6564
+ `| ${header.map(() => "---").join(" | ")} |`,
6565
+ ...rows.map((row) => `| ${row.join(" | ")} |`)
6566
+ ].join(`
6567
+ `);
6568
+ }
6569
+ function renderMergedTable(cells) {
6570
+ const rows = new Map;
6571
+ const coveredPositions = new Set;
6572
+ for (const cell of [...cells].sort((left, right) => left.row - right.row || left.column - right.column)) {
6573
+ const position = `${cell.row}:${cell.column}`;
6574
+ if (coveredPositions.has(position))
6575
+ continue;
6576
+ const row = rows.get(cell.row) ?? [];
6577
+ row.push(cell);
6578
+ rows.set(cell.row, row);
6579
+ for (let rowIndex = cell.row;rowIndex < cell.row + Math.max(1, cell.rowSpan); rowIndex += 1) {
6580
+ for (let columnIndex = cell.column;columnIndex < cell.column + Math.max(1, cell.columnSpan); columnIndex += 1) {
6581
+ if (rowIndex !== cell.row || columnIndex !== cell.column)
6582
+ coveredPositions.add(`${rowIndex}:${columnIndex}`);
6583
+ }
6584
+ }
6585
+ }
6586
+ const renderedRows = [...rows.entries()].sort(([left], [right]) => left - right).map(([row, cellsInRow]) => {
6587
+ const text = cellsInRow.sort((left, right) => left.column - right.column).map((cell) => cell.text).filter(hasRenderableText).join(" | ");
6588
+ return hasRenderableText(text) ? `Row ${row + 1}: ${text}` : "";
6589
+ }).filter((row) => row.length > 0);
6590
+ return ["[Table]", ...renderedRows].join(`
6591
+ `);
6592
+ }
6593
+ function renderCell(cell) {
6594
+ return cell.paragraphs.map(cleanText).filter(hasRenderableText).join("<br>").replaceAll("\\", "\\\\").replace(/\r\n|\r|\n/gu, "<br>").replaceAll("|", "\\|");
6595
+ }
6596
+
6597
+ // src/parser/rhwp-adapter.ts
6598
+ import { readFile as readFile3 } from "node:fs/promises";
6599
+ import initRhwp, { HwpDocument } from "@rhwp/core";
6600
+ function createRhwpExtractor(runtime = defaultRhwpRuntime, overrides = {}) {
6601
+ const limits = resolveLimits(overrides);
6602
+ const initialize = runtime === defaultRhwpRuntime ? initializeDefaultRhwpRuntime : createRuntimeInitializer(runtime);
6603
+ return async (bytes) => {
6604
+ await initialize();
6605
+ const document = runtime.open(bytes);
6606
+ try {
6607
+ return extractDocument(document, limits);
6608
+ } finally {
6609
+ document.free();
6610
+ }
6611
+ };
6612
+ }
6613
+ function createRuntimeInitializer(runtime) {
6614
+ let initialization;
6615
+ return () => {
6616
+ if (initialization === undefined) {
6617
+ const attempt = runtime.initialize();
6618
+ initialization = attempt.catch((error) => {
6619
+ initialization = undefined;
6620
+ throw error;
6621
+ });
6622
+ }
6623
+ return initialization;
6624
+ };
6625
+ }
6626
+ function extractDocument(document, limits) {
6627
+ const paragraphs = [];
6628
+ const paragraphCounts = [];
6629
+ const counters = { characters: 0, tables: 0, cells: 0, cellParagraphs: 0 };
6630
+ const sectionCount = readCount(document.getSectionCount(), "section count");
6631
+ checkBudget("maxSections", sectionCount, limits);
6632
+ for (let sectionIndex = 0;sectionIndex < sectionCount; sectionIndex += 1) {
6633
+ const paragraphCount = readCount(document.getParagraphCount(sectionIndex), `paragraph count for section ${sectionIndex}`);
6634
+ paragraphCounts.push(paragraphCount);
6635
+ checkBudget("maxBodyParagraphs", paragraphs.length + paragraphCount, limits);
6636
+ for (let paragraphIndex = 0;paragraphIndex < paragraphCount; paragraphIndex += 1) {
6637
+ const length = readCount(document.getParagraphLength(sectionIndex, paragraphIndex), `paragraph length at ${sectionIndex}:${paragraphIndex}`);
6638
+ const text = readText(document.getTextRange(sectionIndex, paragraphIndex, 0, length), `paragraph text at ${sectionIndex}:${paragraphIndex}`);
6639
+ counters.characters += [...text].length;
6640
+ checkBudget("maxCharacters", counters.characters, limits);
6641
+ paragraphs.push({
6642
+ sectionIndex,
6643
+ paragraphIndex,
6644
+ text
6645
+ });
6646
+ }
6647
+ }
6648
+ return { paragraphs, tables: extractTables(document, limits, counters, paragraphCounts) };
6649
+ }
6650
+ function extractTables(document, limits, counters, paragraphCounts) {
6651
+ const tables = [];
6652
+ const seen = new Set;
6653
+ const pageCount = readCount(document.pageCount(), "page count");
6654
+ for (let pageIndex = 0;pageIndex < pageCount; pageIndex += 1) {
6655
+ const layout = readJsonRecord(document.getPageControlLayout(pageIndex), `page ${pageIndex} control layout`);
6656
+ const controls = readArray(layout.controls, `controls for page ${pageIndex}`);
6657
+ for (let layoutIndex = 0;layoutIndex < controls.length; layoutIndex += 1) {
6658
+ const control = readRecord(controls[layoutIndex], `control ${layoutIndex} on page ${pageIndex}`);
6659
+ if (typeof control.type !== "string") {
6660
+ throw new TypeError(`rhwp returned an invalid control type at page ${pageIndex}:${layoutIndex}`);
6661
+ }
6662
+ if (control.type !== "table")
6663
+ continue;
6664
+ const label = `table at page ${pageIndex}:${layoutIndex}`;
6665
+ const coordinates = readTableCoordinates(control, label, paragraphCounts, limits);
6666
+ if (coordinates === undefined)
6667
+ continue;
6668
+ const key = `${coordinates.sectionIndex}:${coordinates.parentParagraphIndex}:${coordinates.controlIndex}`;
6669
+ if (seen.has(key))
6670
+ continue;
6671
+ seen.add(key);
6672
+ counters.tables += 1;
6673
+ checkBudget("maxTables", counters.tables, limits);
6674
+ tables.push(extractTable(document, coordinates, limits, counters));
6675
+ }
6676
+ }
6677
+ return tables;
6678
+ }
6679
+ function readTableCoordinates(control, label, paragraphCounts, limits) {
6680
+ const hasSectionIndex = Object.hasOwn(control, "secIdx");
6681
+ const hasParagraphIndex = Object.hasOwn(control, "paraIdx");
6682
+ const hasControlIndex = Object.hasOwn(control, "controlIdx");
6683
+ const coordinateCount = Number(hasSectionIndex) + Number(hasParagraphIndex) + Number(hasControlIndex);
6684
+ if (coordinateCount === 0) {
6685
+ validateDeferredNestedTable(control, label, limits);
6686
+ return;
6687
+ }
6688
+ if (coordinateCount !== 3) {
6689
+ throw new TypeError(`rhwp returned a partial coordinate set for ${label}`);
6690
+ }
6691
+ if (Object.hasOwn(control, "cellPath") || Object.hasOwn(control, "parentParaIdx")) {
6692
+ throw new TypeError(`rhwp returned conflicting top-level and nested context for ${label}`);
6693
+ }
6694
+ const coordinates = {
6695
+ sectionIndex: readCount(control.secIdx, `secIdx for ${label}`),
6696
+ parentParagraphIndex: readCount(control.paraIdx, `paraIdx for ${label}`),
6697
+ controlIndex: readCount(control.controlIdx, `controlIdx for ${label}`)
6698
+ };
6699
+ const paragraphCount = paragraphCounts[coordinates.sectionIndex];
6700
+ if (paragraphCount === undefined) {
6701
+ throw new TypeError(`rhwp returned a section coordinate outside the announced document range for ${label}`);
6702
+ }
6703
+ if (coordinates.parentParagraphIndex >= paragraphCount) {
6704
+ throw new TypeError(`rhwp returned a parent paragraph coordinate outside the announced section range for ${label}`);
6705
+ }
6706
+ return coordinates;
6707
+ }
6708
+ function validateDeferredNestedTable(control, label, limits) {
6709
+ if (!Object.hasOwn(control, "cellPath") && !Object.hasOwn(control, "parentParaIdx")) {
6710
+ validateDeferredNestedLayout(control, label, limits);
6711
+ return;
6712
+ }
6713
+ readCount(control.parentParaIdx, `parentParaIdx for deferred nested ${label}`);
6714
+ const cellPath = readArray(control.cellPath, `cellPath for deferred nested ${label}`);
6715
+ if (cellPath.length === 0) {
6716
+ throw new TypeError(`rhwp returned an empty cellPath for deferred nested ${label}`);
6717
+ }
6718
+ checkBudget("maxCells", cellPath.length, limits);
6719
+ for (let pathIndex = 0;pathIndex < cellPath.length; pathIndex += 1) {
6720
+ const entry = readRecord(cellPath[pathIndex], `cellPath entry ${pathIndex} for deferred nested ${label}`);
6721
+ readCount(entry.controlIndex, `controlIndex in cellPath entry ${pathIndex} for deferred nested ${label}`);
6722
+ readCount(entry.cellIndex, `cellIndex in cellPath entry ${pathIndex} for deferred nested ${label}`);
6723
+ readCount(entry.cellParaIndex, `cellParaIndex in cellPath entry ${pathIndex} for deferred nested ${label}`);
6724
+ }
6725
+ }
6726
+ function validateDeferredNestedLayout(control, label, limits) {
6727
+ readFiniteNumber(control.x, `x for deferred nested ${label}`);
6728
+ readFiniteNumber(control.y, `y for deferred nested ${label}`);
6729
+ readPositiveFiniteNumber(control.w, `w for deferred nested ${label}`);
6730
+ readPositiveFiniteNumber(control.h, `h for deferred nested ${label}`);
6731
+ const rowCount = readPositiveCount(control.rowCount, `rowCount for deferred nested ${label}`);
6732
+ const columnCount = readPositiveCount(control.colCount, `colCount for deferred nested ${label}`);
6733
+ const logicalCellCount = readLogicalCellCount(rowCount, columnCount, limits);
6734
+ readCount(control.plane, `plane for deferred nested ${label}`);
6735
+ readSafeInteger(control.zOrder, `zOrder for deferred nested ${label}`);
6736
+ readCount(control.stableIndex, `stableIndex for deferred nested ${label}`);
6737
+ const cells = readArray(control.cells, `cells for deferred nested ${label}`);
6738
+ if (cells.length === 0) {
6739
+ throw new TypeError(`rhwp returned empty cells for deferred nested ${label}`);
6740
+ }
6741
+ checkBudget("maxCells", cells.length, limits);
6742
+ if (cells.length > logicalCellCount) {
6743
+ throw new TypeError(`rhwp returned too many cells for deferred nested ${label}`);
6744
+ }
6745
+ const anchors = new Set;
6746
+ const cellIndexes = new Set;
6747
+ const coveredCoordinates = new Set;
6748
+ for (let cellIndex = 0;cellIndex < cells.length; cellIndex += 1) {
6749
+ const cell = readRecord(cells[cellIndex], `cell ${cellIndex} for deferred nested ${label}`);
6750
+ readFiniteNumber(cell.x, `x for cell ${cellIndex} in deferred nested ${label}`);
6751
+ readFiniteNumber(cell.y, `y for cell ${cellIndex} in deferred nested ${label}`);
6752
+ readPositiveFiniteNumber(cell.w, `w for cell ${cellIndex} in deferred nested ${label}`);
6753
+ readPositiveFiniteNumber(cell.h, `h for cell ${cellIndex} in deferred nested ${label}`);
6754
+ const row = readCount(cell.row, `row for cell ${cellIndex} in deferred nested ${label}`);
6755
+ const column = readCount(cell.col, `col for cell ${cellIndex} in deferred nested ${label}`);
6756
+ const rowSpan = readPositiveCount(cell.rowSpan, `rowSpan for cell ${cellIndex} in deferred nested ${label}`);
6757
+ const columnSpan = readPositiveCount(cell.colSpan, `colSpan for cell ${cellIndex} in deferred nested ${label}`);
6758
+ const runtimeCellIndex = readCount(cell.cellIdx, `cellIdx for cell ${cellIndex} in deferred nested ${label}`);
6759
+ if (row >= rowCount || rowSpan > rowCount - row) {
6760
+ throw new TypeError(`rhwp returned an out-of-range row span for cell ${cellIndex} in deferred nested ${label}`);
6761
+ }
6762
+ if (column >= columnCount || columnSpan > columnCount - column) {
6763
+ throw new TypeError(`rhwp returned an out-of-range col span for cell ${cellIndex} in deferred nested ${label}`);
6764
+ }
6765
+ if (cellIndexes.has(runtimeCellIndex)) {
6766
+ throw new TypeError(`rhwp returned a duplicate cellIdx for deferred nested ${label}`);
6767
+ }
6768
+ cellIndexes.add(runtimeCellIndex);
6769
+ const anchor = `${row}:${column}`;
6770
+ if (anchors.has(anchor)) {
6771
+ throw new TypeError(`rhwp returned a duplicate cell anchor for deferred nested ${label}`);
6772
+ }
6773
+ anchors.add(anchor);
6774
+ for (let coveredRow = row;coveredRow < row + rowSpan; coveredRow += 1) {
6775
+ for (let coveredColumn = column;coveredColumn < column + columnSpan; coveredColumn += 1) {
6776
+ const coordinate = `${coveredRow}:${coveredColumn}`;
6777
+ if (coveredCoordinates.has(coordinate)) {
6778
+ throw new TypeError(`rhwp returned overlapping cell spans for deferred nested ${label}`);
6779
+ }
6780
+ coveredCoordinates.add(coordinate);
6781
+ }
6782
+ }
6783
+ }
6784
+ }
6785
+ function extractTable(document, coordinates, limits, counters) {
6786
+ const { sectionIndex, parentParagraphIndex, controlIndex } = coordinates;
6787
+ const key = `${sectionIndex}:${parentParagraphIndex}:${controlIndex}`;
6788
+ const dimensions = readJsonRecord(document.getTableDimensions(sectionIndex, parentParagraphIndex, controlIndex), `table dimensions at ${key}`);
6789
+ const rowCount = readPositiveCount(dimensions.rowCount, `rowCount for table ${key}`);
6790
+ const columnCount = readPositiveCount(dimensions.colCount, `colCount for table ${key}`);
6791
+ const cellCount = readCount(dimensions.cellCount, `cellCount for table ${key}`);
6792
+ const logicalCellCount = chargeLogicalCells(rowCount, columnCount, counters, limits);
6793
+ if (cellCount === 0 || cellCount > logicalCellCount) {
6794
+ throw new TypeError(`rhwp returned an invalid cellCount for table ${key}; expected 1..${logicalCellCount}, received ${cellCount}`);
6795
+ }
6796
+ const cellGeometry = [];
6797
+ const anchors = new Set;
6798
+ const coveredCoordinates = new Set;
6799
+ for (let cellIndex = 0;cellIndex < cellCount; cellIndex += 1) {
6800
+ const cellInfo = readJsonRecord(document.getCellInfo(sectionIndex, parentParagraphIndex, controlIndex, cellIndex), `cell metadata at ${key}:${cellIndex}`);
6801
+ const row = readCount(cellInfo.row, `row for cell ${key}:${cellIndex}`);
6802
+ const column = readCount(cellInfo.col, `col for cell ${key}:${cellIndex}`);
6803
+ const rowSpan = readPositiveCount(cellInfo.rowSpan, `rowSpan for cell ${key}:${cellIndex}`);
6804
+ const columnSpan = readPositiveCount(cellInfo.colSpan, `colSpan for cell ${key}:${cellIndex}`);
6805
+ if (row >= rowCount) {
6806
+ throw new TypeError(`rhwp returned a row outside the table range for cell ${key}:${cellIndex}`);
6807
+ }
6808
+ if (column >= columnCount) {
6809
+ throw new TypeError(`rhwp returned a col outside the table range for cell ${key}:${cellIndex}`);
6810
+ }
6811
+ if (rowSpan > rowCount - row) {
6812
+ throw new TypeError(`rhwp returned a rowSpan outside the table range for cell ${key}:${cellIndex}`);
6813
+ }
6814
+ if (columnSpan > columnCount - column) {
6815
+ throw new TypeError(`rhwp returned a colSpan outside the table range for cell ${key}:${cellIndex}`);
6816
+ }
6817
+ const anchor = row * columnCount + column;
6818
+ if (anchors.has(anchor)) {
6819
+ throw new TypeError(`rhwp returned a duplicate cell anchor for table ${key} at ${row}:${column}`);
6820
+ }
6821
+ anchors.add(anchor);
6822
+ for (let coveredRow = row;coveredRow < row + rowSpan; coveredRow += 1) {
6823
+ for (let coveredColumn = column;coveredColumn < column + columnSpan; coveredColumn += 1) {
6824
+ const coordinate = coveredRow * columnCount + coveredColumn;
6825
+ if (coveredCoordinates.has(coordinate)) {
6826
+ throw new TypeError(`rhwp returned overlapping cell spans for table ${key} at ${coveredRow}:${coveredColumn}`);
6827
+ }
6828
+ coveredCoordinates.add(coordinate);
6829
+ }
6830
+ }
6831
+ cellGeometry.push({ row, column, rowSpan, columnSpan });
6832
+ }
6833
+ if (coveredCoordinates.size !== logicalCellCount) {
6834
+ throw new TypeError(`rhwp returned cellCount ${cellCount} that covers ${coveredCoordinates.size} of ${logicalCellCount} logical grid coordinates for table ${key}`);
6835
+ }
6836
+ const cells = [];
6837
+ for (let cellIndex = 0;cellIndex < cellGeometry.length; cellIndex += 1) {
6838
+ const geometry = cellGeometry[cellIndex];
6839
+ if (geometry === undefined)
6840
+ throw new TypeError(`rhwp omitted cell geometry at ${key}:${cellIndex}`);
6841
+ const paragraphCount = readCount(document.getCellParagraphCount(sectionIndex, parentParagraphIndex, controlIndex, cellIndex), `cell paragraph count at ${key}:${cellIndex}`);
6842
+ counters.cellParagraphs += paragraphCount;
6843
+ checkBudget("maxCellParagraphs", counters.cellParagraphs, limits);
6844
+ const paragraphs = [];
6845
+ for (let cellParagraphIndex = 0;cellParagraphIndex < paragraphCount; cellParagraphIndex += 1) {
6846
+ const length = readCount(document.getCellParagraphLength(sectionIndex, parentParagraphIndex, controlIndex, cellIndex, cellParagraphIndex), `cell paragraph length at ${key}:${cellIndex}:${cellParagraphIndex}`);
6847
+ const text = readText(document.getTextInCell(sectionIndex, parentParagraphIndex, controlIndex, cellIndex, cellParagraphIndex, 0, length), `cell paragraph text at ${key}:${cellIndex}:${cellParagraphIndex}`);
6848
+ counters.characters += [...text].length;
6849
+ checkBudget("maxCharacters", counters.characters, limits);
6850
+ paragraphs.push(text);
6851
+ }
6852
+ cells.push({
6853
+ ...geometry,
6854
+ paragraphs
6855
+ });
6856
+ }
6857
+ return {
6858
+ sectionIndex,
6859
+ parentParagraphIndex,
6860
+ controlIndex,
6861
+ rowCount,
6862
+ columnCount,
6863
+ cells
6864
+ };
6865
+ }
6866
+ function chargeLogicalCells(rowCount, columnCount, counters, limits) {
6867
+ const logicalCellCount = readLogicalCellCount(rowCount, columnCount, limits);
6868
+ if (logicalCellCount > limits.maxCells - counters.cells) {
6869
+ throw new HwpExtractionBudgetError("maxCells", counters.cells + logicalCellCount, limits.maxCells);
6870
+ }
6871
+ counters.cells += logicalCellCount;
6872
+ return logicalCellCount;
6873
+ }
6874
+ function readLogicalCellCount(rowCount, columnCount, limits) {
6875
+ if (rowCount > Math.floor(limits.maxCells / columnCount)) {
6876
+ throw new HwpExtractionBudgetError("maxCells", rowCount * columnCount, limits.maxCells);
6877
+ }
6878
+ return rowCount * columnCount;
6879
+ }
6880
+ function resolveLimits(overrides) {
6881
+ const limits = { ...DEFAULT_LIMITS };
6882
+ for (const name of Object.keys(DEFAULT_LIMITS)) {
6883
+ const value = overrides[name];
6884
+ if (value === undefined)
6885
+ continue;
6886
+ if (!Number.isSafeInteger(value) || value <= 0) {
6887
+ throw new TypeError(`${name} must be a positive finite safe integer`);
6888
+ }
6889
+ limits[name] = value;
6890
+ }
6891
+ return limits;
6892
+ }
6893
+ function checkBudget(name, actual, limits) {
6894
+ if (actual > limits[name])
6895
+ throw new HwpExtractionBudgetError(name, actual, limits[name]);
6896
+ }
6897
+ function readCount(value, label) {
6898
+ if (!Number.isSafeInteger(value) || value < 0) {
6899
+ throw new TypeError(`rhwp returned an invalid ${label}; expected a non-negative safe integer`);
6900
+ }
6901
+ return value;
6902
+ }
6903
+ function readPositiveCount(value, label) {
6904
+ const count = readCount(value, label);
6905
+ if (count === 0) {
6906
+ throw new TypeError(`rhwp returned an invalid ${label}; expected a positive safe integer`);
6907
+ }
6908
+ return count;
6909
+ }
6910
+ function readSafeInteger(value, label) {
6911
+ if (!Number.isSafeInteger(value)) {
6912
+ throw new TypeError(`rhwp returned an invalid ${label}; expected a safe integer`);
6913
+ }
6914
+ return value;
6915
+ }
6916
+ function readFiniteNumber(value, label) {
6917
+ if (typeof value !== "number" || !Number.isFinite(value)) {
6918
+ throw new TypeError(`rhwp returned an invalid ${label}; expected a finite number`);
6919
+ }
6920
+ return value;
6921
+ }
6922
+ function readPositiveFiniteNumber(value, label) {
6923
+ const number = readFiniteNumber(value, label);
6924
+ if (number <= 0) {
6925
+ throw new TypeError(`rhwp returned an invalid ${label}; expected a positive finite number`);
6926
+ }
6927
+ return number;
6928
+ }
6929
+ function readText(value, label) {
6930
+ if (typeof value !== "string")
6931
+ throw new TypeError(`rhwp returned invalid ${label}; expected a string`);
6932
+ return value;
6933
+ }
6934
+ function readJsonRecord(value, label) {
6935
+ const text = readText(value, label);
6936
+ let parsed;
6937
+ try {
6938
+ parsed = JSON.parse(text);
6939
+ } catch (cause) {
6940
+ throw new TypeError(`rhwp returned malformed JSON for ${label}`, { cause });
6941
+ }
6942
+ return readRecord(parsed, label);
6943
+ }
6944
+ function readRecord(value, label) {
6945
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
6946
+ throw new TypeError(`rhwp returned invalid ${label}; expected an object`);
6947
+ }
6948
+ return value;
6949
+ }
6950
+ function readArray(value, label) {
6951
+ if (!Array.isArray(value))
6952
+ throw new TypeError(`rhwp returned invalid ${label}; expected an array`);
6953
+ return value;
6954
+ }
6955
+ var DEFAULT_LIMITS, HwpExtractionBudgetError, defaultRhwpRuntime, initializeDefaultRhwpRuntime;
6956
+ var init_rhwp_adapter = __esm(() => {
6957
+ DEFAULT_LIMITS = {
6958
+ maxSections: 256,
6959
+ maxBodyParagraphs: 200000,
6960
+ maxTables: 20000,
6961
+ maxCells: 500000,
6962
+ maxCellParagraphs: 1e6,
6963
+ maxCharacters: 20000000
6964
+ };
6965
+ HwpExtractionBudgetError = class HwpExtractionBudgetError extends Error {
6966
+ code = "HWP_EXTRACTION_BUDGET_EXCEEDED";
6967
+ limit;
6968
+ actual;
6969
+ maximum;
6970
+ constructor(limit, actual, maximum) {
6971
+ super(`HWP extraction exceeded ${limit} budget of ${maximum} (received ${actual})`);
6972
+ this.name = "HwpExtractionBudgetError";
6973
+ this.limit = limit;
6974
+ this.actual = actual;
6975
+ this.maximum = maximum;
6976
+ }
6977
+ };
6978
+ defaultRhwpRuntime = {
6979
+ async initialize() {
6980
+ const rhwpModuleUrl = import.meta.resolve("@rhwp/core");
6981
+ const wasmBytes = await readFile3(new URL("./rhwp_bg.wasm", rhwpModuleUrl));
6982
+ await initRhwp({ module_or_path: wasmBytes });
6983
+ },
6984
+ open(bytes) {
6985
+ return new HwpDocument(bytes);
6986
+ }
6987
+ };
6988
+ initializeDefaultRhwpRuntime = createRuntimeInitializer(defaultRhwpRuntime);
6989
+ });
6990
+
3885
6991
  // src/parser/xml-text.ts
3886
6992
  import { Buffer as Buffer2 } from "node:buffer";
3887
- import { XMLParser } from "fast-xml-parser";
6993
+ import { XMLParser as XMLParser2 } from "fast-xml-parser";
3888
6994
  import JSZip from "jszip";
3889
6995
  async function readZipXmlText(bytes, pathPattern) {
3890
6996
  const zip = await JSZip.loadAsync(bytes);
@@ -3974,7 +7080,7 @@ function isRecord6(value) {
3974
7080
  }
3975
7081
  var parser, MAX_XML_FILES = 64, MAX_XML_BYTES = 5000000, MAX_TEXT_CHUNKS = 20000;
3976
7082
  var init_xml_text = __esm(() => {
3977
- parser = new XMLParser({
7083
+ parser = new XMLParser2({
3978
7084
  ignoreAttributes: false,
3979
7085
  parseTagValue: false,
3980
7086
  processEntities: true,
@@ -3983,19 +7089,29 @@ var init_xml_text = __esm(() => {
3983
7089
  });
3984
7090
 
3985
7091
  // src/parser/hwp.ts
3986
- import { extname as extname2 } from "node:path";
7092
+ import { extname as extname5 } from "node:path";
3987
7093
  var HwpParser;
3988
7094
  var init_hwp = __esm(() => {
3989
7095
  init_errors();
7096
+ init_rhwp_adapter();
3990
7097
  init_text();
3991
7098
  init_xml_text();
3992
7099
  HwpParser = class HwpParser extends Parser {
3993
7100
  name = "hwp";
3994
7101
  extensions = [".hwpx", ".hwp"];
7102
+ extractor;
7103
+ constructor(options = {}) {
7104
+ super();
7105
+ this.extractor = options.extractor ?? createRhwpExtractor(undefined, options.limits);
7106
+ }
3995
7107
  async parse(input) {
3996
7108
  try {
3997
- if (extname2(input.virtualPath).toLowerCase() === ".hwp") {
3998
- throw new Error("legacy HWP5 binary parsing is not supported by the pure JavaScript parser");
7109
+ if (extname5(input.virtualPath).toLowerCase() === ".hwp") {
7110
+ const document = await this.extractor(input.bytes);
7111
+ return {
7112
+ markdown: normalizeMarkdown(renderHwpMarkdown(document)),
7113
+ metadata: { parser: this.name, format: "hwp5" }
7114
+ };
3999
7115
  }
4000
7116
  const chunks = await readZipXmlText(input.bytes, /^(?:Contents|content)\/.*\.xml$/i);
4001
7117
  return {
@@ -4119,7 +7235,7 @@ var init_ocr = __esm(() => {
4119
7235
  });
4120
7236
 
4121
7237
  // src/parser/office.ts
4122
- import { extname as extname3 } from "node:path";
7238
+ import { extname as extname6 } from "node:path";
4123
7239
  function formatMarkdown(parserName, chunks) {
4124
7240
  const markdown = normalizeMarkdown(chunks.filter((chunk) => chunk.trim().length > 0).join(`
4125
7241
 
@@ -4160,7 +7276,7 @@ var init_office = __esm(() => {
4160
7276
  extensions = [".xlsx", ".xls"];
4161
7277
  async parse(input) {
4162
7278
  try {
4163
- if (extname3(input.virtualPath).toLowerCase() === ".xls") {
7279
+ if (extname6(input.virtualPath).toLowerCase() === ".xls") {
4164
7280
  throw new Error("legacy XLS binary parsing is not supported by the pure JavaScript parser");
4165
7281
  }
4166
7282
  const sharedStrings = await readZipXmlText(input.bytes, /^xl\/sharedStrings\.xml$/);
@@ -4174,12 +7290,12 @@ var init_office = __esm(() => {
4174
7290
  });
4175
7291
 
4176
7292
  // src/parser/opendataloader-pdf.ts
4177
- import { mkdtemp as mkdtemp2, readFile, rm, writeFile } from "node:fs/promises";
7293
+ import { mkdtemp as mkdtemp2, readFile as readFile4, rm, writeFile } from "node:fs/promises";
4178
7294
  import { tmpdir as tmpdir2 } from "node:os";
4179
- import { basename as basename5, join as join14, parse as parse3 } from "node:path";
7295
+ import { basename as basename8, join as join17, parse as parse3 } from "node:path";
4180
7296
  import { convert } from "@opendataloader/pdf";
4181
7297
  function markdownOutputPath(outputDir, inputPath) {
4182
- return join14(outputDir, `${parse3(inputPath).name}.md`);
7298
+ return join17(outputDir, `${parse3(inputPath).name}.md`);
4183
7299
  }
4184
7300
  var OpendataloaderPdfParser;
4185
7301
  var init_opendataloader_pdf = __esm(() => {
@@ -4196,9 +7312,9 @@ var init_opendataloader_pdf = __esm(() => {
4196
7312
  this.converter = options.converter ?? convert;
4197
7313
  }
4198
7314
  async parse(input) {
4199
- const tempRoot = await mkdtemp2(join14(tmpdir2(), "autorag-pdf-"));
7315
+ const tempRoot = await mkdtemp2(join17(tmpdir2(), "autorag-pdf-"));
4200
7316
  try {
4201
- const inputPath = input.sourcePath ?? join14(tempRoot, basename5(input.virtualPath));
7317
+ const inputPath = input.sourcePath ?? join17(tempRoot, basename8(input.virtualPath));
4202
7318
  if (!input.sourcePath) {
4203
7319
  await writeFile(inputPath, input.bytes);
4204
7320
  }
@@ -4217,7 +7333,7 @@ var init_opendataloader_pdf = __esm(() => {
4217
7333
  } : {}
4218
7334
  });
4219
7335
  return {
4220
- markdown: normalizeMarkdown(await readFile(markdownOutputPath(tempRoot, inputPath), "utf8")),
7336
+ markdown: normalizeMarkdown(await readFile4(markdownOutputPath(tempRoot, inputPath), "utf8")),
4221
7337
  metadata: { parser: this.name }
4222
7338
  };
4223
7339
  } catch (cause) {
@@ -4243,7 +7359,7 @@ var init_plain_text = __esm(() => {
4243
7359
  });
4244
7360
 
4245
7361
  // src/parser/registry.ts
4246
- import { extname as extname4 } from "node:path";
7362
+ import { extname as extname7 } from "node:path";
4247
7363
 
4248
7364
  class ParserRegistry {
4249
7365
  parsersByExtension = new Map;
@@ -4262,7 +7378,7 @@ class ParserRegistry {
4262
7378
  }
4263
7379
  }
4264
7380
  getForVirtualPath(virtualPath) {
4265
- return this.parsersByExtension.get(normalizeExtension(extname4(virtualPath)));
7381
+ return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
4266
7382
  }
4267
7383
  list() {
4268
7384
  return Array.from(new Set(this.parsersByExtension.values()));
@@ -4282,7 +7398,7 @@ function createDefaultParserRegistry(options = {}) {
4282
7398
  new DocxParser,
4283
7399
  new PptxParser,
4284
7400
  new XlsxParser,
4285
- new HwpParser,
7401
+ new HwpParser(options.hwp),
4286
7402
  new EmlParser
4287
7403
  ];
4288
7404
  if (options.ocr?.enabled)
@@ -4301,9 +7417,9 @@ var init_defaults = __esm(() => {
4301
7417
 
4302
7418
  // src/mirror/sync.ts
4303
7419
  import { randomUUID as randomUUID5 } from "node:crypto";
4304
- import { existsSync as existsSync12, mkdirSync as mkdirSync7, renameSync as renameSync5, rmSync as rmSync3, writeFileSync as writeFileSync5 } from "node:fs";
4305
- import { opendir, readFile as readFile2, stat } from "node:fs/promises";
4306
- import { dirname as dirname7, extname as extname5, resolve as resolve6 } from "node:path";
7420
+ import { existsSync as existsSync13, mkdirSync as mkdirSync8, renameSync as renameSync5, rmSync as rmSync3, writeFileSync as writeFileSync6 } from "node:fs";
7421
+ import { opendir, readFile as readFile5, stat as stat4 } from "node:fs/promises";
7422
+ import { dirname as dirname9, extname as extname8, resolve as resolve6 } from "node:path";
4307
7423
  async function syncParsedMirrors(options) {
4308
7424
  const registry = options.registry ?? createDefaultParserRegistry(options.parserOptions);
4309
7425
  const maxSourceBytes = options.maxSourceBytes ?? DEFAULT_MAX_SOURCE_BYTES;
@@ -4351,11 +7467,11 @@ async function syncParsedMirrors(options) {
4351
7467
  }
4352
7468
  const previousEntry = previous.entries[entry.virtualPath];
4353
7469
  const outputPath = parsedOutputPath(options.root, entry.virtualPath);
4354
- const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName === parser2.name && existsSync12(outputPath);
7470
+ const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName === parser2.name && existsSync13(outputPath);
4355
7471
  if (!unchanged) {
4356
7472
  let parsed;
4357
7473
  try {
4358
- const bytes = await readFile2(entry.sourcePath);
7474
+ const bytes = await readFile5(entry.sourcePath);
4359
7475
  parsed = await parser2.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
4360
7476
  } catch (error) {
4361
7477
  if (!(error instanceof ParseError))
@@ -4475,12 +7591,12 @@ async function collectFiles(sourceRoot, directory, entries, supportedExtensions)
4475
7591
  }
4476
7592
  if (!entry.isFile())
4477
7593
  continue;
4478
- const extension = normalizeExtension2(extname5(entry.name));
7594
+ const extension = normalizeExtension2(extname8(entry.name));
4479
7595
  if (!supportedExtensions.has(extension))
4480
7596
  continue;
4481
7597
  let fileStat;
4482
7598
  try {
4483
- fileStat = await stat(sourcePath, { bigint: true });
7599
+ fileStat = await stat4(sourcePath, { bigint: true });
4484
7600
  } catch {
4485
7601
  continue;
4486
7602
  }
@@ -4500,13 +7616,13 @@ function removePrevious(root, index, virtualPath) {
4500
7616
  return 1;
4501
7617
  }
4502
7618
  function writeAtomic(path, content) {
4503
- mkdirSync7(dirname7(path), { recursive: true });
7619
+ mkdirSync8(dirname9(path), { recursive: true });
4504
7620
  const tmp = `${path}.${randomUUID5()}.tmp`;
4505
- writeFileSync5(tmp, content);
7621
+ writeFileSync6(tmp, content);
4506
7622
  renameSync5(tmp, path);
4507
7623
  }
4508
7624
  function removeFile(path) {
4509
- if (existsSync12(path))
7625
+ if (existsSync13(path))
4510
7626
  rmSync3(path, { force: true });
4511
7627
  }
4512
7628
  var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, SKIP_DIR_NAMES;
@@ -4543,8 +7659,8 @@ var init_sync = __esm(() => {
4543
7659
  });
4544
7660
 
4545
7661
  // src/observability/run-log.ts
4546
- import { appendFileSync, mkdirSync as mkdirSync8 } from "node:fs";
4547
- import { dirname as dirname8 } from "node:path";
7662
+ import { appendFileSync, mkdirSync as mkdirSync9 } from "node:fs";
7663
+ import { dirname as dirname10 } from "node:path";
4548
7664
 
4549
7665
  class AutoRAGRunLogger {
4550
7666
  path;
@@ -4553,7 +7669,7 @@ class AutoRAGRunLogger {
4553
7669
  }
4554
7670
  write(event) {
4555
7671
  try {
4556
- mkdirSync8(dirname8(this.path), { recursive: true });
7672
+ mkdirSync9(dirname10(this.path), { recursive: true });
4557
7673
  appendFileSync(this.path, `${JSON.stringify(event)}
4558
7674
  `, "utf8");
4559
7675
  } catch (error) {}
@@ -4644,9 +7760,9 @@ function methodFailureCode(name) {
4644
7760
  }
4645
7761
 
4646
7762
  // src/retrieval/methods/bm25.ts
4647
- import { createHash as createHash5 } from "node:crypto";
4648
- import { existsSync as existsSync13, mkdirSync as mkdirSync9, readFileSync as readFileSync9, rmSync as rmSync4, writeFileSync as writeFileSync6 } from "node:fs";
4649
- import { dirname as dirname9, join as join15 } from "node:path";
7763
+ import { createHash as createHash7 } from "node:crypto";
7764
+ import { existsSync as existsSync14, mkdirSync as mkdirSync10, readFileSync as readFileSync10, rmSync as rmSync4, writeFileSync as writeFileSync7 } from "node:fs";
7765
+ import { dirname as dirname11, join as join18 } from "node:path";
4650
7766
 
4651
7767
  class BM25Method {
4652
7768
  root;
@@ -4657,16 +7773,16 @@ class BM25Method {
4657
7773
  status;
4658
7774
  constructor(options) {
4659
7775
  this.root = options.root;
4660
- this.indexPath = options.indexPath ?? join15(options.root, BM25_SUBDIR);
7776
+ this.indexPath = options.indexPath ?? join18(options.root, BM25_SUBDIR);
4661
7777
  this.fallback = options.fallback ?? "typescript";
4662
7778
  this.forceEngine = options.forceEngine;
4663
7779
  this.importBinding = options.importBinding ?? (() => import("@pngwasi/node-tantivy-binding"));
4664
7780
  if (options.enabled === false) {
4665
7781
  this.status = { readiness: "disabled", engine: "none", message: "BM25 is disabled" };
4666
7782
  } else {
4667
- const tantivyDir = join15(this.indexPath, TANTIVY_SUBDIR);
4668
- const hasTantivy = existsSync13(join15(tantivyDir, "meta.json"));
4669
- const hasFallback = existsSync13(join15(this.indexPath, FALLBACK_INDEX_FILE));
7783
+ const tantivyDir = join18(this.indexPath, TANTIVY_SUBDIR);
7784
+ const hasTantivy = existsSync14(join18(tantivyDir, "meta.json"));
7785
+ const hasFallback = existsSync14(join18(this.indexPath, FALLBACK_INDEX_FILE));
4670
7786
  if (hasTantivy) {
4671
7787
  this.status = { readiness: "ready", engine: "tantivy" };
4672
7788
  } else if (hasFallback && this.fallback !== "disabled") {
@@ -4732,7 +7848,7 @@ class BM25Method {
4732
7848
  const trimmedQuery = query.trim();
4733
7849
  if (trimmedQuery.length === 0)
4734
7850
  return [];
4735
- const topK = options.topK ?? DEFAULT_TOP_K;
7851
+ const topK = options.topK ?? DEFAULT_TOP_K2;
4736
7852
  if (this.status.readiness === "disabled" || this.status.readiness === "dependency_unavailable") {
4737
7853
  throw new BM25UnavailableError(this.status.readiness, this.status.message ?? "BM25 is unavailable");
4738
7854
  }
@@ -4759,7 +7875,7 @@ class BM25Method {
4759
7875
  }
4760
7876
  const indexDir = this.tantivyIndexPath();
4761
7877
  rmSync4(indexDir, { recursive: true, force: true });
4762
- mkdirSync9(indexDir, { recursive: true });
7878
+ mkdirSync10(indexDir, { recursive: true });
4763
7879
  const schema = new binding.SchemaBuilder().addTextField("virtualPath", { stored: true, indexOption: "basic", tokenizerName: "raw" }).addTextField("chunkId", { stored: true, indexOption: "basic", tokenizerName: "raw" }).addTextField("content", { stored: true, indexOption: "position" }).build();
4764
7880
  const index = new binding.Index(schema, indexDir, false);
4765
7881
  const writer = index.writer(30000000, 1);
@@ -4775,7 +7891,7 @@ class BM25Method {
4775
7891
  async retrieveTantivy(queryText, topK, scope) {
4776
7892
  const binding = await this.importBinding();
4777
7893
  const indexDir = this.tantivyIndexPath();
4778
- if (!existsSync13(indexDir) || !binding.Index.exists(indexDir)) {
7894
+ if (!existsSync14(indexDir) || !binding.Index.exists(indexDir)) {
4779
7895
  this.status = { readiness: "index_missing", engine: "none", message: "Tantivy BM25 index is missing" };
4780
7896
  throw new BM25UnavailableError("index_missing", "Tantivy BM25 index is missing; call refresh() first");
4781
7897
  }
@@ -4820,18 +7936,18 @@ class BM25Method {
4820
7936
  writeFallbackIndex(chunks) {
4821
7937
  const index = { version: 1, chunks };
4822
7938
  const path = this.fallbackIndexPath();
4823
- mkdirSync9(dirname9(path), { recursive: true });
4824
- writeFileSync6(path, `${JSON.stringify(index, null, 2)}
7939
+ mkdirSync10(dirname11(path), { recursive: true });
7940
+ writeFileSync7(path, `${JSON.stringify(index, null, 2)}
4825
7941
  `);
4826
7942
  }
4827
7943
  retrieveFallback(query, topK, scope) {
4828
7944
  const index = this.readFallbackIndex();
4829
- const queryTerms = tokenize(query);
7945
+ const queryTerms = tokenize2(query);
4830
7946
  if (queryTerms.length === 0)
4831
7947
  return [];
4832
7948
  const scopedChunks = index.chunks.filter((chunk) => matchesVirtualPathScope(chunk.virtualPath, scope));
4833
7949
  const documentFrequencies = new Map;
4834
- const tokenized = scopedChunks.map((chunk) => ({ chunk, terms: tokenize(chunk.content) }));
7950
+ const tokenized = scopedChunks.map((chunk) => ({ chunk, terms: tokenize2(chunk.content) }));
4835
7951
  for (const term of new Set(queryTerms)) {
4836
7952
  documentFrequencies.set(term, tokenized.filter(({ terms }) => terms.includes(term)).length);
4837
7953
  }
@@ -4854,19 +7970,19 @@ class BM25Method {
4854
7970
  }
4855
7971
  readFallbackIndex() {
4856
7972
  const path = this.fallbackIndexPath();
4857
- if (!existsSync13(path))
7973
+ if (!existsSync14(path))
4858
7974
  throw new BM25UnavailableError("index_missing", "BM25 fallback index is missing");
4859
- const parsed = JSON.parse(readFileSync9(path, "utf8"));
7975
+ const parsed = JSON.parse(readFileSync10(path, "utf8"));
4860
7976
  return parsed.version === 1 && Array.isArray(parsed.chunks) ? parsed : { version: 1, chunks: [] };
4861
7977
  }
4862
7978
  hasFallbackIndex() {
4863
- return existsSync13(this.fallbackIndexPath());
7979
+ return existsSync14(this.fallbackIndexPath());
4864
7980
  }
4865
7981
  tantivyIndexPath() {
4866
- return join15(this.indexPath, TANTIVY_SUBDIR);
7982
+ return join18(this.indexPath, TANTIVY_SUBDIR);
4867
7983
  }
4868
7984
  fallbackIndexPath() {
4869
- return join15(this.indexPath, FALLBACK_INDEX_FILE);
7985
+ return join18(this.indexPath, FALLBACK_INDEX_FILE);
4870
7986
  }
4871
7987
  syncResult(indexedChunks) {
4872
7988
  return {
@@ -4881,9 +7997,9 @@ function loadChunks(root) {
4881
7997
  const index = loadMirrorIndex(root);
4882
7998
  const chunks = [];
4883
7999
  for (const entry of Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath))) {
4884
- if (!existsSync13(entry.outputPath))
8000
+ if (!existsSync14(entry.outputPath))
4885
8001
  continue;
4886
- const content = normalizeMarkdown(readFileSync9(entry.outputPath, "utf8"));
8002
+ const content = normalizeMarkdown(readFileSync10(entry.outputPath, "utf8"));
4887
8003
  for (const [chunkIndex, chunkContent] of chunkMarkdown(content).entries()) {
4888
8004
  chunks.push({
4889
8005
  id: `bm25:${entry.virtualPath}:${chunkIndex}:${hash(chunkContent)}`,
@@ -4913,7 +8029,7 @@ ${paragraph}`;
4913
8029
  chunks.push(current);
4914
8030
  return chunks;
4915
8031
  }
4916
- function tokenize(value) {
8032
+ function tokenize2(value) {
4917
8033
  return value.toLowerCase().match(/[\p{Letter}\p{Number}_]+/gu) ?? [];
4918
8034
  }
4919
8035
  function bm25Score(queryTerms, documentTerms, documentFrequencies, documentCount, avgDocumentLength) {
@@ -4939,13 +8055,13 @@ function firstString(values) {
4939
8055
  return typeof first === "string" ? first : undefined;
4940
8056
  }
4941
8057
  function hash(value) {
4942
- return createHash5("sha256").update(value).digest("hex").slice(0, 12);
8058
+ return createHash7("sha256").update(value).digest("hex").slice(0, 12);
4943
8059
  }
4944
- var BM25_SUBDIR, TANTIVY_SUBDIR = "tantivy", FALLBACK_INDEX_FILE = "fallback-index.json", DEFAULT_TOP_K = 20, MAX_CHUNK_CHARS = 2400, BM25UnavailableError;
8060
+ var BM25_SUBDIR, TANTIVY_SUBDIR = "tantivy", FALLBACK_INDEX_FILE = "fallback-index.json", DEFAULT_TOP_K2 = 20, MAX_CHUNK_CHARS = 2400, BM25UnavailableError;
4945
8061
  var init_bm25 = __esm(() => {
4946
8062
  init_index_store();
4947
8063
  init_text();
4948
- BM25_SUBDIR = join15(".autorag", "bm25");
8064
+ BM25_SUBDIR = join18(".autorag", "bm25");
4949
8065
  BM25UnavailableError = class BM25UnavailableError extends Error {
4950
8066
  readiness;
4951
8067
  constructor(readiness, message) {
@@ -4979,7 +8095,7 @@ class RetrievalMethodRegistry {
4979
8095
 
4980
8096
  // src/subagents/contracts.ts
4981
8097
  function parseExplorerAssignment(value) {
4982
- const record = asRecord(value, "explorer assignment");
8098
+ const record = asRecord2(value, "explorer assignment");
4983
8099
  const originalQuery = requiredText(record, "originalQuery", "MISSING_ORIGINAL_QUERY", "INVALID_ORIGINAL_QUERY");
4984
8100
  const method = requiredMethod(record, "method", "MISSING_RETRIEVAL_METHOD", "INVALID_RETRIEVAL_METHOD");
4985
8101
  const queryVariant = requiredText(record, "queryVariant", "MISSING_QUERY_VARIANT", "INVALID_QUERY_VARIANT");
@@ -4995,7 +8111,7 @@ function parseExplorerAssignment(value) {
4995
8111
  return { originalQuery, method, queryVariant, queryVariants: [...variantsRaw] };
4996
8112
  }
4997
8113
  function parseEvidenceCandidate(value) {
4998
- const record = asRecord(value, "evidence candidate");
8114
+ const record = asRecord2(value, "evidence candidate");
4999
8115
  const source = requiredText(record, "source", "MISSING_SOURCE", "INVALID_SOURCE");
5000
8116
  const method = requiredMethod(record, "method", "MISSING_EVIDENCE_METHOD", "INVALID_EVIDENCE_METHOD");
5001
8117
  const evidence = requiredText(record, "evidence", "MISSING_EVIDENCE", "INVALID_EVIDENCE");
@@ -5019,7 +8135,7 @@ function parseEvidenceCandidate(value) {
5019
8135
  };
5020
8136
  }
5021
8137
  function parseExplorerReport(value) {
5022
- const record = asRecord(value, "explorer report");
8138
+ const record = asRecord2(value, "explorer report");
5023
8139
  const assignmentRaw = record.assignment;
5024
8140
  if (assignmentRaw === undefined) {
5025
8141
  throw new ContractValidationError("MISSING_ASSIGNMENT", "assignment is required.", "assignment");
@@ -5048,7 +8164,7 @@ function safeParseExplorerReport(value) {
5048
8164
  return safeParse(() => parseExplorerReport(value));
5049
8165
  }
5050
8166
  function parseSourceTemporal(value) {
5051
- const record = asRecord(value, "sourceTemporal");
8167
+ const record = asRecord2(value, "sourceTemporal");
5052
8168
  const status = record.status;
5053
8169
  if (status === undefined) {
5054
8170
  throw new ContractValidationError("INVALID_SOURCE_TEMPORAL", "sourceTemporal.status is required.", "sourceTemporal.status");
@@ -5068,7 +8184,7 @@ function parseSourceTemporal(value) {
5068
8184
  }
5069
8185
  return status === "asOf" ? { status, asOf: date } : status === "updatedAt" ? { status, updatedAt: date } : { status, createdAt: date };
5070
8186
  }
5071
- function asRecord(value, label) {
8187
+ function asRecord2(value, label) {
5072
8188
  if (typeof value !== "object" || value === null || Array.isArray(value)) {
5073
8189
  throw new ContractValidationError("INVALID_OBJECT", `${label} must be an object.`);
5074
8190
  }
@@ -6079,21 +9195,21 @@ var init_model_policy = __esm(() => {
6079
9195
 
6080
9196
  // src/subagents/runtime.ts
6081
9197
  import childProcess from "node:child_process";
6082
- import { createHash as createHash6, randomUUID as randomUUID6 } from "node:crypto";
9198
+ import { createHash as createHash8, randomUUID as randomUUID6 } from "node:crypto";
6083
9199
  import {
6084
9200
  chmodSync,
6085
- existsSync as existsSync14,
9201
+ existsSync as existsSync15,
6086
9202
  linkSync,
6087
- mkdirSync as mkdirSync10,
6088
- readFileSync as readFileSync10,
9203
+ mkdirSync as mkdirSync11,
9204
+ readFileSync as readFileSync11,
6089
9205
  realpathSync as realpathSync3,
6090
9206
  renameSync as renameSync6,
6091
9207
  statSync as statSync2,
6092
9208
  unlinkSync as unlinkSync4,
6093
- writeFileSync as writeFileSync7
9209
+ writeFileSync as writeFileSync8
6094
9210
  } from "node:fs";
6095
9211
  import { createRequire as createRequire2 } from "node:module";
6096
- import { basename as basename6, dirname as dirname10, extname as extname6, isAbsolute as isAbsolute6, join as join16, relative as relative4, resolve as resolve8, sep as sep3 } from "node:path";
9212
+ import { basename as basename9, dirname as dirname12, extname as extname9, isAbsolute as isAbsolute6, join as join19, relative as relative5, resolve as resolve8, sep as sep4 } from "node:path";
6097
9213
  import { fileURLToPath } from "node:url";
6098
9214
  import { InMemoryCredentialStore, InMemoryModelsStore } from "@earendil-works/pi-ai";
6099
9215
  import {
@@ -6105,24 +9221,24 @@ import {
6105
9221
  SettingsManager
6106
9222
  } from "@earendil-works/pi-coding-agent";
6107
9223
  function resolveExplorerToolsExtensionPath() {
6108
- const dir = dirname10(runtimePath);
9224
+ const dir = dirname12(runtimePath);
6109
9225
  const candidates = [
6110
- join16(dir, `explorer-tools-extension${extname6(runtimePath)}`),
6111
- join16(dir, "subagents", "explorer-tools-extension.js"),
6112
- join16(dir, "..", "subagents", "explorer-tools-extension.js")
9226
+ join19(dir, `explorer-tools-extension${extname9(runtimePath)}`),
9227
+ join19(dir, "subagents", "explorer-tools-extension.js"),
9228
+ join19(dir, "..", "subagents", "explorer-tools-extension.js")
6113
9229
  ];
6114
9230
  for (const candidate of candidates) {
6115
- if (existsSync14(candidate))
9231
+ if (existsSync15(candidate))
6116
9232
  return realpathSync3(candidate);
6117
9233
  }
6118
9234
  throw new Error(`AutoRAG explorer tools extension not found; looked in: ${candidates.join(", ")}`);
6119
9235
  }
6120
9236
  function resolvePiSubagentsPackageJson() {
6121
9237
  const require2 = createRequire2(import.meta.url);
6122
- return join16(dirname10(require2.resolve("pi-subagents")), "package.json");
9238
+ return join19(dirname12(require2.resolve("pi-subagents")), "package.json");
6123
9239
  }
6124
9240
  function resolvePiSubagentsExtension() {
6125
- return join16(dirname10(resolvePiSubagentsPackageJson()), "src", "extension", "index.ts");
9241
+ return join19(dirname12(resolvePiSubagentsPackageJson()), "src", "extension", "index.ts");
6126
9242
  }
6127
9243
  function resolveSubagentPiBinary() {
6128
9244
  const configured = process.env.PI_SUBAGENT_PI_BINARY?.trim();
@@ -6130,12 +9246,12 @@ function resolveSubagentPiBinary() {
6130
9246
  return configured;
6131
9247
  const require2 = createRequire2(import.meta.url);
6132
9248
  const subagentsPackage = resolvePiSubagentsPackageJson();
6133
- const packageJsonPath = join16(dirname10(dirname10(subagentsPackage)), "@earendil-works", "pi-coding-agent", "package.json");
9249
+ const packageJsonPath = join19(dirname12(dirname12(subagentsPackage)), "@earendil-works", "pi-coding-agent", "package.json");
6134
9250
  const packageJson = require2(packageJsonPath);
6135
9251
  const bin = typeof packageJson.bin === "string" ? packageJson.bin : packageJson.bin?.pi;
6136
9252
  if (!bin)
6137
9253
  throw new Error("Mandatory pi-subagents runtime could not resolve the Pi CLI binary");
6138
- return join16(dirname10(packageJsonPath), bin);
9254
+ return join19(dirname12(packageJsonPath), bin);
6139
9255
  }
6140
9256
  function asToolDefinition(tool) {
6141
9257
  return {
@@ -6144,12 +9260,12 @@ function asToolDefinition(tool) {
6144
9260
  };
6145
9261
  }
6146
9262
  function defaultAgentDir() {
6147
- return join16(resolveAutoRAGHome(), "pi-agent");
9263
+ return join19(resolveAutoRAGHome(), "pi-agent");
6148
9264
  }
6149
9265
  function ensurePiSettingsFile(agentDir) {
6150
- const settingsPath = join16(agentDir, "settings.json");
9266
+ const settingsPath = join19(agentDir, "settings.json");
6151
9267
  try {
6152
- writeFileSync7(settingsPath, "{}", { encoding: "utf8", flag: "wx" });
9268
+ writeFileSync8(settingsPath, "{}", { encoding: "utf8", flag: "wx" });
6153
9269
  } catch (error) {
6154
9270
  if (typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST")
6155
9271
  return;
@@ -6172,7 +9288,7 @@ function isRecognizedManagedExplorerDefinition(definition) {
6172
9288
  function replaceExplorerDefinitionAtomically(explorerPath) {
6173
9289
  const temporaryPath = `${explorerPath}.${randomUUID6()}.tmp`;
6174
9290
  try {
6175
- writeFileSync7(temporaryPath, AUTORAG_EXPLORER_AGENT_DEFINITION, {
9291
+ writeFileSync8(temporaryPath, AUTORAG_EXPLORER_AGENT_DEFINITION, {
6176
9292
  encoding: "utf8",
6177
9293
  flag: "wx",
6178
9294
  mode: 384
@@ -6180,30 +9296,30 @@ function replaceExplorerDefinitionAtomically(explorerPath) {
6180
9296
  chmodSync(temporaryPath, 384);
6181
9297
  renameSync6(temporaryPath, explorerPath);
6182
9298
  } finally {
6183
- if (existsSync14(temporaryPath))
9299
+ if (existsSync15(temporaryPath))
6184
9300
  unlinkSync4(temporaryPath);
6185
9301
  }
6186
9302
  }
6187
9303
  function ensureMandatoryExplorerAgent(agentDir) {
6188
- const agentsDir = join16(agentDir, "agents");
6189
- const explorerPath = join16(agentsDir, "autorag-explorer.md");
6190
- if (existsSync14(explorerPath)) {
6191
- const existingDefinition = readFileSync10(explorerPath, "utf8");
9304
+ const agentsDir = join19(agentDir, "agents");
9305
+ const explorerPath = join19(agentsDir, "autorag-explorer.md");
9306
+ if (existsSync15(explorerPath)) {
9307
+ const existingDefinition = readFileSync11(explorerPath, "utf8");
6192
9308
  if (existingDefinition === AUTORAG_EXPLORER_AGENT_DEFINITION)
6193
9309
  return;
6194
9310
  if (!isRecognizedManagedExplorerDefinition(existingDefinition)) {
6195
9311
  throw new Error(`AutoRAG persistent explorer definition at ${explorerPath} is not canonical`);
6196
9312
  }
6197
9313
  replaceExplorerDefinitionAtomically(explorerPath);
6198
- if (readFileSync10(explorerPath, "utf8") !== AUTORAG_EXPLORER_AGENT_DEFINITION || (statSync2(explorerPath).mode & 511) !== 384) {
9314
+ if (readFileSync11(explorerPath, "utf8") !== AUTORAG_EXPLORER_AGENT_DEFINITION || (statSync2(explorerPath).mode & 511) !== 384) {
6199
9315
  throw new Error(`AutoRAG persistent explorer definition at ${explorerPath} is not canonical`);
6200
9316
  }
6201
9317
  return;
6202
9318
  }
6203
- mkdirSync10(agentsDir, { recursive: true });
6204
- const temporaryPath = join16(agentsDir, `.autorag-explorer.${randomUUID6()}.tmp`);
9319
+ mkdirSync11(agentsDir, { recursive: true });
9320
+ const temporaryPath = join19(agentsDir, `.autorag-explorer.${randomUUID6()}.tmp`);
6205
9321
  try {
6206
- writeFileSync7(temporaryPath, AUTORAG_EXPLORER_AGENT_DEFINITION, {
9322
+ writeFileSync8(temporaryPath, AUTORAG_EXPLORER_AGENT_DEFINITION, {
6207
9323
  encoding: "utf8",
6208
9324
  flag: "wx",
6209
9325
  mode: 384
@@ -6216,18 +9332,18 @@ function ensureMandatoryExplorerAgent(agentDir) {
6216
9332
  }
6217
9333
  }
6218
9334
  } finally {
6219
- if (existsSync14(temporaryPath))
9335
+ if (existsSync15(temporaryPath))
6220
9336
  unlinkSync4(temporaryPath);
6221
9337
  }
6222
- if (readFileSync10(explorerPath, "utf8") !== AUTORAG_EXPLORER_AGENT_DEFINITION) {
9338
+ if (readFileSync11(explorerPath, "utf8") !== AUTORAG_EXPLORER_AGENT_DEFINITION) {
6223
9339
  throw new Error(`AutoRAG persistent explorer definition at ${explorerPath} is not canonical`);
6224
9340
  }
6225
9341
  }
6226
9342
  function defaultSessionDir(cwd, agentDir) {
6227
9343
  const canonicalCwd = resolve8(cwd);
6228
9344
  const readablePath = canonicalCwd.replace(/^[/\\]/, "").replace(/[/\\:]/g, "-");
6229
- const cwdHash = createHash6("sha256").update(canonicalCwd).digest("hex").slice(0, 12);
6230
- return join16(agentDir, "sessions", `--${readablePath}-${cwdHash}--`);
9345
+ const cwdHash = createHash8("sha256").update(canonicalCwd).digest("hex").slice(0, 12);
9346
+ return join19(agentDir, "sessions", `--${readablePath}-${cwdHash}--`);
6231
9347
  }
6232
9348
  function childEnvironmentFromSpawnOptions(args) {
6233
9349
  const optionsIndex = Array.isArray(args[1]) ? 2 : 1;
@@ -6265,7 +9381,7 @@ function isPiChildSpawn(args, context) {
6265
9381
  return childArgsFromSpawnArguments(args)[0] === context.piBinary;
6266
9382
  }
6267
9383
  function isAsyncSubagentRunnerSpawn(args) {
6268
- return childArgsFromSpawnArguments(args).some((argument) => typeof argument === "string" && basename6(argument) === "subagent-runner.ts");
9384
+ return childArgsFromSpawnArguments(args).some((argument) => typeof argument === "string" && basename9(argument) === "subagent-runner.ts");
6269
9385
  }
6270
9386
  function createScopedChildProcessModule(context) {
6271
9387
  const originalSpawn = childProcess.spawn.bind(childProcess);
@@ -6287,12 +9403,12 @@ function createScopedChildProcessModule(context) {
6287
9403
  return { ...scopedModule, default: scopedModule };
6288
9404
  }
6289
9405
  function isWithinDirectory(root, candidate) {
6290
- const relativePath = relative4(root, candidate);
6291
- return relativePath === "" || relativePath !== ".." && !relativePath.startsWith(`..${sep3}`) && !isAbsolute6(relativePath);
9406
+ const relativePath = relative5(root, candidate);
9407
+ return relativePath === "" || relativePath !== ".." && !relativePath.startsWith(`..${sep4}`) && !isAbsolute6(relativePath);
6292
9408
  }
6293
9409
  async function loadScopedPiSubagentsExtension(extensionPath, environment) {
6294
9410
  const packageJsonPath = resolvePiSubagentsPackageJson();
6295
- const packageRoot = dirname10(packageJsonPath);
9411
+ const packageRoot = dirname12(packageJsonPath);
6296
9412
  const packageRequire = createRequire2(packageJsonPath);
6297
9413
  const jitiModule = packageRequire("jiti");
6298
9414
  if (typeof jitiModule.createJiti !== "function") {
@@ -6300,7 +9416,7 @@ async function loadScopedPiSubagentsExtension(extensionPath, environment) {
6300
9416
  }
6301
9417
  const createJiti = jitiModule.createJiti;
6302
9418
  const baseJiti = createJiti(packageJsonPath, { moduleCache: false, fsCache: false });
6303
- const utilsPath = join16(packageRoot, "src", "shared", "utils.ts");
9419
+ const utilsPath = join19(packageRoot, "src", "shared", "utils.ts");
6304
9420
  const upstreamUtils = await baseJiti.import(utilsPath);
6305
9421
  if (typeof upstreamUtils.getAgentDir !== "function") {
6306
9422
  throw new Error("pi-subagents did not expose its expected getAgentDir utility");
@@ -6330,11 +9446,11 @@ function isJsonObject(value) {
6330
9446
  return typeof value === "object" && value !== null && !Array.isArray(value);
6331
9447
  }
6332
9448
  function readPiModelsConfig(modelsPath) {
6333
- if (!existsSync14(modelsPath))
9449
+ if (!existsSync15(modelsPath))
6334
9450
  return { providers: {} };
6335
9451
  let parsed;
6336
9452
  try {
6337
- parsed = JSON.parse(readFileSync10(modelsPath, "utf8"));
9453
+ parsed = JSON.parse(readFileSync11(modelsPath, "utf8"));
6338
9454
  } catch (error) {
6339
9455
  throw new Error(`AutoRAG could not parse Pi models.json at ${modelsPath}: ${error.message}`, {
6340
9456
  cause: error
@@ -6508,13 +9624,13 @@ async function persistPiModels(modelsPath, roleModels, providerCredentials) {
6508
9624
  }
6509
9625
  }
6510
9626
  temporaryPath = `${modelsPath}.${randomUUID6()}.tmp`;
6511
- writeFileSync7(temporaryPath, content, { encoding: "utf8", flag: "wx", mode: 384 });
9627
+ writeFileSync8(temporaryPath, content, { encoding: "utf8", flag: "wx", mode: 384 });
6512
9628
  await validatePiModelsFile(temporaryPath, roleModels, "generated");
6513
9629
  lock.assertOwned();
6514
9630
  renameSync6(temporaryPath, modelsPath);
6515
9631
  } finally {
6516
9632
  try {
6517
- if (temporaryPath && existsSync14(temporaryPath))
9633
+ if (temporaryPath && existsSync15(temporaryPath))
6518
9634
  unlinkSync4(temporaryPath);
6519
9635
  } finally {
6520
9636
  lock.release();
@@ -6579,7 +9695,7 @@ function explorerEnvironmentIdentity(environment) {
6579
9695
  agentDir: environment.agentDir,
6580
9696
  piBinary: environment.piBinary,
6581
9697
  apiKeyName: environment.apiKeyName,
6582
- apiKeyHash: environment.apiKey === undefined ? undefined : createHash6("sha256").update(environment.apiKey).digest("hex")
9698
+ apiKeyHash: environment.apiKey === undefined ? undefined : createHash8("sha256").update(environment.apiKey).digest("hex")
6583
9699
  });
6584
9700
  }
6585
9701
  function defaultExplorerModel(model) {
@@ -6587,7 +9703,7 @@ function defaultExplorerModel(model) {
6587
9703
  }
6588
9704
  async function createMandatorySubagentSession(options) {
6589
9705
  const agentDir = resolve8(options.agentDir ?? defaultAgentDir());
6590
- mkdirSync10(agentDir, { recursive: true });
9706
+ mkdirSync11(agentDir, { recursive: true });
6591
9707
  ensureMandatoryExplorerAgent(agentDir);
6592
9708
  const piBinary = resolveSubagentPiBinary();
6593
9709
  const extensionPath = options.extensionPath ?? resolvePiSubagentsExtension();
@@ -6609,7 +9725,7 @@ async function createMandatorySubagentSession(options) {
6609
9725
  const explorerRoleModel = activeRoleModel(explorerModel, "explorer");
6610
9726
  const roleModels = [orchestratorRoleModel, explorerRoleModel];
6611
9727
  validateProviderApiKeyEnvNames(roleModels);
6612
- const authPath = join16(agentDir, "auth.json");
9728
+ const authPath = join19(agentDir, "auth.json");
6613
9729
  for (const { reference } of roleModels) {
6614
9730
  if (parentProviderCredentials.has(reference.provider))
6615
9731
  continue;
@@ -6618,7 +9734,7 @@ async function createMandatorySubagentSession(options) {
6618
9734
  continue;
6619
9735
  parentProviderCredentials.set(reference.provider, stored.key);
6620
9736
  }
6621
- const modelsPath = join16(agentDir, "models.json");
9737
+ const modelsPath = join19(agentDir, "models.json");
6622
9738
  const explorerCredential = parentProviderCredentials.get(explorerRoleModel.reference.provider);
6623
9739
  const explorerEnvironment = {
6624
9740
  agentDir,
@@ -6927,12 +10043,12 @@ Required handoff: include temporal metadata.
6927
10043
  });
6928
10044
 
6929
10045
  // src/agent/bash-tool.ts
6930
- import { spawn as spawn4 } from "node:child_process";
10046
+ import { spawn as spawn7 } from "node:child_process";
6931
10047
  import { Type as Type2 } from "typebox";
6932
10048
  function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
6933
10049
  return new Promise((resolve9) => {
6934
10050
  const shell = process.platform === "win32" ? "bash.exe" : "/bin/bash";
6935
- const child = spawn4(shell, ["-c", command], { cwd });
10051
+ const child = spawn7(shell, ["-c", command], { cwd });
6936
10052
  const chunks = [];
6937
10053
  let total = 0;
6938
10054
  let truncated = false;
@@ -6976,7 +10092,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
6976
10092
  });
6977
10093
  }
6978
10094
  function createBashTool(options) {
6979
- const defaultTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
10095
+ const defaultTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
6980
10096
  const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
6981
10097
  return {
6982
10098
  name: BASH_TOOL_NAME,
@@ -7035,7 +10151,7 @@ function createBashTool(options) {
7035
10151
  }
7036
10152
  };
7037
10153
  }
7038
- var BASH_TOOL_NAME = "bash", DEFAULT_TIMEOUT_MS2 = 120000, DEFAULT_MAX_OUTPUT_BYTES = 131072, bashSchema;
10154
+ var BASH_TOOL_NAME = "bash", DEFAULT_TIMEOUT_MS6 = 120000, DEFAULT_MAX_OUTPUT_BYTES = 131072, bashSchema;
7039
10155
  var init_bash_tool = __esm(() => {
7040
10156
  bashSchema = Type2.Object({
7041
10157
  command: Type2.String({
@@ -7585,7 +10701,8 @@ function recordStructuredResultsSession(sessionId, query, details, sessions, mem
7585
10701
  summary: result.summary,
7586
10702
  evidence: result.evidence.map((evidence) => evidence.lineNumber !== undefined ? { excerpt: evidence.excerpt, lineNumber: evidence.lineNumber } : { excerpt: evidence.excerpt }),
7587
10703
  confidence: confidenceFrom(result.confidence),
7588
- feedbackId: `${sessionId}:${result.number}`
10704
+ feedbackId: `${sessionId}:${result.number}`,
10705
+ source: registry.get(result.number)?.source
7589
10706
  }));
7590
10707
  const answer = details.answer;
7591
10708
  const diagnostics = [];
@@ -7992,7 +11109,7 @@ var init_watch_refresh = __esm(() => {
7992
11109
  // src/agent/agent.ts
7993
11110
  import { randomUUID as randomUUID7 } from "node:crypto";
7994
11111
  import { watch as fsWatch, realpathSync as realpathSync4, statSync as statSync3 } from "node:fs";
7995
- import { dirname as dirname11, join as join17, resolve as resolve9 } from "node:path";
11112
+ import { dirname as dirname13, join as join20, resolve as resolve9 } from "node:path";
7996
11113
  import { Agent } from "@earendil-works/pi-agent-core";
7997
11114
 
7998
11115
  class AutoRAGAgent {
@@ -8079,10 +11196,10 @@ class AutoRAGAgent {
8079
11196
  if (options.jikji) {
8080
11197
  this.jikjiClient = new JikjiClient({ ...options.jikji, root: this.workspaceProjectRoot });
8081
11198
  }
8082
- const memPath = memoryPath ?? join17(resolveAutoRAGHome(), "memory.json");
11199
+ const memPath = memoryPath ?? join20(resolveAutoRAGHome(), "memory.json");
8083
11200
  this.memory = new RetrievalMemory({ storagePath: memPath });
8084
11201
  this.memory.load();
8085
- this.runLogger = new AutoRAGRunLogger(join17(dirname11(memPath), "logs", "runs.jsonl"));
11202
+ this.runLogger = new AutoRAGRunLogger(join20(dirname13(memPath), "logs", "runs.jsonl"));
8086
11203
  const checkMemoryTool = createCheckMemoryTool(this.memory);
8087
11204
  const searchBM25Tool = createSearchBM25DocumentsTool(() => this.bm25Method);
8088
11205
  const searchDatasourceTool = createSearchDatasourceDocumentsTool(this);
@@ -8835,7 +11952,7 @@ ${allowedRoots}
8835
11952
  written: 0,
8836
11953
  deleted: 0,
8837
11954
  skipped: 0,
8838
- indexPath: join17(this.workspaceProjectRoot, PARSED_MIRROR_SUBDIR),
11955
+ indexPath: join20(this.workspaceProjectRoot, PARSED_MIRROR_SUBDIR),
8839
11956
  diagnostics
8840
11957
  };
8841
11958
  }
@@ -9946,8 +13063,8 @@ var exports_commands = {};
9946
13063
  __export(exports_commands, {
9947
13064
  runIndex: () => runIndex
9948
13065
  });
9949
- import { existsSync as existsSync15, lstatSync as lstatSync2, rmSync as rmSync5 } from "node:fs";
9950
- import { join as join18, resolve as resolve10, sep as sep4 } from "node:path";
13066
+ import { existsSync as existsSync16, lstatSync as lstatSync2, rmSync as rmSync5 } from "node:fs";
13067
+ import { join as join21, resolve as resolve10, sep as sep5 } from "node:path";
9951
13068
  async function runIndex(ctx) {
9952
13069
  const sub = ctx.positionals[0];
9953
13070
  if (sub !== "reset" && sub !== "rebuild") {
@@ -9966,7 +13083,7 @@ async function runIndex(ctx) {
9966
13083
  const autoragDir = resolve10(config.workspacePath, ".autorag");
9967
13084
  const targets = targetSubdirs.map((subdir) => resolve10(config.workspacePath, subdir));
9968
13085
  for (const dir of [autoragDir, ...targets]) {
9969
- if (existsSync15(dir) && lstatSync2(dir).isSymbolicLink()) {
13086
+ if (existsSync16(dir) && lstatSync2(dir).isSymbolicLink()) {
9970
13087
  ctx.stderr(renderError(new Error("Refusing to reset: an index path is a symlink, not a real directory."), {
9971
13088
  json: ctx.json
9972
13089
  }));
@@ -9981,7 +13098,7 @@ async function runIndex(ctx) {
9981
13098
  }
9982
13099
  if (!ctx.flags.yes) {
9983
13100
  if (ctx.promptYesNo) {
9984
- const ok = await ctx.promptYesNo(`Reset the ${targetNames.join(", ")} indexes under ${join18(config.workspacePath, ".autorag")}?`);
13101
+ const ok = await ctx.promptYesNo(`Reset the ${targetNames.join(", ")} indexes under ${join21(config.workspacePath, ".autorag")}?`);
9985
13102
  if (!ok) {
9986
13103
  ctx.stderr(renderError(new Error("Reset declined."), { json: ctx.json }));
9987
13104
  return 2;
@@ -10040,7 +13157,7 @@ function resolveResetScope(methods) {
10040
13157
  };
10041
13158
  }
10042
13159
  function isWithin(target, base) {
10043
- const prefix = base.endsWith(sep4) ? base : `${base}${sep4}`;
13160
+ const prefix = base.endsWith(sep5) ? base : `${base}${sep5}`;
10044
13161
  return target === base || target.startsWith(prefix);
10045
13162
  }
10046
13163
  var ALL_RESET_TARGETS, ALL_RESET_TARGET_NAMES;
@@ -10131,7 +13248,7 @@ __export(exports_health, {
10131
13248
  });
10132
13249
  import { mkdtempSync, rmSync as rmSync6 } from "node:fs";
10133
13250
  import { tmpdir as tmpdir3 } from "node:os";
10134
- import { join as join19 } from "node:path";
13251
+ import { join as join22 } from "node:path";
10135
13252
  import { completeSimple } from "@earendil-works/pi-ai/compat";
10136
13253
  function defaultResolveConfig(input) {
10137
13254
  return resolveConfigReadOnly({ flags: input.flags, cwd: input.cwd, env: input.env });
@@ -10224,7 +13341,7 @@ function sanitizeMessage(raw) {
10224
13341
  function parseTimeoutMs(flags) {
10225
13342
  const raw = flags["timeout-ms"];
10226
13343
  if (raw === undefined || raw === true)
10227
- return DEFAULT_TIMEOUT_MS3;
13344
+ return DEFAULT_TIMEOUT_MS7;
10228
13345
  const text = typeof raw === "string" ? raw.trim() : "";
10229
13346
  if (text.length === 0)
10230
13347
  return { error: "--timeout-ms requires a value" };
@@ -10332,10 +13449,10 @@ async function defaultOrchestratorProbe(input, signal) {
10332
13449
  }
10333
13450
  }
10334
13451
  async function defaultExplorerProbe(input, signal) {
10335
- const tempRoot = mkdtempSync(join19(tmpdir3(), "autorag-health-"));
13452
+ const tempRoot = mkdtempSync(join22(tmpdir3(), "autorag-health-"));
10336
13453
  try {
10337
- const agentDir = join19(tempRoot, "agent");
10338
- const sessionDir = join19(tempRoot, "sessions");
13454
+ const agentDir = join22(tempRoot, "agent");
13455
+ const sessionDir = join22(tempRoot, "sessions");
10339
13456
  const providerApiKeys = input.providerApiKeys;
10340
13457
  const modelRef = `${input.model.provider}/${input.model.id}`;
10341
13458
  let probeSession;
@@ -10581,7 +13698,7 @@ async function runHealth(ctx, deps = {}) {
10581
13698
  ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
10582
13699
  return exitCodeFor(topCategory);
10583
13700
  }
10584
- var CATEGORY_PRECEDENCE, SECRET_PATTERNS, DEFAULT_TIMEOUT_MS3 = 1e4, AUTH_ERROR_PATTERNS, NETWORK_ERROR_CODES, NETWORK_ERROR_PATTERNS, HEALTH_EXPLORER_SYSTEM_PROMPT = "You are a health probe. Reply with HEALTH_OK. Do not read any files.";
13701
+ var CATEGORY_PRECEDENCE, SECRET_PATTERNS, DEFAULT_TIMEOUT_MS7 = 1e4, AUTH_ERROR_PATTERNS, NETWORK_ERROR_CODES, NETWORK_ERROR_PATTERNS, HEALTH_EXPLORER_SYSTEM_PROMPT = "You are a health probe. Reply with HEALTH_OK. Do not read any files.";
10585
13702
  var init_health = __esm(() => {
10586
13703
  init_runtime();
10587
13704
  init_config();