@qvac/core 0.2.2 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/dist-lib/lib/api/agents.js +2 -3
  2. package/dist-lib/lib/api/chat.d.ts +3 -1
  3. package/dist-lib/lib/api/chat.js +9 -5
  4. package/dist-lib/lib/api/chunks.d.ts +1 -1
  5. package/dist-lib/lib/api/chunks.js +5 -15
  6. package/dist-lib/lib/api/current-mesh.d.ts +16 -1
  7. package/dist-lib/lib/api/current-mesh.js +18 -0
  8. package/dist-lib/lib/api/devices.js +6 -3
  9. package/dist-lib/lib/api/index.d.ts +2 -0
  10. package/dist-lib/lib/api/index.js +4 -2
  11. package/dist-lib/lib/api/inference-calls.d.ts +18 -0
  12. package/dist-lib/lib/api/inference-calls.js +15 -0
  13. package/dist-lib/lib/api/knowledge.js +11 -7
  14. package/dist-lib/lib/api/members.js +2 -3
  15. package/dist-lib/lib/api/rag.js +10 -2
  16. package/dist-lib/lib/api/sync.d.ts +1 -1
  17. package/dist-lib/lib/api/sync.js +2 -1
  18. package/dist-lib/lib/api/tool-activity.js +2 -2
  19. package/dist-lib/lib/api/tool-configs.js +2 -2
  20. package/dist-lib/lib/api/watchable.d.ts +6 -1
  21. package/dist-lib/lib/api/watchable.js +13 -2
  22. package/dist-lib/lib/assistant/harness.d.ts +1 -0
  23. package/dist-lib/lib/assistant/history.d.ts +1 -2
  24. package/dist-lib/lib/assistant/history.js +0 -6
  25. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  26. package/dist-lib/lib/assistant-tree.js +11 -2
  27. package/dist-lib/lib/assistant.d.ts +17 -8
  28. package/dist-lib/lib/assistant.js +172 -125
  29. package/dist-lib/lib/core/chunk.d.ts +1 -0
  30. package/dist-lib/lib/core/chunk.js +32 -0
  31. package/dist-lib/lib/core/contract/devices.js +3 -2
  32. package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
  33. package/dist-lib/lib/core/files/rag/build.js +1 -2
  34. package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
  35. package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
  36. package/dist-lib/lib/core/files/rag/index.js +3 -17
  37. package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
  38. package/dist-lib/lib/core/files/rag/load.js +1 -2
  39. package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
  40. package/dist-lib/lib/core/files/rag/produce.js +1 -3
  41. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
  42. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
  43. package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
  44. package/dist-lib/lib/core/files/rag/similarity.js +6 -0
  45. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  46. package/dist-lib/lib/core/harness-loan.js +1 -1
  47. package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
  48. package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
  49. package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
  50. package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
  51. package/dist-lib/lib/core/local/index.js +6 -0
  52. package/dist-lib/lib/core/notification.js +5 -4
  53. package/dist-lib/lib/core.d.ts +9 -1
  54. package/dist-lib/lib/core.js +98 -16
  55. package/dist-lib/lib/fs-drive-bridge.js +1 -1
  56. package/dist-lib/lib/inference-calls.d.ts +40 -0
  57. package/dist-lib/lib/inference-calls.js +266 -0
  58. package/dist-lib/lib/mesh-scoped-engine.js +5 -0
  59. package/dist-lib/lib/model-download.d.ts +2 -0
  60. package/dist-lib/lib/model-download.js +20 -5
  61. package/dist-lib/lib/oauth/keys.js +1 -0
  62. package/dist-lib/lib/oauth/spotify.js +1 -0
  63. package/dist-lib/lib/oauth/verify.d.ts +8 -0
  64. package/dist-lib/lib/oauth/verify.js +58 -0
  65. package/dist-lib/lib/oauth.d.ts +1 -0
  66. package/dist-lib/lib/oauth.js +1 -0
  67. package/dist-lib/schema/content.js +5 -1
  68. package/dist-lib/schema/engine.js +42 -1
  69. package/dist-lib/schema/qvac.d.ts +3 -0
  70. package/dist-lib/schema/qvac.js +11 -2
  71. package/dist-lib/spec/bind.js +3 -0
  72. package/dist-lib/spec/calls.js +1 -0
  73. package/dist-lib/spec/capabilities.d.ts +2 -0
  74. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  75. package/dist-lib/spec/hrpc/index.js +15 -3
  76. package/dist-lib/spec/hrpc/messages.js +172 -49
  77. package/dist-lib/spec/hyperschema/index.js +172 -49
  78. package/dist-lib/spec/hyperschema/schema.json +127 -1
  79. package/dist-lib/spec/hyperschema/types.d.ts +34 -1
  80. package/dist-lib/spec/local/hyperdb/messages.js +64 -128
  81. package/dist-lib/spec/local/hyperschema/index.js +32 -64
  82. package/dist-lib/spec/local/hyperschema/schema.json +16 -0
  83. package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
  84. package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
  85. package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
  86. package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
  87. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  88. package/package.json +14 -17
  89. package/tui/app.mjs +327 -46
  90. package/tui/lib/knowledge-import.mjs +1 -1
  91. package/tui/lib/render.mjs +17 -1
  92. package/tui/lib/selection.mjs +16 -0
  93. package/tui/lib/usage.mjs +54 -0
  94. package/.claude/skills/asana/SKILL.md +0 -86
  95. package/.claude/skills/compaction/SKILL.md +0 -132
  96. package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
  97. package/.claude/skills/gh-stack/SKILL.md +0 -181
  98. package/.claude/skills/gh-stack/references/commands.md +0 -177
  99. package/.claude/skills/gh-stack/references/stack-design.md +0 -97
  100. package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
  101. package/.claude/skills/js-house-style/SKILL.md +0 -670
  102. package/.claude/skills/pr-create/SKILL.md +0 -201
  103. package/.claude/skills/pr-review/SKILL.md +0 -424
  104. package/CLAUDE.md +0 -43
  105. package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
  106. package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
  107. package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
  108. package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
  109. package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
  110. package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
  111. package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
  112. package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
  113. package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
  114. package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
  115. package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
  116. package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
  117. package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
  118. package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
  119. package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
  120. package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
  121. package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
  122. package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
  123. package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
  124. package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
  125. package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
  126. package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
  127. package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
  128. package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
  129. package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
  130. package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
  131. package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
  132. package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
  133. package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
  134. package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
  135. package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
  136. package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
  137. package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
  138. package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
  139. package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
  140. package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
  141. package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
  142. package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
  143. package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
  144. package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
  145. package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
  146. package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
  147. package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
  148. package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
  149. package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
  150. package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
  151. package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
  152. package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
  153. package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
  154. package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
  155. package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
  156. package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
  157. package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
  158. package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
  159. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
  160. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
  161. package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
  162. package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
  163. package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
  164. package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
  165. package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
  166. package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
  167. package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
  168. package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
  169. package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
  170. package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
  171. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
  172. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
  173. package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
  174. package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
  175. package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
  176. package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
@@ -1,141 +0,0 @@
1
- import b4a from 'b4a';
2
- import { AbortController, AbortSignal } from 'bare-abort-controller';
3
- import { ReadableStream } from 'web-streams-polyfill/dist/ponyfill.mjs';
4
- import { decodeLatin1, decodeUtf16 } from './utf.js';
5
- const g = globalThis;
6
- if (!g.ReadableStream)
7
- g.ReadableStream = ReadableStream;
8
- if (!g.DOMException) {
9
- g.DOMException = class DOMException extends Error {
10
- constructor(message, name = 'Error') {
11
- super(message);
12
- this.name = name;
13
- }
14
- };
15
- }
16
- export class BareTextDecoder {
17
- encoding;
18
- fatal;
19
- ignoreBOM;
20
- constructor(label = 'utf-8', options = {}) {
21
- this.encoding = parseEncoding(label);
22
- this.fatal = options.fatal === true;
23
- this.ignoreBOM = options.ignoreBOM === true;
24
- }
25
- decode(input) {
26
- if (input === undefined)
27
- return '';
28
- const bytes = ArrayBuffer.isView(input)
29
- ? b4a.from(input.buffer, input.byteOffset, input.byteLength)
30
- : b4a.from(input);
31
- return decodeBytes(bytes, this.encoding, { fatal: this.fatal, ignoreBOM: this.ignoreBOM });
32
- }
33
- }
34
- export class BareTextEncoder {
35
- encode(input = '') {
36
- return b4a.from(String(input), 'utf8');
37
- }
38
- }
39
- if (!g.TextDecoder)
40
- g.TextDecoder = BareTextDecoder;
41
- if (!g.TextEncoder)
42
- g.TextEncoder = BareTextEncoder;
43
- export function decodeBytes(bytes, label, options = {}) {
44
- const fatal = options.fatal === true;
45
- const encoding = parseEncoding(label);
46
- const text = encoding === 'utf-16le'
47
- ? utf16(bytes, true, fatal)
48
- : encoding === 'utf-16be'
49
- ? utf16(bytes, false, fatal)
50
- : encoding === 'latin1'
51
- ? decodeLatin1(bytes)
52
- : decodeUtf8(bytes, fatal);
53
- return options.ignoreBOM || text.charCodeAt(0) !== 0xfeff ? text : text.slice(1);
54
- }
55
- function parseEncoding(label) {
56
- const l = String(label).trim().toLowerCase();
57
- if (l.includes('utf-16') || l.includes('utf16') || l === 'ucs-2' || l === 'ucs2') {
58
- return l.includes('be') ? 'utf-16be' : 'utf-16le';
59
- }
60
- if (l.includes('latin1') || l.includes('8859'))
61
- return 'latin1';
62
- return 'utf-8';
63
- }
64
- function utf16(bytes, littleEndian, fatal) {
65
- if (fatal && bytes.length % 2 !== 0)
66
- throw new TypeError('truncated utf-16 sequence');
67
- const text = decodeUtf16(bytes, littleEndian);
68
- if (fatal)
69
- assertPairedSurrogates(text);
70
- return text;
71
- }
72
- function assertPairedSurrogates(text) {
73
- for (let i = 0; i < text.length; i++) {
74
- const unit = text.charCodeAt(i);
75
- if (unit >= 0xd800 && unit <= 0xdbff) {
76
- const low = text.charCodeAt(i + 1);
77
- if (Number.isNaN(low) || low < 0xdc00 || low > 0xdfff) {
78
- throw new TypeError('lone high surrogate');
79
- }
80
- i++;
81
- }
82
- else if (unit >= 0xdc00 && unit <= 0xdfff) {
83
- throw new TypeError('lone low surrogate');
84
- }
85
- }
86
- }
87
- function decodeUtf8(bytes, fatal) {
88
- if (fatal && !isValidUtf8(bytes))
89
- throw new TypeError('invalid utf-8 sequence');
90
- return b4a.toString(bytes, 'utf8');
91
- }
92
- function isValidUtf8(bytes) {
93
- for (let i = 0; i < bytes.length;) {
94
- const b0 = bytes[i];
95
- if (b0 < 0x80) {
96
- i++;
97
- continue;
98
- }
99
- let size;
100
- let min;
101
- let cp;
102
- if ((b0 & 0xe0) === 0xc0) {
103
- size = 2;
104
- min = 0x80;
105
- cp = b0 & 0x1f;
106
- }
107
- else if ((b0 & 0xf0) === 0xe0) {
108
- size = 3;
109
- min = 0x800;
110
- cp = b0 & 0x0f;
111
- }
112
- else if ((b0 & 0xf8) === 0xf0) {
113
- size = 4;
114
- min = 0x10000;
115
- cp = b0 & 0x07;
116
- }
117
- else {
118
- return false;
119
- }
120
- if (i + size > bytes.length)
121
- return false;
122
- for (let j = 1; j < size; j++) {
123
- const bj = bytes[i + j];
124
- if ((bj & 0xc0) !== 0x80)
125
- return false;
126
- cp = (cp << 6) | (bj & 0x3f);
127
- }
128
- if (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
129
- return false;
130
- i += size;
131
- }
132
- return true;
133
- }
134
- if (!g.AbortController) {
135
- g.AbortController = AbortController;
136
- g.AbortSignal = AbortSignal;
137
- }
138
- const clone = globalThis.structuredClone;
139
- if (clone) {
140
- g.structuredClone = (value, options) => clone(value, (options ?? undefined));
141
- }
@@ -1,12 +0,0 @@
1
- import './pdf-platform.ts';
2
- export declare function extractPdfText(bytes: Uint8Array, options?: {
3
- maxChars?: number;
4
- }): Promise<string>;
5
- export interface PdfHead {
6
- readonly text: string;
7
- readonly pagesRead: number;
8
- readonly totalPages: number;
9
- }
10
- export declare function extractPdfHead(bytes: Uint8Array, { maxChars }?: {
11
- maxChars?: number;
12
- }): Promise<PdfHead>;
@@ -1,43 +0,0 @@
1
- import './pdf-platform.js';
2
- import { getDocument } from 'unpdf/pdfjs';
3
- export function extractPdfText(bytes, options = {}) {
4
- return extractPdfHead(bytes, options).then((head) => head.text);
5
- }
6
- export async function extractPdfHead(bytes, { maxChars = Infinity } = {}) {
7
- const pdf = await getDocument({
8
- data: new Uint8Array(bytes),
9
- disableFontFace: true,
10
- isEvalSupported: false,
11
- useSystemFonts: true,
12
- verbosity: 0
13
- }).promise;
14
- try {
15
- const pages = [];
16
- let total = 0;
17
- for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) {
18
- const page = await pdf.getPage(pageNumber);
19
- const textContent = await page.getTextContent();
20
- const pageText = textContent.items.map(textItem).filter(Boolean).join(' ');
21
- pages.push(pageText);
22
- total += pageText.length;
23
- if (total > maxChars)
24
- break;
25
- }
26
- const text = pages
27
- .join('\n\n')
28
- .replace(/[ \t]+\n/g, '\n')
29
- .trim();
30
- return { text, pagesRead: pages.length, totalPages: pdf.numPages };
31
- }
32
- finally {
33
- await pdf.destroy();
34
- }
35
- }
36
- function textItem(item) {
37
- if (!isRecord(item) || typeof item.str !== 'string')
38
- return '';
39
- return item.str.trim();
40
- }
41
- function isRecord(value) {
42
- return typeof value === 'object' && value !== null && !Array.isArray(value);
43
- }
@@ -1 +0,0 @@
1
- export declare function extractRtfText(source: string): string;
@@ -1,15 +0,0 @@
1
- const HEX_ESCAPE = /\\'([0-9a-fA-F]{2})/g;
2
- const CONTROL_WORD = /\\[a-zA-Z]+-?\d* ?/g;
3
- const CONTROL_SYMBOL = /\\[^a-zA-Z0-9]/g;
4
- export function extractRtfText(source) {
5
- return source
6
- .replaceAll('\\par', '\n')
7
- .replaceAll('\\line', '\n')
8
- .replace(HEX_ESCAPE, (_match, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
9
- .replace(CONTROL_WORD, '')
10
- .replace(CONTROL_SYMBOL, '')
11
- .replace(/[{}]/g, '')
12
- .replace(/[ \t]+\n/g, '\n')
13
- .replace(/\n{3,}/g, '\n\n')
14
- .trim();
15
- }
@@ -1 +0,0 @@
1
- export declare function decodeTextBytes(bytes: Uint8Array): string;
@@ -1,91 +0,0 @@
1
- import { decodeLatin1, decodeUtf16 } from './utf.js';
2
- const UTF16_LE_BOM = [0xff, 0xfe];
3
- const UTF16_BE_BOM = [0xfe, 0xff];
4
- const UTF8_BOM = [0xef, 0xbb, 0xbf];
5
- const LATIN1_LABEL = /^(?:iso-8859-1|iso8859-1|iso88591|latin-1|latin1)$/;
6
- const HTML_CHARSET = /<meta\s+[^>]*charset\s*=\s*["']?\s*([A-Za-z0-9._-]+)/i;
7
- const CONTENT_TYPE_CHARSET = /<meta\s+[^>]*http-equiv\s*=\s*["']?content-type["']?[^>]*content\s*=\s*["'][^"']*charset=([A-Za-z0-9._-]+)/i;
8
- export function decodeTextBytes(bytes) {
9
- return decodeWithEncoding(bytes, detectEncoding(bytes) ?? 'utf-8');
10
- }
11
- function detectEncoding(bytes) {
12
- return detectBom(bytes) ?? detectUtf16ByNuls(bytes) ?? detectHtmlCharset(bytes);
13
- }
14
- function detectBom(bytes) {
15
- if (startsWith(bytes, UTF16_LE_BOM))
16
- return 'utf-16le';
17
- if (startsWith(bytes, UTF16_BE_BOM))
18
- return 'utf-16be';
19
- return startsWith(bytes, UTF8_BOM) ? 'utf-8' : null;
20
- }
21
- function startsWith(bytes, prefix) {
22
- return prefix.every((value, index) => bytes[index] === value);
23
- }
24
- function detectUtf16ByNuls(bytes) {
25
- const sampleLength = Math.min(bytes.length, 512);
26
- let evenNuls = 0;
27
- let oddNuls = 0;
28
- for (let index = 0; index < sampleLength; index++) {
29
- if (bytes[index] !== 0)
30
- continue;
31
- if (index % 2 === 0) {
32
- evenNuls++;
33
- }
34
- else {
35
- oddNuls++;
36
- }
37
- }
38
- const threshold = Math.max(8, Math.floor(sampleLength / 8));
39
- if (oddNuls >= threshold && oddNuls > evenNuls * 2)
40
- return 'utf-16le';
41
- return evenNuls >= threshold && evenNuls > oddNuls * 2 ? 'utf-16be' : null;
42
- }
43
- function detectHtmlCharset(bytes) {
44
- const sample = decodeUtf8(bytes.slice(0, 4096));
45
- const match = HTML_CHARSET.exec(sample) ?? CONTENT_TYPE_CHARSET.exec(sample);
46
- return normalizeEncoding(match?.[1] ?? null);
47
- }
48
- function normalizeEncoding(value) {
49
- if (!value)
50
- return null;
51
- const normalized = value.trim().toLowerCase().replace(/_/g, '-');
52
- if (normalized === 'utf8')
53
- return 'utf-8';
54
- if (LATIN1_LABEL.test(normalized))
55
- return 'latin1';
56
- if (normalized === 'utf16' || normalized === 'utf-16')
57
- return 'utf-16le';
58
- if (normalized === 'utf16le')
59
- return 'utf-16le';
60
- if (normalized === 'utf16be')
61
- return 'utf-16be';
62
- return normalized;
63
- }
64
- function decodeWithEncoding(bytes, encoding) {
65
- const normalized = normalizeEncoding(encoding) ?? 'utf-8';
66
- if (normalized === 'utf-16le')
67
- return decodeUtf16(bytes, true).replace(/^\uFEFF/, '');
68
- if (normalized === 'utf-16be')
69
- return decodeUtf16(bytes, false).replace(/^\uFEFF/, '');
70
- if (normalized === 'latin1')
71
- return decodeLatin1(bytes);
72
- return decodeWithTextDecoder(bytes, normalized);
73
- }
74
- function decodeUtf8(bytes) {
75
- return decodeWithTextDecoder(bytes, 'utf-8');
76
- }
77
- const TextDecoderCtor = globalThis
78
- .TextDecoder;
79
- function decodeWithTextDecoder(bytes, encoding) {
80
- if (TextDecoderCtor) {
81
- try {
82
- return new TextDecoderCtor(encoding).decode(bytes).replace(/^\uFEFF/, '');
83
- }
84
- catch {
85
- return new TextDecoderCtor('utf-8').decode(bytes).replace(/^\uFEFF/, '');
86
- }
87
- }
88
- return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
89
- .toString('utf8')
90
- .replace(/^\uFEFF/, '');
91
- }
@@ -1,36 +0,0 @@
1
- export type RagDocumentInput = {
2
- readonly id?: string;
3
- readonly blobId?: string;
4
- readonly title: string;
5
- readonly content: string;
6
- readonly sourcePath?: string;
7
- readonly sourceUrl?: string;
8
- readonly mimeType?: string;
9
- readonly artifactId?: string;
10
- readonly attachmentId?: string;
11
- readonly messageId?: string;
12
- readonly chatId?: string;
13
- };
14
- export type RagFileInput = {
15
- readonly id?: string;
16
- readonly blobId?: string;
17
- readonly title?: string;
18
- readonly fileName?: string;
19
- readonly mimeType?: string;
20
- readonly sourcePath?: string;
21
- readonly sourceUrl?: string;
22
- readonly bytes?: Uint8Array;
23
- readonly artifactId?: string;
24
- readonly attachmentId?: string;
25
- readonly messageId?: string;
26
- readonly chatId?: string;
27
- };
28
- export declare const RAG_EXTRACT_ERROR_CODE: {
29
- readonly unsupportedFile: 'RAG_UNSUPPORTED_FILE';
30
- readonly extractionFailed: 'RAG_EXTRACTION_FAILED';
31
- };
32
- export type RagExtractErrorCode = (typeof RAG_EXTRACT_ERROR_CODE)[keyof typeof RAG_EXTRACT_ERROR_CODE];
33
- export declare class RagExtractError extends Error {
34
- readonly code: RagExtractErrorCode;
35
- constructor(code: RagExtractErrorCode, message: string);
36
- }
@@ -1,12 +0,0 @@
1
- export const RAG_EXTRACT_ERROR_CODE = {
2
- unsupportedFile: 'RAG_UNSUPPORTED_FILE',
3
- extractionFailed: 'RAG_EXTRACTION_FAILED'
4
- };
5
- export class RagExtractError extends Error {
6
- code;
7
- constructor(code, message) {
8
- super(message);
9
- this.name = 'RagExtractError';
10
- this.code = code;
11
- }
12
- }
@@ -1,2 +0,0 @@
1
- export declare function decodeUtf16(bytes: Uint8Array, littleEndian: boolean): string;
2
- export declare function decodeLatin1(bytes: Uint8Array): string;
@@ -1,18 +0,0 @@
1
- import b4a from 'b4a';
2
- export function decodeUtf16(bytes, littleEndian) {
3
- let out = '';
4
- const chunk = [];
5
- for (let i = 0; i + 1 < bytes.length; i += 2) {
6
- const a = bytes[i];
7
- const b = bytes[i + 1];
8
- chunk.push(littleEndian ? a | (b << 8) : (a << 8) | b);
9
- if (chunk.length >= 8192) {
10
- out += String.fromCharCode(...chunk);
11
- chunk.length = 0;
12
- }
13
- }
14
- return out + String.fromCharCode(...chunk);
15
- }
16
- export function decodeLatin1(bytes) {
17
- return b4a.toString(b4a.from(bytes), 'latin1');
18
- }
@@ -1,2 +0,0 @@
1
- import type { RagSource } from './sources.ts';
2
- export declare function citedByPhraseOverlap(answer: string, records: ReadonlyArray<RagSource>): string[];
@@ -1,16 +0,0 @@
1
- import { similarity } from 'asenion';
2
- const MIN_PHRASE_TOKENS = 4;
3
- export function citedByPhraseOverlap(answer, records) {
4
- const cited = [];
5
- const seen = new Set();
6
- for (const record of records) {
7
- const key = record.title.toLowerCase();
8
- if (seen.has(key))
9
- continue;
10
- if (similarity(answer, record.excerpt, { phrase: MIN_PHRASE_TOKENS }) === 0)
11
- continue;
12
- seen.add(key);
13
- cited.push(record.title);
14
- }
15
- return cited;
16
- }
@@ -1,4 +0,0 @@
1
- import type { VectorHit } from '../vector/types.ts';
2
- export declare function selectCoverage(hits: ReadonlyArray<VectorHit>, { budgetTokens }?: {
3
- readonly budgetTokens?: number;
4
- }): ReadonlyArray<VectorHit>;
@@ -1,48 +0,0 @@
1
- import { parseRagMetadata } from '../chunk/metadata.js';
2
- import { estimateRagTokens } from '../chunk/tokenizer.js';
3
- const DEFAULT_COVERAGE_BUDGET_TOKENS = 3072;
4
- export function selectCoverage(hits, { budgetTokens = DEFAULT_COVERAGE_BUDGET_TOKENS } = {}) {
5
- const documents = groupByDocument(hits);
6
- if (documents.length === 0)
7
- return [];
8
- const perDocument = Math.max(1, Math.floor(budgetTokens / documents.length));
9
- const selected = [];
10
- for (const document of documents)
11
- selected.push(...stride(document, perDocument));
12
- return selected;
13
- }
14
- function stride(document, budgetTokens) {
15
- const costs = document.chunks.map((chunk) => estimateRagTokens(chunk.hit.content));
16
- const total = costs.reduce((sum, cost) => sum + cost, 0);
17
- if (total <= budgetTokens)
18
- return document.chunks.map((chunk) => chunk.hit);
19
- const last = document.chunks.length - 1;
20
- const average = Math.max(1, Math.round(total / document.chunks.length));
21
- const seats = Math.min(document.chunks.length, Math.max(1, Math.floor(budgetTokens / average)));
22
- const step = seats > 1 ? last / (seats - 1) : 0;
23
- const taken = [];
24
- let used = 0;
25
- for (let seat = 0; seat < seats; seat++) {
26
- const at = Math.round(seat * step);
27
- const cost = costs[at] ?? 0;
28
- if (used + cost > budgetTokens && taken.length > 0)
29
- continue;
30
- taken.push(document.chunks[at].hit);
31
- used += cost;
32
- }
33
- return taken;
34
- }
35
- function groupByDocument(hits) {
36
- const byDocument = new Map();
37
- for (const hit of hits) {
38
- const metadata = parseRagMetadata(hit.metadataJson);
39
- const documentId = metadata?.documentId ?? hit.id;
40
- const chunks = byDocument.get(documentId) ?? [];
41
- chunks.push({ index: metadata?.chunkIndex ?? 0, hit });
42
- byDocument.set(documentId, chunks);
43
- }
44
- return [...byDocument].map(([documentId, chunks]) => ({
45
- documentId,
46
- chunks: chunks.sort((left, right) => left.index - right.index)
47
- }));
48
- }
@@ -1,10 +0,0 @@
1
- import type { VectorCandidate, VectorHit } from '../vector/types.ts';
2
- export type RagScopeFilters = {
3
- readonly chatId?: string;
4
- readonly documentIds?: ReadonlyArray<string>;
5
- readonly attachmentIds?: ReadonlyArray<string>;
6
- readonly liveSourceIds?: ReadonlyArray<string>;
7
- };
8
- export declare function scopeFilterFor(filters: RagScopeFilters): ((candidate: VectorCandidate) => boolean) | undefined;
9
- export declare function filterHits(hits: ReadonlyArray<VectorHit>, filters: RagScopeFilters): ReadonlyArray<VectorHit>;
10
- export declare function applyPerSourceCap(hits: ReadonlyArray<VectorHit>, limit: number): ReadonlyArray<VectorHit>;
@@ -1,37 +0,0 @@
1
- import { hitMatchesFilters, parseRagMetadata } from '../chunk/metadata.js';
2
- export function scopeFilterFor(filters) {
3
- const liveSourceIds = filters.liveSourceIds ? new Set(filters.liveSourceIds) : undefined;
4
- if (!filters.chatId &&
5
- !filters.documentIds?.length &&
6
- !filters.attachmentIds?.length &&
7
- !liveSourceIds) {
8
- return undefined;
9
- }
10
- return (candidate) => hitMatchesFilters(candidate, filters, liveSourceIds);
11
- }
12
- export function filterHits(hits, filters) {
13
- const filter = scopeFilterFor(filters);
14
- return filter ? hits.filter(filter) : hits;
15
- }
16
- export function applyPerSourceCap(hits, limit) {
17
- if (hits.length <= limit)
18
- return hits;
19
- const seen = new Set();
20
- const firstPass = [];
21
- const leftovers = [];
22
- for (const hit of hits) {
23
- const key = sourceKey(hit);
24
- if (seen.has(key)) {
25
- leftovers.push(hit);
26
- }
27
- else {
28
- seen.add(key);
29
- firstPass.push(hit);
30
- }
31
- }
32
- return [...firstPass, ...leftovers].slice(0, limit);
33
- }
34
- function sourceKey(hit) {
35
- const metadata = parseRagMetadata(hit.metadataJson);
36
- return metadata?.documentId ?? `__chunk:${hit.id}`;
37
- }
@@ -1,5 +0,0 @@
1
- import '../../../../logger.ts';
2
- import type { VectorHit } from '../vector/types.ts';
3
- export declare const RAG_MIN_SIMILARITY = 0.4;
4
- export declare function applySimilarityFloor(hits: ReadonlyArray<VectorHit>, targetDocumentId: string | null): ReadonlyArray<VectorHit>;
5
- export declare function logSimilarityScores(candidates: ReadonlyArray<VectorHit>, kept: ReadonlyArray<VectorHit>): void;
@@ -1,21 +0,0 @@
1
- import '../../../../logger.js';
2
- import { parseRagMetadata } from '../chunk/metadata.js';
3
- export const RAG_MIN_SIMILARITY = 0.4;
4
- export function applySimilarityFloor(hits, targetDocumentId) {
5
- if (targetDocumentId)
6
- return hits;
7
- return hits.filter((hit) => hit.score >= RAG_MIN_SIMILARITY);
8
- }
9
- export function logSimilarityScores(candidates, kept) {
10
- if (candidates.length === 0) {
11
- logger.debug('rag', 'semantic search returned 0 hits');
12
- return;
13
- }
14
- const keptIds = new Set(kept.map((hit) => hit.id));
15
- for (const [index, hit] of candidates.entries()) {
16
- const source = parseRagMetadata(hit.metadataJson)?.title ?? hit.id;
17
- const percent = (hit.score * 100).toFixed(1);
18
- const status = keptIds.has(hit.id) ? 'kept' : 'dropped';
19
- logger.debug('rag', `similarity #${index + 1} ${percent}% (${status}) - ${source}`);
20
- }
21
- }
@@ -1,6 +0,0 @@
1
- type FormatHit = {
2
- readonly content: string;
3
- readonly metadataJson: string | null;
4
- };
5
- export declare function formatRagContext(hits: ReadonlyArray<FormatHit>, availableFiles?: ReadonlyArray<string>): string;
6
- export {};
@@ -1,21 +0,0 @@
1
- import { parseRagMetadata } from '../chunk/metadata.js';
2
- export function formatRagContext(hits, availableFiles = []) {
3
- const manifest = availableFiles.length > 0 ? [`Available files in this chat: ${availableFiles.join(', ')}`] : [];
4
- return [
5
- 'The following knowledge excerpts are the primary source for answering the user.',
6
- 'Each excerpt is tagged with its source file name and chunk index in a [Source N: filename, ...] label.',
7
- 'The app renders source attribution separately. Do not echo the [Source N: ...] labels. When attribution helps, name only the files you used naturally in prose.',
8
- ...manifest,
9
- 'If the user references a specific file or document by name, prioritise excerpts whose source matches that file.',
10
- 'If the user references a file that has no excerpts below, tell the user that file is not in the retrieved knowledge - do not invent an answer from other files.',
11
- 'If the excerpts collectively do not contain the answer, say the knowledge base did not contain it.',
12
- ...hits.flatMap((hit, index) => ['', formatHitLabel(hit, index), hit.content])
13
- ].join('\n');
14
- }
15
- function formatHitLabel(hit, index) {
16
- const metadata = hit.metadataJson ? parseRagMetadata(hit.metadataJson) : null;
17
- const title = metadata?.title ?? 'document';
18
- const chunk = metadata ? `chunk ${metadata.chunkIndex}` : `chunk ${index}`;
19
- const attachment = metadata?.attachmentId ? `, attachmentId=${metadata.attachmentId}` : '';
20
- return `[Source ${index + 1}: ${title}, ${chunk}${attachment}]`;
21
- }
@@ -1,2 +0,0 @@
1
- import type { RagKnowledgeQuery, RagKnowledgeResult, RagRetrievalDeps } from './types.ts';
2
- export declare function retrieveRagKnowledge(query: RagKnowledgeQuery, deps: RagRetrievalDeps): Promise<RagKnowledgeResult>;
@@ -1,54 +0,0 @@
1
- import { applySimilarityFloor, logSimilarityScores } from './floor.js';
2
- import { searchRagChunks } from './search.js';
3
- import { DEFAULT_CHUNK_BUDGET_TOKENS, MAX_SELECTED_CHUNKS } from './selection.js';
4
- import { resolveTargetDocument } from './target.js';
5
- import { isWholeDocumentQuery } from './whole-document-query.js';
6
- export async function retrieveRagKnowledge(query, deps) {
7
- if (query.scopeDocumentIds?.length) {
8
- const hits = await searchRagChunks({
9
- query: query.query,
10
- chatId: query.chatId,
11
- liveSourceIds: query.liveSourceIds,
12
- coverDocuments: query.scopeDocumentIds.length > 1,
13
- ...scopedShape(query.query, query.scopeDocumentIds)
14
- }, deps);
15
- logSimilarityScores(hits, hits);
16
- return { hits, targetDocumentId: null, searchedCount: hits.length };
17
- }
18
- const targetDocumentId = resolveTargetDocument(query.query, query.documents);
19
- if (!targetDocumentId && isWholeDocumentQuery(query.query)) {
20
- const hits = await searchRagChunks({
21
- query: query.query,
22
- chatId: query.chatId,
23
- liveSourceIds: query.liveSourceIds,
24
- documentIds: query.documents.map((document) => document.documentId),
25
- coverage: true,
26
- chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS
27
- }, deps);
28
- return { hits, targetDocumentId, searchedCount: hits.length };
29
- }
30
- const { hits: searched, scoped } = await searchKnowledge(query, deps, targetDocumentId);
31
- const hits = applySimilarityFloor(searched, scoped ? targetDocumentId : null);
32
- logSimilarityScores(searched, hits);
33
- return { hits, targetDocumentId, searchedCount: searched.length };
34
- }
35
- async function searchKnowledge(query, deps, targetDocumentId) {
36
- const base = { query: query.query, chatId: query.chatId, liveSourceIds: query.liveSourceIds };
37
- if (!targetDocumentId) {
38
- const hits = await searchRagChunks({ ...base, documentFirst: true, chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS }, deps);
39
- return { hits, scoped: false };
40
- }
41
- const scopedHits = await searchRagChunks({ ...base, ...scopedShape(query.query, [targetDocumentId]) }, deps);
42
- if (scopedHits.length > 0)
43
- return { hits: scopedHits, scoped: true };
44
- const hits = await searchRagChunks({ ...base, documentFirst: true, chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS }, deps);
45
- return { hits, scoped: false };
46
- }
47
- function scopedShape(query, documentIds) {
48
- return {
49
- documentIds,
50
- chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS,
51
- ...(documentIds.length === 1 ? { perDocChunkCap: MAX_SELECTED_CHUNKS } : {}),
52
- ...(isWholeDocumentQuery(query) ? { coverage: true } : { documentFirst: true })
53
- };
54
- }
@@ -1,16 +0,0 @@
1
- export declare const RAG_FAILED_NOTICE = "Knowledge search failed and was not used.";
2
- export declare const RAG_EMPTY_NOTICE = "Knowledge is enabled, but no relevant excerpts were found.";
3
- export declare const RAG_UNAVAILABLE_NOTICE = "Knowledge is still being indexed and was not used yet.";
4
- export declare const RAG_KNOWLEDGE_OFF_NOTICE = "Knowledge is off for this agent, so the attached document was not used. Enable Knowledge to let the agent read it.";
5
- export declare const RAG_TRUNCATED_NOTICE = "Only the beginning of the attached document was used. Enable Knowledge to let the agent search all of it.";
6
- export declare function ragUnreadableNotice(files: ReadonlyArray<string>): string;
7
- export declare function chooseRagNotice(input: {
8
- readonly errorMessage?: string | null;
9
- readonly hitCount: number;
10
- readonly searchedCount: number;
11
- readonly indexEmpty: boolean;
12
- readonly hasKnowledge: boolean;
13
- readonly pendingReferenced?: boolean;
14
- readonly failedFiles?: ReadonlyArray<string>;
15
- readonly pendingFiles?: ReadonlyArray<string>;
16
- }): string | null;