@qvac/core 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist-lib/lib/api/chat.d.ts +3 -1
  2. package/dist-lib/lib/api/chat.js +6 -2
  3. package/dist-lib/lib/api/index.d.ts +2 -0
  4. package/dist-lib/lib/api/index.js +3 -1
  5. package/dist-lib/lib/api/inference-calls.d.ts +18 -0
  6. package/dist-lib/lib/api/inference-calls.js +15 -0
  7. package/dist-lib/lib/api/rag.js +10 -2
  8. package/dist-lib/lib/assistant/harness.d.ts +1 -0
  9. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  10. package/dist-lib/lib/assistant-tree.js +11 -2
  11. package/dist-lib/lib/assistant.d.ts +14 -1
  12. package/dist-lib/lib/assistant.js +107 -15
  13. package/dist-lib/lib/core/chunk.js +5 -0
  14. package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
  15. package/dist-lib/lib/core/files/rag/build.js +1 -2
  16. package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
  17. package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
  18. package/dist-lib/lib/core/files/rag/index.js +3 -17
  19. package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
  20. package/dist-lib/lib/core/files/rag/load.js +1 -2
  21. package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
  22. package/dist-lib/lib/core/files/rag/produce.js +1 -3
  23. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
  24. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
  25. package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
  26. package/dist-lib/lib/core/files/rag/similarity.js +6 -0
  27. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  28. package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
  29. package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
  30. package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
  31. package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
  32. package/dist-lib/lib/core/notification.js +5 -4
  33. package/dist-lib/lib/core.d.ts +8 -1
  34. package/dist-lib/lib/core.js +78 -4
  35. package/dist-lib/lib/inference-calls.d.ts +40 -0
  36. package/dist-lib/lib/inference-calls.js +266 -0
  37. package/dist-lib/schema/content.js +5 -1
  38. package/dist-lib/schema/engine.js +42 -1
  39. package/dist-lib/schema/qvac.js +4 -2
  40. package/dist-lib/spec/bind.js +3 -0
  41. package/dist-lib/spec/calls.js +1 -0
  42. package/dist-lib/spec/capabilities.d.ts +2 -0
  43. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  44. package/dist-lib/spec/hrpc/index.js +15 -3
  45. package/dist-lib/spec/hrpc/messages.js +172 -49
  46. package/dist-lib/spec/hyperschema/index.js +172 -49
  47. package/dist-lib/spec/hyperschema/schema.json +127 -1
  48. package/dist-lib/spec/hyperschema/types.d.ts +34 -1
  49. package/dist-lib/spec/local/hyperdb/messages.js +64 -128
  50. package/dist-lib/spec/local/hyperschema/index.js +32 -64
  51. package/dist-lib/spec/local/hyperschema/schema.json +16 -0
  52. package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
  53. package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
  54. package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
  55. package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
  56. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  57. package/package.json +14 -17
  58. package/tui/lib/knowledge-import.mjs +1 -1
  59. package/.claude/skills/asana/SKILL.md +0 -86
  60. package/.claude/skills/compaction/SKILL.md +0 -136
  61. package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
  62. package/.claude/skills/gh-stack/SKILL.md +0 -181
  63. package/.claude/skills/gh-stack/references/commands.md +0 -177
  64. package/.claude/skills/gh-stack/references/stack-design.md +0 -97
  65. package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
  66. package/.claude/skills/js-house-style/SKILL.md +0 -671
  67. package/.claude/skills/pr-create/SKILL.md +0 -243
  68. package/.claude/skills/pr-review/SKILL.md +0 -428
  69. package/.claude/skills/release/SKILL.md +0 -122
  70. package/CLAUDE.md +0 -48
  71. package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
  72. package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
  73. package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
  74. package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
  75. package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
  76. package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
  77. package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
  78. package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
  79. package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
  80. package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
  81. package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
  82. package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
  83. package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
  84. package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
  85. package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
  86. package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
  87. package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
  88. package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
  89. package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
  90. package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
  91. package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
  92. package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
  93. package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
  94. package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
  95. package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
  96. package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
  97. package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
  98. package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
  99. package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
  100. package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
  101. package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
  102. package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
  103. package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
  104. package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
  105. package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
  106. package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
  107. package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
  108. package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
  109. package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
  110. package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
  111. package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
  112. package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
  113. package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
  114. package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
  115. package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
  116. package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
  117. package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
  118. package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
  119. package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
  120. package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
  121. package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
  122. package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
  123. package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
  124. package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
  125. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
  126. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
  127. package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
  128. package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
  129. package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
  130. package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
  131. package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
  132. package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
  133. package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
  134. package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
  135. package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
  136. package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
  137. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
  138. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
  139. package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
  140. package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
  141. package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
  142. package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
@@ -1,132 +0,0 @@
1
- import { formatHtmlTables } from './html-table-text.js';
2
- const DROP_BLOCKS = /<(script|style|noscript|template|svg|head)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
3
- const COMMENT = /<!--[\s\S]*?-->/g;
4
- const HEADING_OPEN = /<h([1-6])\b[^>]*>/gi;
5
- const HEADING_CLOSE = /<\/h[1-6]\s*>/gi;
6
- const LIST_ITEM_OPEN = /<li\b[^>]*>/gi;
7
- const LIST_ITEM_CLOSE = /<\/li\s*>/gi;
8
- const TABLE_CELL_OPEN = /<t[hd]\b[^>]*>/gi;
9
- const TABLE_CELL_CLOSE = /<\/t[hd]\s*>/gi;
10
- const TABLE_ROW_OPEN = /<tr\b[^>]*>/gi;
11
- const TABLE_ROW_CLOSE = /<\/tr\s*>/gi;
12
- const BREAK_TAG = /<br\s*\/?>/gi;
13
- const BLOCK_CONTAINER_TAGS = 'p|div|section|article|header|footer|main|aside|nav|blockquote|pre|figure|figcaption|ul|ol|table|thead|tbody|tfoot|address|form|fieldset';
14
- const BLOCK_CLOSE = new RegExp(`</(${BLOCK_CONTAINER_TAGS})\\s*>`, 'gi');
15
- const BLOCK_OPEN = new RegExp(`<(${BLOCK_CONTAINER_TAGS}|hr)\\b[^>]*>`, 'gi');
16
- const STRAY_CLOSE_BRACKET = />/g;
17
- const NAMED_ENTITIES = new Map([
18
- ['amp', '&'],
19
- ['lt', '<'],
20
- ['gt', '>'],
21
- ['quot', '"'],
22
- ['apos', "'"],
23
- ['nbsp', ' '],
24
- ['copy', '©'],
25
- ['reg', '®'],
26
- ['trade', '™'],
27
- ['hellip', '…'],
28
- ['mdash', '—'],
29
- ['ndash', '–'],
30
- ['lsquo', '‘'],
31
- ['rsquo', '’'],
32
- ['ldquo', '“'],
33
- ['rdquo', '”'],
34
- ['middot', '·'],
35
- ['bull', '•'],
36
- ['deg', '°'],
37
- ['times', '×'],
38
- ['divide', '÷'],
39
- ['plusmn', '±'],
40
- ['laquo', '«'],
41
- ['raquo', '»'],
42
- ['euro', '€'],
43
- ['pound', '£'],
44
- ['yen', '¥'],
45
- ['cent', '¢'],
46
- ['sect', '§'],
47
- ['para', '¶'],
48
- ['iexcl', '¡'],
49
- ['iquest', '¿'],
50
- ['dagger', '†'],
51
- ['Dagger', '‡'],
52
- ['sbquo', '‚'],
53
- ['bdquo', '„'],
54
- ['prime', '′'],
55
- ['Prime', '″'],
56
- ['frac12', '½'],
57
- ['frac14', '¼'],
58
- ['frac34', '¾']
59
- ]);
60
- const NUMERIC_ENTITY = /&#(x[0-9a-fA-F]+|[0-9]+);/g;
61
- const NAMED_ENTITY = /&([a-zA-Z][a-zA-Z0-9]+);/g;
62
- export function extractHtmlText(source) {
63
- const structured = markStructure(formatHtmlTables(dropNonContent(source)));
64
- const text = stripTagsFully(structured);
65
- return normalizeWhitespace(decodeEntities(text));
66
- }
67
- function dropNonContent(source) {
68
- let previous = source;
69
- let current = source.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
70
- while (current !== previous) {
71
- previous = current;
72
- current = current.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
73
- }
74
- return current;
75
- }
76
- function stripTagsFully(source) {
77
- let output = '';
78
- let cursor = 0;
79
- while (cursor < source.length) {
80
- const open = source.indexOf('<', cursor);
81
- if (open === -1) {
82
- return output + stripStrayCloseBrackets(source.slice(cursor));
83
- }
84
- output += stripStrayCloseBrackets(source.slice(cursor, open));
85
- const close = source.indexOf('>', open + 1);
86
- if (close === -1) {
87
- return `${output} `;
88
- }
89
- cursor = close + 1;
90
- }
91
- return output;
92
- }
93
- function stripStrayCloseBrackets(text) {
94
- return text.replace(STRAY_CLOSE_BRACKET, ' ');
95
- }
96
- function markStructure(html) {
97
- return html
98
- .replace(HEADING_OPEN, (_match, level) => `\n\n${'#'.repeat(Number(level))} `)
99
- .replace(HEADING_CLOSE, '\n\n')
100
- .replace(TABLE_CELL_OPEN, ' | ')
101
- .replace(TABLE_CELL_CLOSE, '')
102
- .replace(TABLE_ROW_OPEN, '\n')
103
- .replace(TABLE_ROW_CLOSE, '')
104
- .replace(LIST_ITEM_OPEN, '\n- ')
105
- .replace(LIST_ITEM_CLOSE, '')
106
- .replace(BREAK_TAG, '\n')
107
- .replace(BLOCK_OPEN, '\n\n')
108
- .replace(BLOCK_CLOSE, '\n\n');
109
- }
110
- function decodeEntities(text) {
111
- return text
112
- .replace(NUMERIC_ENTITY, (_match, code) => {
113
- const codePoint = code.startsWith('x') || code.startsWith('X')
114
- ? Number.parseInt(code.slice(1), 16)
115
- : Number.parseInt(code, 10);
116
- return Number.isFinite(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff
117
- ? String.fromCodePoint(codePoint)
118
- : '';
119
- })
120
- .replace(NAMED_ENTITY, (match, name) => NAMED_ENTITIES.get(name) ?? match);
121
- }
122
- function normalizeWhitespace(text) {
123
- return text
124
- .replace(/\r\n?/g, '\n')
125
- .replace(/[ \t\f\v]+/g, ' ')
126
- .replace(/[ \t]+\n/g, '\n')
127
- .replace(/\n[ \t]+/g, '\n')
128
- .replace(/^\| ?/gm, '')
129
- .replace(/ \|\s*$/gm, '')
130
- .replace(/\n{3,}/g, '\n\n')
131
- .trim();
132
- }
@@ -1,17 +0,0 @@
1
- export declare class BareTextDecoder {
2
- readonly encoding: string;
3
- readonly fatal: boolean;
4
- readonly ignoreBOM: boolean;
5
- constructor(label?: string, options?: {
6
- fatal?: boolean;
7
- ignoreBOM?: boolean;
8
- });
9
- decode(input?: ArrayBufferView | ArrayBuffer): string;
10
- }
11
- export declare class BareTextEncoder {
12
- encode(input?: string): Uint8Array;
13
- }
14
- export declare function decodeBytes(bytes: Uint8Array, label: string, options?: {
15
- fatal?: boolean;
16
- ignoreBOM?: boolean;
17
- }): string;
@@ -1,141 +0,0 @@
1
- import b4a from 'b4a';
2
- import { AbortController, AbortSignal } from 'bare-abort-controller';
3
- import { ReadableStream } from 'web-streams-polyfill/dist/ponyfill.mjs';
4
- import { decodeLatin1, decodeUtf16 } from './utf.js';
5
- const g = globalThis;
6
- if (!g.ReadableStream)
7
- g.ReadableStream = ReadableStream;
8
- if (!g.DOMException) {
9
- g.DOMException = class DOMException extends Error {
10
- constructor(message, name = 'Error') {
11
- super(message);
12
- this.name = name;
13
- }
14
- };
15
- }
16
- export class BareTextDecoder {
17
- encoding;
18
- fatal;
19
- ignoreBOM;
20
- constructor(label = 'utf-8', options = {}) {
21
- this.encoding = parseEncoding(label);
22
- this.fatal = options.fatal === true;
23
- this.ignoreBOM = options.ignoreBOM === true;
24
- }
25
- decode(input) {
26
- if (input === undefined)
27
- return '';
28
- const bytes = ArrayBuffer.isView(input)
29
- ? b4a.from(input.buffer, input.byteOffset, input.byteLength)
30
- : b4a.from(input);
31
- return decodeBytes(bytes, this.encoding, { fatal: this.fatal, ignoreBOM: this.ignoreBOM });
32
- }
33
- }
34
- export class BareTextEncoder {
35
- encode(input = '') {
36
- return b4a.from(String(input), 'utf8');
37
- }
38
- }
39
- if (!g.TextDecoder)
40
- g.TextDecoder = BareTextDecoder;
41
- if (!g.TextEncoder)
42
- g.TextEncoder = BareTextEncoder;
43
- export function decodeBytes(bytes, label, options = {}) {
44
- const fatal = options.fatal === true;
45
- const encoding = parseEncoding(label);
46
- const text = encoding === 'utf-16le'
47
- ? utf16(bytes, true, fatal)
48
- : encoding === 'utf-16be'
49
- ? utf16(bytes, false, fatal)
50
- : encoding === 'latin1'
51
- ? decodeLatin1(bytes)
52
- : decodeUtf8(bytes, fatal);
53
- return options.ignoreBOM || text.charCodeAt(0) !== 0xfeff ? text : text.slice(1);
54
- }
55
- function parseEncoding(label) {
56
- const l = String(label).trim().toLowerCase();
57
- if (l.includes('utf-16') || l.includes('utf16') || l === 'ucs-2' || l === 'ucs2') {
58
- return l.includes('be') ? 'utf-16be' : 'utf-16le';
59
- }
60
- if (l.includes('latin1') || l.includes('8859'))
61
- return 'latin1';
62
- return 'utf-8';
63
- }
64
- function utf16(bytes, littleEndian, fatal) {
65
- if (fatal && bytes.length % 2 !== 0)
66
- throw new TypeError('truncated utf-16 sequence');
67
- const text = decodeUtf16(bytes, littleEndian);
68
- if (fatal)
69
- assertPairedSurrogates(text);
70
- return text;
71
- }
72
- function assertPairedSurrogates(text) {
73
- for (let i = 0; i < text.length; i++) {
74
- const unit = text.charCodeAt(i);
75
- if (unit >= 0xd800 && unit <= 0xdbff) {
76
- const low = text.charCodeAt(i + 1);
77
- if (Number.isNaN(low) || low < 0xdc00 || low > 0xdfff) {
78
- throw new TypeError('lone high surrogate');
79
- }
80
- i++;
81
- }
82
- else if (unit >= 0xdc00 && unit <= 0xdfff) {
83
- throw new TypeError('lone low surrogate');
84
- }
85
- }
86
- }
87
- function decodeUtf8(bytes, fatal) {
88
- if (fatal && !isValidUtf8(bytes))
89
- throw new TypeError('invalid utf-8 sequence');
90
- return b4a.toString(bytes, 'utf8');
91
- }
92
- function isValidUtf8(bytes) {
93
- for (let i = 0; i < bytes.length;) {
94
- const b0 = bytes[i];
95
- if (b0 < 0x80) {
96
- i++;
97
- continue;
98
- }
99
- let size;
100
- let min;
101
- let cp;
102
- if ((b0 & 0xe0) === 0xc0) {
103
- size = 2;
104
- min = 0x80;
105
- cp = b0 & 0x1f;
106
- }
107
- else if ((b0 & 0xf0) === 0xe0) {
108
- size = 3;
109
- min = 0x800;
110
- cp = b0 & 0x0f;
111
- }
112
- else if ((b0 & 0xf8) === 0xf0) {
113
- size = 4;
114
- min = 0x10000;
115
- cp = b0 & 0x07;
116
- }
117
- else {
118
- return false;
119
- }
120
- if (i + size > bytes.length)
121
- return false;
122
- for (let j = 1; j < size; j++) {
123
- const bj = bytes[i + j];
124
- if ((bj & 0xc0) !== 0x80)
125
- return false;
126
- cp = (cp << 6) | (bj & 0x3f);
127
- }
128
- if (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
129
- return false;
130
- i += size;
131
- }
132
- return true;
133
- }
134
- if (!g.AbortController) {
135
- g.AbortController = AbortController;
136
- g.AbortSignal = AbortSignal;
137
- }
138
- const clone = globalThis.structuredClone;
139
- if (clone) {
140
- g.structuredClone = (value, options) => clone(value, (options ?? undefined));
141
- }
@@ -1,12 +0,0 @@
1
- import './pdf-platform.ts';
2
- export declare function extractPdfText(bytes: Uint8Array, options?: {
3
- maxChars?: number;
4
- }): Promise<string>;
5
- export interface PdfHead {
6
- readonly text: string;
7
- readonly pagesRead: number;
8
- readonly totalPages: number;
9
- }
10
- export declare function extractPdfHead(bytes: Uint8Array, { maxChars }?: {
11
- maxChars?: number;
12
- }): Promise<PdfHead>;
@@ -1,43 +0,0 @@
1
- import './pdf-platform.js';
2
- import { getDocument } from 'unpdf/pdfjs';
3
- export function extractPdfText(bytes, options = {}) {
4
- return extractPdfHead(bytes, options).then((head) => head.text);
5
- }
6
- export async function extractPdfHead(bytes, { maxChars = Infinity } = {}) {
7
- const pdf = await getDocument({
8
- data: new Uint8Array(bytes),
9
- disableFontFace: true,
10
- isEvalSupported: false,
11
- useSystemFonts: true,
12
- verbosity: 0
13
- }).promise;
14
- try {
15
- const pages = [];
16
- let total = 0;
17
- for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) {
18
- const page = await pdf.getPage(pageNumber);
19
- const textContent = await page.getTextContent();
20
- const pageText = textContent.items.map(textItem).filter(Boolean).join(' ');
21
- pages.push(pageText);
22
- total += pageText.length;
23
- if (total > maxChars)
24
- break;
25
- }
26
- const text = pages
27
- .join('\n\n')
28
- .replace(/[ \t]+\n/g, '\n')
29
- .trim();
30
- return { text, pagesRead: pages.length, totalPages: pdf.numPages };
31
- }
32
- finally {
33
- await pdf.destroy();
34
- }
35
- }
36
- function textItem(item) {
37
- if (!isRecord(item) || typeof item.str !== 'string')
38
- return '';
39
- return item.str.trim();
40
- }
41
- function isRecord(value) {
42
- return typeof value === 'object' && value !== null && !Array.isArray(value);
43
- }
@@ -1 +0,0 @@
1
- export declare function extractRtfText(source: string): string;
@@ -1,15 +0,0 @@
1
- const HEX_ESCAPE = /\\'([0-9a-fA-F]{2})/g;
2
- const CONTROL_WORD = /\\[a-zA-Z]+-?\d* ?/g;
3
- const CONTROL_SYMBOL = /\\[^a-zA-Z0-9]/g;
4
- export function extractRtfText(source) {
5
- return source
6
- .replaceAll('\\par', '\n')
7
- .replaceAll('\\line', '\n')
8
- .replace(HEX_ESCAPE, (_match, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
9
- .replace(CONTROL_WORD, '')
10
- .replace(CONTROL_SYMBOL, '')
11
- .replace(/[{}]/g, '')
12
- .replace(/[ \t]+\n/g, '\n')
13
- .replace(/\n{3,}/g, '\n\n')
14
- .trim();
15
- }
@@ -1 +0,0 @@
1
- export declare function decodeTextBytes(bytes: Uint8Array): string;
@@ -1,91 +0,0 @@
1
- import { decodeLatin1, decodeUtf16 } from './utf.js';
2
- const UTF16_LE_BOM = [0xff, 0xfe];
3
- const UTF16_BE_BOM = [0xfe, 0xff];
4
- const UTF8_BOM = [0xef, 0xbb, 0xbf];
5
- const LATIN1_LABEL = /^(?:iso-8859-1|iso8859-1|iso88591|latin-1|latin1)$/;
6
- const HTML_CHARSET = /<meta\s+[^>]*charset\s*=\s*["']?\s*([A-Za-z0-9._-]+)/i;
7
- const CONTENT_TYPE_CHARSET = /<meta\s+[^>]*http-equiv\s*=\s*["']?content-type["']?[^>]*content\s*=\s*["'][^"']*charset=([A-Za-z0-9._-]+)/i;
8
- export function decodeTextBytes(bytes) {
9
- return decodeWithEncoding(bytes, detectEncoding(bytes) ?? 'utf-8');
10
- }
11
- function detectEncoding(bytes) {
12
- return detectBom(bytes) ?? detectUtf16ByNuls(bytes) ?? detectHtmlCharset(bytes);
13
- }
14
- function detectBom(bytes) {
15
- if (startsWith(bytes, UTF16_LE_BOM))
16
- return 'utf-16le';
17
- if (startsWith(bytes, UTF16_BE_BOM))
18
- return 'utf-16be';
19
- return startsWith(bytes, UTF8_BOM) ? 'utf-8' : null;
20
- }
21
- function startsWith(bytes, prefix) {
22
- return prefix.every((value, index) => bytes[index] === value);
23
- }
24
- function detectUtf16ByNuls(bytes) {
25
- const sampleLength = Math.min(bytes.length, 512);
26
- let evenNuls = 0;
27
- let oddNuls = 0;
28
- for (let index = 0; index < sampleLength; index++) {
29
- if (bytes[index] !== 0)
30
- continue;
31
- if (index % 2 === 0) {
32
- evenNuls++;
33
- }
34
- else {
35
- oddNuls++;
36
- }
37
- }
38
- const threshold = Math.max(8, Math.floor(sampleLength / 8));
39
- if (oddNuls >= threshold && oddNuls > evenNuls * 2)
40
- return 'utf-16le';
41
- return evenNuls >= threshold && evenNuls > oddNuls * 2 ? 'utf-16be' : null;
42
- }
43
- function detectHtmlCharset(bytes) {
44
- const sample = decodeUtf8(bytes.slice(0, 4096));
45
- const match = HTML_CHARSET.exec(sample) ?? CONTENT_TYPE_CHARSET.exec(sample);
46
- return normalizeEncoding(match?.[1] ?? null);
47
- }
48
- function normalizeEncoding(value) {
49
- if (!value)
50
- return null;
51
- const normalized = value.trim().toLowerCase().replace(/_/g, '-');
52
- if (normalized === 'utf8')
53
- return 'utf-8';
54
- if (LATIN1_LABEL.test(normalized))
55
- return 'latin1';
56
- if (normalized === 'utf16' || normalized === 'utf-16')
57
- return 'utf-16le';
58
- if (normalized === 'utf16le')
59
- return 'utf-16le';
60
- if (normalized === 'utf16be')
61
- return 'utf-16be';
62
- return normalized;
63
- }
64
- function decodeWithEncoding(bytes, encoding) {
65
- const normalized = normalizeEncoding(encoding) ?? 'utf-8';
66
- if (normalized === 'utf-16le')
67
- return decodeUtf16(bytes, true).replace(/^\uFEFF/, '');
68
- if (normalized === 'utf-16be')
69
- return decodeUtf16(bytes, false).replace(/^\uFEFF/, '');
70
- if (normalized === 'latin1')
71
- return decodeLatin1(bytes);
72
- return decodeWithTextDecoder(bytes, normalized);
73
- }
74
- function decodeUtf8(bytes) {
75
- return decodeWithTextDecoder(bytes, 'utf-8');
76
- }
77
- const TextDecoderCtor = globalThis
78
- .TextDecoder;
79
- function decodeWithTextDecoder(bytes, encoding) {
80
- if (TextDecoderCtor) {
81
- try {
82
- return new TextDecoderCtor(encoding).decode(bytes).replace(/^\uFEFF/, '');
83
- }
84
- catch {
85
- return new TextDecoderCtor('utf-8').decode(bytes).replace(/^\uFEFF/, '');
86
- }
87
- }
88
- return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
89
- .toString('utf8')
90
- .replace(/^\uFEFF/, '');
91
- }
@@ -1,36 +0,0 @@
1
- export type RagDocumentInput = {
2
- readonly id?: string;
3
- readonly blobId?: string;
4
- readonly title: string;
5
- readonly content: string;
6
- readonly sourcePath?: string;
7
- readonly sourceUrl?: string;
8
- readonly mimeType?: string;
9
- readonly artifactId?: string;
10
- readonly attachmentId?: string;
11
- readonly messageId?: string;
12
- readonly chatId?: string;
13
- };
14
- export type RagFileInput = {
15
- readonly id?: string;
16
- readonly blobId?: string;
17
- readonly title?: string;
18
- readonly fileName?: string;
19
- readonly mimeType?: string;
20
- readonly sourcePath?: string;
21
- readonly sourceUrl?: string;
22
- readonly bytes?: Uint8Array;
23
- readonly artifactId?: string;
24
- readonly attachmentId?: string;
25
- readonly messageId?: string;
26
- readonly chatId?: string;
27
- };
28
- export declare const RAG_EXTRACT_ERROR_CODE: {
29
- readonly unsupportedFile: 'RAG_UNSUPPORTED_FILE';
30
- readonly extractionFailed: 'RAG_EXTRACTION_FAILED';
31
- };
32
- export type RagExtractErrorCode = (typeof RAG_EXTRACT_ERROR_CODE)[keyof typeof RAG_EXTRACT_ERROR_CODE];
33
- export declare class RagExtractError extends Error {
34
- readonly code: RagExtractErrorCode;
35
- constructor(code: RagExtractErrorCode, message: string);
36
- }
@@ -1,12 +0,0 @@
1
- export const RAG_EXTRACT_ERROR_CODE = {
2
- unsupportedFile: 'RAG_UNSUPPORTED_FILE',
3
- extractionFailed: 'RAG_EXTRACTION_FAILED'
4
- };
5
- export class RagExtractError extends Error {
6
- code;
7
- constructor(code, message) {
8
- super(message);
9
- this.name = 'RagExtractError';
10
- this.code = code;
11
- }
12
- }
@@ -1,2 +0,0 @@
1
- export declare function decodeUtf16(bytes: Uint8Array, littleEndian: boolean): string;
2
- export declare function decodeLatin1(bytes: Uint8Array): string;
@@ -1,18 +0,0 @@
1
- import b4a from 'b4a';
2
- export function decodeUtf16(bytes, littleEndian) {
3
- let out = '';
4
- const chunk = [];
5
- for (let i = 0; i + 1 < bytes.length; i += 2) {
6
- const a = bytes[i];
7
- const b = bytes[i + 1];
8
- chunk.push(littleEndian ? a | (b << 8) : (a << 8) | b);
9
- if (chunk.length >= 8192) {
10
- out += String.fromCharCode(...chunk);
11
- chunk.length = 0;
12
- }
13
- }
14
- return out + String.fromCharCode(...chunk);
15
- }
16
- export function decodeLatin1(bytes) {
17
- return b4a.toString(b4a.from(bytes), 'latin1');
18
- }
@@ -1,2 +0,0 @@
1
- import type { RagSource } from './sources.ts';
2
- export declare function citedByPhraseOverlap(answer: string, records: ReadonlyArray<RagSource>): string[];
@@ -1,16 +0,0 @@
1
- import { similarity } from 'asenion';
2
- const MIN_PHRASE_TOKENS = 4;
3
- export function citedByPhraseOverlap(answer, records) {
4
- const cited = [];
5
- const seen = new Set();
6
- for (const record of records) {
7
- const key = record.title.toLowerCase();
8
- if (seen.has(key))
9
- continue;
10
- if (similarity(answer, record.excerpt, { phrase: MIN_PHRASE_TOKENS }) === 0)
11
- continue;
12
- seen.add(key);
13
- cited.push(record.title);
14
- }
15
- return cited;
16
- }
@@ -1,4 +0,0 @@
1
- import type { VectorHit } from '../vector/types.ts';
2
- export declare function selectCoverage(hits: ReadonlyArray<VectorHit>, { budgetTokens }?: {
3
- readonly budgetTokens?: number;
4
- }): ReadonlyArray<VectorHit>;
@@ -1,48 +0,0 @@
1
- import { parseRagMetadata } from '../chunk/metadata.js';
2
- import { estimateRagTokens } from '../chunk/tokenizer.js';
3
- const DEFAULT_COVERAGE_BUDGET_TOKENS = 3072;
4
- export function selectCoverage(hits, { budgetTokens = DEFAULT_COVERAGE_BUDGET_TOKENS } = {}) {
5
- const documents = groupByDocument(hits);
6
- if (documents.length === 0)
7
- return [];
8
- const perDocument = Math.max(1, Math.floor(budgetTokens / documents.length));
9
- const selected = [];
10
- for (const document of documents)
11
- selected.push(...stride(document, perDocument));
12
- return selected;
13
- }
14
- function stride(document, budgetTokens) {
15
- const costs = document.chunks.map((chunk) => estimateRagTokens(chunk.hit.content));
16
- const total = costs.reduce((sum, cost) => sum + cost, 0);
17
- if (total <= budgetTokens)
18
- return document.chunks.map((chunk) => chunk.hit);
19
- const last = document.chunks.length - 1;
20
- const average = Math.max(1, Math.round(total / document.chunks.length));
21
- const seats = Math.min(document.chunks.length, Math.max(1, Math.floor(budgetTokens / average)));
22
- const step = seats > 1 ? last / (seats - 1) : 0;
23
- const taken = [];
24
- let used = 0;
25
- for (let seat = 0; seat < seats; seat++) {
26
- const at = Math.round(seat * step);
27
- const cost = costs[at] ?? 0;
28
- if (used + cost > budgetTokens && taken.length > 0)
29
- continue;
30
- taken.push(document.chunks[at].hit);
31
- used += cost;
32
- }
33
- return taken;
34
- }
35
- function groupByDocument(hits) {
36
- const byDocument = new Map();
37
- for (const hit of hits) {
38
- const metadata = parseRagMetadata(hit.metadataJson);
39
- const documentId = metadata?.documentId ?? hit.id;
40
- const chunks = byDocument.get(documentId) ?? [];
41
- chunks.push({ index: metadata?.chunkIndex ?? 0, hit });
42
- byDocument.set(documentId, chunks);
43
- }
44
- return [...byDocument].map(([documentId, chunks]) => ({
45
- documentId,
46
- chunks: chunks.sort((left, right) => left.index - right.index)
47
- }));
48
- }
@@ -1,10 +0,0 @@
1
- import type { VectorCandidate, VectorHit } from '../vector/types.ts';
2
- export type RagScopeFilters = {
3
- readonly chatId?: string;
4
- readonly documentIds?: ReadonlyArray<string>;
5
- readonly attachmentIds?: ReadonlyArray<string>;
6
- readonly liveSourceIds?: ReadonlyArray<string>;
7
- };
8
- export declare function scopeFilterFor(filters: RagScopeFilters): ((candidate: VectorCandidate) => boolean) | undefined;
9
- export declare function filterHits(hits: ReadonlyArray<VectorHit>, filters: RagScopeFilters): ReadonlyArray<VectorHit>;
10
- export declare function applyPerSourceCap(hits: ReadonlyArray<VectorHit>, limit: number): ReadonlyArray<VectorHit>;
@@ -1,37 +0,0 @@
1
- import { hitMatchesFilters, parseRagMetadata } from '../chunk/metadata.js';
2
- export function scopeFilterFor(filters) {
3
- const liveSourceIds = filters.liveSourceIds ? new Set(filters.liveSourceIds) : undefined;
4
- if (!filters.chatId &&
5
- !filters.documentIds?.length &&
6
- !filters.attachmentIds?.length &&
7
- !liveSourceIds) {
8
- return undefined;
9
- }
10
- return (candidate) => hitMatchesFilters(candidate, filters, liveSourceIds);
11
- }
12
- export function filterHits(hits, filters) {
13
- const filter = scopeFilterFor(filters);
14
- return filter ? hits.filter(filter) : hits;
15
- }
16
- export function applyPerSourceCap(hits, limit) {
17
- if (hits.length <= limit)
18
- return hits;
19
- const seen = new Set();
20
- const firstPass = [];
21
- const leftovers = [];
22
- for (const hit of hits) {
23
- const key = sourceKey(hit);
24
- if (seen.has(key)) {
25
- leftovers.push(hit);
26
- }
27
- else {
28
- seen.add(key);
29
- firstPass.push(hit);
30
- }
31
- }
32
- return [...firstPass, ...leftovers].slice(0, limit);
33
- }
34
- function sourceKey(hit) {
35
- const metadata = parseRagMetadata(hit.metadataJson);
36
- return metadata?.documentId ?? `__chunk:${hit.id}`;
37
- }
@@ -1,5 +0,0 @@
1
- import '../../../../logger.ts';
2
- import type { VectorHit } from '../vector/types.ts';
3
- export declare const RAG_MIN_SIMILARITY = 0.4;
4
- export declare function applySimilarityFloor(hits: ReadonlyArray<VectorHit>, targetDocumentId: string | null): ReadonlyArray<VectorHit>;
5
- export declare function logSimilarityScores(candidates: ReadonlyArray<VectorHit>, kept: ReadonlyArray<VectorHit>): void;