@qvac/core 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-lib/lib/api/chat.d.ts +3 -1
- package/dist-lib/lib/api/chat.js +6 -2
- package/dist-lib/lib/api/index.d.ts +2 -0
- package/dist-lib/lib/api/index.js +3 -1
- package/dist-lib/lib/api/inference-calls.d.ts +18 -0
- package/dist-lib/lib/api/inference-calls.js +15 -0
- package/dist-lib/lib/api/rag.js +10 -2
- package/dist-lib/lib/assistant/harness.d.ts +1 -0
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +11 -2
- package/dist-lib/lib/assistant.d.ts +14 -1
- package/dist-lib/lib/assistant.js +107 -15
- package/dist-lib/lib/core/chunk.js +5 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
- package/dist-lib/lib/core/files/rag/build.js +1 -2
- package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
- package/dist-lib/lib/core/files/rag/index.js +3 -17
- package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
- package/dist-lib/lib/core/files/rag/load.js +1 -2
- package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -3
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
- package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
- package/dist-lib/lib/core/files/rag/similarity.js +6 -0
- package/dist-lib/lib/core/files/upload.d.ts +1 -1
- package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
- package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
- package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
- package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
- package/dist-lib/lib/core/notification.js +5 -4
- package/dist-lib/lib/core.d.ts +8 -1
- package/dist-lib/lib/core.js +78 -4
- package/dist-lib/lib/inference-calls.d.ts +40 -0
- package/dist-lib/lib/inference-calls.js +266 -0
- package/dist-lib/schema/content.js +5 -1
- package/dist-lib/schema/engine.js +42 -1
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/bind.js +3 -0
- package/dist-lib/spec/calls.js +1 -0
- package/dist-lib/spec/capabilities.d.ts +2 -0
- package/dist-lib/spec/hrpc/hrpc.json +13 -0
- package/dist-lib/spec/hrpc/index.js +15 -3
- package/dist-lib/spec/hrpc/messages.js +172 -49
- package/dist-lib/spec/hyperschema/index.js +172 -49
- package/dist-lib/spec/hyperschema/schema.json +127 -1
- package/dist-lib/spec/hyperschema/types.d.ts +34 -1
- package/dist-lib/spec/local/hyperdb/messages.js +64 -128
- package/dist-lib/spec/local/hyperschema/index.js +32 -64
- package/dist-lib/spec/local/hyperschema/schema.json +16 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
- package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
- package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +14 -17
- package/tui/lib/knowledge-import.mjs +1 -1
- package/.claude/skills/asana/SKILL.md +0 -86
- package/.claude/skills/compaction/SKILL.md +0 -136
- package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
- package/.claude/skills/gh-stack/SKILL.md +0 -181
- package/.claude/skills/gh-stack/references/commands.md +0 -177
- package/.claude/skills/gh-stack/references/stack-design.md +0 -97
- package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
- package/.claude/skills/js-house-style/SKILL.md +0 -671
- package/.claude/skills/pr-create/SKILL.md +0 -243
- package/.claude/skills/pr-review/SKILL.md +0 -428
- package/.claude/skills/release/SKILL.md +0 -122
- package/CLAUDE.md +0 -48
- package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
- package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
- package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
- package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
- package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
- package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
- package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
- package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
- package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
- package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
- package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
- package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
- package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
- package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
- package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
- package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
- package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
- package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
- package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
- package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
- package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
- package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
- package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
- package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
- package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
- package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
- package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
- package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
- package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
- package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
- package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
- package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
- package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
- package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
- package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
- package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
- package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
- package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
- package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
- package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
- package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
- package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
- package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
- package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
- package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
- package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
- package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
- package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
- package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
- package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
- package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
- package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
- package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
- package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
- package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
- package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
import { formatHtmlTables } from './html-table-text.js';
|
|
2
|
-
const DROP_BLOCKS = /<(script|style|noscript|template|svg|head)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
|
|
3
|
-
const COMMENT = /<!--[\s\S]*?-->/g;
|
|
4
|
-
const HEADING_OPEN = /<h([1-6])\b[^>]*>/gi;
|
|
5
|
-
const HEADING_CLOSE = /<\/h[1-6]\s*>/gi;
|
|
6
|
-
const LIST_ITEM_OPEN = /<li\b[^>]*>/gi;
|
|
7
|
-
const LIST_ITEM_CLOSE = /<\/li\s*>/gi;
|
|
8
|
-
const TABLE_CELL_OPEN = /<t[hd]\b[^>]*>/gi;
|
|
9
|
-
const TABLE_CELL_CLOSE = /<\/t[hd]\s*>/gi;
|
|
10
|
-
const TABLE_ROW_OPEN = /<tr\b[^>]*>/gi;
|
|
11
|
-
const TABLE_ROW_CLOSE = /<\/tr\s*>/gi;
|
|
12
|
-
const BREAK_TAG = /<br\s*\/?>/gi;
|
|
13
|
-
const BLOCK_CONTAINER_TAGS = 'p|div|section|article|header|footer|main|aside|nav|blockquote|pre|figure|figcaption|ul|ol|table|thead|tbody|tfoot|address|form|fieldset';
|
|
14
|
-
const BLOCK_CLOSE = new RegExp(`</(${BLOCK_CONTAINER_TAGS})\\s*>`, 'gi');
|
|
15
|
-
const BLOCK_OPEN = new RegExp(`<(${BLOCK_CONTAINER_TAGS}|hr)\\b[^>]*>`, 'gi');
|
|
16
|
-
const STRAY_CLOSE_BRACKET = />/g;
|
|
17
|
-
const NAMED_ENTITIES = new Map([
|
|
18
|
-
['amp', '&'],
|
|
19
|
-
['lt', '<'],
|
|
20
|
-
['gt', '>'],
|
|
21
|
-
['quot', '"'],
|
|
22
|
-
['apos', "'"],
|
|
23
|
-
['nbsp', ' '],
|
|
24
|
-
['copy', '©'],
|
|
25
|
-
['reg', '®'],
|
|
26
|
-
['trade', '™'],
|
|
27
|
-
['hellip', '…'],
|
|
28
|
-
['mdash', '—'],
|
|
29
|
-
['ndash', '–'],
|
|
30
|
-
['lsquo', '‘'],
|
|
31
|
-
['rsquo', '’'],
|
|
32
|
-
['ldquo', '“'],
|
|
33
|
-
['rdquo', '”'],
|
|
34
|
-
['middot', '·'],
|
|
35
|
-
['bull', '•'],
|
|
36
|
-
['deg', '°'],
|
|
37
|
-
['times', '×'],
|
|
38
|
-
['divide', '÷'],
|
|
39
|
-
['plusmn', '±'],
|
|
40
|
-
['laquo', '«'],
|
|
41
|
-
['raquo', '»'],
|
|
42
|
-
['euro', '€'],
|
|
43
|
-
['pound', '£'],
|
|
44
|
-
['yen', '¥'],
|
|
45
|
-
['cent', '¢'],
|
|
46
|
-
['sect', '§'],
|
|
47
|
-
['para', '¶'],
|
|
48
|
-
['iexcl', '¡'],
|
|
49
|
-
['iquest', '¿'],
|
|
50
|
-
['dagger', '†'],
|
|
51
|
-
['Dagger', '‡'],
|
|
52
|
-
['sbquo', '‚'],
|
|
53
|
-
['bdquo', '„'],
|
|
54
|
-
['prime', '′'],
|
|
55
|
-
['Prime', '″'],
|
|
56
|
-
['frac12', '½'],
|
|
57
|
-
['frac14', '¼'],
|
|
58
|
-
['frac34', '¾']
|
|
59
|
-
]);
|
|
60
|
-
const NUMERIC_ENTITY = /&#(x[0-9a-fA-F]+|[0-9]+);/g;
|
|
61
|
-
const NAMED_ENTITY = /&([a-zA-Z][a-zA-Z0-9]+);/g;
|
|
62
|
-
export function extractHtmlText(source) {
|
|
63
|
-
const structured = markStructure(formatHtmlTables(dropNonContent(source)));
|
|
64
|
-
const text = stripTagsFully(structured);
|
|
65
|
-
return normalizeWhitespace(decodeEntities(text));
|
|
66
|
-
}
|
|
67
|
-
function dropNonContent(source) {
|
|
68
|
-
let previous = source;
|
|
69
|
-
let current = source.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
|
|
70
|
-
while (current !== previous) {
|
|
71
|
-
previous = current;
|
|
72
|
-
current = current.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
|
|
73
|
-
}
|
|
74
|
-
return current;
|
|
75
|
-
}
|
|
76
|
-
function stripTagsFully(source) {
|
|
77
|
-
let output = '';
|
|
78
|
-
let cursor = 0;
|
|
79
|
-
while (cursor < source.length) {
|
|
80
|
-
const open = source.indexOf('<', cursor);
|
|
81
|
-
if (open === -1) {
|
|
82
|
-
return output + stripStrayCloseBrackets(source.slice(cursor));
|
|
83
|
-
}
|
|
84
|
-
output += stripStrayCloseBrackets(source.slice(cursor, open));
|
|
85
|
-
const close = source.indexOf('>', open + 1);
|
|
86
|
-
if (close === -1) {
|
|
87
|
-
return `${output} `;
|
|
88
|
-
}
|
|
89
|
-
cursor = close + 1;
|
|
90
|
-
}
|
|
91
|
-
return output;
|
|
92
|
-
}
|
|
93
|
-
function stripStrayCloseBrackets(text) {
|
|
94
|
-
return text.replace(STRAY_CLOSE_BRACKET, ' ');
|
|
95
|
-
}
|
|
96
|
-
function markStructure(html) {
|
|
97
|
-
return html
|
|
98
|
-
.replace(HEADING_OPEN, (_match, level) => `\n\n${'#'.repeat(Number(level))} `)
|
|
99
|
-
.replace(HEADING_CLOSE, '\n\n')
|
|
100
|
-
.replace(TABLE_CELL_OPEN, ' | ')
|
|
101
|
-
.replace(TABLE_CELL_CLOSE, '')
|
|
102
|
-
.replace(TABLE_ROW_OPEN, '\n')
|
|
103
|
-
.replace(TABLE_ROW_CLOSE, '')
|
|
104
|
-
.replace(LIST_ITEM_OPEN, '\n- ')
|
|
105
|
-
.replace(LIST_ITEM_CLOSE, '')
|
|
106
|
-
.replace(BREAK_TAG, '\n')
|
|
107
|
-
.replace(BLOCK_OPEN, '\n\n')
|
|
108
|
-
.replace(BLOCK_CLOSE, '\n\n');
|
|
109
|
-
}
|
|
110
|
-
function decodeEntities(text) {
|
|
111
|
-
return text
|
|
112
|
-
.replace(NUMERIC_ENTITY, (_match, code) => {
|
|
113
|
-
const codePoint = code.startsWith('x') || code.startsWith('X')
|
|
114
|
-
? Number.parseInt(code.slice(1), 16)
|
|
115
|
-
: Number.parseInt(code, 10);
|
|
116
|
-
return Number.isFinite(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff
|
|
117
|
-
? String.fromCodePoint(codePoint)
|
|
118
|
-
: '';
|
|
119
|
-
})
|
|
120
|
-
.replace(NAMED_ENTITY, (match, name) => NAMED_ENTITIES.get(name) ?? match);
|
|
121
|
-
}
|
|
122
|
-
function normalizeWhitespace(text) {
|
|
123
|
-
return text
|
|
124
|
-
.replace(/\r\n?/g, '\n')
|
|
125
|
-
.replace(/[ \t\f\v]+/g, ' ')
|
|
126
|
-
.replace(/[ \t]+\n/g, '\n')
|
|
127
|
-
.replace(/\n[ \t]+/g, '\n')
|
|
128
|
-
.replace(/^\| ?/gm, '')
|
|
129
|
-
.replace(/ \|\s*$/gm, '')
|
|
130
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
131
|
-
.trim();
|
|
132
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
export declare class BareTextDecoder {
|
|
2
|
-
readonly encoding: string;
|
|
3
|
-
readonly fatal: boolean;
|
|
4
|
-
readonly ignoreBOM: boolean;
|
|
5
|
-
constructor(label?: string, options?: {
|
|
6
|
-
fatal?: boolean;
|
|
7
|
-
ignoreBOM?: boolean;
|
|
8
|
-
});
|
|
9
|
-
decode(input?: ArrayBufferView | ArrayBuffer): string;
|
|
10
|
-
}
|
|
11
|
-
export declare class BareTextEncoder {
|
|
12
|
-
encode(input?: string): Uint8Array;
|
|
13
|
-
}
|
|
14
|
-
export declare function decodeBytes(bytes: Uint8Array, label: string, options?: {
|
|
15
|
-
fatal?: boolean;
|
|
16
|
-
ignoreBOM?: boolean;
|
|
17
|
-
}): string;
|
|
@@ -1,141 +0,0 @@
|
|
|
1
|
-
import b4a from 'b4a';
|
|
2
|
-
import { AbortController, AbortSignal } from 'bare-abort-controller';
|
|
3
|
-
import { ReadableStream } from 'web-streams-polyfill/dist/ponyfill.mjs';
|
|
4
|
-
import { decodeLatin1, decodeUtf16 } from './utf.js';
|
|
5
|
-
const g = globalThis;
|
|
6
|
-
if (!g.ReadableStream)
|
|
7
|
-
g.ReadableStream = ReadableStream;
|
|
8
|
-
if (!g.DOMException) {
|
|
9
|
-
g.DOMException = class DOMException extends Error {
|
|
10
|
-
constructor(message, name = 'Error') {
|
|
11
|
-
super(message);
|
|
12
|
-
this.name = name;
|
|
13
|
-
}
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
export class BareTextDecoder {
|
|
17
|
-
encoding;
|
|
18
|
-
fatal;
|
|
19
|
-
ignoreBOM;
|
|
20
|
-
constructor(label = 'utf-8', options = {}) {
|
|
21
|
-
this.encoding = parseEncoding(label);
|
|
22
|
-
this.fatal = options.fatal === true;
|
|
23
|
-
this.ignoreBOM = options.ignoreBOM === true;
|
|
24
|
-
}
|
|
25
|
-
decode(input) {
|
|
26
|
-
if (input === undefined)
|
|
27
|
-
return '';
|
|
28
|
-
const bytes = ArrayBuffer.isView(input)
|
|
29
|
-
? b4a.from(input.buffer, input.byteOffset, input.byteLength)
|
|
30
|
-
: b4a.from(input);
|
|
31
|
-
return decodeBytes(bytes, this.encoding, { fatal: this.fatal, ignoreBOM: this.ignoreBOM });
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
export class BareTextEncoder {
|
|
35
|
-
encode(input = '') {
|
|
36
|
-
return b4a.from(String(input), 'utf8');
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
if (!g.TextDecoder)
|
|
40
|
-
g.TextDecoder = BareTextDecoder;
|
|
41
|
-
if (!g.TextEncoder)
|
|
42
|
-
g.TextEncoder = BareTextEncoder;
|
|
43
|
-
export function decodeBytes(bytes, label, options = {}) {
|
|
44
|
-
const fatal = options.fatal === true;
|
|
45
|
-
const encoding = parseEncoding(label);
|
|
46
|
-
const text = encoding === 'utf-16le'
|
|
47
|
-
? utf16(bytes, true, fatal)
|
|
48
|
-
: encoding === 'utf-16be'
|
|
49
|
-
? utf16(bytes, false, fatal)
|
|
50
|
-
: encoding === 'latin1'
|
|
51
|
-
? decodeLatin1(bytes)
|
|
52
|
-
: decodeUtf8(bytes, fatal);
|
|
53
|
-
return options.ignoreBOM || text.charCodeAt(0) !== 0xfeff ? text : text.slice(1);
|
|
54
|
-
}
|
|
55
|
-
function parseEncoding(label) {
|
|
56
|
-
const l = String(label).trim().toLowerCase();
|
|
57
|
-
if (l.includes('utf-16') || l.includes('utf16') || l === 'ucs-2' || l === 'ucs2') {
|
|
58
|
-
return l.includes('be') ? 'utf-16be' : 'utf-16le';
|
|
59
|
-
}
|
|
60
|
-
if (l.includes('latin1') || l.includes('8859'))
|
|
61
|
-
return 'latin1';
|
|
62
|
-
return 'utf-8';
|
|
63
|
-
}
|
|
64
|
-
function utf16(bytes, littleEndian, fatal) {
|
|
65
|
-
if (fatal && bytes.length % 2 !== 0)
|
|
66
|
-
throw new TypeError('truncated utf-16 sequence');
|
|
67
|
-
const text = decodeUtf16(bytes, littleEndian);
|
|
68
|
-
if (fatal)
|
|
69
|
-
assertPairedSurrogates(text);
|
|
70
|
-
return text;
|
|
71
|
-
}
|
|
72
|
-
function assertPairedSurrogates(text) {
|
|
73
|
-
for (let i = 0; i < text.length; i++) {
|
|
74
|
-
const unit = text.charCodeAt(i);
|
|
75
|
-
if (unit >= 0xd800 && unit <= 0xdbff) {
|
|
76
|
-
const low = text.charCodeAt(i + 1);
|
|
77
|
-
if (Number.isNaN(low) || low < 0xdc00 || low > 0xdfff) {
|
|
78
|
-
throw new TypeError('lone high surrogate');
|
|
79
|
-
}
|
|
80
|
-
i++;
|
|
81
|
-
}
|
|
82
|
-
else if (unit >= 0xdc00 && unit <= 0xdfff) {
|
|
83
|
-
throw new TypeError('lone low surrogate');
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
function decodeUtf8(bytes, fatal) {
|
|
88
|
-
if (fatal && !isValidUtf8(bytes))
|
|
89
|
-
throw new TypeError('invalid utf-8 sequence');
|
|
90
|
-
return b4a.toString(bytes, 'utf8');
|
|
91
|
-
}
|
|
92
|
-
function isValidUtf8(bytes) {
|
|
93
|
-
for (let i = 0; i < bytes.length;) {
|
|
94
|
-
const b0 = bytes[i];
|
|
95
|
-
if (b0 < 0x80) {
|
|
96
|
-
i++;
|
|
97
|
-
continue;
|
|
98
|
-
}
|
|
99
|
-
let size;
|
|
100
|
-
let min;
|
|
101
|
-
let cp;
|
|
102
|
-
if ((b0 & 0xe0) === 0xc0) {
|
|
103
|
-
size = 2;
|
|
104
|
-
min = 0x80;
|
|
105
|
-
cp = b0 & 0x1f;
|
|
106
|
-
}
|
|
107
|
-
else if ((b0 & 0xf0) === 0xe0) {
|
|
108
|
-
size = 3;
|
|
109
|
-
min = 0x800;
|
|
110
|
-
cp = b0 & 0x0f;
|
|
111
|
-
}
|
|
112
|
-
else if ((b0 & 0xf8) === 0xf0) {
|
|
113
|
-
size = 4;
|
|
114
|
-
min = 0x10000;
|
|
115
|
-
cp = b0 & 0x07;
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
return false;
|
|
119
|
-
}
|
|
120
|
-
if (i + size > bytes.length)
|
|
121
|
-
return false;
|
|
122
|
-
for (let j = 1; j < size; j++) {
|
|
123
|
-
const bj = bytes[i + j];
|
|
124
|
-
if ((bj & 0xc0) !== 0x80)
|
|
125
|
-
return false;
|
|
126
|
-
cp = (cp << 6) | (bj & 0x3f);
|
|
127
|
-
}
|
|
128
|
-
if (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
|
|
129
|
-
return false;
|
|
130
|
-
i += size;
|
|
131
|
-
}
|
|
132
|
-
return true;
|
|
133
|
-
}
|
|
134
|
-
if (!g.AbortController) {
|
|
135
|
-
g.AbortController = AbortController;
|
|
136
|
-
g.AbortSignal = AbortSignal;
|
|
137
|
-
}
|
|
138
|
-
const clone = globalThis.structuredClone;
|
|
139
|
-
if (clone) {
|
|
140
|
-
g.structuredClone = (value, options) => clone(value, (options ?? undefined));
|
|
141
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import './pdf-platform.ts';
|
|
2
|
-
export declare function extractPdfText(bytes: Uint8Array, options?: {
|
|
3
|
-
maxChars?: number;
|
|
4
|
-
}): Promise<string>;
|
|
5
|
-
export interface PdfHead {
|
|
6
|
-
readonly text: string;
|
|
7
|
-
readonly pagesRead: number;
|
|
8
|
-
readonly totalPages: number;
|
|
9
|
-
}
|
|
10
|
-
export declare function extractPdfHead(bytes: Uint8Array, { maxChars }?: {
|
|
11
|
-
maxChars?: number;
|
|
12
|
-
}): Promise<PdfHead>;
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import './pdf-platform.js';
|
|
2
|
-
import { getDocument } from 'unpdf/pdfjs';
|
|
3
|
-
export function extractPdfText(bytes, options = {}) {
|
|
4
|
-
return extractPdfHead(bytes, options).then((head) => head.text);
|
|
5
|
-
}
|
|
6
|
-
export async function extractPdfHead(bytes, { maxChars = Infinity } = {}) {
|
|
7
|
-
const pdf = await getDocument({
|
|
8
|
-
data: new Uint8Array(bytes),
|
|
9
|
-
disableFontFace: true,
|
|
10
|
-
isEvalSupported: false,
|
|
11
|
-
useSystemFonts: true,
|
|
12
|
-
verbosity: 0
|
|
13
|
-
}).promise;
|
|
14
|
-
try {
|
|
15
|
-
const pages = [];
|
|
16
|
-
let total = 0;
|
|
17
|
-
for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) {
|
|
18
|
-
const page = await pdf.getPage(pageNumber);
|
|
19
|
-
const textContent = await page.getTextContent();
|
|
20
|
-
const pageText = textContent.items.map(textItem).filter(Boolean).join(' ');
|
|
21
|
-
pages.push(pageText);
|
|
22
|
-
total += pageText.length;
|
|
23
|
-
if (total > maxChars)
|
|
24
|
-
break;
|
|
25
|
-
}
|
|
26
|
-
const text = pages
|
|
27
|
-
.join('\n\n')
|
|
28
|
-
.replace(/[ \t]+\n/g, '\n')
|
|
29
|
-
.trim();
|
|
30
|
-
return { text, pagesRead: pages.length, totalPages: pdf.numPages };
|
|
31
|
-
}
|
|
32
|
-
finally {
|
|
33
|
-
await pdf.destroy();
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
function textItem(item) {
|
|
37
|
-
if (!isRecord(item) || typeof item.str !== 'string')
|
|
38
|
-
return '';
|
|
39
|
-
return item.str.trim();
|
|
40
|
-
}
|
|
41
|
-
function isRecord(value) {
|
|
42
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
43
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function extractRtfText(source: string): string;
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
const HEX_ESCAPE = /\\'([0-9a-fA-F]{2})/g;
|
|
2
|
-
const CONTROL_WORD = /\\[a-zA-Z]+-?\d* ?/g;
|
|
3
|
-
const CONTROL_SYMBOL = /\\[^a-zA-Z0-9]/g;
|
|
4
|
-
export function extractRtfText(source) {
|
|
5
|
-
return source
|
|
6
|
-
.replaceAll('\\par', '\n')
|
|
7
|
-
.replaceAll('\\line', '\n')
|
|
8
|
-
.replace(HEX_ESCAPE, (_match, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
|
|
9
|
-
.replace(CONTROL_WORD, '')
|
|
10
|
-
.replace(CONTROL_SYMBOL, '')
|
|
11
|
-
.replace(/[{}]/g, '')
|
|
12
|
-
.replace(/[ \t]+\n/g, '\n')
|
|
13
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
14
|
-
.trim();
|
|
15
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function decodeTextBytes(bytes: Uint8Array): string;
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import { decodeLatin1, decodeUtf16 } from './utf.js';
|
|
2
|
-
const UTF16_LE_BOM = [0xff, 0xfe];
|
|
3
|
-
const UTF16_BE_BOM = [0xfe, 0xff];
|
|
4
|
-
const UTF8_BOM = [0xef, 0xbb, 0xbf];
|
|
5
|
-
const LATIN1_LABEL = /^(?:iso-8859-1|iso8859-1|iso88591|latin-1|latin1)$/;
|
|
6
|
-
const HTML_CHARSET = /<meta\s+[^>]*charset\s*=\s*["']?\s*([A-Za-z0-9._-]+)/i;
|
|
7
|
-
const CONTENT_TYPE_CHARSET = /<meta\s+[^>]*http-equiv\s*=\s*["']?content-type["']?[^>]*content\s*=\s*["'][^"']*charset=([A-Za-z0-9._-]+)/i;
|
|
8
|
-
export function decodeTextBytes(bytes) {
|
|
9
|
-
return decodeWithEncoding(bytes, detectEncoding(bytes) ?? 'utf-8');
|
|
10
|
-
}
|
|
11
|
-
function detectEncoding(bytes) {
|
|
12
|
-
return detectBom(bytes) ?? detectUtf16ByNuls(bytes) ?? detectHtmlCharset(bytes);
|
|
13
|
-
}
|
|
14
|
-
function detectBom(bytes) {
|
|
15
|
-
if (startsWith(bytes, UTF16_LE_BOM))
|
|
16
|
-
return 'utf-16le';
|
|
17
|
-
if (startsWith(bytes, UTF16_BE_BOM))
|
|
18
|
-
return 'utf-16be';
|
|
19
|
-
return startsWith(bytes, UTF8_BOM) ? 'utf-8' : null;
|
|
20
|
-
}
|
|
21
|
-
function startsWith(bytes, prefix) {
|
|
22
|
-
return prefix.every((value, index) => bytes[index] === value);
|
|
23
|
-
}
|
|
24
|
-
function detectUtf16ByNuls(bytes) {
|
|
25
|
-
const sampleLength = Math.min(bytes.length, 512);
|
|
26
|
-
let evenNuls = 0;
|
|
27
|
-
let oddNuls = 0;
|
|
28
|
-
for (let index = 0; index < sampleLength; index++) {
|
|
29
|
-
if (bytes[index] !== 0)
|
|
30
|
-
continue;
|
|
31
|
-
if (index % 2 === 0) {
|
|
32
|
-
evenNuls++;
|
|
33
|
-
}
|
|
34
|
-
else {
|
|
35
|
-
oddNuls++;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
const threshold = Math.max(8, Math.floor(sampleLength / 8));
|
|
39
|
-
if (oddNuls >= threshold && oddNuls > evenNuls * 2)
|
|
40
|
-
return 'utf-16le';
|
|
41
|
-
return evenNuls >= threshold && evenNuls > oddNuls * 2 ? 'utf-16be' : null;
|
|
42
|
-
}
|
|
43
|
-
function detectHtmlCharset(bytes) {
|
|
44
|
-
const sample = decodeUtf8(bytes.slice(0, 4096));
|
|
45
|
-
const match = HTML_CHARSET.exec(sample) ?? CONTENT_TYPE_CHARSET.exec(sample);
|
|
46
|
-
return normalizeEncoding(match?.[1] ?? null);
|
|
47
|
-
}
|
|
48
|
-
function normalizeEncoding(value) {
|
|
49
|
-
if (!value)
|
|
50
|
-
return null;
|
|
51
|
-
const normalized = value.trim().toLowerCase().replace(/_/g, '-');
|
|
52
|
-
if (normalized === 'utf8')
|
|
53
|
-
return 'utf-8';
|
|
54
|
-
if (LATIN1_LABEL.test(normalized))
|
|
55
|
-
return 'latin1';
|
|
56
|
-
if (normalized === 'utf16' || normalized === 'utf-16')
|
|
57
|
-
return 'utf-16le';
|
|
58
|
-
if (normalized === 'utf16le')
|
|
59
|
-
return 'utf-16le';
|
|
60
|
-
if (normalized === 'utf16be')
|
|
61
|
-
return 'utf-16be';
|
|
62
|
-
return normalized;
|
|
63
|
-
}
|
|
64
|
-
function decodeWithEncoding(bytes, encoding) {
|
|
65
|
-
const normalized = normalizeEncoding(encoding) ?? 'utf-8';
|
|
66
|
-
if (normalized === 'utf-16le')
|
|
67
|
-
return decodeUtf16(bytes, true).replace(/^\uFEFF/, '');
|
|
68
|
-
if (normalized === 'utf-16be')
|
|
69
|
-
return decodeUtf16(bytes, false).replace(/^\uFEFF/, '');
|
|
70
|
-
if (normalized === 'latin1')
|
|
71
|
-
return decodeLatin1(bytes);
|
|
72
|
-
return decodeWithTextDecoder(bytes, normalized);
|
|
73
|
-
}
|
|
74
|
-
function decodeUtf8(bytes) {
|
|
75
|
-
return decodeWithTextDecoder(bytes, 'utf-8');
|
|
76
|
-
}
|
|
77
|
-
const TextDecoderCtor = globalThis
|
|
78
|
-
.TextDecoder;
|
|
79
|
-
function decodeWithTextDecoder(bytes, encoding) {
|
|
80
|
-
if (TextDecoderCtor) {
|
|
81
|
-
try {
|
|
82
|
-
return new TextDecoderCtor(encoding).decode(bytes).replace(/^\uFEFF/, '');
|
|
83
|
-
}
|
|
84
|
-
catch {
|
|
85
|
-
return new TextDecoderCtor('utf-8').decode(bytes).replace(/^\uFEFF/, '');
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
89
|
-
.toString('utf8')
|
|
90
|
-
.replace(/^\uFEFF/, '');
|
|
91
|
-
}
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
export type RagDocumentInput = {
|
|
2
|
-
readonly id?: string;
|
|
3
|
-
readonly blobId?: string;
|
|
4
|
-
readonly title: string;
|
|
5
|
-
readonly content: string;
|
|
6
|
-
readonly sourcePath?: string;
|
|
7
|
-
readonly sourceUrl?: string;
|
|
8
|
-
readonly mimeType?: string;
|
|
9
|
-
readonly artifactId?: string;
|
|
10
|
-
readonly attachmentId?: string;
|
|
11
|
-
readonly messageId?: string;
|
|
12
|
-
readonly chatId?: string;
|
|
13
|
-
};
|
|
14
|
-
export type RagFileInput = {
|
|
15
|
-
readonly id?: string;
|
|
16
|
-
readonly blobId?: string;
|
|
17
|
-
readonly title?: string;
|
|
18
|
-
readonly fileName?: string;
|
|
19
|
-
readonly mimeType?: string;
|
|
20
|
-
readonly sourcePath?: string;
|
|
21
|
-
readonly sourceUrl?: string;
|
|
22
|
-
readonly bytes?: Uint8Array;
|
|
23
|
-
readonly artifactId?: string;
|
|
24
|
-
readonly attachmentId?: string;
|
|
25
|
-
readonly messageId?: string;
|
|
26
|
-
readonly chatId?: string;
|
|
27
|
-
};
|
|
28
|
-
export declare const RAG_EXTRACT_ERROR_CODE: {
|
|
29
|
-
readonly unsupportedFile: 'RAG_UNSUPPORTED_FILE';
|
|
30
|
-
readonly extractionFailed: 'RAG_EXTRACTION_FAILED';
|
|
31
|
-
};
|
|
32
|
-
export type RagExtractErrorCode = (typeof RAG_EXTRACT_ERROR_CODE)[keyof typeof RAG_EXTRACT_ERROR_CODE];
|
|
33
|
-
export declare class RagExtractError extends Error {
|
|
34
|
-
readonly code: RagExtractErrorCode;
|
|
35
|
-
constructor(code: RagExtractErrorCode, message: string);
|
|
36
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
export const RAG_EXTRACT_ERROR_CODE = {
|
|
2
|
-
unsupportedFile: 'RAG_UNSUPPORTED_FILE',
|
|
3
|
-
extractionFailed: 'RAG_EXTRACTION_FAILED'
|
|
4
|
-
};
|
|
5
|
-
export class RagExtractError extends Error {
|
|
6
|
-
code;
|
|
7
|
-
constructor(code, message) {
|
|
8
|
-
super(message);
|
|
9
|
-
this.name = 'RagExtractError';
|
|
10
|
-
this.code = code;
|
|
11
|
-
}
|
|
12
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import b4a from 'b4a';
|
|
2
|
-
export function decodeUtf16(bytes, littleEndian) {
|
|
3
|
-
let out = '';
|
|
4
|
-
const chunk = [];
|
|
5
|
-
for (let i = 0; i + 1 < bytes.length; i += 2) {
|
|
6
|
-
const a = bytes[i];
|
|
7
|
-
const b = bytes[i + 1];
|
|
8
|
-
chunk.push(littleEndian ? a | (b << 8) : (a << 8) | b);
|
|
9
|
-
if (chunk.length >= 8192) {
|
|
10
|
-
out += String.fromCharCode(...chunk);
|
|
11
|
-
chunk.length = 0;
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
|
-
return out + String.fromCharCode(...chunk);
|
|
15
|
-
}
|
|
16
|
-
export function decodeLatin1(bytes) {
|
|
17
|
-
return b4a.toString(b4a.from(bytes), 'latin1');
|
|
18
|
-
}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { similarity } from 'asenion';
|
|
2
|
-
const MIN_PHRASE_TOKENS = 4;
|
|
3
|
-
export function citedByPhraseOverlap(answer, records) {
|
|
4
|
-
const cited = [];
|
|
5
|
-
const seen = new Set();
|
|
6
|
-
for (const record of records) {
|
|
7
|
-
const key = record.title.toLowerCase();
|
|
8
|
-
if (seen.has(key))
|
|
9
|
-
continue;
|
|
10
|
-
if (similarity(answer, record.excerpt, { phrase: MIN_PHRASE_TOKENS }) === 0)
|
|
11
|
-
continue;
|
|
12
|
-
seen.add(key);
|
|
13
|
-
cited.push(record.title);
|
|
14
|
-
}
|
|
15
|
-
return cited;
|
|
16
|
-
}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
2
|
-
import { estimateRagTokens } from '../chunk/tokenizer.js';
|
|
3
|
-
const DEFAULT_COVERAGE_BUDGET_TOKENS = 3072;
|
|
4
|
-
export function selectCoverage(hits, { budgetTokens = DEFAULT_COVERAGE_BUDGET_TOKENS } = {}) {
|
|
5
|
-
const documents = groupByDocument(hits);
|
|
6
|
-
if (documents.length === 0)
|
|
7
|
-
return [];
|
|
8
|
-
const perDocument = Math.max(1, Math.floor(budgetTokens / documents.length));
|
|
9
|
-
const selected = [];
|
|
10
|
-
for (const document of documents)
|
|
11
|
-
selected.push(...stride(document, perDocument));
|
|
12
|
-
return selected;
|
|
13
|
-
}
|
|
14
|
-
function stride(document, budgetTokens) {
|
|
15
|
-
const costs = document.chunks.map((chunk) => estimateRagTokens(chunk.hit.content));
|
|
16
|
-
const total = costs.reduce((sum, cost) => sum + cost, 0);
|
|
17
|
-
if (total <= budgetTokens)
|
|
18
|
-
return document.chunks.map((chunk) => chunk.hit);
|
|
19
|
-
const last = document.chunks.length - 1;
|
|
20
|
-
const average = Math.max(1, Math.round(total / document.chunks.length));
|
|
21
|
-
const seats = Math.min(document.chunks.length, Math.max(1, Math.floor(budgetTokens / average)));
|
|
22
|
-
const step = seats > 1 ? last / (seats - 1) : 0;
|
|
23
|
-
const taken = [];
|
|
24
|
-
let used = 0;
|
|
25
|
-
for (let seat = 0; seat < seats; seat++) {
|
|
26
|
-
const at = Math.round(seat * step);
|
|
27
|
-
const cost = costs[at] ?? 0;
|
|
28
|
-
if (used + cost > budgetTokens && taken.length > 0)
|
|
29
|
-
continue;
|
|
30
|
-
taken.push(document.chunks[at].hit);
|
|
31
|
-
used += cost;
|
|
32
|
-
}
|
|
33
|
-
return taken;
|
|
34
|
-
}
|
|
35
|
-
function groupByDocument(hits) {
|
|
36
|
-
const byDocument = new Map();
|
|
37
|
-
for (const hit of hits) {
|
|
38
|
-
const metadata = parseRagMetadata(hit.metadataJson);
|
|
39
|
-
const documentId = metadata?.documentId ?? hit.id;
|
|
40
|
-
const chunks = byDocument.get(documentId) ?? [];
|
|
41
|
-
chunks.push({ index: metadata?.chunkIndex ?? 0, hit });
|
|
42
|
-
byDocument.set(documentId, chunks);
|
|
43
|
-
}
|
|
44
|
-
return [...byDocument].map(([documentId, chunks]) => ({
|
|
45
|
-
documentId,
|
|
46
|
-
chunks: chunks.sort((left, right) => left.index - right.index)
|
|
47
|
-
}));
|
|
48
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { VectorCandidate, VectorHit } from '../vector/types.ts';
|
|
2
|
-
export type RagScopeFilters = {
|
|
3
|
-
readonly chatId?: string;
|
|
4
|
-
readonly documentIds?: ReadonlyArray<string>;
|
|
5
|
-
readonly attachmentIds?: ReadonlyArray<string>;
|
|
6
|
-
readonly liveSourceIds?: ReadonlyArray<string>;
|
|
7
|
-
};
|
|
8
|
-
export declare function scopeFilterFor(filters: RagScopeFilters): ((candidate: VectorCandidate) => boolean) | undefined;
|
|
9
|
-
export declare function filterHits(hits: ReadonlyArray<VectorHit>, filters: RagScopeFilters): ReadonlyArray<VectorHit>;
|
|
10
|
-
export declare function applyPerSourceCap(hits: ReadonlyArray<VectorHit>, limit: number): ReadonlyArray<VectorHit>;
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import { hitMatchesFilters, parseRagMetadata } from '../chunk/metadata.js';
|
|
2
|
-
export function scopeFilterFor(filters) {
|
|
3
|
-
const liveSourceIds = filters.liveSourceIds ? new Set(filters.liveSourceIds) : undefined;
|
|
4
|
-
if (!filters.chatId &&
|
|
5
|
-
!filters.documentIds?.length &&
|
|
6
|
-
!filters.attachmentIds?.length &&
|
|
7
|
-
!liveSourceIds) {
|
|
8
|
-
return undefined;
|
|
9
|
-
}
|
|
10
|
-
return (candidate) => hitMatchesFilters(candidate, filters, liveSourceIds);
|
|
11
|
-
}
|
|
12
|
-
export function filterHits(hits, filters) {
|
|
13
|
-
const filter = scopeFilterFor(filters);
|
|
14
|
-
return filter ? hits.filter(filter) : hits;
|
|
15
|
-
}
|
|
16
|
-
export function applyPerSourceCap(hits, limit) {
|
|
17
|
-
if (hits.length <= limit)
|
|
18
|
-
return hits;
|
|
19
|
-
const seen = new Set();
|
|
20
|
-
const firstPass = [];
|
|
21
|
-
const leftovers = [];
|
|
22
|
-
for (const hit of hits) {
|
|
23
|
-
const key = sourceKey(hit);
|
|
24
|
-
if (seen.has(key)) {
|
|
25
|
-
leftovers.push(hit);
|
|
26
|
-
}
|
|
27
|
-
else {
|
|
28
|
-
seen.add(key);
|
|
29
|
-
firstPass.push(hit);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
return [...firstPass, ...leftovers].slice(0, limit);
|
|
33
|
-
}
|
|
34
|
-
function sourceKey(hit) {
|
|
35
|
-
const metadata = parseRagMetadata(hit.metadataJson);
|
|
36
|
-
return metadata?.documentId ?? `__chunk:${hit.id}`;
|
|
37
|
-
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import '../../../../logger.ts';
|
|
2
|
-
import type { VectorHit } from '../vector/types.ts';
|
|
3
|
-
export declare const RAG_MIN_SIMILARITY = 0.4;
|
|
4
|
-
export declare function applySimilarityFloor(hits: ReadonlyArray<VectorHit>, targetDocumentId: string | null): ReadonlyArray<VectorHit>;
|
|
5
|
-
export declare function logSimilarityScores(candidates: ReadonlyArray<VectorHit>, kept: ReadonlyArray<VectorHit>): void;
|