@qvac/core 0.2.2 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/dist-lib/lib/api/agents.js +2 -3
  2. package/dist-lib/lib/api/chat.d.ts +3 -1
  3. package/dist-lib/lib/api/chat.js +9 -5
  4. package/dist-lib/lib/api/chunks.d.ts +1 -1
  5. package/dist-lib/lib/api/chunks.js +5 -15
  6. package/dist-lib/lib/api/current-mesh.d.ts +16 -1
  7. package/dist-lib/lib/api/current-mesh.js +18 -0
  8. package/dist-lib/lib/api/devices.js +6 -3
  9. package/dist-lib/lib/api/index.d.ts +2 -0
  10. package/dist-lib/lib/api/index.js +4 -2
  11. package/dist-lib/lib/api/inference-calls.d.ts +18 -0
  12. package/dist-lib/lib/api/inference-calls.js +15 -0
  13. package/dist-lib/lib/api/knowledge.js +11 -7
  14. package/dist-lib/lib/api/members.js +2 -3
  15. package/dist-lib/lib/api/rag.js +10 -2
  16. package/dist-lib/lib/api/sync.d.ts +1 -1
  17. package/dist-lib/lib/api/sync.js +2 -1
  18. package/dist-lib/lib/api/tool-activity.js +2 -2
  19. package/dist-lib/lib/api/tool-configs.js +2 -2
  20. package/dist-lib/lib/api/watchable.d.ts +6 -1
  21. package/dist-lib/lib/api/watchable.js +13 -2
  22. package/dist-lib/lib/assistant/harness.d.ts +1 -0
  23. package/dist-lib/lib/assistant/history.d.ts +1 -2
  24. package/dist-lib/lib/assistant/history.js +0 -6
  25. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  26. package/dist-lib/lib/assistant-tree.js +11 -2
  27. package/dist-lib/lib/assistant.d.ts +17 -8
  28. package/dist-lib/lib/assistant.js +172 -125
  29. package/dist-lib/lib/core/chunk.d.ts +1 -0
  30. package/dist-lib/lib/core/chunk.js +32 -0
  31. package/dist-lib/lib/core/contract/devices.js +3 -2
  32. package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
  33. package/dist-lib/lib/core/files/rag/build.js +1 -2
  34. package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
  35. package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
  36. package/dist-lib/lib/core/files/rag/index.js +3 -17
  37. package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
  38. package/dist-lib/lib/core/files/rag/load.js +1 -2
  39. package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
  40. package/dist-lib/lib/core/files/rag/produce.js +1 -3
  41. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
  42. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
  43. package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
  44. package/dist-lib/lib/core/files/rag/similarity.js +6 -0
  45. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  46. package/dist-lib/lib/core/harness-loan.js +1 -1
  47. package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
  48. package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
  49. package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
  50. package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
  51. package/dist-lib/lib/core/local/index.js +6 -0
  52. package/dist-lib/lib/core/notification.js +5 -4
  53. package/dist-lib/lib/core.d.ts +9 -1
  54. package/dist-lib/lib/core.js +98 -16
  55. package/dist-lib/lib/fs-drive-bridge.js +1 -1
  56. package/dist-lib/lib/inference-calls.d.ts +40 -0
  57. package/dist-lib/lib/inference-calls.js +266 -0
  58. package/dist-lib/lib/mesh-scoped-engine.js +5 -0
  59. package/dist-lib/lib/model-download.d.ts +2 -0
  60. package/dist-lib/lib/model-download.js +20 -5
  61. package/dist-lib/lib/oauth/keys.js +1 -0
  62. package/dist-lib/lib/oauth/spotify.js +1 -0
  63. package/dist-lib/lib/oauth/verify.d.ts +8 -0
  64. package/dist-lib/lib/oauth/verify.js +58 -0
  65. package/dist-lib/lib/oauth.d.ts +1 -0
  66. package/dist-lib/lib/oauth.js +1 -0
  67. package/dist-lib/schema/content.js +5 -1
  68. package/dist-lib/schema/engine.js +42 -1
  69. package/dist-lib/schema/qvac.d.ts +3 -0
  70. package/dist-lib/schema/qvac.js +11 -2
  71. package/dist-lib/spec/bind.js +3 -0
  72. package/dist-lib/spec/calls.js +1 -0
  73. package/dist-lib/spec/capabilities.d.ts +2 -0
  74. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  75. package/dist-lib/spec/hrpc/index.js +15 -3
  76. package/dist-lib/spec/hrpc/messages.js +172 -49
  77. package/dist-lib/spec/hyperschema/index.js +172 -49
  78. package/dist-lib/spec/hyperschema/schema.json +127 -1
  79. package/dist-lib/spec/hyperschema/types.d.ts +34 -1
  80. package/dist-lib/spec/local/hyperdb/messages.js +64 -128
  81. package/dist-lib/spec/local/hyperschema/index.js +32 -64
  82. package/dist-lib/spec/local/hyperschema/schema.json +16 -0
  83. package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
  84. package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
  85. package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
  86. package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
  87. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  88. package/package.json +14 -17
  89. package/tui/app.mjs +327 -46
  90. package/tui/lib/knowledge-import.mjs +1 -1
  91. package/tui/lib/render.mjs +17 -1
  92. package/tui/lib/selection.mjs +16 -0
  93. package/tui/lib/usage.mjs +54 -0
  94. package/.claude/skills/asana/SKILL.md +0 -86
  95. package/.claude/skills/compaction/SKILL.md +0 -132
  96. package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
  97. package/.claude/skills/gh-stack/SKILL.md +0 -181
  98. package/.claude/skills/gh-stack/references/commands.md +0 -177
  99. package/.claude/skills/gh-stack/references/stack-design.md +0 -97
  100. package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
  101. package/.claude/skills/js-house-style/SKILL.md +0 -670
  102. package/.claude/skills/pr-create/SKILL.md +0 -201
  103. package/.claude/skills/pr-review/SKILL.md +0 -424
  104. package/CLAUDE.md +0 -43
  105. package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
  106. package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
  107. package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
  108. package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
  109. package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
  110. package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
  111. package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
  112. package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
  113. package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
  114. package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
  115. package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
  116. package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
  117. package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
  118. package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
  119. package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
  120. package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
  121. package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
  122. package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
  123. package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
  124. package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
  125. package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
  126. package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
  127. package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
  128. package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
  129. package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
  130. package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
  131. package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
  132. package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
  133. package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
  134. package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
  135. package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
  136. package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
  137. package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
  138. package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
  139. package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
  140. package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
  141. package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
  142. package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
  143. package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
  144. package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
  145. package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
  146. package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
  147. package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
  148. package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
  149. package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
  150. package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
  151. package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
  152. package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
  153. package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
  154. package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
  155. package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
  156. package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
  157. package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
  158. package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
  159. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
  160. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
  161. package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
  162. package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
  163. package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
  164. package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
  165. package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
  166. package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
  167. package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
  168. package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
  169. package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
  170. package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
  171. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
  172. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
  173. package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
  174. package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
  175. package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
  176. package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
@@ -1,22 +0,0 @@
1
- export declare const RAG_DOCUMENT_KIND: {
2
- readonly pdf: 'pdf';
3
- readonly office: 'office';
4
- readonly rtf: 'rtf';
5
- readonly html: 'html';
6
- readonly csv: 'csv';
7
- readonly tsv: 'tsv';
8
- readonly text: 'text';
9
- readonly unsupported: 'unsupported';
10
- };
11
- export type RagDocumentKind = (typeof RAG_DOCUMENT_KIND)[keyof typeof RAG_DOCUMENT_KIND];
12
- export declare const RAG_ACCEPT_EXTENSIONS: readonly ['.pdf', '.docx', '.doc', '.docm', '.dot', '.dotx', '.odt', '.fodt', '.ott', '.txt', '.md', '.markdown', '.rtf', '.html', '.htm', '.xhtml', '.log', '.json', '.jsonl', '.csv', '.tsv', '.xml', '.yaml', '.yml'];
13
- export declare function classifyRagDocument(input: {
14
- readonly fileName?: string;
15
- readonly mimeType?: string;
16
- readonly bytes?: Uint8Array;
17
- }): RagDocumentKind;
18
- export declare function isSupportedRagDocument(input: {
19
- readonly fileName?: string;
20
- readonly mimeType?: string;
21
- readonly bytes?: Uint8Array;
22
- }): boolean;
@@ -1,152 +0,0 @@
1
- export const RAG_DOCUMENT_KIND = {
2
- pdf: 'pdf',
3
- office: 'office',
4
- rtf: 'rtf',
5
- html: 'html',
6
- csv: 'csv',
7
- tsv: 'tsv',
8
- text: 'text',
9
- unsupported: 'unsupported'
10
- };
11
- const PDF_EXTENSIONS = new Set(['.pdf']);
12
- const OFFICE_EXTENSIONS = new Set([
13
- '.docx',
14
- '.doc',
15
- '.docm',
16
- '.dot',
17
- '.dotx',
18
- '.odt',
19
- '.fodt',
20
- '.ott'
21
- ]);
22
- const OFFICE_MIME_TYPES = new Set([
23
- 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
24
- 'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
25
- 'application/msword',
26
- 'application/vnd.oasis.opendocument.text',
27
- 'application/vnd.oasis.opendocument.text-template'
28
- ]);
29
- const RTF_EXTENSIONS = new Set(['.rtf']);
30
- const HTML_EXTENSIONS = new Set(['.html', '.htm', '.xhtml']);
31
- const CSV_EXTENSIONS = new Set(['.csv']);
32
- const TSV_EXTENSIONS = new Set(['.tsv']);
33
- const TEXT_EXTENSIONS = new Set([
34
- '.json',
35
- '.jsonl',
36
- '.log',
37
- '.markdown',
38
- '.md',
39
- '.mdown',
40
- '.mkd',
41
- '.text',
42
- '.txt',
43
- '.xml',
44
- '.yaml',
45
- '.yml'
46
- ]);
47
- const TEXT_MIME_TYPES = new Set([
48
- 'application/json',
49
- 'application/ld+json',
50
- 'application/rtf',
51
- 'application/x-ndjson',
52
- 'application/xml',
53
- 'application/yaml',
54
- 'text/rtf'
55
- ]);
56
- export const RAG_ACCEPT_EXTENSIONS = [
57
- '.pdf',
58
- '.docx',
59
- '.doc',
60
- '.docm',
61
- '.dot',
62
- '.dotx',
63
- '.odt',
64
- '.fodt',
65
- '.ott',
66
- '.txt',
67
- '.md',
68
- '.markdown',
69
- '.rtf',
70
- '.html',
71
- '.htm',
72
- '.xhtml',
73
- '.log',
74
- '.json',
75
- '.jsonl',
76
- '.csv',
77
- '.tsv',
78
- '.xml',
79
- '.yaml',
80
- '.yml'
81
- ];
82
- export function classifyRagDocument(input) {
83
- const extension = extensionForName(input.fileName);
84
- const mimeType = normalizeMimeType(input.mimeType);
85
- const explicit = kindFromExtension(extension) ?? kindFromMimeType(mimeType);
86
- if (explicit)
87
- return explicit;
88
- return input.bytes && looksLikeText(input.bytes)
89
- ? RAG_DOCUMENT_KIND.text
90
- : RAG_DOCUMENT_KIND.unsupported;
91
- }
92
- export function isSupportedRagDocument(input) {
93
- return classifyRagDocument(input) !== RAG_DOCUMENT_KIND.unsupported;
94
- }
95
- function extensionForName(fileName) {
96
- if (!fileName)
97
- return null;
98
- const index = fileName.lastIndexOf('.');
99
- return index >= 0 ? fileName.slice(index).toLowerCase() : null;
100
- }
101
- function normalizeMimeType(mimeType) {
102
- return mimeType?.split(';')[0]?.trim().toLowerCase() ?? '';
103
- }
104
- function kindFromExtension(extension) {
105
- if (!extension)
106
- return null;
107
- if (PDF_EXTENSIONS.has(extension))
108
- return RAG_DOCUMENT_KIND.pdf;
109
- if (OFFICE_EXTENSIONS.has(extension))
110
- return RAG_DOCUMENT_KIND.office;
111
- if (RTF_EXTENSIONS.has(extension))
112
- return RAG_DOCUMENT_KIND.rtf;
113
- if (HTML_EXTENSIONS.has(extension))
114
- return RAG_DOCUMENT_KIND.html;
115
- if (CSV_EXTENSIONS.has(extension))
116
- return RAG_DOCUMENT_KIND.csv;
117
- if (TSV_EXTENSIONS.has(extension))
118
- return RAG_DOCUMENT_KIND.tsv;
119
- return TEXT_EXTENSIONS.has(extension) ? RAG_DOCUMENT_KIND.text : null;
120
- }
121
- function kindFromMimeType(mimeType) {
122
- if (mimeType === 'application/pdf')
123
- return RAG_DOCUMENT_KIND.pdf;
124
- if (OFFICE_MIME_TYPES.has(mimeType))
125
- return RAG_DOCUMENT_KIND.office;
126
- if (mimeType === 'application/rtf' || mimeType === 'text/rtf')
127
- return RAG_DOCUMENT_KIND.rtf;
128
- if (mimeType === 'text/html' || mimeType === 'application/xhtml+xml') {
129
- return RAG_DOCUMENT_KIND.html;
130
- }
131
- if (mimeType === 'text/csv' || mimeType === 'application/csv')
132
- return RAG_DOCUMENT_KIND.csv;
133
- if (mimeType === 'text/tab-separated-values')
134
- return RAG_DOCUMENT_KIND.tsv;
135
- return mimeType.startsWith('text/') || TEXT_MIME_TYPES.has(mimeType)
136
- ? RAG_DOCUMENT_KIND.text
137
- : null;
138
- }
139
- function looksLikeText(bytes) {
140
- const length = Math.min(bytes.byteLength, 4096);
141
- if (length === 0)
142
- return true;
143
- let suspicious = 0;
144
- for (let index = 0; index < length; index++) {
145
- const byte = bytes[index] ?? 0;
146
- if (byte === 0)
147
- return false;
148
- if (byte < 32 && byte !== 9 && byte !== 10 && byte !== 12 && byte !== 13)
149
- suspicious++;
150
- }
151
- return suspicious / length < 0.02;
152
- }
@@ -1,17 +0,0 @@
1
- import type { AbortSignal } from 'bare-abort-controller';
2
- import { type RagDocumentInput, type RagFileInput } from './types.ts';
3
- export type RagTextExtractor = (bytes: Uint8Array) => Promise<string>;
4
- export type RagFileExtractor = (bytes: Uint8Array, fileName: string, opts?: {
5
- readonly signal?: AbortSignal;
6
- }) => Promise<string>;
7
- export type RagExtractors = {
8
- readonly pdf?: RagTextExtractor;
9
- readonly office?: RagFileExtractor;
10
- };
11
- export type RagExtractOptions = {
12
- readonly extractors?: RagExtractors;
13
- readonly signal?: AbortSignal;
14
- };
15
- export declare function extractRagDocumentFromBytes(input: RagFileInput & {
16
- readonly bytes: Uint8Array;
17
- }, opts?: RagExtractOptions): Promise<RagDocumentInput>;
@@ -1,81 +0,0 @@
1
- import { extractCsvText } from './csv-text.js';
2
- import { classifyRagDocument, RAG_DOCUMENT_KIND } from './document-types.js';
3
- import { extractHtmlText } from './html-text.js';
4
- import { extractPdfText } from './pdf-text.js';
5
- import { extractRtfText } from './rtf-text.js';
6
- import { decodeTextBytes } from './text-decoding.js';
7
- import { RAG_EXTRACT_ERROR_CODE, RagExtractError } from './types.js';
8
- export async function extractRagDocumentFromBytes(input, opts = {}) {
9
- const fileName = input.fileName ?? input.title ?? input.sourcePath ?? 'document';
10
- const kind = classifyRagDocument({
11
- fileName,
12
- bytes: input.bytes,
13
- ...(input.mimeType ? { mimeType: input.mimeType } : {})
14
- });
15
- if (kind === RAG_DOCUMENT_KIND.unsupported) {
16
- throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.unsupportedFile, `Unsupported document type: ${fileName}`);
17
- }
18
- try {
19
- const content = await extractText(kind, input.bytes, fileName, opts);
20
- return buildDocumentInput(input, fileName, content);
21
- }
22
- catch (e) {
23
- if (e instanceof RagExtractError)
24
- throw e;
25
- throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.extractionFailed, errorMessage(e));
26
- }
27
- }
28
- function buildDocumentInput(input, fileName, content) {
29
- return {
30
- title: input.title ?? fileNameFromPath(fileName),
31
- content,
32
- ...(input.id !== undefined ? { id: input.id } : {}),
33
- ...(input.blobId !== undefined ? { blobId: input.blobId } : {}),
34
- ...(input.sourcePath !== undefined ? { sourcePath: input.sourcePath } : {}),
35
- ...(input.sourceUrl !== undefined ? { sourceUrl: input.sourceUrl } : {}),
36
- ...(input.mimeType !== undefined ? { mimeType: input.mimeType } : {}),
37
- ...(input.artifactId !== undefined ? { artifactId: input.artifactId } : {}),
38
- ...(input.attachmentId !== undefined ? { attachmentId: input.attachmentId } : {}),
39
- ...(input.messageId !== undefined ? { messageId: input.messageId } : {}),
40
- ...(input.chatId !== undefined ? { chatId: input.chatId } : {})
41
- };
42
- }
43
- async function extractText(kind, bytes, fileName, opts) {
44
- if (kind === RAG_DOCUMENT_KIND.pdf) {
45
- const extractPdf = opts.extractors?.pdf ?? extractPdfText;
46
- try {
47
- return await extractPdf(bytes);
48
- }
49
- catch (e) {
50
- throw new Error(`PDF extraction failed: ${errorMessage(e)}`);
51
- }
52
- }
53
- if (kind === RAG_DOCUMENT_KIND.office) {
54
- const extractOffice = opts.extractors?.office;
55
- if (!extractOffice) {
56
- throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.unsupportedFile, 'Office document extraction is not available on this device');
57
- }
58
- try {
59
- return await extractOffice(bytes, fileName, { signal: opts.signal });
60
- }
61
- catch (e) {
62
- throw new Error(`Office document extraction failed: ${errorMessage(e)}`);
63
- }
64
- }
65
- const text = decodeTextBytes(bytes);
66
- if (kind === RAG_DOCUMENT_KIND.rtf)
67
- return extractRtfText(text);
68
- if (kind === RAG_DOCUMENT_KIND.html)
69
- return extractHtmlText(text);
70
- if (kind === RAG_DOCUMENT_KIND.csv)
71
- return extractCsvText(text);
72
- if (kind === RAG_DOCUMENT_KIND.tsv)
73
- return extractCsvText(text, '\t');
74
- return text;
75
- }
76
- function fileNameFromPath(path) {
77
- return path.split(/[\\/]/).filter(Boolean).pop() ?? path;
78
- }
79
- function errorMessage(error) {
80
- return error instanceof Error ? error.message : String(error);
81
- }
@@ -1 +0,0 @@
1
- export declare function formatHtmlTables(source: string): string;
@@ -1,125 +0,0 @@
1
- const TABLE_BLOCK = /<table\b[^>]*>[\s\S]*?<\/table\s*>/gi;
2
- const CAPTION_BLOCK = /<caption\b[^>]*>([\s\S]*?)<\/caption\s*>/i;
3
- const TABLE_TAG = /<\/?(tr|th|td)\b[^>]*>/gi;
4
- const DROP_BLOCKS = /<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
5
- const COMMENT = /<!--[\s\S]*?-->/g;
6
- const BREAK_TAG = /<br\s*\/?>/gi;
7
- const TAG = /<[^>]*>/g;
8
- export function formatHtmlTables(source) {
9
- return source.replace(TABLE_BLOCK, (table) => `\n\n${formatTable(table)}\n\n`);
10
- }
11
- function formatTable(table) {
12
- const caption = extractCaption(table);
13
- const rowsText = formatTableRows(table.replace(CAPTION_BLOCK, ' '));
14
- return caption ? joinTableParts(caption, rowsText) : rowsText;
15
- }
16
- function extractCaption(table) {
17
- const caption = normalizeCell(CAPTION_BLOCK.exec(table)?.[1] ?? '');
18
- return caption.length > 0 ? caption : null;
19
- }
20
- function joinTableParts(caption, rowsText) {
21
- return rowsText.length > 0 ? `${caption}\n\n${rowsText}` : caption;
22
- }
23
- function formatTableRows(table) {
24
- const rows = parseRows(table).filter((row) => row.some((cell) => cell.text.length > 0));
25
- const first = rows[0];
26
- if (!first)
27
- return normalizeCell(table);
28
- if (!isColumnHeaderRow(first, rows.length))
29
- return formatFlatRows(rows);
30
- const headers = first.map((cell, index) => cell.text || `column_${index + 1}`);
31
- const dataRows = rows
32
- .slice(1)
33
- .map((row) => formatDataRow(headers, row))
34
- .filter((line) => line.length > 0);
35
- return dataRows.length > 0
36
- ? [`Columns: ${headers.join(', ')}`, ...dataRows].join('\n\n')
37
- : formatFlatRows([first]);
38
- }
39
- function parseRows(table) {
40
- const rows = [];
41
- let row = null;
42
- let cell = null;
43
- let cursor = 0;
44
- for (const match of table.matchAll(TABLE_TAG)) {
45
- if (cell) {
46
- cell.html += table.slice(cursor, match.index);
47
- }
48
- ;
49
- ({ row, cell } = applyTableTag(rows, row, cell, match[0] ?? ''));
50
- cursor = match.index + match[0].length;
51
- }
52
- if (cell) {
53
- cell.html += table.slice(cursor);
54
- }
55
- finishRow(rows, row, cell);
56
- return rows;
57
- }
58
- function applyTableTag(rows, row, cell, source) {
59
- const tag = parseTableTag(source);
60
- if (tag?.name === 'tr') {
61
- return tag.closing ? finishRow(rows, row, cell) : startRow(rows, row, cell);
62
- }
63
- if (tag?.name === 'th' || tag?.name === 'td') {
64
- return tag.closing ? finishCell(row, cell) : startCell(row, cell, tag.name === 'th');
65
- }
66
- return { row, cell };
67
- }
68
- function startRow(rows, row, cell) {
69
- finishRow(rows, row, cell);
70
- return { row: [], cell: null };
71
- }
72
- function startCell(row, cell, header) {
73
- const state = finishCell(row, cell);
74
- return { row: state.row ?? [], cell: { html: '', header } };
75
- }
76
- function finishRow(rows, row, cell) {
77
- const state = finishCell(row, cell);
78
- if (state.row && state.row.length > 0) {
79
- rows.push(state.row);
80
- }
81
- return { row: null, cell: null };
82
- }
83
- function finishCell(row, cell) {
84
- if (!cell)
85
- return { row, cell: null };
86
- const nextRow = row ?? [];
87
- nextRow.push({ text: normalizeCell(cell.html), header: cell.header });
88
- return { row: nextRow, cell: null };
89
- }
90
- function parseTableTag(tag) {
91
- const match = /^<\s*(\/)?\s*(tr|th|td)\b/i.exec(tag);
92
- return match ? { name: match[2] ?? '', closing: Boolean(match[1]) } : null;
93
- }
94
- function isColumnHeaderRow(row, rowCount) {
95
- return rowCount > 1 && row.length >= 2 && row.every((cell) => cell.header);
96
- }
97
- function formatDataRow(headers, row) {
98
- const parts = [];
99
- const columnCount = Math.max(headers.length, row.length);
100
- for (let index = 0; index < columnCount; index++) {
101
- const value = row[index]?.text ?? '';
102
- if (value.length === 0)
103
- continue;
104
- parts.push(`${headers[index] ?? `column_${index + 1}`}: ${value}`);
105
- }
106
- return parts.join(' | ');
107
- }
108
- function formatFlatRows(rows) {
109
- return rows
110
- .map((row) => row
111
- .map((cell) => cell.text)
112
- .filter((cell) => cell.length > 0)
113
- .join(' | '))
114
- .filter((line) => line.length > 0)
115
- .join('\n');
116
- }
117
- function normalizeCell(html) {
118
- return html
119
- .replace(DROP_BLOCKS, ' ')
120
- .replace(COMMENT, ' ')
121
- .replace(BREAK_TAG, ' ')
122
- .replace(TAG, ' ')
123
- .replace(/\s+/g, ' ')
124
- .trim();
125
- }
@@ -1 +0,0 @@
1
- export declare function extractHtmlText(source: string): string;
@@ -1,132 +0,0 @@
1
- import { formatHtmlTables } from './html-table-text.js';
2
- const DROP_BLOCKS = /<(script|style|noscript|template|svg|head)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
3
- const COMMENT = /<!--[\s\S]*?-->/g;
4
- const HEADING_OPEN = /<h([1-6])\b[^>]*>/gi;
5
- const HEADING_CLOSE = /<\/h[1-6]\s*>/gi;
6
- const LIST_ITEM_OPEN = /<li\b[^>]*>/gi;
7
- const LIST_ITEM_CLOSE = /<\/li\s*>/gi;
8
- const TABLE_CELL_OPEN = /<t[hd]\b[^>]*>/gi;
9
- const TABLE_CELL_CLOSE = /<\/t[hd]\s*>/gi;
10
- const TABLE_ROW_OPEN = /<tr\b[^>]*>/gi;
11
- const TABLE_ROW_CLOSE = /<\/tr\s*>/gi;
12
- const BREAK_TAG = /<br\s*\/?>/gi;
13
- const BLOCK_CONTAINER_TAGS = 'p|div|section|article|header|footer|main|aside|nav|blockquote|pre|figure|figcaption|ul|ol|table|thead|tbody|tfoot|address|form|fieldset';
14
- const BLOCK_CLOSE = new RegExp(`</(${BLOCK_CONTAINER_TAGS})\\s*>`, 'gi');
15
- const BLOCK_OPEN = new RegExp(`<(${BLOCK_CONTAINER_TAGS}|hr)\\b[^>]*>`, 'gi');
16
- const STRAY_CLOSE_BRACKET = />/g;
17
- const NAMED_ENTITIES = new Map([
18
- ['amp', '&'],
19
- ['lt', '<'],
20
- ['gt', '>'],
21
- ['quot', '"'],
22
- ['apos', "'"],
23
- ['nbsp', ' '],
24
- ['copy', '©'],
25
- ['reg', '®'],
26
- ['trade', '™'],
27
- ['hellip', '…'],
28
- ['mdash', '—'],
29
- ['ndash', '–'],
30
- ['lsquo', '‘'],
31
- ['rsquo', '’'],
32
- ['ldquo', '“'],
33
- ['rdquo', '”'],
34
- ['middot', '·'],
35
- ['bull', '•'],
36
- ['deg', '°'],
37
- ['times', '×'],
38
- ['divide', '÷'],
39
- ['plusmn', '±'],
40
- ['laquo', '«'],
41
- ['raquo', '»'],
42
- ['euro', '€'],
43
- ['pound', '£'],
44
- ['yen', '¥'],
45
- ['cent', '¢'],
46
- ['sect', '§'],
47
- ['para', '¶'],
48
- ['iexcl', '¡'],
49
- ['iquest', '¿'],
50
- ['dagger', '†'],
51
- ['Dagger', '‡'],
52
- ['sbquo', '‚'],
53
- ['bdquo', '„'],
54
- ['prime', '′'],
55
- ['Prime', '″'],
56
- ['frac12', '½'],
57
- ['frac14', '¼'],
58
- ['frac34', '¾']
59
- ]);
60
- const NUMERIC_ENTITY = /&#(x[0-9a-fA-F]+|[0-9]+);/g;
61
- const NAMED_ENTITY = /&([a-zA-Z][a-zA-Z0-9]+);/g;
62
- export function extractHtmlText(source) {
63
- const structured = markStructure(formatHtmlTables(dropNonContent(source)));
64
- const text = stripTagsFully(structured);
65
- return normalizeWhitespace(decodeEntities(text));
66
- }
67
- function dropNonContent(source) {
68
- let previous = source;
69
- let current = source.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
70
- while (current !== previous) {
71
- previous = current;
72
- current = current.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
73
- }
74
- return current;
75
- }
76
- function stripTagsFully(source) {
77
- let output = '';
78
- let cursor = 0;
79
- while (cursor < source.length) {
80
- const open = source.indexOf('<', cursor);
81
- if (open === -1) {
82
- return output + stripStrayCloseBrackets(source.slice(cursor));
83
- }
84
- output += stripStrayCloseBrackets(source.slice(cursor, open));
85
- const close = source.indexOf('>', open + 1);
86
- if (close === -1) {
87
- return `${output} `;
88
- }
89
- cursor = close + 1;
90
- }
91
- return output;
92
- }
93
- function stripStrayCloseBrackets(text) {
94
- return text.replace(STRAY_CLOSE_BRACKET, ' ');
95
- }
96
- function markStructure(html) {
97
- return html
98
- .replace(HEADING_OPEN, (_match, level) => `\n\n${'#'.repeat(Number(level))} `)
99
- .replace(HEADING_CLOSE, '\n\n')
100
- .replace(TABLE_CELL_OPEN, ' | ')
101
- .replace(TABLE_CELL_CLOSE, '')
102
- .replace(TABLE_ROW_OPEN, '\n')
103
- .replace(TABLE_ROW_CLOSE, '')
104
- .replace(LIST_ITEM_OPEN, '\n- ')
105
- .replace(LIST_ITEM_CLOSE, '')
106
- .replace(BREAK_TAG, '\n')
107
- .replace(BLOCK_OPEN, '\n\n')
108
- .replace(BLOCK_CLOSE, '\n\n');
109
- }
110
- function decodeEntities(text) {
111
- return text
112
- .replace(NUMERIC_ENTITY, (_match, code) => {
113
- const codePoint = code.startsWith('x') || code.startsWith('X')
114
- ? Number.parseInt(code.slice(1), 16)
115
- : Number.parseInt(code, 10);
116
- return Number.isFinite(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff
117
- ? String.fromCodePoint(codePoint)
118
- : '';
119
- })
120
- .replace(NAMED_ENTITY, (match, name) => NAMED_ENTITIES.get(name) ?? match);
121
- }
122
- function normalizeWhitespace(text) {
123
- return text
124
- .replace(/\r\n?/g, '\n')
125
- .replace(/[ \t\f\v]+/g, ' ')
126
- .replace(/[ \t]+\n/g, '\n')
127
- .replace(/\n[ \t]+/g, '\n')
128
- .replace(/^\| ?/gm, '')
129
- .replace(/ \|\s*$/gm, '')
130
- .replace(/\n{3,}/g, '\n\n')
131
- .trim();
132
- }
@@ -1,17 +0,0 @@
1
- export declare class BareTextDecoder {
2
- readonly encoding: string;
3
- readonly fatal: boolean;
4
- readonly ignoreBOM: boolean;
5
- constructor(label?: string, options?: {
6
- fatal?: boolean;
7
- ignoreBOM?: boolean;
8
- });
9
- decode(input?: ArrayBufferView | ArrayBuffer): string;
10
- }
11
- export declare class BareTextEncoder {
12
- encode(input?: string): Uint8Array;
13
- }
14
- export declare function decodeBytes(bytes: Uint8Array, label: string, options?: {
15
- fatal?: boolean;
16
- ignoreBOM?: boolean;
17
- }): string;