@qvac/core 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-lib/lib/api/chat.d.ts +3 -1
- package/dist-lib/lib/api/chat.js +6 -2
- package/dist-lib/lib/api/index.d.ts +2 -0
- package/dist-lib/lib/api/index.js +3 -1
- package/dist-lib/lib/api/inference-calls.d.ts +18 -0
- package/dist-lib/lib/api/inference-calls.js +15 -0
- package/dist-lib/lib/api/rag.js +10 -2
- package/dist-lib/lib/assistant/harness.d.ts +1 -0
- package/dist-lib/lib/assistant/history.d.ts +10 -1
- package/dist-lib/lib/assistant/history.js +36 -1
- package/dist-lib/lib/assistant/message.d.ts +2 -0
- package/dist-lib/lib/assistant/message.js +11 -0
- package/dist-lib/lib/assistant/skill-encode.js +27 -1
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +11 -2
- package/dist-lib/lib/assistant.d.ts +19 -3
- package/dist-lib/lib/assistant.js +148 -23
- package/dist-lib/lib/core/chunk.js +5 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
- package/dist-lib/lib/core/files/rag/build.js +1 -2
- package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
- package/dist-lib/lib/core/files/rag/index.js +3 -17
- package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
- package/dist-lib/lib/core/files/rag/load.js +1 -2
- package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -3
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
- package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
- package/dist-lib/lib/core/files/rag/similarity.js +6 -0
- package/dist-lib/lib/core/files/upload.d.ts +1 -1
- package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
- package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
- package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
- package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
- package/dist-lib/lib/core/notification.js +5 -4
- package/dist-lib/lib/core/skill-secrets.js +3 -6
- package/dist-lib/lib/core.d.ts +9 -1
- package/dist-lib/lib/core.js +129 -13
- package/dist-lib/lib/credential-pusher.d.ts +2 -3
- package/dist-lib/lib/credential-pusher.js +67 -39
- package/dist-lib/lib/inference-calls.d.ts +40 -0
- package/dist-lib/lib/inference-calls.js +266 -0
- package/dist-lib/lib/integrations/index.d.ts +1 -2
- package/dist-lib/lib/integrations/index.js +0 -1
- package/dist-lib/lib/integrations/setup-routes.js +0 -15
- package/dist-lib/lib/oauth/flow-core.js +1 -1
- package/dist-lib/lib/oauth/flow.d.ts +4 -2
- package/dist-lib/lib/oauth/flow.js +3 -4
- package/dist-lib/lib/oauth/host.d.ts +3 -1
- package/dist-lib/lib/oauth/host.js +3 -1
- package/dist-lib/lib/oauth/keys.d.ts +21 -20
- package/dist-lib/lib/oauth/keys.js +59 -85
- package/dist-lib/lib/oauth/loopback.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.js +32 -19
- package/dist-lib/lib/oauth/loopback.stub.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.stub.js +2 -2
- package/dist-lib/lib/oauth/providers.d.ts +3 -5
- package/dist-lib/lib/oauth/providers.js +20 -67
- package/dist-lib/lib/oauth/redirect-types.d.ts +1 -0
- package/dist-lib/lib/oauth/registry.d.ts +10 -0
- package/dist-lib/lib/oauth/registry.js +25 -0
- package/dist-lib/lib/oauth/resolve.js +1 -1
- package/dist-lib/lib/oauth/token-exchange-core.js +2 -1
- package/dist-lib/lib/oauth/types.d.ts +1 -1
- package/dist-lib/lib/oauth.client.d.ts +3 -1
- package/dist-lib/lib/oauth.client.js +3 -1
- package/dist-lib/lib/oauth.d.ts +4 -7
- package/dist-lib/lib/oauth.js +4 -7
- package/dist-lib/lib/skill-types.d.ts +12 -0
- package/dist-lib/schema/content.js +5 -1
- package/dist-lib/schema/engine.js +63 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/bind.js +3 -0
- package/dist-lib/spec/calls.js +1 -0
- package/dist-lib/spec/capabilities.d.ts +2 -0
- package/dist-lib/spec/hrpc/hrpc.json +13 -0
- package/dist-lib/spec/hrpc/index.js +15 -3
- package/dist-lib/spec/hrpc/messages.js +284 -52
- package/dist-lib/spec/hyperschema/index.js +284 -52
- package/dist-lib/spec/hyperschema/schema.json +210 -1
- package/dist-lib/spec/hyperschema/types.d.ts +51 -1
- package/dist-lib/spec/local/hyperdb/messages.js +64 -128
- package/dist-lib/spec/local/hyperschema/index.js +32 -64
- package/dist-lib/spec/local/hyperschema/schema.json +16 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
- package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
- package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +15 -18
- package/tui/app.mjs +98 -155
- package/tui/lib/knowledge-import.mjs +1 -1
- package/tui/lib/save-file.mjs +1 -1
- package/.claude/skills/asana/SKILL.md +0 -86
- package/.claude/skills/compaction/SKILL.md +0 -136
- package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
- package/.claude/skills/gh-stack/SKILL.md +0 -181
- package/.claude/skills/gh-stack/references/commands.md +0 -177
- package/.claude/skills/gh-stack/references/stack-design.md +0 -97
- package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
- package/.claude/skills/js-house-style/SKILL.md +0 -671
- package/.claude/skills/pr-create/SKILL.md +0 -243
- package/.claude/skills/pr-review/SKILL.md +0 -428
- package/.claude/skills/release/SKILL.md +0 -122
- package/CLAUDE.md +0 -48
- package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
- package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
- package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
- package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
- package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
- package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
- package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
- package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
- package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
- package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
- package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
- package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
- package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
- package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
- package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
- package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
- package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
- package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
- package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
- package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
- package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
- package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
- package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
- package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
- package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
- package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
- package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
- package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
- package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
- package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
- package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
- package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
- package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
- package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
- package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
- package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
- package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
- package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
- package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
- package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
- package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
- package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
- package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
- package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
- package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
- package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
- package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
- package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
- package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
- package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
- package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
- package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
- package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
- package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
- package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
- package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
- package/dist-lib/lib/integrations/oauth-providers.d.ts +0 -14
- package/dist-lib/lib/integrations/oauth-providers.js +0 -116
- package/dist-lib/lib/oauth/asana.d.ts +0 -35
- package/dist-lib/lib/oauth/asana.js +0 -71
- package/dist-lib/lib/oauth/google.d.ts +0 -37
- package/dist-lib/lib/oauth/google.js +0 -77
- package/dist-lib/lib/oauth/notion.d.ts +0 -34
- package/dist-lib/lib/oauth/notion.js +0 -82
- package/dist-lib/lib/oauth/spotify.d.ts +0 -32
- package/dist-lib/lib/oauth/spotify.js +0 -74
- package/dist-lib/lib/oauth/verify.d.ts +0 -8
- package/dist-lib/lib/oauth/verify.js +0 -58
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
const OPTIONAL_STRING_KEYS = [
|
|
2
|
-
'sourcePath',
|
|
3
|
-
'sourceUrl',
|
|
4
|
-
'mimeType',
|
|
5
|
-
'artifactId',
|
|
6
|
-
'attachmentId',
|
|
7
|
-
'messageId',
|
|
8
|
-
'chatId'
|
|
9
|
-
];
|
|
10
|
-
export function attachmentDocumentId(attachmentId) {
|
|
11
|
-
return `attachment-${attachmentId}`;
|
|
12
|
-
}
|
|
13
|
-
export function buildRagChunkMetadata(input) {
|
|
14
|
-
return {
|
|
15
|
-
documentId: input.documentId,
|
|
16
|
-
title: input.document.title,
|
|
17
|
-
sourcePath: input.document.sourcePath ?? null,
|
|
18
|
-
sourceUrl: input.document.sourceUrl ?? null,
|
|
19
|
-
mimeType: input.document.mimeType ?? null,
|
|
20
|
-
artifactId: input.document.artifactId ?? null,
|
|
21
|
-
attachmentId: input.document.attachmentId ?? null,
|
|
22
|
-
messageId: input.document.messageId ?? null,
|
|
23
|
-
chatId: input.document.chatId ?? null,
|
|
24
|
-
chunkIndex: input.chunkIndex,
|
|
25
|
-
tokenCount: input.tokenCount,
|
|
26
|
-
indexedAt: new Date().toISOString()
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
export function parseRagMetadata(metadataJson) {
|
|
30
|
-
if (!metadataJson)
|
|
31
|
-
return null;
|
|
32
|
-
let parsed;
|
|
33
|
-
try {
|
|
34
|
-
parsed = JSON.parse(metadataJson);
|
|
35
|
-
}
|
|
36
|
-
catch {
|
|
37
|
-
return null;
|
|
38
|
-
}
|
|
39
|
-
if (typeof parsed !== 'object' || parsed === null)
|
|
40
|
-
return null;
|
|
41
|
-
const value = parsed;
|
|
42
|
-
const { documentId, title, indexedAt, chunkIndex, tokenCount } = value;
|
|
43
|
-
if (typeof documentId !== 'string' || typeof title !== 'string')
|
|
44
|
-
return null;
|
|
45
|
-
if (typeof indexedAt !== 'string')
|
|
46
|
-
return null;
|
|
47
|
-
if (!isNumber(chunkIndex) || !isNumber(tokenCount))
|
|
48
|
-
return null;
|
|
49
|
-
for (const key of OPTIONAL_STRING_KEYS) {
|
|
50
|
-
const optional = value[key];
|
|
51
|
-
if (optional !== undefined && optional !== null && typeof optional !== 'string')
|
|
52
|
-
return null;
|
|
53
|
-
}
|
|
54
|
-
return {
|
|
55
|
-
documentId,
|
|
56
|
-
title,
|
|
57
|
-
sourcePath: stringOrNull(value.sourcePath),
|
|
58
|
-
sourceUrl: stringOrNull(value.sourceUrl),
|
|
59
|
-
mimeType: stringOrNull(value.mimeType),
|
|
60
|
-
artifactId: stringOrNull(value.artifactId),
|
|
61
|
-
attachmentId: stringOrNull(value.attachmentId),
|
|
62
|
-
messageId: stringOrNull(value.messageId),
|
|
63
|
-
chatId: stringOrNull(value.chatId),
|
|
64
|
-
chunkIndex,
|
|
65
|
-
tokenCount,
|
|
66
|
-
indexedAt
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
export function hitMatchesFilters(hit, filters, liveSourceIds) {
|
|
70
|
-
const metadata = parseRagMetadata(hit.metadataJson);
|
|
71
|
-
if (!metadata)
|
|
72
|
-
return false;
|
|
73
|
-
if (filters.chatId && metadata.chatId !== filters.chatId)
|
|
74
|
-
return false;
|
|
75
|
-
if (filters.documentIds?.length && !filters.documentIds.includes(metadata.documentId)) {
|
|
76
|
-
return false;
|
|
77
|
-
}
|
|
78
|
-
if (filters.attachmentIds?.length &&
|
|
79
|
-
(!metadata.attachmentId || !filters.attachmentIds.includes(metadata.attachmentId))) {
|
|
80
|
-
return false;
|
|
81
|
-
}
|
|
82
|
-
return !(liveSourceIds && metadata.attachmentId && !liveSourceIds.has(metadata.attachmentId));
|
|
83
|
-
}
|
|
84
|
-
function isNumber(value) {
|
|
85
|
-
return typeof value === 'number' && !Number.isNaN(value);
|
|
86
|
-
}
|
|
87
|
-
function stringOrNull(value) {
|
|
88
|
-
return typeof value === 'string' ? value : null;
|
|
89
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
const TOKEN_PATTERN = /'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
|
|
2
|
-
export function tokenizeForRag(text) {
|
|
3
|
-
const tokens = [];
|
|
4
|
-
TOKEN_PATTERN.lastIndex = 0;
|
|
5
|
-
let match = TOKEN_PATTERN.exec(text);
|
|
6
|
-
while (match) {
|
|
7
|
-
tokens.push({
|
|
8
|
-
text: match[0],
|
|
9
|
-
start: match.index,
|
|
10
|
-
end: match.index + match[0].length
|
|
11
|
-
});
|
|
12
|
-
match = TOKEN_PATTERN.exec(text);
|
|
13
|
-
}
|
|
14
|
-
return tokens;
|
|
15
|
-
}
|
|
16
|
-
export function estimateRagTokens(text) {
|
|
17
|
-
return tokenizeForRag(text).length;
|
|
18
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function extractCsvText(source: string, delimiter?: string): string;
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
const FIELD_SEPARATOR = ' | ';
|
|
2
|
-
const KEY_VALUE_SEPARATOR = ': ';
|
|
3
|
-
const DEFAULT_DELIMITER = ',';
|
|
4
|
-
export function extractCsvText(source, delimiter = DEFAULT_DELIMITER) {
|
|
5
|
-
const rows = parseCsv(source, delimiter);
|
|
6
|
-
if (rows.length === 0)
|
|
7
|
-
return '';
|
|
8
|
-
const headerRow = rows[0];
|
|
9
|
-
if (!headerRow || headerRow.length === 0)
|
|
10
|
-
return '';
|
|
11
|
-
const headers = headerRow
|
|
12
|
-
.map(normalizeCell)
|
|
13
|
-
.map((value, index) => (value.length > 0 ? value : `column_${index + 1}`));
|
|
14
|
-
const dataRows = rows.slice(1);
|
|
15
|
-
if (dataRows.length === 0 || headers.length < 2) {
|
|
16
|
-
return formatFlat(rows);
|
|
17
|
-
}
|
|
18
|
-
const headerLine = `Columns: ${headers.join(', ')}`;
|
|
19
|
-
const formatted = dataRows
|
|
20
|
-
.filter((row) => row.some((cell) => cell.trim().length > 0))
|
|
21
|
-
.map((row) => formatRow(headers, row));
|
|
22
|
-
return [headerLine, ...formatted].join('\n').trim();
|
|
23
|
-
}
|
|
24
|
-
function formatFlat(rows) {
|
|
25
|
-
return rows
|
|
26
|
-
.map((row) => row
|
|
27
|
-
.map(normalizeCell)
|
|
28
|
-
.filter((cell) => cell.length > 0)
|
|
29
|
-
.join(', '))
|
|
30
|
-
.filter((line) => line.length > 0)
|
|
31
|
-
.join('\n')
|
|
32
|
-
.trim();
|
|
33
|
-
}
|
|
34
|
-
function formatRow(headers, row) {
|
|
35
|
-
const parts = [];
|
|
36
|
-
for (let index = 0; index < headers.length; index++) {
|
|
37
|
-
const value = normalizeCell(row[index] ?? '');
|
|
38
|
-
if (value.length === 0)
|
|
39
|
-
continue;
|
|
40
|
-
const header = headers[index] ?? `column_${index + 1}`;
|
|
41
|
-
parts.push(`${header}${KEY_VALUE_SEPARATOR}${value}`);
|
|
42
|
-
}
|
|
43
|
-
return parts.join(FIELD_SEPARATOR);
|
|
44
|
-
}
|
|
45
|
-
function normalizeCell(value) {
|
|
46
|
-
return value.replace(/\s+/g, ' ').trim();
|
|
47
|
-
}
|
|
48
|
-
function parseCsv(source, delimiter) {
|
|
49
|
-
const state = {
|
|
50
|
-
rows: [],
|
|
51
|
-
current: [],
|
|
52
|
-
field: '',
|
|
53
|
-
inQuotes: false,
|
|
54
|
-
index: 0,
|
|
55
|
-
delimiter
|
|
56
|
-
};
|
|
57
|
-
while (state.index < source.length) {
|
|
58
|
-
if (state.inQuotes) {
|
|
59
|
-
consumeQuotedChar(state, source);
|
|
60
|
-
}
|
|
61
|
-
else {
|
|
62
|
-
consumeUnquotedChar(state, source);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
flushFinalField(state);
|
|
66
|
-
return state.rows.filter((row) => row.length > 1 || (row[0] ?? '').length > 0);
|
|
67
|
-
}
|
|
68
|
-
function consumeQuotedChar(state, source) {
|
|
69
|
-
const char = source[state.index];
|
|
70
|
-
if (char !== '"') {
|
|
71
|
-
state.field += char;
|
|
72
|
-
state.index += 1;
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
if (source[state.index + 1] === '"') {
|
|
76
|
-
state.field += '"';
|
|
77
|
-
state.index += 2;
|
|
78
|
-
return;
|
|
79
|
-
}
|
|
80
|
-
state.inQuotes = false;
|
|
81
|
-
state.index += 1;
|
|
82
|
-
}
|
|
83
|
-
function consumeUnquotedChar(state, source) {
|
|
84
|
-
const char = source[state.index];
|
|
85
|
-
if (char === '"' && state.field.length === 0) {
|
|
86
|
-
state.inQuotes = true;
|
|
87
|
-
state.index += 1;
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
90
|
-
if (char === state.delimiter) {
|
|
91
|
-
finishField(state);
|
|
92
|
-
state.index += 1;
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
if (char === '\n' || char === '\r') {
|
|
96
|
-
finishRow(state);
|
|
97
|
-
state.index += char === '\r' && source[state.index + 1] === '\n' ? 2 : 1;
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
state.field += char;
|
|
101
|
-
state.index += 1;
|
|
102
|
-
}
|
|
103
|
-
function finishField(state) {
|
|
104
|
-
state.current.push(state.field);
|
|
105
|
-
state.field = '';
|
|
106
|
-
}
|
|
107
|
-
function finishRow(state) {
|
|
108
|
-
finishField(state);
|
|
109
|
-
state.rows.push(state.current);
|
|
110
|
-
state.current = [];
|
|
111
|
-
}
|
|
112
|
-
function flushFinalField(state) {
|
|
113
|
-
if (state.field.length === 0 && state.current.length === 0)
|
|
114
|
-
return;
|
|
115
|
-
state.current.push(state.field);
|
|
116
|
-
state.rows.push(state.current);
|
|
117
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
export declare const RAG_DOCUMENT_KIND: {
|
|
2
|
-
readonly pdf: 'pdf';
|
|
3
|
-
readonly office: 'office';
|
|
4
|
-
readonly rtf: 'rtf';
|
|
5
|
-
readonly html: 'html';
|
|
6
|
-
readonly csv: 'csv';
|
|
7
|
-
readonly tsv: 'tsv';
|
|
8
|
-
readonly text: 'text';
|
|
9
|
-
readonly unsupported: 'unsupported';
|
|
10
|
-
};
|
|
11
|
-
export type RagDocumentKind = (typeof RAG_DOCUMENT_KIND)[keyof typeof RAG_DOCUMENT_KIND];
|
|
12
|
-
export declare const RAG_ACCEPT_EXTENSIONS: readonly ['.pdf', '.docx', '.doc', '.docm', '.dot', '.dotx', '.odt', '.fodt', '.ott', '.txt', '.md', '.markdown', '.rtf', '.html', '.htm', '.xhtml', '.log', '.json', '.jsonl', '.csv', '.tsv', '.xml', '.yaml', '.yml'];
|
|
13
|
-
export declare function classifyRagDocument(input: {
|
|
14
|
-
readonly fileName?: string;
|
|
15
|
-
readonly mimeType?: string;
|
|
16
|
-
readonly bytes?: Uint8Array;
|
|
17
|
-
}): RagDocumentKind;
|
|
18
|
-
export declare function isSupportedRagDocument(input: {
|
|
19
|
-
readonly fileName?: string;
|
|
20
|
-
readonly mimeType?: string;
|
|
21
|
-
readonly bytes?: Uint8Array;
|
|
22
|
-
}): boolean;
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
export const RAG_DOCUMENT_KIND = {
|
|
2
|
-
pdf: 'pdf',
|
|
3
|
-
office: 'office',
|
|
4
|
-
rtf: 'rtf',
|
|
5
|
-
html: 'html',
|
|
6
|
-
csv: 'csv',
|
|
7
|
-
tsv: 'tsv',
|
|
8
|
-
text: 'text',
|
|
9
|
-
unsupported: 'unsupported'
|
|
10
|
-
};
|
|
11
|
-
const PDF_EXTENSIONS = new Set(['.pdf']);
|
|
12
|
-
const OFFICE_EXTENSIONS = new Set([
|
|
13
|
-
'.docx',
|
|
14
|
-
'.doc',
|
|
15
|
-
'.docm',
|
|
16
|
-
'.dot',
|
|
17
|
-
'.dotx',
|
|
18
|
-
'.odt',
|
|
19
|
-
'.fodt',
|
|
20
|
-
'.ott'
|
|
21
|
-
]);
|
|
22
|
-
const OFFICE_MIME_TYPES = new Set([
|
|
23
|
-
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
24
|
-
'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
|
|
25
|
-
'application/msword',
|
|
26
|
-
'application/vnd.oasis.opendocument.text',
|
|
27
|
-
'application/vnd.oasis.opendocument.text-template'
|
|
28
|
-
]);
|
|
29
|
-
const RTF_EXTENSIONS = new Set(['.rtf']);
|
|
30
|
-
const HTML_EXTENSIONS = new Set(['.html', '.htm', '.xhtml']);
|
|
31
|
-
const CSV_EXTENSIONS = new Set(['.csv']);
|
|
32
|
-
const TSV_EXTENSIONS = new Set(['.tsv']);
|
|
33
|
-
const TEXT_EXTENSIONS = new Set([
|
|
34
|
-
'.json',
|
|
35
|
-
'.jsonl',
|
|
36
|
-
'.log',
|
|
37
|
-
'.markdown',
|
|
38
|
-
'.md',
|
|
39
|
-
'.mdown',
|
|
40
|
-
'.mkd',
|
|
41
|
-
'.text',
|
|
42
|
-
'.txt',
|
|
43
|
-
'.xml',
|
|
44
|
-
'.yaml',
|
|
45
|
-
'.yml'
|
|
46
|
-
]);
|
|
47
|
-
const TEXT_MIME_TYPES = new Set([
|
|
48
|
-
'application/json',
|
|
49
|
-
'application/ld+json',
|
|
50
|
-
'application/rtf',
|
|
51
|
-
'application/x-ndjson',
|
|
52
|
-
'application/xml',
|
|
53
|
-
'application/yaml',
|
|
54
|
-
'text/rtf'
|
|
55
|
-
]);
|
|
56
|
-
export const RAG_ACCEPT_EXTENSIONS = [
|
|
57
|
-
'.pdf',
|
|
58
|
-
'.docx',
|
|
59
|
-
'.doc',
|
|
60
|
-
'.docm',
|
|
61
|
-
'.dot',
|
|
62
|
-
'.dotx',
|
|
63
|
-
'.odt',
|
|
64
|
-
'.fodt',
|
|
65
|
-
'.ott',
|
|
66
|
-
'.txt',
|
|
67
|
-
'.md',
|
|
68
|
-
'.markdown',
|
|
69
|
-
'.rtf',
|
|
70
|
-
'.html',
|
|
71
|
-
'.htm',
|
|
72
|
-
'.xhtml',
|
|
73
|
-
'.log',
|
|
74
|
-
'.json',
|
|
75
|
-
'.jsonl',
|
|
76
|
-
'.csv',
|
|
77
|
-
'.tsv',
|
|
78
|
-
'.xml',
|
|
79
|
-
'.yaml',
|
|
80
|
-
'.yml'
|
|
81
|
-
];
|
|
82
|
-
export function classifyRagDocument(input) {
|
|
83
|
-
const extension = extensionForName(input.fileName);
|
|
84
|
-
const mimeType = normalizeMimeType(input.mimeType);
|
|
85
|
-
const explicit = kindFromExtension(extension) ?? kindFromMimeType(mimeType);
|
|
86
|
-
if (explicit)
|
|
87
|
-
return explicit;
|
|
88
|
-
return input.bytes && looksLikeText(input.bytes)
|
|
89
|
-
? RAG_DOCUMENT_KIND.text
|
|
90
|
-
: RAG_DOCUMENT_KIND.unsupported;
|
|
91
|
-
}
|
|
92
|
-
export function isSupportedRagDocument(input) {
|
|
93
|
-
return classifyRagDocument(input) !== RAG_DOCUMENT_KIND.unsupported;
|
|
94
|
-
}
|
|
95
|
-
function extensionForName(fileName) {
|
|
96
|
-
if (!fileName)
|
|
97
|
-
return null;
|
|
98
|
-
const index = fileName.lastIndexOf('.');
|
|
99
|
-
return index >= 0 ? fileName.slice(index).toLowerCase() : null;
|
|
100
|
-
}
|
|
101
|
-
function normalizeMimeType(mimeType) {
|
|
102
|
-
return mimeType?.split(';')[0]?.trim().toLowerCase() ?? '';
|
|
103
|
-
}
|
|
104
|
-
function kindFromExtension(extension) {
|
|
105
|
-
if (!extension)
|
|
106
|
-
return null;
|
|
107
|
-
if (PDF_EXTENSIONS.has(extension))
|
|
108
|
-
return RAG_DOCUMENT_KIND.pdf;
|
|
109
|
-
if (OFFICE_EXTENSIONS.has(extension))
|
|
110
|
-
return RAG_DOCUMENT_KIND.office;
|
|
111
|
-
if (RTF_EXTENSIONS.has(extension))
|
|
112
|
-
return RAG_DOCUMENT_KIND.rtf;
|
|
113
|
-
if (HTML_EXTENSIONS.has(extension))
|
|
114
|
-
return RAG_DOCUMENT_KIND.html;
|
|
115
|
-
if (CSV_EXTENSIONS.has(extension))
|
|
116
|
-
return RAG_DOCUMENT_KIND.csv;
|
|
117
|
-
if (TSV_EXTENSIONS.has(extension))
|
|
118
|
-
return RAG_DOCUMENT_KIND.tsv;
|
|
119
|
-
return TEXT_EXTENSIONS.has(extension) ? RAG_DOCUMENT_KIND.text : null;
|
|
120
|
-
}
|
|
121
|
-
function kindFromMimeType(mimeType) {
|
|
122
|
-
if (mimeType === 'application/pdf')
|
|
123
|
-
return RAG_DOCUMENT_KIND.pdf;
|
|
124
|
-
if (OFFICE_MIME_TYPES.has(mimeType))
|
|
125
|
-
return RAG_DOCUMENT_KIND.office;
|
|
126
|
-
if (mimeType === 'application/rtf' || mimeType === 'text/rtf')
|
|
127
|
-
return RAG_DOCUMENT_KIND.rtf;
|
|
128
|
-
if (mimeType === 'text/html' || mimeType === 'application/xhtml+xml') {
|
|
129
|
-
return RAG_DOCUMENT_KIND.html;
|
|
130
|
-
}
|
|
131
|
-
if (mimeType === 'text/csv' || mimeType === 'application/csv')
|
|
132
|
-
return RAG_DOCUMENT_KIND.csv;
|
|
133
|
-
if (mimeType === 'text/tab-separated-values')
|
|
134
|
-
return RAG_DOCUMENT_KIND.tsv;
|
|
135
|
-
return mimeType.startsWith('text/') || TEXT_MIME_TYPES.has(mimeType)
|
|
136
|
-
? RAG_DOCUMENT_KIND.text
|
|
137
|
-
: null;
|
|
138
|
-
}
|
|
139
|
-
function looksLikeText(bytes) {
|
|
140
|
-
const length = Math.min(bytes.byteLength, 4096);
|
|
141
|
-
if (length === 0)
|
|
142
|
-
return true;
|
|
143
|
-
let suspicious = 0;
|
|
144
|
-
for (let index = 0; index < length; index++) {
|
|
145
|
-
const byte = bytes[index] ?? 0;
|
|
146
|
-
if (byte === 0)
|
|
147
|
-
return false;
|
|
148
|
-
if (byte < 32 && byte !== 9 && byte !== 10 && byte !== 12 && byte !== 13)
|
|
149
|
-
suspicious++;
|
|
150
|
-
}
|
|
151
|
-
return suspicious / length < 0.02;
|
|
152
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import { type RagDocumentInput, type RagFileInput } from './types.ts';
|
|
3
|
-
export type RagTextExtractor = (bytes: Uint8Array) => Promise<string>;
|
|
4
|
-
export type RagFileExtractor = (bytes: Uint8Array, fileName: string, opts?: {
|
|
5
|
-
readonly signal?: AbortSignal;
|
|
6
|
-
}) => Promise<string>;
|
|
7
|
-
export type RagExtractors = {
|
|
8
|
-
readonly pdf?: RagTextExtractor;
|
|
9
|
-
readonly office?: RagFileExtractor;
|
|
10
|
-
};
|
|
11
|
-
export type RagExtractOptions = {
|
|
12
|
-
readonly extractors?: RagExtractors;
|
|
13
|
-
readonly signal?: AbortSignal;
|
|
14
|
-
};
|
|
15
|
-
export declare function extractRagDocumentFromBytes(input: RagFileInput & {
|
|
16
|
-
readonly bytes: Uint8Array;
|
|
17
|
-
}, opts?: RagExtractOptions): Promise<RagDocumentInput>;
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
import { extractCsvText } from './csv-text.js';
|
|
2
|
-
import { classifyRagDocument, RAG_DOCUMENT_KIND } from './document-types.js';
|
|
3
|
-
import { extractHtmlText } from './html-text.js';
|
|
4
|
-
import { extractPdfText } from './pdf-text.js';
|
|
5
|
-
import { extractRtfText } from './rtf-text.js';
|
|
6
|
-
import { decodeTextBytes } from './text-decoding.js';
|
|
7
|
-
import { RAG_EXTRACT_ERROR_CODE, RagExtractError } from './types.js';
|
|
8
|
-
export async function extractRagDocumentFromBytes(input, opts = {}) {
|
|
9
|
-
const fileName = input.fileName ?? input.title ?? input.sourcePath ?? 'document';
|
|
10
|
-
const kind = classifyRagDocument({
|
|
11
|
-
fileName,
|
|
12
|
-
bytes: input.bytes,
|
|
13
|
-
...(input.mimeType ? { mimeType: input.mimeType } : {})
|
|
14
|
-
});
|
|
15
|
-
if (kind === RAG_DOCUMENT_KIND.unsupported) {
|
|
16
|
-
throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.unsupportedFile, `Unsupported document type: ${fileName}`);
|
|
17
|
-
}
|
|
18
|
-
try {
|
|
19
|
-
const content = await extractText(kind, input.bytes, fileName, opts);
|
|
20
|
-
return buildDocumentInput(input, fileName, content);
|
|
21
|
-
}
|
|
22
|
-
catch (e) {
|
|
23
|
-
if (e instanceof RagExtractError)
|
|
24
|
-
throw e;
|
|
25
|
-
throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.extractionFailed, errorMessage(e));
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
function buildDocumentInput(input, fileName, content) {
|
|
29
|
-
return {
|
|
30
|
-
title: input.title ?? fileNameFromPath(fileName),
|
|
31
|
-
content,
|
|
32
|
-
...(input.id !== undefined ? { id: input.id } : {}),
|
|
33
|
-
...(input.blobId !== undefined ? { blobId: input.blobId } : {}),
|
|
34
|
-
...(input.sourcePath !== undefined ? { sourcePath: input.sourcePath } : {}),
|
|
35
|
-
...(input.sourceUrl !== undefined ? { sourceUrl: input.sourceUrl } : {}),
|
|
36
|
-
...(input.mimeType !== undefined ? { mimeType: input.mimeType } : {}),
|
|
37
|
-
...(input.artifactId !== undefined ? { artifactId: input.artifactId } : {}),
|
|
38
|
-
...(input.attachmentId !== undefined ? { attachmentId: input.attachmentId } : {}),
|
|
39
|
-
...(input.messageId !== undefined ? { messageId: input.messageId } : {}),
|
|
40
|
-
...(input.chatId !== undefined ? { chatId: input.chatId } : {})
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
async function extractText(kind, bytes, fileName, opts) {
|
|
44
|
-
if (kind === RAG_DOCUMENT_KIND.pdf) {
|
|
45
|
-
const extractPdf = opts.extractors?.pdf ?? extractPdfText;
|
|
46
|
-
try {
|
|
47
|
-
return await extractPdf(bytes);
|
|
48
|
-
}
|
|
49
|
-
catch (e) {
|
|
50
|
-
throw new Error(`PDF extraction failed: ${errorMessage(e)}`);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
if (kind === RAG_DOCUMENT_KIND.office) {
|
|
54
|
-
const extractOffice = opts.extractors?.office;
|
|
55
|
-
if (!extractOffice) {
|
|
56
|
-
throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.unsupportedFile, 'Office document extraction is not available on this device');
|
|
57
|
-
}
|
|
58
|
-
try {
|
|
59
|
-
return await extractOffice(bytes, fileName, { signal: opts.signal });
|
|
60
|
-
}
|
|
61
|
-
catch (e) {
|
|
62
|
-
throw new Error(`Office document extraction failed: ${errorMessage(e)}`);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
const text = decodeTextBytes(bytes);
|
|
66
|
-
if (kind === RAG_DOCUMENT_KIND.rtf)
|
|
67
|
-
return extractRtfText(text);
|
|
68
|
-
if (kind === RAG_DOCUMENT_KIND.html)
|
|
69
|
-
return extractHtmlText(text);
|
|
70
|
-
if (kind === RAG_DOCUMENT_KIND.csv)
|
|
71
|
-
return extractCsvText(text);
|
|
72
|
-
if (kind === RAG_DOCUMENT_KIND.tsv)
|
|
73
|
-
return extractCsvText(text, '\t');
|
|
74
|
-
return text;
|
|
75
|
-
}
|
|
76
|
-
function fileNameFromPath(path) {
|
|
77
|
-
return path.split(/[\\/]/).filter(Boolean).pop() ?? path;
|
|
78
|
-
}
|
|
79
|
-
function errorMessage(error) {
|
|
80
|
-
return error instanceof Error ? error.message : String(error);
|
|
81
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function formatHtmlTables(source: string): string;
|
|
@@ -1,125 +0,0 @@
|
|
|
1
|
-
const TABLE_BLOCK = /<table\b[^>]*>[\s\S]*?<\/table\s*>/gi;
|
|
2
|
-
const CAPTION_BLOCK = /<caption\b[^>]*>([\s\S]*?)<\/caption\s*>/i;
|
|
3
|
-
const TABLE_TAG = /<\/?(tr|th|td)\b[^>]*>/gi;
|
|
4
|
-
const DROP_BLOCKS = /<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
|
|
5
|
-
const COMMENT = /<!--[\s\S]*?-->/g;
|
|
6
|
-
const BREAK_TAG = /<br\s*\/?>/gi;
|
|
7
|
-
const TAG = /<[^>]*>/g;
|
|
8
|
-
export function formatHtmlTables(source) {
|
|
9
|
-
return source.replace(TABLE_BLOCK, (table) => `\n\n${formatTable(table)}\n\n`);
|
|
10
|
-
}
|
|
11
|
-
function formatTable(table) {
|
|
12
|
-
const caption = extractCaption(table);
|
|
13
|
-
const rowsText = formatTableRows(table.replace(CAPTION_BLOCK, ' '));
|
|
14
|
-
return caption ? joinTableParts(caption, rowsText) : rowsText;
|
|
15
|
-
}
|
|
16
|
-
function extractCaption(table) {
|
|
17
|
-
const caption = normalizeCell(CAPTION_BLOCK.exec(table)?.[1] ?? '');
|
|
18
|
-
return caption.length > 0 ? caption : null;
|
|
19
|
-
}
|
|
20
|
-
function joinTableParts(caption, rowsText) {
|
|
21
|
-
return rowsText.length > 0 ? `${caption}\n\n${rowsText}` : caption;
|
|
22
|
-
}
|
|
23
|
-
function formatTableRows(table) {
|
|
24
|
-
const rows = parseRows(table).filter((row) => row.some((cell) => cell.text.length > 0));
|
|
25
|
-
const first = rows[0];
|
|
26
|
-
if (!first)
|
|
27
|
-
return normalizeCell(table);
|
|
28
|
-
if (!isColumnHeaderRow(first, rows.length))
|
|
29
|
-
return formatFlatRows(rows);
|
|
30
|
-
const headers = first.map((cell, index) => cell.text || `column_${index + 1}`);
|
|
31
|
-
const dataRows = rows
|
|
32
|
-
.slice(1)
|
|
33
|
-
.map((row) => formatDataRow(headers, row))
|
|
34
|
-
.filter((line) => line.length > 0);
|
|
35
|
-
return dataRows.length > 0
|
|
36
|
-
? [`Columns: ${headers.join(', ')}`, ...dataRows].join('\n\n')
|
|
37
|
-
: formatFlatRows([first]);
|
|
38
|
-
}
|
|
39
|
-
function parseRows(table) {
|
|
40
|
-
const rows = [];
|
|
41
|
-
let row = null;
|
|
42
|
-
let cell = null;
|
|
43
|
-
let cursor = 0;
|
|
44
|
-
for (const match of table.matchAll(TABLE_TAG)) {
|
|
45
|
-
if (cell) {
|
|
46
|
-
cell.html += table.slice(cursor, match.index);
|
|
47
|
-
}
|
|
48
|
-
;
|
|
49
|
-
({ row, cell } = applyTableTag(rows, row, cell, match[0] ?? ''));
|
|
50
|
-
cursor = match.index + match[0].length;
|
|
51
|
-
}
|
|
52
|
-
if (cell) {
|
|
53
|
-
cell.html += table.slice(cursor);
|
|
54
|
-
}
|
|
55
|
-
finishRow(rows, row, cell);
|
|
56
|
-
return rows;
|
|
57
|
-
}
|
|
58
|
-
function applyTableTag(rows, row, cell, source) {
|
|
59
|
-
const tag = parseTableTag(source);
|
|
60
|
-
if (tag?.name === 'tr') {
|
|
61
|
-
return tag.closing ? finishRow(rows, row, cell) : startRow(rows, row, cell);
|
|
62
|
-
}
|
|
63
|
-
if (tag?.name === 'th' || tag?.name === 'td') {
|
|
64
|
-
return tag.closing ? finishCell(row, cell) : startCell(row, cell, tag.name === 'th');
|
|
65
|
-
}
|
|
66
|
-
return { row, cell };
|
|
67
|
-
}
|
|
68
|
-
function startRow(rows, row, cell) {
|
|
69
|
-
finishRow(rows, row, cell);
|
|
70
|
-
return { row: [], cell: null };
|
|
71
|
-
}
|
|
72
|
-
function startCell(row, cell, header) {
|
|
73
|
-
const state = finishCell(row, cell);
|
|
74
|
-
return { row: state.row ?? [], cell: { html: '', header } };
|
|
75
|
-
}
|
|
76
|
-
function finishRow(rows, row, cell) {
|
|
77
|
-
const state = finishCell(row, cell);
|
|
78
|
-
if (state.row && state.row.length > 0) {
|
|
79
|
-
rows.push(state.row);
|
|
80
|
-
}
|
|
81
|
-
return { row: null, cell: null };
|
|
82
|
-
}
|
|
83
|
-
function finishCell(row, cell) {
|
|
84
|
-
if (!cell)
|
|
85
|
-
return { row, cell: null };
|
|
86
|
-
const nextRow = row ?? [];
|
|
87
|
-
nextRow.push({ text: normalizeCell(cell.html), header: cell.header });
|
|
88
|
-
return { row: nextRow, cell: null };
|
|
89
|
-
}
|
|
90
|
-
function parseTableTag(tag) {
|
|
91
|
-
const match = /^<\s*(\/)?\s*(tr|th|td)\b/i.exec(tag);
|
|
92
|
-
return match ? { name: match[2] ?? '', closing: Boolean(match[1]) } : null;
|
|
93
|
-
}
|
|
94
|
-
function isColumnHeaderRow(row, rowCount) {
|
|
95
|
-
return rowCount > 1 && row.length >= 2 && row.every((cell) => cell.header);
|
|
96
|
-
}
|
|
97
|
-
function formatDataRow(headers, row) {
|
|
98
|
-
const parts = [];
|
|
99
|
-
const columnCount = Math.max(headers.length, row.length);
|
|
100
|
-
for (let index = 0; index < columnCount; index++) {
|
|
101
|
-
const value = row[index]?.text ?? '';
|
|
102
|
-
if (value.length === 0)
|
|
103
|
-
continue;
|
|
104
|
-
parts.push(`${headers[index] ?? `column_${index + 1}`}: ${value}`);
|
|
105
|
-
}
|
|
106
|
-
return parts.join(' | ');
|
|
107
|
-
}
|
|
108
|
-
function formatFlatRows(rows) {
|
|
109
|
-
return rows
|
|
110
|
-
.map((row) => row
|
|
111
|
-
.map((cell) => cell.text)
|
|
112
|
-
.filter((cell) => cell.length > 0)
|
|
113
|
-
.join(' | '))
|
|
114
|
-
.filter((line) => line.length > 0)
|
|
115
|
-
.join('\n');
|
|
116
|
-
}
|
|
117
|
-
function normalizeCell(html) {
|
|
118
|
-
return html
|
|
119
|
-
.replace(DROP_BLOCKS, ' ')
|
|
120
|
-
.replace(COMMENT, ' ')
|
|
121
|
-
.replace(BREAK_TAG, ' ')
|
|
122
|
-
.replace(TAG, ' ')
|
|
123
|
-
.replace(/\s+/g, ' ')
|
|
124
|
-
.trim();
|
|
125
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function extractHtmlText(source: string): string;
|