@qvac/core 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-lib/lib/api/agents.js +2 -3
- package/dist-lib/lib/api/chat.d.ts +3 -1
- package/dist-lib/lib/api/chat.js +9 -5
- package/dist-lib/lib/api/chunks.d.ts +1 -1
- package/dist-lib/lib/api/chunks.js +5 -15
- package/dist-lib/lib/api/current-mesh.d.ts +16 -1
- package/dist-lib/lib/api/current-mesh.js +18 -0
- package/dist-lib/lib/api/devices.js +6 -3
- package/dist-lib/lib/api/index.d.ts +2 -0
- package/dist-lib/lib/api/index.js +4 -2
- package/dist-lib/lib/api/inference-calls.d.ts +18 -0
- package/dist-lib/lib/api/inference-calls.js +15 -0
- package/dist-lib/lib/api/knowledge.js +11 -7
- package/dist-lib/lib/api/members.js +2 -3
- package/dist-lib/lib/api/rag.js +10 -2
- package/dist-lib/lib/api/sync.d.ts +1 -1
- package/dist-lib/lib/api/sync.js +2 -1
- package/dist-lib/lib/api/tool-activity.js +2 -2
- package/dist-lib/lib/api/tool-configs.js +2 -2
- package/dist-lib/lib/api/watchable.d.ts +6 -1
- package/dist-lib/lib/api/watchable.js +13 -2
- package/dist-lib/lib/assistant/harness.d.ts +1 -0
- package/dist-lib/lib/assistant/history.d.ts +1 -2
- package/dist-lib/lib/assistant/history.js +0 -6
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +11 -2
- package/dist-lib/lib/assistant.d.ts +17 -8
- package/dist-lib/lib/assistant.js +172 -125
- package/dist-lib/lib/core/chunk.d.ts +1 -0
- package/dist-lib/lib/core/chunk.js +32 -0
- package/dist-lib/lib/core/contract/devices.js +3 -2
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
- package/dist-lib/lib/core/files/rag/build.js +1 -2
- package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
- package/dist-lib/lib/core/files/rag/index.js +3 -17
- package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
- package/dist-lib/lib/core/files/rag/load.js +1 -2
- package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -3
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
- package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
- package/dist-lib/lib/core/files/rag/similarity.js +6 -0
- package/dist-lib/lib/core/files/upload.d.ts +1 -1
- package/dist-lib/lib/core/harness-loan.js +1 -1
- package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
- package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
- package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
- package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
- package/dist-lib/lib/core/local/index.js +6 -0
- package/dist-lib/lib/core/notification.js +5 -4
- package/dist-lib/lib/core.d.ts +9 -1
- package/dist-lib/lib/core.js +98 -16
- package/dist-lib/lib/fs-drive-bridge.js +1 -1
- package/dist-lib/lib/inference-calls.d.ts +40 -0
- package/dist-lib/lib/inference-calls.js +266 -0
- package/dist-lib/lib/mesh-scoped-engine.js +5 -0
- package/dist-lib/lib/model-download.d.ts +2 -0
- package/dist-lib/lib/model-download.js +20 -5
- package/dist-lib/lib/oauth/keys.js +1 -0
- package/dist-lib/lib/oauth/spotify.js +1 -0
- package/dist-lib/lib/oauth/verify.d.ts +8 -0
- package/dist-lib/lib/oauth/verify.js +58 -0
- package/dist-lib/lib/oauth.d.ts +1 -0
- package/dist-lib/lib/oauth.js +1 -0
- package/dist-lib/schema/content.js +5 -1
- package/dist-lib/schema/engine.js +42 -1
- package/dist-lib/schema/qvac.d.ts +3 -0
- package/dist-lib/schema/qvac.js +11 -2
- package/dist-lib/spec/bind.js +3 -0
- package/dist-lib/spec/calls.js +1 -0
- package/dist-lib/spec/capabilities.d.ts +2 -0
- package/dist-lib/spec/hrpc/hrpc.json +13 -0
- package/dist-lib/spec/hrpc/index.js +15 -3
- package/dist-lib/spec/hrpc/messages.js +172 -49
- package/dist-lib/spec/hyperschema/index.js +172 -49
- package/dist-lib/spec/hyperschema/schema.json +127 -1
- package/dist-lib/spec/hyperschema/types.d.ts +34 -1
- package/dist-lib/spec/local/hyperdb/messages.js +64 -128
- package/dist-lib/spec/local/hyperschema/index.js +32 -64
- package/dist-lib/spec/local/hyperschema/schema.json +16 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
- package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
- package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +14 -17
- package/tui/app.mjs +327 -46
- package/tui/lib/knowledge-import.mjs +1 -1
- package/tui/lib/render.mjs +17 -1
- package/tui/lib/selection.mjs +16 -0
- package/tui/lib/usage.mjs +54 -0
- package/.claude/skills/asana/SKILL.md +0 -86
- package/.claude/skills/compaction/SKILL.md +0 -132
- package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
- package/.claude/skills/gh-stack/SKILL.md +0 -181
- package/.claude/skills/gh-stack/references/commands.md +0 -177
- package/.claude/skills/gh-stack/references/stack-design.md +0 -97
- package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
- package/.claude/skills/js-house-style/SKILL.md +0 -670
- package/.claude/skills/pr-create/SKILL.md +0 -201
- package/.claude/skills/pr-review/SKILL.md +0 -424
- package/CLAUDE.md +0 -43
- package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
- package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
- package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
- package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
- package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
- package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
- package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
- package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
- package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
- package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
- package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
- package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
- package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
- package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
- package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
- package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
- package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
- package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
- package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
- package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
- package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
- package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
- package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
- package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
- package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
- package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
- package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
- package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
- package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
- package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
- package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
- package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
- package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
- package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
- package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
- package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
- package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
- package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
- package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
- package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
- package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
- package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
- package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
- package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
- package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
- package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
- package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
- package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
- package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
- package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
- package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
- package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
- package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
- package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
- package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
- package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
- package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
import b4a from 'b4a';
|
|
2
|
-
import { ARTIFACT_FORMAT_VERSION, assertRagArtifactPayload } from './types.js';
|
|
3
|
-
const MAGIC = b4a.from(ARTIFACT_FORMAT_VERSION, 'ascii');
|
|
4
|
-
const HEADER_LENGTH_BYTES = 4;
|
|
5
|
-
export function encodeRagArtifact(payload) {
|
|
6
|
-
assertRagArtifactPayload(payload);
|
|
7
|
-
assertPayloadDimensions(payload);
|
|
8
|
-
const sections = buildSections(payload);
|
|
9
|
-
const header = b4a.from(JSON.stringify(sections.header), 'utf8');
|
|
10
|
-
const headerLength = b4a.alloc(HEADER_LENGTH_BYTES);
|
|
11
|
-
b4a.writeUInt32LE(headerLength, header.byteLength, 0);
|
|
12
|
-
return b4a.concat([
|
|
13
|
-
MAGIC,
|
|
14
|
-
headerLength,
|
|
15
|
-
header,
|
|
16
|
-
sections.text,
|
|
17
|
-
sections.metadata,
|
|
18
|
-
sections.embeddings
|
|
19
|
-
]);
|
|
20
|
-
}
|
|
21
|
-
export function decodeRagArtifact(bytes) {
|
|
22
|
-
assertMagic(bytes);
|
|
23
|
-
const headerLengthOffset = MAGIC.byteLength;
|
|
24
|
-
const headerLength = b4a.readUInt32LE(bytes, headerLengthOffset);
|
|
25
|
-
const headerStart = headerLengthOffset + HEADER_LENGTH_BYTES;
|
|
26
|
-
const headerEnd = headerStart + headerLength;
|
|
27
|
-
let parsed;
|
|
28
|
-
try {
|
|
29
|
-
parsed = JSON.parse(b4a.toString(bytes.subarray(headerStart, headerEnd), 'utf8'));
|
|
30
|
-
}
|
|
31
|
-
catch {
|
|
32
|
-
throw new Error('RAG artifact header is not valid JSON');
|
|
33
|
-
}
|
|
34
|
-
const header = parseArtifactHeader(parsed);
|
|
35
|
-
const textStart = headerEnd;
|
|
36
|
-
const metadataStart = textStart + header.textByteLength;
|
|
37
|
-
const embeddingStart = metadataStart + header.metadataByteLength;
|
|
38
|
-
assertExpectedLength(bytes, header, embeddingStart);
|
|
39
|
-
return {
|
|
40
|
-
modelName: header.modelName,
|
|
41
|
-
dimensions: header.dimensions,
|
|
42
|
-
chunks: header.chunks.map((chunk, index) => decodeChunk(bytes, header, chunk, index, textStart, metadataStart, embeddingStart))
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
function buildSections(payload) {
|
|
46
|
-
const textBuffers = [];
|
|
47
|
-
const metadataBuffers = [];
|
|
48
|
-
const chunks = [];
|
|
49
|
-
let contentOffset = 0;
|
|
50
|
-
let metadataOffset = 0;
|
|
51
|
-
for (const chunk of payload.chunks) {
|
|
52
|
-
const content = b4a.from(chunk.content, 'utf8');
|
|
53
|
-
const metadata = b4a.from(chunk.metadataJson, 'utf8');
|
|
54
|
-
chunks.push({
|
|
55
|
-
id: chunk.id,
|
|
56
|
-
contentOffset,
|
|
57
|
-
contentLength: content.byteLength,
|
|
58
|
-
metadataOffset,
|
|
59
|
-
metadataLength: metadata.byteLength
|
|
60
|
-
});
|
|
61
|
-
textBuffers.push(content);
|
|
62
|
-
metadataBuffers.push(metadata);
|
|
63
|
-
contentOffset += content.byteLength;
|
|
64
|
-
metadataOffset += metadata.byteLength;
|
|
65
|
-
}
|
|
66
|
-
return {
|
|
67
|
-
header: {
|
|
68
|
-
modelName: payload.modelName,
|
|
69
|
-
dimensions: payload.dimensions,
|
|
70
|
-
chunkCount: payload.chunks.length,
|
|
71
|
-
textByteLength: contentOffset,
|
|
72
|
-
metadataByteLength: metadataOffset,
|
|
73
|
-
chunks
|
|
74
|
-
},
|
|
75
|
-
text: b4a.concat(textBuffers),
|
|
76
|
-
metadata: b4a.concat(metadataBuffers),
|
|
77
|
-
embeddings: encodeEmbeddings(payload)
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
function encodeEmbeddings(payload) {
|
|
81
|
-
const embeddings = b4a.alloc(payload.chunks.length * payload.dimensions * 4);
|
|
82
|
-
let offset = 0;
|
|
83
|
-
for (const chunk of payload.chunks) {
|
|
84
|
-
for (const value of chunk.embedding) {
|
|
85
|
-
b4a.writeFloatLE(embeddings, finite(value), offset);
|
|
86
|
-
offset += 4;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return embeddings;
|
|
90
|
-
}
|
|
91
|
-
function decodeChunk(bytes, header, chunk, index, textStart, metadataStart, embeddingStart) {
|
|
92
|
-
return {
|
|
93
|
-
id: chunk.id,
|
|
94
|
-
content: b4a.toString(bytes.subarray(textStart + chunk.contentOffset, textStart + chunk.contentOffset + chunk.contentLength), 'utf8'),
|
|
95
|
-
metadataJson: b4a.toString(bytes.subarray(metadataStart + chunk.metadataOffset, metadataStart + chunk.metadataOffset + chunk.metadataLength), 'utf8'),
|
|
96
|
-
embedding: decodeEmbedding(bytes, header.dimensions, embeddingStart, index)
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
function decodeEmbedding(bytes, dimensions, embeddingStart, chunkIndex) {
|
|
100
|
-
const values = [];
|
|
101
|
-
let offset = embeddingStart + chunkIndex * dimensions * 4;
|
|
102
|
-
for (let index = 0; index < dimensions; index++) {
|
|
103
|
-
const value = b4a.readFloatLE(bytes, offset);
|
|
104
|
-
if (!Number.isFinite(value)) {
|
|
105
|
-
throw new Error('RAG artifact has a non-finite embedding value');
|
|
106
|
-
}
|
|
107
|
-
values.push(value);
|
|
108
|
-
offset += 4;
|
|
109
|
-
}
|
|
110
|
-
return values;
|
|
111
|
-
}
|
|
112
|
-
function parseArtifactHeader(value) {
|
|
113
|
-
const header = value;
|
|
114
|
-
if (!header ||
|
|
115
|
-
typeof header.modelName !== 'string' ||
|
|
116
|
-
!isNonNegativeInteger(header.dimensions) ||
|
|
117
|
-
header.dimensions === 0 ||
|
|
118
|
-
!isNonNegativeInteger(header.chunkCount) ||
|
|
119
|
-
!isNonNegativeInteger(header.textByteLength) ||
|
|
120
|
-
!isNonNegativeInteger(header.metadataByteLength) ||
|
|
121
|
-
!Array.isArray(header.chunks) ||
|
|
122
|
-
!header.chunks.every(isChunkHeader)) {
|
|
123
|
-
throw new Error('RAG artifact header is malformed');
|
|
124
|
-
}
|
|
125
|
-
const parsed = header;
|
|
126
|
-
for (const chunk of parsed.chunks) {
|
|
127
|
-
if (chunk.contentOffset + chunk.contentLength > parsed.textByteLength ||
|
|
128
|
-
chunk.metadataOffset + chunk.metadataLength > parsed.metadataByteLength) {
|
|
129
|
-
throw new Error('RAG artifact chunk range is out of bounds');
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
return parsed;
|
|
133
|
-
}
|
|
134
|
-
function isChunkHeader(value) {
|
|
135
|
-
const chunk = value;
|
|
136
|
-
return Boolean(chunk &&
|
|
137
|
-
typeof chunk.id === 'string' &&
|
|
138
|
-
isNonNegativeInteger(chunk.contentOffset) &&
|
|
139
|
-
isNonNegativeInteger(chunk.contentLength) &&
|
|
140
|
-
isNonNegativeInteger(chunk.metadataOffset) &&
|
|
141
|
-
isNonNegativeInteger(chunk.metadataLength));
|
|
142
|
-
}
|
|
143
|
-
function isNonNegativeInteger(value) {
|
|
144
|
-
return typeof value === 'number' && Number.isInteger(value) && value >= 0;
|
|
145
|
-
}
|
|
146
|
-
function assertPayloadDimensions(payload) {
|
|
147
|
-
for (const chunk of payload.chunks) {
|
|
148
|
-
if (chunk.embedding.length !== payload.dimensions) {
|
|
149
|
-
throw new Error(`Chunk ${chunk.id} has ${chunk.embedding.length} dimensions`);
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
function assertMagic(bytes) {
|
|
154
|
-
if (bytes.byteLength < MAGIC.byteLength + HEADER_LENGTH_BYTES) {
|
|
155
|
-
throw new Error('RAG artifact is truncated');
|
|
156
|
-
}
|
|
157
|
-
if (!b4a.equals(bytes.subarray(0, MAGIC.byteLength), MAGIC)) {
|
|
158
|
-
throw new Error('RAG artifact has an unsupported format');
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
function assertExpectedLength(bytes, header, embeddingStart) {
|
|
162
|
-
const expected = embeddingStart + header.chunkCount * header.dimensions * 4;
|
|
163
|
-
if (bytes.byteLength !== expected || header.chunkCount !== header.chunks.length) {
|
|
164
|
-
throw new Error('RAG artifact length does not match its header');
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
function finite(value) {
|
|
168
|
-
if (!Number.isFinite(value)) {
|
|
169
|
-
throw new Error('Embedding value must be finite');
|
|
170
|
-
}
|
|
171
|
-
return value;
|
|
172
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export type ArtifactIdInput = {
|
|
2
|
-
readonly attachmentId: string;
|
|
3
|
-
readonly sourceBlobId: string;
|
|
4
|
-
readonly sourceHash: string;
|
|
5
|
-
readonly embeddingModel: string;
|
|
6
|
-
};
|
|
7
|
-
export declare function contentHash(bytes: Buffer): string;
|
|
8
|
-
export declare function documentIndexArtifactId(input: ArtifactIdInput): string;
|
|
9
|
-
export declare function pipelineVersionFor(model: string): string;
|
|
10
|
-
export declare function stableHashString(value: string): string;
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import b4a from 'b4a';
|
|
2
|
-
import { CHUNKER_VERSION, EMBEDDER_CODE_VERSION, EXTRACTOR_VERSION } from './types.js';
|
|
3
|
-
export function contentHash(bytes) {
|
|
4
|
-
return stableHashBytes(bytes);
|
|
5
|
-
}
|
|
6
|
-
export function documentIndexArtifactId(input) {
|
|
7
|
-
return [
|
|
8
|
-
'docidx',
|
|
9
|
-
stableHashString([
|
|
10
|
-
input.attachmentId,
|
|
11
|
-
input.sourceBlobId,
|
|
12
|
-
input.sourceHash,
|
|
13
|
-
input.embeddingModel,
|
|
14
|
-
EMBEDDER_CODE_VERSION,
|
|
15
|
-
EXTRACTOR_VERSION,
|
|
16
|
-
CHUNKER_VERSION
|
|
17
|
-
].join('\u0000'))
|
|
18
|
-
].join('-');
|
|
19
|
-
}
|
|
20
|
-
export function pipelineVersionFor(model) {
|
|
21
|
-
return `${model}|${EMBEDDER_CODE_VERSION}|${EXTRACTOR_VERSION}|${CHUNKER_VERSION}`;
|
|
22
|
-
}
|
|
23
|
-
export function stableHashString(value) {
|
|
24
|
-
return stableHashBytes(b4a.from(value, 'utf8'));
|
|
25
|
-
}
|
|
26
|
-
function stableHashBytes(bytes) {
|
|
27
|
-
let high = 0x811c9dc5;
|
|
28
|
-
let low = 0x01000193;
|
|
29
|
-
for (const byte of bytes) {
|
|
30
|
-
high ^= byte;
|
|
31
|
-
high = Math.imul(high, 0x01000193);
|
|
32
|
-
low ^= byte + 0x9e3779b9;
|
|
33
|
-
low = Math.imul(low, 0x85ebca6b);
|
|
34
|
-
}
|
|
35
|
-
return `${hex32(high)}${hex32(low)}`;
|
|
36
|
-
}
|
|
37
|
-
function hex32(value) {
|
|
38
|
-
return (value >>> 0).toString(16).padStart(8, '0');
|
|
39
|
-
}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
export declare const DOCUMENT_INDEX_ARTIFACT_STATUS: {
|
|
2
|
-
readonly pending: 'pending';
|
|
3
|
-
readonly indexing: 'indexing';
|
|
4
|
-
readonly ready: 'ready';
|
|
5
|
-
readonly failed: 'failed';
|
|
6
|
-
};
|
|
7
|
-
export type DocumentIndexArtifactStatus = (typeof DOCUMENT_INDEX_ARTIFACT_STATUS)[keyof typeof DOCUMENT_INDEX_ARTIFACT_STATUS];
|
|
8
|
-
export declare const EXTRACTOR_VERSION = "document-extractor-v1";
|
|
9
|
-
export declare const CHUNKER_VERSION = "rag-chunker-v1";
|
|
10
|
-
export declare const ARTIFACT_FORMAT_VERSION = "QVRAGIDX1";
|
|
11
|
-
export declare const EMBEDDER_CODE_VERSION = "embedder-v1";
|
|
12
|
-
export type RagArtifactChunk = {
|
|
13
|
-
readonly id: string;
|
|
14
|
-
readonly content: string;
|
|
15
|
-
readonly metadataJson: string;
|
|
16
|
-
readonly embedding: ReadonlyArray<number>;
|
|
17
|
-
};
|
|
18
|
-
export type RagArtifactPayload = {
|
|
19
|
-
readonly modelName: string;
|
|
20
|
-
readonly dimensions: number;
|
|
21
|
-
readonly chunks: ReadonlyArray<RagArtifactChunk>;
|
|
22
|
-
};
|
|
23
|
-
export declare function isDocumentIndexArtifactStatus(value: unknown): value is DocumentIndexArtifactStatus;
|
|
24
|
-
export declare function assertRagArtifactPayload(payload: RagArtifactPayload): void;
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
export const DOCUMENT_INDEX_ARTIFACT_STATUS = {
|
|
2
|
-
pending: 'pending',
|
|
3
|
-
indexing: 'indexing',
|
|
4
|
-
ready: 'ready',
|
|
5
|
-
failed: 'failed'
|
|
6
|
-
};
|
|
7
|
-
export const EXTRACTOR_VERSION = 'document-extractor-v1';
|
|
8
|
-
export const CHUNKER_VERSION = 'rag-chunker-v1';
|
|
9
|
-
export const ARTIFACT_FORMAT_VERSION = 'QVRAGIDX1';
|
|
10
|
-
export const EMBEDDER_CODE_VERSION = 'embedder-v1';
|
|
11
|
-
export function isDocumentIndexArtifactStatus(value) {
|
|
12
|
-
return typeof value === 'string' && value in DOCUMENT_INDEX_ARTIFACT_STATUS;
|
|
13
|
-
}
|
|
14
|
-
export function assertRagArtifactPayload(payload) {
|
|
15
|
-
if (typeof payload.modelName !== 'string' || payload.modelName.length === 0) {
|
|
16
|
-
throw new Error('RAG artifact payload needs a model name');
|
|
17
|
-
}
|
|
18
|
-
if (!Number.isInteger(payload.dimensions) || payload.dimensions <= 0) {
|
|
19
|
-
throw new Error('RAG artifact payload needs positive integer dimensions');
|
|
20
|
-
}
|
|
21
|
-
if (!Array.isArray(payload.chunks)) {
|
|
22
|
-
throw new Error('RAG artifact payload needs a chunk array');
|
|
23
|
-
}
|
|
24
|
-
for (const chunk of payload.chunks) {
|
|
25
|
-
if (chunk === null || typeof chunk !== 'object') {
|
|
26
|
-
throw new Error('RAG artifact chunk must be an object');
|
|
27
|
-
}
|
|
28
|
-
if (typeof chunk.id !== 'string' || chunk.id.length === 0) {
|
|
29
|
-
throw new Error('RAG artifact chunk needs an id');
|
|
30
|
-
}
|
|
31
|
-
if (typeof chunk.content !== 'string' || typeof chunk.metadataJson !== 'string') {
|
|
32
|
-
throw new Error(`RAG artifact chunk ${chunk.id} needs string content and metadata`);
|
|
33
|
-
}
|
|
34
|
-
if (!Array.isArray(chunk.embedding) || !chunk.embedding.every(isNumber)) {
|
|
35
|
-
throw new Error(`RAG artifact chunk ${chunk.id} needs a numeric embedding`);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
function isNumber(value) {
|
|
40
|
-
return typeof value === 'number';
|
|
41
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { RagDocumentInput } from '../extract/types.ts';
|
|
2
|
-
export type RagPreparedChunk = {
|
|
3
|
-
readonly id: string;
|
|
4
|
-
readonly content: string;
|
|
5
|
-
readonly tokenCount: number;
|
|
6
|
-
readonly metadataJson: string;
|
|
7
|
-
};
|
|
8
|
-
export type ChunkOptions = {
|
|
9
|
-
readonly chunkSize?: number;
|
|
10
|
-
readonly chunkOverlap?: number;
|
|
11
|
-
};
|
|
12
|
-
export declare function chunkRagDocuments(documents: ReadonlyArray<RagDocumentInput>, options: ChunkOptions): ReadonlyArray<RagPreparedChunk>;
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
import { buildRagChunkMetadata } from './metadata.js';
|
|
2
|
-
import { tokenizeForRag } from './tokenizer.js';
|
|
3
|
-
const DEFAULT_CHUNK_SIZE = 256;
|
|
4
|
-
const DEFAULT_CHUNK_OVERLAP = 50;
|
|
5
|
-
export function chunkRagDocuments(documents, options) {
|
|
6
|
-
return documents.flatMap((document) => chunkDocument(document, normalizeChunkOptions(options)));
|
|
7
|
-
}
|
|
8
|
-
function chunkDocument(document, options) {
|
|
9
|
-
const tokens = tokenizeForRag(document.content);
|
|
10
|
-
if (tokens.length === 0) {
|
|
11
|
-
return [];
|
|
12
|
-
}
|
|
13
|
-
const chunks = [];
|
|
14
|
-
const paragraphEnds = paragraphEndIndexes(tokens);
|
|
15
|
-
for (let start = skipBreakTokens(tokens, 0); start < tokens.length;) {
|
|
16
|
-
const end = chunkEnd(start, tokens.length, paragraphEnds, options.chunkSize);
|
|
17
|
-
const first = tokens[start];
|
|
18
|
-
const last = tokens[end - 1];
|
|
19
|
-
if (!first || !last) {
|
|
20
|
-
break;
|
|
21
|
-
}
|
|
22
|
-
const content = document.content.slice(first.start, last.end).trim();
|
|
23
|
-
chunks.push(buildChunk(document, chunks.length, content, end - start));
|
|
24
|
-
if (end === tokens.length) {
|
|
25
|
-
break;
|
|
26
|
-
}
|
|
27
|
-
start = skipBreakTokens(tokens, nextStart(start, end, options.chunkOverlap));
|
|
28
|
-
}
|
|
29
|
-
return chunks;
|
|
30
|
-
}
|
|
31
|
-
function paragraphEndIndexes(tokens) {
|
|
32
|
-
const ends = [];
|
|
33
|
-
let paragraphStart = 0;
|
|
34
|
-
let breakStart = null;
|
|
35
|
-
let newlineCount = 0;
|
|
36
|
-
tokens.forEach((token, index) => {
|
|
37
|
-
const newlines = countNewlines(token.text);
|
|
38
|
-
if (newlines === 0 && token.text.trim().length > 0) {
|
|
39
|
-
breakStart = null;
|
|
40
|
-
newlineCount = 0;
|
|
41
|
-
return;
|
|
42
|
-
}
|
|
43
|
-
if (newlines === 0) {
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
breakStart ??= index;
|
|
47
|
-
newlineCount += newlines;
|
|
48
|
-
if (newlineCount < 2) {
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
if (paragraphStart < breakStart) {
|
|
52
|
-
ends.push(breakStart);
|
|
53
|
-
}
|
|
54
|
-
paragraphStart = index + 1;
|
|
55
|
-
breakStart = null;
|
|
56
|
-
newlineCount = 0;
|
|
57
|
-
});
|
|
58
|
-
return [...ends, tokens.length];
|
|
59
|
-
}
|
|
60
|
-
function chunkEnd(start, tokenCount, paragraphEnds, chunkSize) {
|
|
61
|
-
const maxEnd = Math.min(start + chunkSize, tokenCount);
|
|
62
|
-
return paragraphEnds.filter((end) => end > start && end <= maxEnd).at(-1) ?? maxEnd;
|
|
63
|
-
}
|
|
64
|
-
function nextStart(start, end, overlap) {
|
|
65
|
-
const overlapped = end - overlap;
|
|
66
|
-
return overlapped > start ? overlapped : end;
|
|
67
|
-
}
|
|
68
|
-
function skipBreakTokens(tokens, start) {
|
|
69
|
-
let next = start;
|
|
70
|
-
let token = tokens[next];
|
|
71
|
-
while (token && isParagraphSeparatorToken(token.text)) {
|
|
72
|
-
next++;
|
|
73
|
-
token = tokens[next];
|
|
74
|
-
}
|
|
75
|
-
return next;
|
|
76
|
-
}
|
|
77
|
-
function isParagraphSeparatorToken(text) {
|
|
78
|
-
return text.trim().length === 0 && text.includes('\n');
|
|
79
|
-
}
|
|
80
|
-
function countNewlines(text) {
|
|
81
|
-
return [...text].filter((char) => char === '\n').length;
|
|
82
|
-
}
|
|
83
|
-
function buildChunk(document, chunkIndex, content, tokenCount) {
|
|
84
|
-
const documentId = document.id ?? stableDocumentId(document);
|
|
85
|
-
return {
|
|
86
|
-
id: chunkId(documentId, document.chatId, chunkIndex),
|
|
87
|
-
content,
|
|
88
|
-
tokenCount,
|
|
89
|
-
metadataJson: JSON.stringify(buildRagChunkMetadata({ document, documentId, chunkIndex, tokenCount }))
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
function chunkId(documentId, chatId, chunkIndex) {
|
|
93
|
-
return chatId ? `${documentId}:${chatId}:${chunkIndex}` : `${documentId}:${chunkIndex}`;
|
|
94
|
-
}
|
|
95
|
-
function normalizeChunkOptions(options) {
|
|
96
|
-
const chunkSize = positiveInteger(options.chunkSize, DEFAULT_CHUNK_SIZE);
|
|
97
|
-
const requestedOverlap = nonNegativeInteger(options.chunkOverlap, DEFAULT_CHUNK_OVERLAP);
|
|
98
|
-
const chunkOverlap = Math.min(requestedOverlap, Math.max(0, chunkSize - 1));
|
|
99
|
-
return { chunkSize, chunkOverlap };
|
|
100
|
-
}
|
|
101
|
-
function positiveInteger(value, fallback) {
|
|
102
|
-
return Number.isInteger(value) && value !== undefined && value > 0 ? value : fallback;
|
|
103
|
-
}
|
|
104
|
-
function nonNegativeInteger(value, fallback) {
|
|
105
|
-
return Number.isInteger(value) && value !== undefined && value >= 0 ? value : fallback;
|
|
106
|
-
}
|
|
107
|
-
function stableDocumentId(document) {
|
|
108
|
-
return `doc-${fnv1a(`${document.title}\n${document.sourcePath ?? ''}\n${document.content}`)}`;
|
|
109
|
-
}
|
|
110
|
-
function fnv1a(value) {
|
|
111
|
-
let hash = 0x811c9dc5;
|
|
112
|
-
for (let i = 0; i < value.length; i++) {
|
|
113
|
-
hash ^= value.charCodeAt(i);
|
|
114
|
-
hash = Math.imul(hash, 0x01000193);
|
|
115
|
-
}
|
|
116
|
-
return (hash >>> 0).toString(16).padStart(8, '0');
|
|
117
|
-
}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import type { RagDocumentInput } from '../extract/types.ts';
|
|
2
|
-
export type RagChunkMetadata = {
|
|
3
|
-
readonly documentId: string;
|
|
4
|
-
readonly title: string;
|
|
5
|
-
readonly sourcePath: string | null;
|
|
6
|
-
readonly sourceUrl: string | null;
|
|
7
|
-
readonly mimeType: string | null;
|
|
8
|
-
readonly artifactId: string | null;
|
|
9
|
-
readonly attachmentId: string | null;
|
|
10
|
-
readonly messageId: string | null;
|
|
11
|
-
readonly chatId: string | null;
|
|
12
|
-
readonly chunkIndex: number;
|
|
13
|
-
readonly tokenCount: number;
|
|
14
|
-
readonly indexedAt: string;
|
|
15
|
-
};
|
|
16
|
-
export declare function attachmentDocumentId(attachmentId: string): string;
|
|
17
|
-
export declare function buildRagChunkMetadata(input: {
|
|
18
|
-
readonly document: RagDocumentInput;
|
|
19
|
-
readonly documentId: string;
|
|
20
|
-
readonly chunkIndex: number;
|
|
21
|
-
readonly tokenCount: number;
|
|
22
|
-
}): RagChunkMetadata;
|
|
23
|
-
export declare function parseRagMetadata(metadataJson: string | null): RagChunkMetadata | null;
|
|
24
|
-
export declare function hitMatchesFilters(hit: {
|
|
25
|
-
readonly metadataJson: string | null;
|
|
26
|
-
}, filters: {
|
|
27
|
-
readonly chatId?: string;
|
|
28
|
-
readonly documentIds?: ReadonlyArray<string>;
|
|
29
|
-
readonly attachmentIds?: ReadonlyArray<string>;
|
|
30
|
-
}, liveSourceIds?: ReadonlySet<string>): boolean;
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
const OPTIONAL_STRING_KEYS = [
|
|
2
|
-
'sourcePath',
|
|
3
|
-
'sourceUrl',
|
|
4
|
-
'mimeType',
|
|
5
|
-
'artifactId',
|
|
6
|
-
'attachmentId',
|
|
7
|
-
'messageId',
|
|
8
|
-
'chatId'
|
|
9
|
-
];
|
|
10
|
-
export function attachmentDocumentId(attachmentId) {
|
|
11
|
-
return `attachment-${attachmentId}`;
|
|
12
|
-
}
|
|
13
|
-
export function buildRagChunkMetadata(input) {
|
|
14
|
-
return {
|
|
15
|
-
documentId: input.documentId,
|
|
16
|
-
title: input.document.title,
|
|
17
|
-
sourcePath: input.document.sourcePath ?? null,
|
|
18
|
-
sourceUrl: input.document.sourceUrl ?? null,
|
|
19
|
-
mimeType: input.document.mimeType ?? null,
|
|
20
|
-
artifactId: input.document.artifactId ?? null,
|
|
21
|
-
attachmentId: input.document.attachmentId ?? null,
|
|
22
|
-
messageId: input.document.messageId ?? null,
|
|
23
|
-
chatId: input.document.chatId ?? null,
|
|
24
|
-
chunkIndex: input.chunkIndex,
|
|
25
|
-
tokenCount: input.tokenCount,
|
|
26
|
-
indexedAt: new Date().toISOString()
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
export function parseRagMetadata(metadataJson) {
|
|
30
|
-
if (!metadataJson)
|
|
31
|
-
return null;
|
|
32
|
-
let parsed;
|
|
33
|
-
try {
|
|
34
|
-
parsed = JSON.parse(metadataJson);
|
|
35
|
-
}
|
|
36
|
-
catch {
|
|
37
|
-
return null;
|
|
38
|
-
}
|
|
39
|
-
if (typeof parsed !== 'object' || parsed === null)
|
|
40
|
-
return null;
|
|
41
|
-
const value = parsed;
|
|
42
|
-
const { documentId, title, indexedAt, chunkIndex, tokenCount } = value;
|
|
43
|
-
if (typeof documentId !== 'string' || typeof title !== 'string')
|
|
44
|
-
return null;
|
|
45
|
-
if (typeof indexedAt !== 'string')
|
|
46
|
-
return null;
|
|
47
|
-
if (!isNumber(chunkIndex) || !isNumber(tokenCount))
|
|
48
|
-
return null;
|
|
49
|
-
for (const key of OPTIONAL_STRING_KEYS) {
|
|
50
|
-
const optional = value[key];
|
|
51
|
-
if (optional !== undefined && optional !== null && typeof optional !== 'string')
|
|
52
|
-
return null;
|
|
53
|
-
}
|
|
54
|
-
return {
|
|
55
|
-
documentId,
|
|
56
|
-
title,
|
|
57
|
-
sourcePath: stringOrNull(value.sourcePath),
|
|
58
|
-
sourceUrl: stringOrNull(value.sourceUrl),
|
|
59
|
-
mimeType: stringOrNull(value.mimeType),
|
|
60
|
-
artifactId: stringOrNull(value.artifactId),
|
|
61
|
-
attachmentId: stringOrNull(value.attachmentId),
|
|
62
|
-
messageId: stringOrNull(value.messageId),
|
|
63
|
-
chatId: stringOrNull(value.chatId),
|
|
64
|
-
chunkIndex,
|
|
65
|
-
tokenCount,
|
|
66
|
-
indexedAt
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
export function hitMatchesFilters(hit, filters, liveSourceIds) {
|
|
70
|
-
const metadata = parseRagMetadata(hit.metadataJson);
|
|
71
|
-
if (!metadata)
|
|
72
|
-
return false;
|
|
73
|
-
if (filters.chatId && metadata.chatId !== filters.chatId)
|
|
74
|
-
return false;
|
|
75
|
-
if (filters.documentIds?.length && !filters.documentIds.includes(metadata.documentId)) {
|
|
76
|
-
return false;
|
|
77
|
-
}
|
|
78
|
-
if (filters.attachmentIds?.length &&
|
|
79
|
-
(!metadata.attachmentId || !filters.attachmentIds.includes(metadata.attachmentId))) {
|
|
80
|
-
return false;
|
|
81
|
-
}
|
|
82
|
-
return !(liveSourceIds && metadata.attachmentId && !liveSourceIds.has(metadata.attachmentId));
|
|
83
|
-
}
|
|
84
|
-
function isNumber(value) {
|
|
85
|
-
return typeof value === 'number' && !Number.isNaN(value);
|
|
86
|
-
}
|
|
87
|
-
function stringOrNull(value) {
|
|
88
|
-
return typeof value === 'string' ? value : null;
|
|
89
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
const TOKEN_PATTERN = /'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
|
|
2
|
-
export function tokenizeForRag(text) {
|
|
3
|
-
const tokens = [];
|
|
4
|
-
TOKEN_PATTERN.lastIndex = 0;
|
|
5
|
-
let match = TOKEN_PATTERN.exec(text);
|
|
6
|
-
while (match) {
|
|
7
|
-
tokens.push({
|
|
8
|
-
text: match[0],
|
|
9
|
-
start: match.index,
|
|
10
|
-
end: match.index + match[0].length
|
|
11
|
-
});
|
|
12
|
-
match = TOKEN_PATTERN.exec(text);
|
|
13
|
-
}
|
|
14
|
-
return tokens;
|
|
15
|
-
}
|
|
16
|
-
export function estimateRagTokens(text) {
|
|
17
|
-
return tokenizeForRag(text).length;
|
|
18
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function extractCsvText(source: string, delimiter?: string): string;
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
const FIELD_SEPARATOR = ' | ';
|
|
2
|
-
const KEY_VALUE_SEPARATOR = ': ';
|
|
3
|
-
const DEFAULT_DELIMITER = ',';
|
|
4
|
-
export function extractCsvText(source, delimiter = DEFAULT_DELIMITER) {
|
|
5
|
-
const rows = parseCsv(source, delimiter);
|
|
6
|
-
if (rows.length === 0)
|
|
7
|
-
return '';
|
|
8
|
-
const headerRow = rows[0];
|
|
9
|
-
if (!headerRow || headerRow.length === 0)
|
|
10
|
-
return '';
|
|
11
|
-
const headers = headerRow
|
|
12
|
-
.map(normalizeCell)
|
|
13
|
-
.map((value, index) => (value.length > 0 ? value : `column_${index + 1}`));
|
|
14
|
-
const dataRows = rows.slice(1);
|
|
15
|
-
if (dataRows.length === 0 || headers.length < 2) {
|
|
16
|
-
return formatFlat(rows);
|
|
17
|
-
}
|
|
18
|
-
const headerLine = `Columns: ${headers.join(', ')}`;
|
|
19
|
-
const formatted = dataRows
|
|
20
|
-
.filter((row) => row.some((cell) => cell.trim().length > 0))
|
|
21
|
-
.map((row) => formatRow(headers, row));
|
|
22
|
-
return [headerLine, ...formatted].join('\n').trim();
|
|
23
|
-
}
|
|
24
|
-
function formatFlat(rows) {
|
|
25
|
-
return rows
|
|
26
|
-
.map((row) => row
|
|
27
|
-
.map(normalizeCell)
|
|
28
|
-
.filter((cell) => cell.length > 0)
|
|
29
|
-
.join(', '))
|
|
30
|
-
.filter((line) => line.length > 0)
|
|
31
|
-
.join('\n')
|
|
32
|
-
.trim();
|
|
33
|
-
}
|
|
34
|
-
function formatRow(headers, row) {
|
|
35
|
-
const parts = [];
|
|
36
|
-
for (let index = 0; index < headers.length; index++) {
|
|
37
|
-
const value = normalizeCell(row[index] ?? '');
|
|
38
|
-
if (value.length === 0)
|
|
39
|
-
continue;
|
|
40
|
-
const header = headers[index] ?? `column_${index + 1}`;
|
|
41
|
-
parts.push(`${header}${KEY_VALUE_SEPARATOR}${value}`);
|
|
42
|
-
}
|
|
43
|
-
return parts.join(FIELD_SEPARATOR);
|
|
44
|
-
}
|
|
45
|
-
function normalizeCell(value) {
|
|
46
|
-
return value.replace(/\s+/g, ' ').trim();
|
|
47
|
-
}
|
|
48
|
-
function parseCsv(source, delimiter) {
|
|
49
|
-
const state = {
|
|
50
|
-
rows: [],
|
|
51
|
-
current: [],
|
|
52
|
-
field: '',
|
|
53
|
-
inQuotes: false,
|
|
54
|
-
index: 0,
|
|
55
|
-
delimiter
|
|
56
|
-
};
|
|
57
|
-
while (state.index < source.length) {
|
|
58
|
-
if (state.inQuotes) {
|
|
59
|
-
consumeQuotedChar(state, source);
|
|
60
|
-
}
|
|
61
|
-
else {
|
|
62
|
-
consumeUnquotedChar(state, source);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
flushFinalField(state);
|
|
66
|
-
return state.rows.filter((row) => row.length > 1 || (row[0] ?? '').length > 0);
|
|
67
|
-
}
|
|
68
|
-
function consumeQuotedChar(state, source) {
|
|
69
|
-
const char = source[state.index];
|
|
70
|
-
if (char !== '"') {
|
|
71
|
-
state.field += char;
|
|
72
|
-
state.index += 1;
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
if (source[state.index + 1] === '"') {
|
|
76
|
-
state.field += '"';
|
|
77
|
-
state.index += 2;
|
|
78
|
-
return;
|
|
79
|
-
}
|
|
80
|
-
state.inQuotes = false;
|
|
81
|
-
state.index += 1;
|
|
82
|
-
}
|
|
83
|
-
function consumeUnquotedChar(state, source) {
|
|
84
|
-
const char = source[state.index];
|
|
85
|
-
if (char === '"' && state.field.length === 0) {
|
|
86
|
-
state.inQuotes = true;
|
|
87
|
-
state.index += 1;
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
90
|
-
if (char === state.delimiter) {
|
|
91
|
-
finishField(state);
|
|
92
|
-
state.index += 1;
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
if (char === '\n' || char === '\r') {
|
|
96
|
-
finishRow(state);
|
|
97
|
-
state.index += char === '\r' && source[state.index + 1] === '\n' ? 2 : 1;
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
state.field += char;
|
|
101
|
-
state.index += 1;
|
|
102
|
-
}
|
|
103
|
-
function finishField(state) {
|
|
104
|
-
state.current.push(state.field);
|
|
105
|
-
state.field = '';
|
|
106
|
-
}
|
|
107
|
-
function finishRow(state) {
|
|
108
|
-
finishField(state);
|
|
109
|
-
state.rows.push(state.current);
|
|
110
|
-
state.current = [];
|
|
111
|
-
}
|
|
112
|
-
function flushFinalField(state) {
|
|
113
|
-
if (state.field.length === 0 && state.current.length === 0)
|
|
114
|
-
return;
|
|
115
|
-
state.current.push(state.field);
|
|
116
|
-
state.rows.push(state.current);
|
|
117
|
-
}
|