@anvia/core 1.0.0-rc.1 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +344 -85
- package/dist/agent/index.d.ts +33 -18
- package/dist/agent/index.js +42 -17
- package/dist/agent-CXScUOpX.d.ts +150 -0
- package/dist/{chunk-R2LJSQUW.js → chunk-3CMWKK32.js} +260 -104
- package/dist/chunk-3CMWKK32.js.map +1 -0
- package/dist/chunk-3RM57ZT2.js +31 -0
- package/dist/chunk-3RM57ZT2.js.map +1 -0
- package/dist/chunk-5MK3BISB.js +704 -0
- package/dist/chunk-5MK3BISB.js.map +1 -0
- package/dist/chunk-AI5JMUW7.js +45 -0
- package/dist/chunk-AI5JMUW7.js.map +1 -0
- package/dist/{chunk-FNQB2JEH.js → chunk-AZB6N7P4.js} +94 -65
- package/dist/chunk-AZB6N7P4.js.map +1 -0
- package/dist/chunk-BEYBU7VJ.js +53 -0
- package/dist/chunk-BEYBU7VJ.js.map +1 -0
- package/dist/chunk-BY4OMNIU.js +4699 -0
- package/dist/chunk-BY4OMNIU.js.map +1 -0
- package/dist/chunk-D2ECOWN5.js +515 -0
- package/dist/chunk-D2ECOWN5.js.map +1 -0
- package/dist/{chunk-A3UBYTJT.js → chunk-FN7HLLAZ.js} +60 -6
- package/dist/chunk-FN7HLLAZ.js.map +1 -0
- package/dist/chunk-GKDERULJ.js +50 -0
- package/dist/chunk-GKDERULJ.js.map +1 -0
- package/dist/chunk-GQL5KRAH.js +331 -0
- package/dist/chunk-GQL5KRAH.js.map +1 -0
- package/dist/{chunk-5XKDW36Z.js → chunk-KFR4CDGK.js} +70 -75
- package/dist/chunk-KFR4CDGK.js.map +1 -0
- package/dist/chunk-KV6QCRQ4.js +171 -0
- package/dist/chunk-KV6QCRQ4.js.map +1 -0
- package/dist/{chunk-DMPUP4R3.js → chunk-LXMAFD23.js} +5 -5
- package/dist/{chunk-I6G7BC42.js → chunk-MGNTXGTX.js} +2 -6
- package/dist/chunk-MGNTXGTX.js.map +1 -0
- package/dist/chunk-PK5LKOFL.js +239 -0
- package/dist/chunk-PK5LKOFL.js.map +1 -0
- package/dist/chunk-QOUPWOGW.js +29 -0
- package/dist/chunk-QOUPWOGW.js.map +1 -0
- package/dist/chunk-TRQ3XRFD.js +54 -0
- package/dist/chunk-TRQ3XRFD.js.map +1 -0
- package/dist/chunk-VKYZZXP5.js +323 -0
- package/dist/chunk-VKYZZXP5.js.map +1 -0
- package/dist/chunk-ZCPOIDAQ.js +127 -0
- package/dist/chunk-ZCPOIDAQ.js.map +1 -0
- package/dist/client-DvUatElp.d.ts +84 -0
- package/dist/completion/index.d.ts +5 -4
- package/dist/completion/index.js +25 -28
- package/dist/documents/index.d.ts +34 -0
- package/dist/documents/index.js +315 -0
- package/dist/documents/index.js.map +1 -0
- package/dist/{dynamic-tools-i9woysFa.d.ts → dynamic-tools-AEBOTgu1.d.ts} +21 -9
- package/dist/embeddings/index.d.ts +11 -9
- package/dist/embeddings/index.js +3 -3
- package/dist/evals/index.d.ts +53 -25
- package/dist/evals/index.js +153 -103
- package/dist/evals/index.js.map +1 -1
- package/dist/extractor/index.d.ts +23 -26
- package/dist/extractor/index.js +7 -8
- package/dist/guardrails/index.d.ts +7 -129
- package/dist/guardrails/index.js +1 -1
- package/dist/image-generation/index.d.ts +18 -16
- package/dist/image-generation/index.js +2 -2
- package/dist/index.d.ts +26 -23
- package/dist/index.js +71 -48
- package/dist/interactions-csssHYcN.d.ts +174 -0
- package/dist/internal/agent.d.ts +36 -29
- package/dist/internal/agent.js +48 -15
- package/dist/internal/agent.js.map +1 -1
- package/dist/mcp/index.d.ts +19 -13
- package/dist/mcp/index.js +14 -348
- package/dist/mcp/index.js.map +1 -1
- package/dist/memory/index.d.ts +20 -6
- package/dist/memory/index.js +10 -10
- package/dist/message-schema-B2M7AYzR.d.ts +52 -0
- package/dist/model-call-options-CZkSw_xN.d.ts +6 -0
- package/dist/model-listing/index.d.ts +3 -1
- package/dist/observability/index.d.ts +21 -6
- package/dist/observability/index.js +8 -5
- package/dist/observability/index.js.map +1 -1
- package/dist/pipeline/index.d.ts +106 -57
- package/dist/pipeline/index.js +419 -236
- package/dist/pipeline/index.js.map +1 -1
- package/dist/{retry-D3Ruy-ba.d.ts → retry-CjvSlKGW.d.ts} +2 -1
- package/dist/skills/index.d.ts +5 -4
- package/dist/skills/index.js +8 -7
- package/dist/speech-generation/index.d.ts +37 -0
- package/dist/speech-generation/index.js +8 -0
- package/dist/text-C_6eKrbC.d.ts +32 -0
- package/dist/{think-tool-DZqZj2LV.d.ts → think-tool-D8frXxwV.d.ts} +16 -2
- package/dist/tool/index.d.ts +9 -7
- package/dist/tool/index.js +12 -10
- package/dist/{tool-C_8GudXn.d.ts → tool-r62gf3QY.d.ts} +17 -8
- package/dist/transcription/index.d.ts +23 -14
- package/dist/transcription/index.js +2 -2
- package/dist/types-C3gv9EX6.d.ts +130 -0
- package/dist/{index-DxB74ybJ.d.ts → types-CElwurHZ.d.ts} +139 -88
- package/dist/types-CIPaIbZo.d.ts +105 -0
- package/dist/types-Cr4uiYo5.d.ts +99 -0
- package/dist/types-D9rKQ_as.d.ts +341 -0
- package/dist/types-DbolvLtZ.d.ts +155 -0
- package/dist/{types-DhQfwiZn.d.ts → types-RT3cuyRk.d.ts} +1 -1
- package/dist/vector-store/index.d.ts +33 -34
- package/dist/vector-store/index.js +7 -6
- package/package.json +10 -14
- package/dist/agent-DGYY_onS.d.ts +0 -145
- package/dist/audio-generation/index.d.ts +0 -31
- package/dist/audio-generation/index.js +0 -8
- package/dist/chunk-5XKDW36Z.js.map +0 -1
- package/dist/chunk-6WGGTJE6.js +0 -139
- package/dist/chunk-6WGGTJE6.js.map +0 -1
- package/dist/chunk-A3UBYTJT.js.map +0 -1
- package/dist/chunk-ADH7NNCS.js +0 -512
- package/dist/chunk-ADH7NNCS.js.map +0 -1
- package/dist/chunk-DZ3CKYTH.js +0 -267
- package/dist/chunk-DZ3CKYTH.js.map +0 -1
- package/dist/chunk-FNQB2JEH.js.map +0 -1
- package/dist/chunk-I6G7BC42.js.map +0 -1
- package/dist/chunk-IL3ZSNJI.js +0 -3748
- package/dist/chunk-IL3ZSNJI.js.map +0 -1
- package/dist/chunk-J6GNQART.js +0 -122
- package/dist/chunk-J6GNQART.js.map +0 -1
- package/dist/chunk-M3IOWB4Y.js +0 -280
- package/dist/chunk-M3IOWB4Y.js.map +0 -1
- package/dist/chunk-QY5GZ7HR.js +0 -39
- package/dist/chunk-QY5GZ7HR.js.map +0 -1
- package/dist/chunk-R2LJSQUW.js.map +0 -1
- package/dist/chunk-RFDVKCIE.js +0 -12
- package/dist/chunk-RFDVKCIE.js.map +0 -1
- package/dist/chunk-RMUBRRSK.js +0 -34
- package/dist/chunk-RMUBRRSK.js.map +0 -1
- package/dist/chunk-RXKHOYEH.js +0 -466
- package/dist/chunk-RXKHOYEH.js.map +0 -1
- package/dist/chunk-XDRSPSR7.js +0 -38
- package/dist/chunk-XDRSPSR7.js.map +0 -1
- package/dist/chunk-XWUC7CIT.js +0 -1
- package/dist/chunk-XWUC7CIT.js.map +0 -1
- package/dist/errors-CYKUcU89.d.ts +0 -17
- package/dist/json-Zo7K3xmt.d.ts +0 -40
- package/dist/loaders/index.d.ts +0 -86
- package/dist/loaders/index.js +0 -299
- package/dist/loaders/index.js.map +0 -1
- package/dist/middleware-KynLg_RW.d.ts +0 -56
- package/dist/types-B3A4YxaA.d.ts +0 -354
- package/dist/types-BBEGKB9m.d.ts +0 -119
- package/dist/types-DT3nEemY.d.ts +0 -79
- package/dist/types-DulRCYdB.d.ts +0 -70
- package/dist/types-p55Jc-sr.d.ts +0 -47
- package/dist/ui/index.d.ts +0 -107
- package/dist/ui/index.js +0 -10
- package/dist/ui/index.js.map +0 -1
- /package/dist/{chunk-DMPUP4R3.js.map → chunk-LXMAFD23.js.map} +0 -0
- /package/dist/{audio-generation → speech-generation}/index.js.map +0 -0
package/dist/completion/index.js
CHANGED
|
@@ -1,56 +1,53 @@
|
|
|
1
1
|
import {
|
|
2
2
|
formatDocument,
|
|
3
3
|
normalizeDocuments
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-MGNTXGTX.js";
|
|
5
5
|
import {
|
|
6
|
-
createCompletion,
|
|
7
|
-
createCompletionStream,
|
|
8
|
-
createParsedCompletion,
|
|
9
|
-
isStreamingCompletionModel
|
|
10
|
-
} from "../chunk-M3IOWB4Y.js";
|
|
11
|
-
import "../chunk-RFDVKCIE.js";
|
|
12
|
-
import {
|
|
13
|
-
AssistantContent,
|
|
14
6
|
CompletionCapabilityError,
|
|
15
|
-
Message,
|
|
16
|
-
ToolContent,
|
|
17
7
|
Usage,
|
|
18
|
-
UserContent,
|
|
19
8
|
assertCompletionRequestSupported,
|
|
20
9
|
calculateContextUsage,
|
|
10
|
+
generateCompletion,
|
|
21
11
|
getAssistantGenerationMetadata,
|
|
22
|
-
isJsonValue,
|
|
23
12
|
isProviderTool,
|
|
24
|
-
|
|
13
|
+
isStreamingCompletionModel,
|
|
25
14
|
reasoningDisplayText,
|
|
26
|
-
|
|
27
|
-
|
|
15
|
+
resolveModelContextLimits,
|
|
16
|
+
streamCompletion,
|
|
28
17
|
textFromAssistantContent,
|
|
29
18
|
withContextUsage
|
|
30
|
-
} from "../chunk-
|
|
31
|
-
import
|
|
19
|
+
} from "../chunk-D2ECOWN5.js";
|
|
20
|
+
import {
|
|
21
|
+
createMessageSchema,
|
|
22
|
+
isJsonValue,
|
|
23
|
+
isMessage,
|
|
24
|
+
messageSchema,
|
|
25
|
+
messagesSchema,
|
|
26
|
+
parseMessage,
|
|
27
|
+
parseMessages
|
|
28
|
+
} from "../chunk-PK5LKOFL.js";
|
|
29
|
+
import "../chunk-FN7HLLAZ.js";
|
|
32
30
|
export {
|
|
33
|
-
AssistantContent,
|
|
34
31
|
CompletionCapabilityError,
|
|
35
|
-
Message,
|
|
36
|
-
ToolContent,
|
|
37
32
|
Usage,
|
|
38
|
-
UserContent,
|
|
39
33
|
assertCompletionRequestSupported,
|
|
40
34
|
calculateContextUsage,
|
|
41
|
-
|
|
42
|
-
createCompletionStream,
|
|
43
|
-
createParsedCompletion,
|
|
35
|
+
createMessageSchema,
|
|
44
36
|
formatDocument,
|
|
37
|
+
generateCompletion,
|
|
45
38
|
getAssistantGenerationMetadata,
|
|
46
39
|
isJsonValue,
|
|
40
|
+
isMessage,
|
|
47
41
|
isProviderTool,
|
|
48
42
|
isStreamingCompletionModel,
|
|
49
|
-
|
|
43
|
+
messageSchema,
|
|
44
|
+
messagesSchema,
|
|
50
45
|
normalizeDocuments,
|
|
46
|
+
parseMessage,
|
|
47
|
+
parseMessages,
|
|
51
48
|
reasoningDisplayText,
|
|
52
|
-
|
|
53
|
-
|
|
49
|
+
resolveModelContextLimits,
|
|
50
|
+
streamCompletion,
|
|
54
51
|
textFromAssistantContent,
|
|
55
52
|
withContextUsage
|
|
56
53
|
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
type TextChunk = Readonly<{
|
|
2
|
+
index: number;
|
|
3
|
+
text: string;
|
|
4
|
+
start: number;
|
|
5
|
+
end: number;
|
|
6
|
+
}>;
|
|
7
|
+
type ChunkTextOptions = Readonly<{
|
|
8
|
+
text: string;
|
|
9
|
+
strategy: "fixed";
|
|
10
|
+
maxSize: number;
|
|
11
|
+
overlap?: number;
|
|
12
|
+
}> | Readonly<{
|
|
13
|
+
text: string;
|
|
14
|
+
strategy: "recursive";
|
|
15
|
+
maxSize: number;
|
|
16
|
+
overlap?: number;
|
|
17
|
+
separators: readonly string[];
|
|
18
|
+
}>;
|
|
19
|
+
declare function chunkText(options: ChunkTextOptions): readonly TextChunk[];
|
|
20
|
+
|
|
21
|
+
type PdfTextPage = Readonly<{
|
|
22
|
+
pageNumber: number;
|
|
23
|
+
text: string;
|
|
24
|
+
}>;
|
|
25
|
+
type ExtractPdfTextOptions = Readonly<{
|
|
26
|
+
data: Uint8Array;
|
|
27
|
+
abortSignal?: AbortSignal;
|
|
28
|
+
}>;
|
|
29
|
+
type ExtractPdfTextResult = Readonly<{
|
|
30
|
+
pages: readonly PdfTextPage[];
|
|
31
|
+
}>;
|
|
32
|
+
declare function extractPdfText(options: ExtractPdfTextOptions): Promise<ExtractPdfTextResult>;
|
|
33
|
+
|
|
34
|
+
export { type ChunkTextOptions, type ExtractPdfTextOptions, type ExtractPdfTextResult, type PdfTextPage, type TextChunk, chunkText, extractPdfText };
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
// src/documents/chunk-text.ts
|
|
2
|
+
function chunkText(options) {
|
|
3
|
+
const validated = validateOptions(options);
|
|
4
|
+
if (validated.text.length === 0) {
|
|
5
|
+
return [];
|
|
6
|
+
}
|
|
7
|
+
if (validated.strategy === "fixed") {
|
|
8
|
+
return chunkFixed(validated.text, validated.maxSize, validated.overlap);
|
|
9
|
+
}
|
|
10
|
+
const ranges = resolveRecursiveRanges(
|
|
11
|
+
validated.text,
|
|
12
|
+
{ start: 0, end: validated.text.length },
|
|
13
|
+
validated.maxSize,
|
|
14
|
+
validated.separators
|
|
15
|
+
);
|
|
16
|
+
return applyRecursiveOverlap(validated.text, ranges, validated.maxSize, validated.overlap);
|
|
17
|
+
}
|
|
18
|
+
function validateOptions(options) {
|
|
19
|
+
if (typeof options !== "object" || options === null) {
|
|
20
|
+
throw new TypeError("chunkText options must be an object");
|
|
21
|
+
}
|
|
22
|
+
const value = options;
|
|
23
|
+
if (typeof value.text !== "string") {
|
|
24
|
+
throw new TypeError("chunkText text must be a string");
|
|
25
|
+
}
|
|
26
|
+
if (!Number.isSafeInteger(value.maxSize) || value.maxSize <= 0) {
|
|
27
|
+
throw new RangeError("chunkText maxSize must be a positive safe integer");
|
|
28
|
+
}
|
|
29
|
+
const maxSize = value.maxSize;
|
|
30
|
+
const overlap = value.overlap === void 0 ? 0 : value.overlap;
|
|
31
|
+
if (!Number.isSafeInteger(overlap) || overlap < 0 || overlap >= maxSize) {
|
|
32
|
+
throw new RangeError(
|
|
33
|
+
"chunkText overlap must be a non-negative safe integer smaller than maxSize"
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
if (value.strategy === "fixed") {
|
|
37
|
+
if ("separators" in value) {
|
|
38
|
+
throw new TypeError("chunkText fixed strategy does not accept separators");
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
text: value.text,
|
|
42
|
+
strategy: "fixed",
|
|
43
|
+
maxSize,
|
|
44
|
+
overlap
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
if (value.strategy !== "recursive") {
|
|
48
|
+
throw new TypeError('chunkText strategy must be either "fixed" or "recursive"');
|
|
49
|
+
}
|
|
50
|
+
if (!Array.isArray(value.separators) || value.separators.length === 0) {
|
|
51
|
+
throw new TypeError("chunkText recursive separators must be a non-empty array");
|
|
52
|
+
}
|
|
53
|
+
const separators = value.separators;
|
|
54
|
+
const seen = /* @__PURE__ */ new Set();
|
|
55
|
+
for (const separator of separators) {
|
|
56
|
+
if (typeof separator !== "string" || separator.length === 0) {
|
|
57
|
+
throw new TypeError("chunkText recursive separators must contain non-empty strings");
|
|
58
|
+
}
|
|
59
|
+
if (seen.has(separator)) {
|
|
60
|
+
throw new TypeError("chunkText recursive separators must not contain duplicates");
|
|
61
|
+
}
|
|
62
|
+
seen.add(separator);
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
text: value.text,
|
|
66
|
+
strategy: "recursive",
|
|
67
|
+
maxSize,
|
|
68
|
+
overlap,
|
|
69
|
+
separators
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function chunkFixed(text, maxSize, overlap) {
|
|
73
|
+
const chunks = [];
|
|
74
|
+
let start = 0;
|
|
75
|
+
while (start < text.length) {
|
|
76
|
+
const end = Math.min(start + maxSize, text.length);
|
|
77
|
+
chunks.push(createChunk(text, chunks.length, start, end));
|
|
78
|
+
if (end === text.length) {
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
start = end - overlap;
|
|
82
|
+
}
|
|
83
|
+
return chunks;
|
|
84
|
+
}
|
|
85
|
+
function resolveRecursiveRanges(text, initialRange, maxSize, separators) {
|
|
86
|
+
const resolved = [];
|
|
87
|
+
const pending = [
|
|
88
|
+
{ range: initialRange, separatorIndex: 0 }
|
|
89
|
+
];
|
|
90
|
+
while (pending.length > 0) {
|
|
91
|
+
const current = pending.pop();
|
|
92
|
+
if (current === void 0) {
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
if (current.range.end - current.range.start <= maxSize) {
|
|
96
|
+
resolved.push(current.range);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
let pieces;
|
|
100
|
+
let nextSeparatorIndex = separators.length;
|
|
101
|
+
for (let index = current.separatorIndex; index < separators.length; index += 1) {
|
|
102
|
+
const candidate = splitAfterSeparator(text, current.range, separators[index]);
|
|
103
|
+
if (candidate.length > 1) {
|
|
104
|
+
pieces = candidate;
|
|
105
|
+
nextSeparatorIndex = index + 1;
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
if (pieces === void 0) {
|
|
110
|
+
resolved.push(...hardSplit(current.range, maxSize));
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
for (let index = pieces.length - 1; index >= 0; index -= 1) {
|
|
114
|
+
pending.push({ range: pieces[index], separatorIndex: nextSeparatorIndex });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return resolved;
|
|
118
|
+
}
|
|
119
|
+
function splitAfterSeparator(text, range, separator) {
|
|
120
|
+
const ranges = [];
|
|
121
|
+
let start = range.start;
|
|
122
|
+
let searchFrom = range.start;
|
|
123
|
+
while (searchFrom < range.end) {
|
|
124
|
+
const separatorStart = text.indexOf(separator, searchFrom);
|
|
125
|
+
if (separatorStart < 0 || separatorStart + separator.length > range.end) {
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
const end = separatorStart + separator.length;
|
|
129
|
+
if (end > start) {
|
|
130
|
+
ranges.push({ start, end });
|
|
131
|
+
}
|
|
132
|
+
start = end;
|
|
133
|
+
searchFrom = end;
|
|
134
|
+
}
|
|
135
|
+
if (start < range.end) {
|
|
136
|
+
ranges.push({ start, end: range.end });
|
|
137
|
+
}
|
|
138
|
+
return ranges;
|
|
139
|
+
}
|
|
140
|
+
function hardSplit(range, maxSize) {
|
|
141
|
+
const ranges = [];
|
|
142
|
+
for (let start = range.start; start < range.end; start += maxSize) {
|
|
143
|
+
ranges.push({ start, end: Math.min(start + maxSize, range.end) });
|
|
144
|
+
}
|
|
145
|
+
return ranges;
|
|
146
|
+
}
|
|
147
|
+
function applyRecursiveOverlap(text, ranges, maxSize, overlap) {
|
|
148
|
+
const boundaries = ranges.map((range) => range.end);
|
|
149
|
+
const chunks = [];
|
|
150
|
+
let start = 0;
|
|
151
|
+
let previousEnd = 0;
|
|
152
|
+
while (start < text.length) {
|
|
153
|
+
const limit = Math.min(start + maxSize, text.length);
|
|
154
|
+
const boundary = findLastBoundaryAtMost(boundaries, limit);
|
|
155
|
+
const end = boundary !== void 0 && boundary > previousEnd ? boundary : limit;
|
|
156
|
+
chunks.push(createChunk(text, chunks.length, start, end));
|
|
157
|
+
if (end === text.length) {
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
previousEnd = end;
|
|
161
|
+
start = Math.max(0, end - overlap);
|
|
162
|
+
}
|
|
163
|
+
return chunks;
|
|
164
|
+
}
|
|
165
|
+
function findLastBoundaryAtMost(boundaries, limit) {
|
|
166
|
+
let low = 0;
|
|
167
|
+
let high = boundaries.length;
|
|
168
|
+
while (low < high) {
|
|
169
|
+
const middle = low + Math.floor((high - low) / 2);
|
|
170
|
+
if (boundaries[middle] <= limit) {
|
|
171
|
+
low = middle + 1;
|
|
172
|
+
} else {
|
|
173
|
+
high = middle;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return boundaries[low - 1];
|
|
177
|
+
}
|
|
178
|
+
function createChunk(text, index, start, end) {
|
|
179
|
+
return { index, text: text.slice(start, end), start, end };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// src/documents/pdf.ts
|
|
183
|
+
async function extractPdfText(options) {
|
|
184
|
+
if (typeof options !== "object" || options === null) {
|
|
185
|
+
throw new TypeError("extractPdfText options must be an object");
|
|
186
|
+
}
|
|
187
|
+
if (!(options.data instanceof Uint8Array)) {
|
|
188
|
+
throw new TypeError("extractPdfText data must be a Uint8Array");
|
|
189
|
+
}
|
|
190
|
+
if (options.abortSignal !== void 0 && !isAbortSignal(options.abortSignal)) {
|
|
191
|
+
throw new TypeError("extractPdfText abortSignal must be an AbortSignal");
|
|
192
|
+
}
|
|
193
|
+
const { abortSignal } = options;
|
|
194
|
+
abortSignal?.throwIfAborted();
|
|
195
|
+
const data = new Uint8Array(options.data);
|
|
196
|
+
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
|
197
|
+
abortSignal?.throwIfAborted();
|
|
198
|
+
const loadingTask = pdfjs.getDocument({ data });
|
|
199
|
+
let destroyPromise;
|
|
200
|
+
const destroy = () => {
|
|
201
|
+
destroyPromise ??= loadingTask.destroy();
|
|
202
|
+
return destroyPromise;
|
|
203
|
+
};
|
|
204
|
+
const abort = createAbortPromise(abortSignal, destroy);
|
|
205
|
+
return runWithCleanup(
|
|
206
|
+
async () => {
|
|
207
|
+
const document = await abort.wait(loadingTask.promise);
|
|
208
|
+
const pages = [];
|
|
209
|
+
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
|
|
210
|
+
abortSignal?.throwIfAborted();
|
|
211
|
+
const page = await abort.wait(document.getPage(pageNumber));
|
|
212
|
+
const content = await abort.wait(page.getTextContent());
|
|
213
|
+
pages.push({ pageNumber, text: extractPageText(content.items) });
|
|
214
|
+
}
|
|
215
|
+
return { pages };
|
|
216
|
+
},
|
|
217
|
+
async () => {
|
|
218
|
+
abort.dispose();
|
|
219
|
+
await destroy();
|
|
220
|
+
}
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
async function runWithCleanup(operation, cleanup) {
|
|
224
|
+
let outcome;
|
|
225
|
+
try {
|
|
226
|
+
outcome = { ok: true, value: await operation() };
|
|
227
|
+
} catch (error) {
|
|
228
|
+
outcome = { ok: false, error };
|
|
229
|
+
}
|
|
230
|
+
let cleanupError;
|
|
231
|
+
let cleanupFailed = false;
|
|
232
|
+
try {
|
|
233
|
+
await cleanup();
|
|
234
|
+
} catch (error) {
|
|
235
|
+
cleanupFailed = true;
|
|
236
|
+
cleanupError = error;
|
|
237
|
+
}
|
|
238
|
+
if (!outcome.ok) {
|
|
239
|
+
if (cleanupFailed) {
|
|
240
|
+
throw new AggregateError(
|
|
241
|
+
[outcome.error, cleanupError],
|
|
242
|
+
"PDF extraction and cleanup both failed",
|
|
243
|
+
{ cause: outcome.error }
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
throw outcome.error;
|
|
247
|
+
}
|
|
248
|
+
if (cleanupFailed) {
|
|
249
|
+
throw cleanupError;
|
|
250
|
+
}
|
|
251
|
+
return outcome.value;
|
|
252
|
+
}
|
|
253
|
+
function isAbortSignal(value) {
|
|
254
|
+
if (typeof value !== "object" || value === null) {
|
|
255
|
+
return false;
|
|
256
|
+
}
|
|
257
|
+
const candidate = value;
|
|
258
|
+
return typeof candidate.aborted === "boolean" && typeof candidate.addEventListener === "function" && typeof candidate.removeEventListener === "function" && typeof candidate.throwIfAborted === "function";
|
|
259
|
+
}
|
|
260
|
+
function extractPageText(items) {
|
|
261
|
+
let text = "";
|
|
262
|
+
for (const item of items) {
|
|
263
|
+
if (typeof item !== "object" || item === null) {
|
|
264
|
+
throw new TypeError("PDF text content contains a malformed item");
|
|
265
|
+
}
|
|
266
|
+
if (!("str" in item)) {
|
|
267
|
+
if ("type" in item && (item.type === "beginMarkedContent" || item.type === "beginMarkedContentProps" || item.type === "endMarkedContent")) {
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
throw new TypeError("PDF text content contains an unknown non-text item");
|
|
271
|
+
}
|
|
272
|
+
if (typeof item.str !== "string") {
|
|
273
|
+
throw new TypeError("PDF text content item str must be a string");
|
|
274
|
+
}
|
|
275
|
+
if (!("hasEOL" in item) || typeof item.hasEOL !== "boolean") {
|
|
276
|
+
throw new TypeError("PDF text content item hasEOL must be a boolean");
|
|
277
|
+
}
|
|
278
|
+
text += item.str;
|
|
279
|
+
if (item.hasEOL) {
|
|
280
|
+
text += "\n";
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return text;
|
|
284
|
+
}
|
|
285
|
+
function createAbortPromise(abortSignal, destroy) {
|
|
286
|
+
if (abortSignal === void 0) {
|
|
287
|
+
return {
|
|
288
|
+
wait: (promise) => promise,
|
|
289
|
+
dispose: () => void 0
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
let rejectAbort;
|
|
293
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
294
|
+
rejectAbort = reject;
|
|
295
|
+
});
|
|
296
|
+
const onAbort = () => {
|
|
297
|
+
void destroy();
|
|
298
|
+
rejectAbort?.(
|
|
299
|
+
abortSignal.reason ?? new DOMException("The operation was aborted", "AbortError")
|
|
300
|
+
);
|
|
301
|
+
};
|
|
302
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
303
|
+
if (abortSignal.aborted) {
|
|
304
|
+
onAbort();
|
|
305
|
+
}
|
|
306
|
+
return {
|
|
307
|
+
wait: (promise) => Promise.race([promise, aborted]),
|
|
308
|
+
dispose: () => abortSignal.removeEventListener("abort", onAbort)
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
export {
|
|
312
|
+
chunkText,
|
|
313
|
+
extractPdfText
|
|
314
|
+
};
|
|
315
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/documents/chunk-text.ts","../../src/documents/pdf.ts"],"sourcesContent":["export type TextChunk = Readonly<{\n index: number;\n text: string;\n start: number;\n end: number;\n}>;\n\nexport type ChunkTextOptions =\n | Readonly<{\n text: string;\n strategy: \"fixed\";\n maxSize: number;\n overlap?: number;\n }>\n | Readonly<{\n text: string;\n strategy: \"recursive\";\n maxSize: number;\n overlap?: number;\n separators: readonly string[];\n }>;\n\ntype TextRange = Readonly<{\n start: number;\n end: number;\n}>;\n\nexport function chunkText(options: ChunkTextOptions): readonly TextChunk[] {\n const validated = validateOptions(options);\n if (validated.text.length === 0) {\n return [];\n }\n\n if (validated.strategy === \"fixed\") {\n return chunkFixed(validated.text, validated.maxSize, validated.overlap);\n }\n\n const ranges = resolveRecursiveRanges(\n validated.text,\n { start: 0, end: validated.text.length },\n validated.maxSize,\n validated.separators,\n );\n return applyRecursiveOverlap(validated.text, ranges, validated.maxSize, validated.overlap);\n}\n\nfunction validateOptions(options: ChunkTextOptions):\n | Readonly<{\n text: string;\n strategy: \"fixed\";\n maxSize: number;\n overlap: number;\n }>\n | Readonly<{\n text: string;\n strategy: \"recursive\";\n maxSize: number;\n overlap: number;\n separators: readonly string[];\n }> {\n if (typeof options !== \"object\" || options === null) {\n throw new TypeError(\"chunkText options must be an object\");\n }\n\n const value = options as unknown as Record<string, unknown>;\n if (typeof value.text !== \"string\") {\n throw new TypeError(\"chunkText text must be a string\");\n }\n if (!Number.isSafeInteger(value.maxSize) || (value.maxSize as number) <= 0) {\n throw new RangeError(\"chunkText maxSize must be a positive safe integer\");\n }\n\n const maxSize = value.maxSize as number;\n const overlap = value.overlap === undefined ? 0 : value.overlap;\n if (!Number.isSafeInteger(overlap) || (overlap as number) < 0 || (overlap as number) >= maxSize) {\n throw new RangeError(\n \"chunkText overlap must be a non-negative safe integer smaller than maxSize\",\n );\n }\n\n if (value.strategy === \"fixed\") {\n if (\"separators\" in value) {\n throw new TypeError(\"chunkText fixed strategy does not accept separators\");\n }\n return {\n text: value.text,\n strategy: \"fixed\",\n maxSize,\n overlap: overlap as number,\n };\n }\n\n if (value.strategy !== \"recursive\") {\n throw new TypeError('chunkText strategy must be either \"fixed\" or \"recursive\"');\n }\n if (!Array.isArray(value.separators) || value.separators.length === 0) {\n throw new TypeError(\"chunkText recursive separators must be a non-empty array\");\n }\n\n const separators = value.separators as unknown[];\n const seen = new Set<string>();\n for (const separator of separators) {\n if (typeof separator !== \"string\" || separator.length === 0) {\n throw new TypeError(\"chunkText recursive separators must contain non-empty strings\");\n }\n if (seen.has(separator)) {\n throw new TypeError(\"chunkText recursive separators must not contain duplicates\");\n }\n seen.add(separator);\n }\n\n return {\n text: value.text,\n strategy: \"recursive\",\n maxSize,\n overlap: overlap as number,\n separators: separators as string[],\n };\n}\n\nfunction chunkFixed(text: string, maxSize: number, overlap: number): readonly TextChunk[] {\n const chunks: TextChunk[] = [];\n let start = 0;\n\n while (start < text.length) {\n const end = Math.min(start + maxSize, text.length);\n chunks.push(createChunk(text, chunks.length, start, end));\n if (end === text.length) {\n break;\n }\n start = end - overlap;\n }\n\n return chunks;\n}\n\nfunction resolveRecursiveRanges(\n text: string,\n initialRange: TextRange,\n maxSize: number,\n separators: readonly string[],\n): readonly TextRange[] {\n const resolved: TextRange[] = [];\n const pending: Array<Readonly<{ range: TextRange; separatorIndex: number }>> = [\n { range: initialRange, separatorIndex: 0 },\n ];\n\n while (pending.length > 0) {\n const current = pending.pop();\n if (current === undefined) {\n break;\n }\n if (current.range.end - current.range.start <= maxSize) {\n resolved.push(current.range);\n continue;\n }\n\n let pieces: readonly TextRange[] | undefined;\n let nextSeparatorIndex = separators.length;\n for (let index = current.separatorIndex; index < separators.length; index += 1) {\n const candidate = splitAfterSeparator(text, current.range, separators[index] as string);\n if (candidate.length > 1) {\n pieces = candidate;\n nextSeparatorIndex = index + 1;\n break;\n }\n }\n\n if (pieces === undefined) {\n resolved.push(...hardSplit(current.range, maxSize));\n continue;\n }\n\n for (let index = pieces.length - 1; index >= 0; index -= 1) {\n pending.push({ range: pieces[index] as TextRange, separatorIndex: nextSeparatorIndex });\n }\n }\n\n return resolved;\n}\n\nfunction splitAfterSeparator(text: string, range: TextRange, separator: string): TextRange[] {\n const ranges: TextRange[] = [];\n let start = range.start;\n let searchFrom = range.start;\n\n while (searchFrom < range.end) {\n const separatorStart = text.indexOf(separator, searchFrom);\n if (separatorStart < 0 || separatorStart + separator.length > range.end) {\n break;\n }\n const end = separatorStart + separator.length;\n if (end > start) {\n ranges.push({ start, end });\n }\n start = end;\n searchFrom = end;\n }\n\n if (start < range.end) {\n ranges.push({ start, end: range.end });\n }\n return ranges;\n}\n\nfunction hardSplit(range: TextRange, maxSize: number): readonly TextRange[] {\n const ranges: TextRange[] = [];\n for (let start = range.start; start < range.end; start += maxSize) {\n ranges.push({ start, end: Math.min(start + maxSize, range.end) });\n }\n return ranges;\n}\n\nfunction applyRecursiveOverlap(\n text: string,\n ranges: readonly TextRange[],\n maxSize: number,\n overlap: number,\n): readonly TextChunk[] {\n const boundaries = ranges.map((range) => range.end);\n const chunks: TextChunk[] = [];\n let start = 0;\n let previousEnd = 0;\n\n while (start < text.length) {\n const limit = Math.min(start + maxSize, text.length);\n const boundary = findLastBoundaryAtMost(boundaries, limit);\n const end = boundary !== undefined && boundary > previousEnd ? boundary : limit;\n\n chunks.push(createChunk(text, chunks.length, start, end));\n if (end === text.length) {\n break;\n }\n previousEnd = end;\n start = Math.max(0, end - overlap);\n }\n\n return chunks;\n}\n\nfunction findLastBoundaryAtMost(boundaries: readonly number[], limit: number): number | undefined {\n let low = 0;\n let high = boundaries.length;\n while (low < high) {\n const middle = low + Math.floor((high - low) / 2);\n if ((boundaries[middle] as number) <= limit) {\n low = middle + 1;\n } else {\n high = middle;\n }\n }\n return boundaries[low - 1];\n}\n\nfunction createChunk(text: string, index: number, start: number, end: number): TextChunk {\n return { index, text: text.slice(start, end), start, end };\n}\n","export type PdfTextPage = Readonly<{\n pageNumber: number;\n text: string;\n}>;\n\nexport type ExtractPdfTextOptions = Readonly<{\n data: Uint8Array;\n abortSignal?: AbortSignal;\n}>;\n\nexport type ExtractPdfTextResult = Readonly<{\n pages: readonly PdfTextPage[];\n}>;\n\nexport async function extractPdfText(\n options: ExtractPdfTextOptions,\n): Promise<ExtractPdfTextResult> {\n if (typeof options !== \"object\" || options === null) {\n throw new TypeError(\"extractPdfText options must be an object\");\n }\n if (!(options.data instanceof Uint8Array)) {\n throw new TypeError(\"extractPdfText data must be a Uint8Array\");\n }\n if (options.abortSignal !== undefined && !isAbortSignal(options.abortSignal)) {\n throw new TypeError(\"extractPdfText abortSignal must be an AbortSignal\");\n }\n\n const { abortSignal } = options;\n abortSignal?.throwIfAborted();\n const data = new Uint8Array(options.data);\n const pdfjs = await import(\"pdfjs-dist/legacy/build/pdf.mjs\");\n abortSignal?.throwIfAborted();\n\n const loadingTask = pdfjs.getDocument({ data });\n let destroyPromise: Promise<void> | undefined;\n const destroy = (): Promise<void> => {\n destroyPromise ??= loadingTask.destroy();\n return destroyPromise;\n };\n const abort = createAbortPromise(abortSignal, destroy);\n\n return runWithCleanup(\n async () => {\n const document = await abort.wait(loadingTask.promise);\n const pages: PdfTextPage[] = [];\n for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {\n abortSignal?.throwIfAborted();\n const page = await abort.wait(document.getPage(pageNumber));\n const content = await abort.wait(page.getTextContent());\n pages.push({ pageNumber, text: extractPageText(content.items) });\n }\n return { pages };\n },\n async () => {\n abort.dispose();\n await destroy();\n },\n );\n}\n\nasync function runWithCleanup<Result>(\n operation: () => Promise<Result>,\n cleanup: () => Promise<void>,\n): Promise<Result> {\n let outcome: { ok: true; value: Result } | { ok: false; error: unknown };\n try {\n outcome = { ok: true, value: await operation() };\n } catch (error) {\n outcome = { ok: false, error };\n }\n\n let cleanupError: unknown;\n let cleanupFailed = false;\n try {\n await cleanup();\n } catch (error) {\n cleanupFailed = true;\n cleanupError = error;\n }\n\n if (!outcome.ok) {\n if (cleanupFailed) {\n throw new AggregateError(\n [outcome.error, cleanupError],\n \"PDF extraction and cleanup both failed\",\n { cause: outcome.error },\n );\n }\n throw outcome.error;\n }\n if (cleanupFailed) {\n throw cleanupError;\n }\n return outcome.value;\n}\n\nfunction isAbortSignal(value: unknown): value is AbortSignal {\n if (typeof value !== \"object\" || value === null) {\n return false;\n }\n const candidate = value as Partial<AbortSignal>;\n return (\n typeof candidate.aborted === \"boolean\" &&\n typeof candidate.addEventListener === \"function\" &&\n typeof candidate.removeEventListener === \"function\" &&\n typeof candidate.throwIfAborted === \"function\"\n );\n}\n\nfunction extractPageText(items: readonly unknown[]): string {\n let text = \"\";\n for (const item of items) {\n if (typeof item !== \"object\" || item === null) {\n throw new TypeError(\"PDF text content contains a malformed item\");\n }\n if (!(\"str\" in item)) {\n if (\n \"type\" in item &&\n (item.type === \"beginMarkedContent\" ||\n item.type === \"beginMarkedContentProps\" ||\n item.type === \"endMarkedContent\")\n ) {\n continue;\n }\n throw new TypeError(\"PDF text content contains an unknown non-text item\");\n }\n if (typeof item.str !== \"string\") {\n throw new TypeError(\"PDF text content item str must be a string\");\n }\n if (!(\"hasEOL\" in item) || typeof item.hasEOL !== \"boolean\") {\n throw new TypeError(\"PDF text content item hasEOL must be a boolean\");\n }\n text += item.str;\n if (item.hasEOL) {\n text += \"\\n\";\n }\n }\n return text;\n}\n\nfunction createAbortPromise(\n abortSignal: AbortSignal | undefined,\n destroy: () => Promise<void>,\n): {\n wait<T>(promise: Promise<T>): Promise<T>;\n dispose(): void;\n} {\n if (abortSignal === undefined) {\n return {\n wait: (promise) => promise,\n dispose: () => undefined,\n };\n }\n\n let rejectAbort: ((reason?: unknown) => void) | undefined;\n const aborted = new Promise<never>((_resolve, reject) => {\n rejectAbort = reject;\n });\n const onAbort = () => {\n void destroy();\n rejectAbort?.(\n abortSignal.reason ?? new DOMException(\"The operation was aborted\", \"AbortError\"),\n );\n };\n abortSignal.addEventListener(\"abort\", onAbort, { once: true });\n if (abortSignal.aborted) {\n onAbort();\n }\n\n return {\n wait: (promise) => Promise.race([promise, aborted]),\n dispose: () => abortSignal.removeEventListener(\"abort\", onAbort),\n };\n}\n"],"mappings":";AA2BO,SAAS,UAAU,SAAiD;AACzE,QAAM,YAAY,gBAAgB,OAAO;AACzC,MAAI,UAAU,KAAK,WAAW,GAAG;AAC/B,WAAO,CAAC;AAAA,EACV;AAEA,MAAI,UAAU,aAAa,SAAS;AAClC,WAAO,WAAW,UAAU,MAAM,UAAU,SAAS,UAAU,OAAO;AAAA,EACxE;AAEA,QAAM,SAAS;AAAA,IACb,UAAU;AAAA,IACV,EAAE,OAAO,GAAG,KAAK,UAAU,KAAK,OAAO;AAAA,IACvC,UAAU;AAAA,IACV,UAAU;AAAA,EACZ;AACA,SAAO,sBAAsB,UAAU,MAAM,QAAQ,UAAU,SAAS,UAAU,OAAO;AAC3F;AAEA,SAAS,gBAAgB,SAalB;AACL,MAAI,OAAO,YAAY,YAAY,YAAY,MAAM;AACnD,UAAM,IAAI,UAAU,qCAAqC;AAAA,EAC3D;AAEA,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,SAAS,UAAU;AAClC,UAAM,IAAI,UAAU,iCAAiC;AAAA,EACvD;AACA,MAAI,CAAC,OAAO,cAAc,MAAM,OAAO,KAAM,MAAM,WAAsB,GAAG;AAC1E,UAAM,IAAI,WAAW,mDAAmD;AAAA,EAC1E;AAEA,QAAM,UAAU,MAAM;AACtB,QAAM,UAAU,MAAM,YAAY,SAAY,IAAI,MAAM;AACxD,MAAI,CAAC,OAAO,cAAc,OAAO,KAAM,UAAqB,KAAM,WAAsB,SAAS;AAC/F,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM,aAAa,SAAS;AAC9B,QAAI,gBAAgB,OAAO;AACzB,YAAM,IAAI,UAAU,qDAAqD;AAAA,IAC3E;AACA,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,UAAU;AAAA,MACV;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM,aAAa,aAAa;AAClC,UAAM,IAAI,UAAU,0DAA0D;AAAA,EAChF;AACA,MAAI,CAAC,MAAM,QAAQ,MAAM,UAAU,KAAK,MAAM,WAAW,WAAW,GAAG;AACrE,UAAM,IAAI,UAAU,0DAA0D;AAAA,EAChF;AAEA,QAAM,aAAa,MAAM;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,aAAa,YAAY;AAClC,QAAI,OAAO,cAAc,YAAY,UAAU,WAAW,GAAG;AAC3D,YAAM,IAAI,UAAU,+DAA+D;AAAA,IACrF;AACA,QAAI,KAAK,IAAI,SAAS,GAAG;AACvB,YAAM,IAAI,UAAU,4DAA4D;AAAA,IAClF;AACA,SAAK,IAAI,SAAS;AAAA,EACpB;AAEA,SAAO;AAAA,IACL,MAAM,MAAM;AAAA,IACZ,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,WAAW,MAAc,SAAiB,SAAuC;AACxF,QAAM,SAAsB,CAAC;AAC7B,MAAI,QAAQ;AAEZ,SAAO,QAAQ,KAAK,QAAQ;AAC1B,UAAM,MAAM,KAAK,IAAI,QAAQ,SAAS,KAAK,MAAM;AACjD,WAAO,KAAK,YAAY,MAAM,OAAO,QAAQ,OAAO,GAAG,CAAC;AACxD,QAAI,QAAQ,KAAK,QAAQ;AACvB;AAAA,IACF;AACA,YAAQ,MAAM;AAAA,EAChB;AAEA,SAAO;AACT;AAEA,SAAS,uBACP,MACA,cACA,SACA,YACsB;AACtB,QAAM,WAAwB,CAAC;AAC/B,QAAM,UAAyE;AAAA,IAC7E,EAAE,OAAO,cAAc,gBAAgB,EAAE;AAAA,EAC3C;AAEA,SAAO,QAAQ,SAAS,GAAG;AACzB,UAAM,UAAU,QAAQ,IAAI;AAC5B,QAAI,YAAY,QAAW;AACzB;AAAA,IACF;AACA,QAAI,QAAQ,MAAM,MAAM,QAAQ,MAAM,SAAS,SAAS;AACtD,eAAS,KAAK,QAAQ,KAAK;AAC3B;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,qBAAqB,WAAW;AACpC,aAAS,QAAQ,QAAQ,gBAAgB,QAAQ,WAAW,QAAQ,SAAS,GAAG;AAC9E,YAAM,YAAY,oBAAoB,MAAM,QAAQ,OAAO,WAAW,KAAK,CAAW;AACtF,UAAI,UAAU,SAAS,GAAG;AACxB,iBAAS;AACT,6BAAqB,QAAQ;AAC7B;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,QAAW;AACxB,eAAS,KAAK,GAAG,UAAU,QAAQ,OAAO,OAAO,CAAC;AAClD;AAAA,IACF;AAEA,aAAS,QAAQ,OAAO,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG;AAC1D,cAAQ,KAAK,EAAE,OAAO,OAAO,KAAK,GAAgB,gBAAgB,mBAAmB,CAAC;AAAA,IACxF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,MAAc,OAAkB,WAAgC;AAC3F,QAAM,SAAsB,CAAC;AAC7B,MAAI,QAAQ,MAAM;AAClB,MAAI,aAAa,MAAM;AAEvB,SAAO,aAAa,MAAM,KAAK;AAC7B,UAAM,iBAAiB,KAAK,QAAQ,WAAW,UAAU;AACzD,QAAI,iBAAiB,KAAK,iBAAiB,UAAU,SAAS,MAAM,KAAK;AACvE;AAAA,IACF;AACA,UAAM,MAAM,iBAAiB,UAAU;AACvC,QAAI,MAAM,OAAO;AACf,aAAO,KAAK,EAAE,OAAO,IAAI,CAAC;AAAA,IAC5B;AACA,YAAQ;AACR,iBAAa;AAAA,EACf;AAEA,MAAI,QAAQ,MAAM,KAAK;AACrB,WAAO,KAAK,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,EACvC;AACA,SAAO;AACT;AAEA,SAAS,UAAU,OAAkB,SAAuC;AAC1E,QAAM,SAAsB,CAAC;AAC7B,WAAS,QAAQ,MAAM,OAAO,QAAQ,MAAM,KAAK,SAAS,SAAS;AACjE,WAAO,KAAK,EAAE,OAAO,KAAK,KAAK,IAAI,QAAQ,SAAS,MAAM,GAAG,EAAE,CAAC;AAAA,EAClE;AACA,SAAO;AACT;AAEA,SAAS,sBACP,MACA,QACA,SACA,SACsB;AACtB,QAAM,aAAa,OAAO,IAAI,CAAC,UAAU,MAAM,GAAG;AAClD,QAAM,SAAsB,CAAC;AAC7B,MAAI,QAAQ;AACZ,MAAI,cAAc;AAElB,SAAO,QAAQ,KAAK,QAAQ;AAC1B,UAAM,QAAQ,KAAK,IAAI,QAAQ,SAAS,KAAK,MAAM;AACnD,UAAM,WAAW,uBAAuB,YAAY,KAAK;AACzD,UAAM,MAAM,aAAa,UAAa,WAAW,cAAc,WAAW;AAE1E,WAAO,KAAK,YAAY,MAAM,OAAO,QAAQ,OAAO,GAAG,CAAC;AACxD,QAAI,QAAQ,KAAK,QAAQ;AACvB;AAAA,IACF;AACA,kBAAc;AACd,YAAQ,KAAK,IAAI,GAAG,MAAM,OAAO;AAAA,EACnC;AAEA,SAAO;AACT;AAEA,SAAS,uBAAuB,YAA+B,OAAmC;AAChG,MAAI,MAAM;AACV,MAAI,OAAO,WAAW;AACtB,SAAO,MAAM,MAAM;AACjB,UAAM,SAAS,MAAM,KAAK,OAAO,OAAO,OAAO,CAAC;AAChD,QAAK,WAAW,MAAM,KAAgB,OAAO;AAC3C,YAAM,SAAS;AAAA,IACjB,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO,WAAW,MAAM,CAAC;AAC3B;AAEA,SAAS,YAAY,MAAc,OAAe,OAAe,KAAwB;AACvF,SAAO,EAAE,OAAO,MAAM,KAAK,MAAM,OAAO,GAAG,GAAG,OAAO,IAAI;AAC3D;;;AClPA,eAAsB,eACpB,SAC+B;AAC/B,MAAI,OAAO,YAAY,YAAY,YAAY,MAAM;AACnD,UAAM,IAAI,UAAU,0CAA0C;AAAA,EAChE;AACA,MAAI,EAAE,QAAQ,gBAAgB,aAAa;AACzC,UAAM,IAAI,UAAU,0CAA0C;AAAA,EAChE;AACA,MAAI,QAAQ,gBAAgB,UAAa,CAAC,cAAc,QAAQ,WAAW,GAAG;AAC5E,UAAM,IAAI,UAAU,mDAAmD;AAAA,EACzE;AAEA,QAAM,EAAE,YAAY,IAAI;AACxB,eAAa,eAAe;AAC5B,QAAM,OAAO,IAAI,WAAW,QAAQ,IAAI;AACxC,QAAM,QAAQ,MAAM,OAAO,iCAAiC;AAC5D,eAAa,eAAe;AAE5B,QAAM,cAAc,MAAM,YAAY,EAAE,KAAK,CAAC;AAC9C,MAAI;AACJ,QAAM,UAAU,MAAqB;AACnC,uBAAmB,YAAY,QAAQ;AACvC,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,mBAAmB,aAAa,OAAO;AAErD,SAAO;AAAA,IACL,YAAY;AACV,YAAM,WAAW,MAAM,MAAM,KAAK,YAAY,OAAO;AACrD,YAAM,QAAuB,CAAC;AAC9B,eAAS,aAAa,GAAG,cAAc,SAAS,UAAU,cAAc,GAAG;AACzE,qBAAa,eAAe;AAC5B,cAAM,OAAO,MAAM,MAAM,KAAK,SAAS,QAAQ,UAAU,CAAC;AAC1D,cAAM,UAAU,MAAM,MAAM,KAAK,KAAK,eAAe,CAAC;AACtD,cAAM,KAAK,EAAE,YAAY,MAAM,gBAAgB,QAAQ,KAAK,EAAE,CAAC;AAAA,MACjE;AACA,aAAO,EAAE,MAAM;AAAA,IACjB;AAAA,IACA,YAAY;AACV,YAAM,QAAQ;AACd,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;AAEA,eAAe,eACb,WACA,SACiB;AACjB,MAAI;AACJ,MAAI;AACF,cAAU,EAAE,IAAI,MAAM,OAAO,MAAM,UAAU,EAAE;AAAA,EACjD,SAAS,OAAO;AACd,cAAU,EAAE,IAAI,OAAO,MAAM;AAAA,EAC/B;AAEA,MAAI;AACJ,MAAI,gBAAgB;AACpB,MAAI;AACF,UAAM,QAAQ;AAAA,EAChB,SAAS,OAAO;AACd,oBAAgB;AAChB,mBAAe;AAAA,EACjB;AAEA,MAAI,CAAC,QAAQ,IAAI;AACf,QAAI,eAAe;AACjB,YAAM,IAAI;AAAA,QACR,CAAC,QAAQ,OAAO,YAAY;AAAA,QAC5B;AAAA,QACA,EAAE,OAAO,QAAQ,MAAM;AAAA,MACzB;AAAA,IACF;AACA,UAAM,QAAQ;AAAA,EAChB;AACA,MAAI,eAAe;AACjB,UAAM;AAAA,EACR;AACA,SAAO,QAAQ;AACjB;AAEA,SAAS,cAAc,OAAsC;AAC3D,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,YAAY;AAClB,SACE,OAAO,UAAU,YAAY,aAC7B,OAAO,UAAU,qBAAqB,cACtC,OAAO,UAAU,wBAAwB,cACzC,OAAO,UAAU,mBAAmB;AAExC;AAEA,SAAS,gBAAgB,OAAmC;AAC1D,MAAI,OAAO;AACX,aAAW,QAAQ,OAAO;AACxB,QAAI,OAAO,SAAS,YAAY,SAAS,MAAM;AAC7C,YAAM,IAAI,UAAU,4CAA4C;AAAA,IAClE;AACA,QAAI,EAAE,SAAS,OAAO;AACpB,UACE,UAAU,SACT,KAAK,SAAS,wBACb,KAAK,SAAS,6BACd,KAAK,SAAS,qBAChB;AACA;AAAA,MACF;AACA,YAAM,IAAI,UAAU,oDAAoD;AAAA,IAC1E;AACA,QAAI,OAAO,KAAK,QAAQ,UAAU;AAChC,YAAM,IAAI,UAAU,4CAA4C;AAAA,IAClE;AACA,QAAI,EAAE,YAAY,SAAS,OAAO,KAAK,WAAW,WAAW;AAC3D,YAAM,IAAI,UAAU,gDAAgD;AAAA,IACtE;AACA,YAAQ,KAAK;AACb,QAAI,KAAK,QAAQ;AACf,cAAQ;AAAA,IACV;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,mBACP,aACA,SAIA;AACA,MAAI,gBAAgB,QAAW;AAC7B,WAAO;AAAA,MACL,MAAM,CAAC,YAAY;AAAA,MACnB,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,MAAI;AACJ,QAAM,UAAU,IAAI,QAAe,CAAC,UAAU,WAAW;AACvD,kBAAc;AAAA,EAChB,CAAC;AACD,QAAM,UAAU,MAAM;AACpB,SAAK,QAAQ;AACb;AAAA,MACE,YAAY,UAAU,IAAI,aAAa,6BAA6B,YAAY;AAAA,IAClF;AAAA,EACF;AACA,cAAY,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAC7D,MAAI,YAAY,SAAS;AACvB,YAAQ;AAAA,EACV;AAEA,SAAO;AAAA,IACL,MAAM,CAAC,YAAY,QAAQ,KAAK,CAAC,SAAS,OAAO,CAAC;AAAA,IAClD,SAAS,MAAM,YAAY,oBAAoB,SAAS,OAAO;AAAA,EACjE;AACF;","names":[]}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { b as RetrySetting } from './retry-CjvSlKGW.js';
|
|
2
|
+
import { v as ToolDefinition } from './types-D9rKQ_as.js';
|
|
3
|
+
import { V as VectorMetadata, E as EmbeddingModel, a as EmbeddedDocument } from './types-Cr4uiYo5.js';
|
|
4
|
+
import { A as AnyTool } from './tool-r62gf3QY.js';
|
|
5
|
+
import { V as VectorFilter, a as VectorSearchResult, b as VectorInspectRequest, c as VectorInspectPage } from './types-C3gv9EX6.js';
|
|
5
6
|
|
|
6
7
|
type ToolSearchDocument<Metadata extends VectorMetadata = VectorMetadata> = {
|
|
7
8
|
toolName: string;
|
|
@@ -10,24 +11,35 @@ type ToolSearchDocument<Metadata extends VectorMetadata = VectorMetadata> = {
|
|
|
10
11
|
metadata?: Metadata | undefined;
|
|
11
12
|
};
|
|
12
13
|
type EmbedToolsOptions<Metadata extends VectorMetadata = VectorMetadata> = {
|
|
14
|
+
model: EmbeddingModel;
|
|
15
|
+
tools: readonly AnyTool[];
|
|
13
16
|
content?: ((tool: AnyTool, definition: ToolDefinition) => string | string[]) | undefined;
|
|
14
17
|
metadata?: ((tool: AnyTool, definition: ToolDefinition) => Metadata | undefined) | undefined;
|
|
15
18
|
concurrency?: number | undefined;
|
|
19
|
+
retries?: RetrySetting | undefined;
|
|
20
|
+
abortSignal?: AbortSignal | undefined;
|
|
16
21
|
};
|
|
17
22
|
type CreateToolIndexOptions<Metadata extends VectorMetadata = VectorMetadata> = EmbedToolsOptions<Metadata> & {
|
|
18
23
|
topK: number;
|
|
19
|
-
|
|
24
|
+
minScore?: number | undefined;
|
|
20
25
|
filter?: VectorFilter | undefined;
|
|
21
26
|
};
|
|
22
|
-
interface ToolIndex<Metadata extends VectorMetadata = VectorMetadata>
|
|
27
|
+
interface ToolIndex<Metadata extends VectorMetadata = VectorMetadata> {
|
|
23
28
|
readonly kind: "tool-index";
|
|
24
29
|
readonly tools: readonly AnyTool[];
|
|
25
30
|
readonly topK: number;
|
|
26
|
-
readonly
|
|
31
|
+
readonly minScore?: number | undefined;
|
|
27
32
|
readonly filter?: VectorFilter | undefined;
|
|
33
|
+
search(options: {
|
|
34
|
+
query: string;
|
|
35
|
+
abortSignal?: AbortSignal | undefined;
|
|
36
|
+
}): Promise<Array<VectorSearchResult<ToolSearchDocument<Metadata>, Metadata>>>;
|
|
37
|
+
inspect?(request: VectorInspectRequest): Promise<VectorInspectPage<ToolSearchDocument<Metadata>, Metadata>>;
|
|
28
38
|
}
|
|
29
|
-
declare function embedTools<Metadata extends VectorMetadata = VectorMetadata>(
|
|
30
|
-
|
|
39
|
+
declare function embedTools<Metadata extends VectorMetadata = VectorMetadata>(options: EmbedToolsOptions<Metadata>): Promise<{
|
|
40
|
+
documents: Array<EmbeddedDocument<ToolSearchDocument<Metadata>, Metadata>>;
|
|
41
|
+
}>;
|
|
42
|
+
declare function createToolIndex<Metadata extends VectorMetadata = VectorMetadata>(options: CreateToolIndexOptions<Metadata>): Promise<ToolIndex<Metadata>>;
|
|
31
43
|
declare function isToolIndex(value: unknown): value is ToolIndex;
|
|
32
44
|
|
|
33
45
|
export { type CreateToolIndexOptions as C, type EmbedToolsOptions as E, type ToolIndex as T, type ToolSearchDocument as a, createToolIndex as c, embedTools as e, isToolIndex as i };
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
export {
|
|
1
|
+
export { M as ModelCallOptions } from '../model-call-options-CZkSw_xN.js';
|
|
2
|
+
export { R as RetryContext, a as RetryOptions, b as RetrySetting } from '../retry-CjvSlKGW.js';
|
|
3
|
+
import { V as VectorMetadata, d as EmbedDocumentsOptions, e as EmbedDocumentsResult, f as EmbedHybridDocumentsOptions, g as EmbedSparseQueryOptions, h as EmbedSparseQueryResult, i as EmbedSparseTextsOptions, j as EmbedSparseTextsResult, k as EmbedTextOptions, l as EmbedTextResult, m as EmbedTextsOptions, n as EmbedTextsResult } from '../types-Cr4uiYo5.js';
|
|
4
|
+
export { o as EmbedDocumentsBaseOptions, a as EmbeddedDocument, p as Embedding, E as EmbeddingModel, q as EmbeddingOperationOptions, r as SparseEmbedding, c as SparseEmbeddingModel, S as SparseVector, b as VectorMetadataValue } from '../types-Cr4uiYo5.js';
|
|
3
5
|
|
|
4
6
|
declare function dotProduct(left: number[], right: number[]): number;
|
|
5
7
|
declare function cosineSimilarity(left: number[], right: number[]): number;
|
|
@@ -8,11 +10,11 @@ declare function euclideanDistance(left: number[], right: number[]): number;
|
|
|
8
10
|
declare function manhattanDistance(left: number[], right: number[]): number;
|
|
9
11
|
declare function chebyshevDistance(left: number[], right: number[]): number;
|
|
10
12
|
|
|
11
|
-
declare function embedText(
|
|
12
|
-
declare function embedTexts(
|
|
13
|
-
declare function embedSparseTexts(
|
|
14
|
-
declare function embedSparseQuery(
|
|
15
|
-
declare function embedDocuments<T, Metadata extends VectorMetadata = VectorMetadata>(
|
|
16
|
-
declare function
|
|
13
|
+
declare function embedText(options: EmbedTextOptions): Promise<EmbedTextResult>;
|
|
14
|
+
declare function embedTexts(options: EmbedTextsOptions): Promise<EmbedTextsResult>;
|
|
15
|
+
declare function embedSparseTexts(options: EmbedSparseTextsOptions): Promise<EmbedSparseTextsResult>;
|
|
16
|
+
declare function embedSparseQuery(options: EmbedSparseQueryOptions): Promise<EmbedSparseQueryResult>;
|
|
17
|
+
declare function embedDocuments<T, Metadata extends VectorMetadata = VectorMetadata>(options: EmbedDocumentsOptions<T, Metadata>): Promise<EmbedDocumentsResult<T, Metadata>>;
|
|
18
|
+
declare function embedDocuments<T, Metadata extends VectorMetadata = VectorMetadata>(options: EmbedHybridDocumentsOptions<T, Metadata>): Promise<EmbedDocumentsResult<T, Metadata>>;
|
|
17
19
|
|
|
18
|
-
export { EmbedDocumentsOptions, EmbedHybridDocumentsOptions,
|
|
20
|
+
export { EmbedDocumentsOptions, EmbedDocumentsResult, EmbedHybridDocumentsOptions, EmbedSparseQueryOptions, EmbedSparseQueryResult, EmbedSparseTextsOptions, EmbedSparseTextsResult, EmbedTextOptions, EmbedTextResult, EmbedTextsOptions, EmbedTextsResult, VectorMetadata, angularDistance, chebyshevDistance, cosineSimilarity, dotProduct, embedDocuments, embedSparseQuery, embedSparseTexts, embedText, embedTexts, euclideanDistance, manhattanDistance };
|
package/dist/embeddings/index.js
CHANGED
|
@@ -4,21 +4,21 @@ import {
|
|
|
4
4
|
cosineSimilarity,
|
|
5
5
|
dotProduct,
|
|
6
6
|
embedDocuments,
|
|
7
|
-
embedHybridDocuments,
|
|
8
7
|
embedSparseQuery,
|
|
9
8
|
embedSparseTexts,
|
|
10
9
|
embedText,
|
|
11
10
|
embedTexts,
|
|
12
11
|
euclideanDistance,
|
|
13
12
|
manhattanDistance
|
|
14
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-GQL5KRAH.js";
|
|
14
|
+
import "../chunk-3RM57ZT2.js";
|
|
15
|
+
import "../chunk-FN7HLLAZ.js";
|
|
15
16
|
export {
|
|
16
17
|
angularDistance,
|
|
17
18
|
chebyshevDistance,
|
|
18
19
|
cosineSimilarity,
|
|
19
20
|
dotProduct,
|
|
20
21
|
embedDocuments,
|
|
21
|
-
embedHybridDocuments,
|
|
22
22
|
embedSparseQuery,
|
|
23
23
|
embedSparseTexts,
|
|
24
24
|
embedText,
|