@speakeasy-api/docs-mcp-core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/chunk-cache.d.ts +17 -0
- package/dist/chunk-cache.d.ts.map +1 -0
- package/dist/chunk-cache.js +123 -0
- package/dist/chunk-cache.js.map +1 -0
- package/dist/chunking.d.ts +6 -0
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +102 -11
- package/dist/chunking.js.map +1 -1
- package/dist/embedding-cache.d.ts +2 -2
- package/dist/embedding-cache.d.ts.map +1 -1
- package/dist/embedding-cache.js +26 -2
- package/dist/embedding-cache.js.map +1 -1
- package/dist/embedding.d.ts +41 -1
- package/dist/embedding.d.ts.map +1 -1
- package/dist/embedding.js +269 -1
- package/dist/embedding.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/lancedb.d.ts +18 -0
- package/dist/lancedb.d.ts.map +1 -1
- package/dist/lancedb.js +37 -5
- package/dist/lancedb.js.map +1 -1
- package/dist/manifest-schema.d.ts +63 -0
- package/dist/manifest-schema.d.ts.map +1 -0
- package/dist/manifest-schema.js +68 -0
- package/dist/manifest-schema.js.map +1 -0
- package/dist/manifest.d.ts +6 -1
- package/dist/manifest.d.ts.map +1 -1
- package/dist/manifest.js +26 -0
- package/dist/manifest.js.map +1 -1
- package/dist/metadata.d.ts +6 -1
- package/dist/metadata.d.ts.map +1 -1
- package/dist/metadata.js +15 -3
- package/dist/metadata.js.map +1 -1
- package/dist/search-common.d.ts +6 -0
- package/dist/search-common.d.ts.map +1 -1
- package/dist/search-common.js +26 -0
- package/dist/search-common.js.map +1 -1
- package/dist/search.d.ts +4 -1
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +21 -3
- package/dist/search.js.map +1 -1
- package/dist/types.d.ts +28 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +8 -3
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ Core retrieval and indexing primitives for [Speakeasy Docs MCP](https://github.c
|
|
|
10
10
|
- Hybrid search combining full-text search with semantic vector search via Reciprocal Rank Fusion
|
|
11
11
|
- LanceDB integration with memory-mapped IO for fast retrieval
|
|
12
12
|
- Embedding generation and caching (OpenAI or deterministic hash)
|
|
13
|
-
- Manifest resolution and metadata extraction from `.mcp
|
|
13
|
+
- Manifest resolution and metadata extraction from `.docs-mcp.json` files
|
|
14
14
|
|
|
15
15
|
## Installation
|
|
16
16
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Chunk, ChunkingStrategy } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Compute a fingerprint for a file's chunking inputs.
|
|
4
|
+
* Changes when the markdown, strategy, or metadata change.
|
|
5
|
+
*/
|
|
6
|
+
export declare function computeChunkFingerprint(markdown: string, strategy: ChunkingStrategy, metadata: Record<string, string>): string;
|
|
7
|
+
export interface PreviousIndexReader {
|
|
8
|
+
fingerprints: Map<string, string>;
|
|
9
|
+
getChunks(filepath: string): Promise<Chunk[]>;
|
|
10
|
+
close(): void;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Open the previous `.lancedb/` index and extract per-file fingerprints.
|
|
14
|
+
* Returns null if the index doesn't exist or lacks the `file_fingerprint` column.
|
|
15
|
+
*/
|
|
16
|
+
export declare function loadChunksFromPreviousIndex(dbPath: string, tableName?: string): Promise<PreviousIndexReader | null>;
|
|
17
|
+
//# sourceMappingURL=chunk-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-cache.d.ts","sourceRoot":"","sources":["../src/chunk-cache.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAK1D;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,MAAM,CAIR;AAED,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9C,KAAK,IAAI,IAAI,CAAC;CACf;AAED;;;GAGG;AACH,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAkFrC"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { connect } from "@lancedb/lancedb";
|
|
2
|
+
import { sha256hex } from "./embedding.js";
|
|
3
|
+
const DEFAULT_TABLE_NAME = "chunks";
|
|
4
|
+
/**
|
|
5
|
+
* Compute a fingerprint for a file's chunking inputs.
|
|
6
|
+
* Changes when the markdown, strategy, or metadata change.
|
|
7
|
+
*/
|
|
8
|
+
export function computeChunkFingerprint(markdown, strategy, metadata) {
|
|
9
|
+
return sha256hex([JSON.stringify(strategy), JSON.stringify(metadata), markdown].join("\0"));
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Open the previous `.lancedb/` index and extract per-file fingerprints.
|
|
13
|
+
* Returns null if the index doesn't exist or lacks the `file_fingerprint` column.
|
|
14
|
+
*/
|
|
15
|
+
export async function loadChunksFromPreviousIndex(dbPath, tableName) {
|
|
16
|
+
const table = tableName ?? DEFAULT_TABLE_NAME;
|
|
17
|
+
let db;
|
|
18
|
+
try {
|
|
19
|
+
db = await connect(dbPath);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
try {
|
|
25
|
+
const tableNames = await db.tableNames();
|
|
26
|
+
if (!tableNames.includes(table)) {
|
|
27
|
+
db.close();
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
const tbl = await db.openTable(table);
|
|
31
|
+
// Probe for file_fingerprint column by fetching a single row
|
|
32
|
+
const probeRows = await tbl
|
|
33
|
+
.query()
|
|
34
|
+
.select(["filepath", "file_fingerprint"])
|
|
35
|
+
.limit(1)
|
|
36
|
+
.toArray();
|
|
37
|
+
if (probeRows.length === 0) {
|
|
38
|
+
// Empty table — nothing to cache from
|
|
39
|
+
tbl.close();
|
|
40
|
+
db.close();
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
const probeRow = probeRows[0];
|
|
44
|
+
if (!("file_fingerprint" in probeRow) ||
|
|
45
|
+
typeof probeRow.file_fingerprint !== "string") {
|
|
46
|
+
// Old-format index without fingerprints
|
|
47
|
+
tbl.close();
|
|
48
|
+
db.close();
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
// Load all fingerprints (lightweight: only two string columns)
|
|
52
|
+
const fpRows = await tbl
|
|
53
|
+
.query()
|
|
54
|
+
.select(["filepath", "file_fingerprint"])
|
|
55
|
+
.toArray();
|
|
56
|
+
const fingerprints = new Map();
|
|
57
|
+
for (const row of fpRows) {
|
|
58
|
+
const filepath = row.filepath;
|
|
59
|
+
const fp = row.file_fingerprint;
|
|
60
|
+
if (fp) {
|
|
61
|
+
fingerprints.set(filepath, fp);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
fingerprints,
|
|
66
|
+
async getChunks(filepath) {
|
|
67
|
+
const escaped = filepath.replace(/'/g, "''");
|
|
68
|
+
const rows = await tbl
|
|
69
|
+
.query()
|
|
70
|
+
.where(`filepath = '${escaped}'`)
|
|
71
|
+
.toArray();
|
|
72
|
+
return rows
|
|
73
|
+
.map((row) => row)
|
|
74
|
+
.sort((a, b) => toNumber(a.chunk_index) - toNumber(b.chunk_index))
|
|
75
|
+
.map((row) => rowToChunk(row));
|
|
76
|
+
},
|
|
77
|
+
close() {
|
|
78
|
+
tbl.close();
|
|
79
|
+
db.close();
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
db.close();
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function toNumber(value) {
|
|
89
|
+
if (typeof value === "number")
|
|
90
|
+
return value;
|
|
91
|
+
if (typeof value === "bigint")
|
|
92
|
+
return Number(value);
|
|
93
|
+
const parsed = Number(value);
|
|
94
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
95
|
+
}
|
|
96
|
+
function rowToChunk(row) {
|
|
97
|
+
let metadata = {};
|
|
98
|
+
if (typeof row.metadata_json === "string" && row.metadata_json.trim()) {
|
|
99
|
+
try {
|
|
100
|
+
const parsed = JSON.parse(row.metadata_json);
|
|
101
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
102
|
+
if (typeof value === "string") {
|
|
103
|
+
metadata[key] = value;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
metadata = {};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
chunk_id: String(row.chunk_id ?? ""),
|
|
113
|
+
filepath: String(row.filepath ?? ""),
|
|
114
|
+
heading: String(row.heading ?? ""),
|
|
115
|
+
heading_level: toNumber(row.heading_level),
|
|
116
|
+
content: String(row.content ?? ""),
|
|
117
|
+
content_text: String(row.content_text ?? ""),
|
|
118
|
+
breadcrumb: String(row.breadcrumb ?? ""),
|
|
119
|
+
chunk_index: toNumber(row.chunk_index),
|
|
120
|
+
metadata,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=chunk-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-cache.js","sourceRoot":"","sources":["../src/chunk-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAI3C,MAAM,kBAAkB,GAAG,QAAQ,CAAC;AAEpC;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACrC,QAAgB,EAChB,QAA0B,EAC1B,QAAgC;IAEhC,OAAO,SAAS,CACd,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1E,CAAC;AACJ,CAAC;AAQD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,MAAc,EACd,SAAkB;IAElB,MAAM,KAAK,GAAG,SAAS,IAAI,kBAAkB,CAAC;IAE9C,IAAI,EAAE,CAAC;IACP,IAAI,CAAC;QACH,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,SAAS,GAAG,MAAM,GAAG;aACxB,KAAK,EAAE;aACP,MAAM,CAAC,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;aACxC,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAC;QAEb,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,sCAAsC;YACtC,GAAG,CAAC,KAAK,EAAE,CAAC;YACZ,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;QAC/B,IACE,CAAC,CAAC,kBAAkB,IAAI,QAAQ,CAAC;YACjC,OAAO,QAAQ,CAAC,gBAAgB,KAAK,QAAQ,EAC7C,CAAC;YACD,wCAAwC;YACxC,GAAG,CAAC,KAAK,EAAE,CAAC;YACZ,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,+DAA+D;QAC/D,MAAM,MAAM,GAAG,MAAM,GAAG;aACrB,KAAK,EAAE;aACP,MAAM,CAAC,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;aACxC,OAAO,EAAE,CAAC;QAEb,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAkB,CAAC;YACxC,MAAM,EAAE,GAAG,GAAG,CAAC,gBAA0B,CAAC;YAC1C,IAAI,EAAE,EAAE,CAAC;gBACP,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,OAAO;YACL,YAAY;YACZ,KAAK,CAAC,SAAS,CAAC,QAAgB;gBAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC7C,MAAM,IAAI,GAAG,MAAM,GAAG;qBACnB,KAAK,EAAE;qBACP,KAAK,CAAC,eAAe,OAAO,GAAG,CAAC;qBAChC,OAAO,EAAE,CAAC;gBAEb,OAAO,IAAI;qBACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAe,CAAC;qBAC7B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;qBACjE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YACnC,CAAC;YACD,KAAK;gBACH,GAAG,CAAC,KAAK,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,CAAC;YACb,CAAC;SACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,UAAU,CAAC,GAAa;IAC/B,IAAI,QAAQ,GAA2B,EAAE,CAAC;IAC1C,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACtE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAA4B,CAAC;YACxE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;QACpC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;QACpC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QAClC,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;QAC5C,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;QACxC,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;QACtC,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
package/dist/chunking.d.ts
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
1
|
import type { BuildChunksInput, Chunk } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Default maximum chunk size in characters (~6,700 tokens at ~3 chars/token),
|
|
4
|
+
* well under OpenAI's 8,191-token embedding limit. Applied when no explicit
|
|
5
|
+
* `max_chunk_size` is configured.
|
|
6
|
+
*/
|
|
7
|
+
export declare const DEFAULT_MAX_CHUNK_SIZE = 20000;
|
|
2
8
|
export declare function buildChunks(input: BuildChunksInput): Chunk[];
|
|
3
9
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAoB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAoB,MAAM,YAAY,CAAC;AAmC5E;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,QAAS,CAAC;AAI7C,wBAAgB,WAAW,CAAC,KAAK,EAAE,gBAAgB,GAAG,KAAK,EAAE,CAiC5D"}
|
package/dist/chunking.js
CHANGED
|
@@ -5,6 +5,12 @@ const CHUNK_LEVEL_MAP = {
|
|
|
5
5
|
h2: 2,
|
|
6
6
|
h3: 3
|
|
7
7
|
};
|
|
8
|
+
/**
|
|
9
|
+
* Default maximum chunk size in characters (~6,700 tokens at ~3 chars/token),
|
|
10
|
+
* well under OpenAI's 8,191-token embedding limit. Applied when no explicit
|
|
11
|
+
* `max_chunk_size` is configured.
|
|
12
|
+
*/
|
|
13
|
+
export const DEFAULT_MAX_CHUNK_SIZE = 20_000;
|
|
8
14
|
// ─── Public API ──────────────────────────────────────────────────
|
|
9
15
|
export function buildChunks(input) {
|
|
10
16
|
if (!input.filepath.trim()) {
|
|
@@ -172,24 +178,19 @@ function slugify(value) {
|
|
|
172
178
|
}
|
|
173
179
|
// ─── AST-safe size rules ─────────────────────────────────────────
|
|
174
180
|
function applySizeRules(segments, strategy) {
|
|
175
|
-
const max = strategy.max_chunk_size;
|
|
181
|
+
const max = strategy.max_chunk_size ?? DEFAULT_MAX_CHUNK_SIZE;
|
|
176
182
|
const min = strategy.min_chunk_size;
|
|
177
|
-
// Phase 1: split oversized segments
|
|
183
|
+
// Phase 1: split oversized segments — try recursive heading refinement first,
|
|
184
|
+
// then fall back to AST node boundary splitting.
|
|
178
185
|
const expanded = [];
|
|
179
186
|
for (const segment of segments) {
|
|
180
187
|
const contentLength = rawMarkdown(segment.nodes, segment.fullMarkdown).length;
|
|
181
|
-
if (
|
|
188
|
+
if (contentLength <= max) {
|
|
182
189
|
expanded.push(segment);
|
|
183
190
|
continue;
|
|
184
191
|
}
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
expanded.push({
|
|
188
|
-
...segment,
|
|
189
|
-
nodes: groupNodes,
|
|
190
|
-
part: partIndex + 1
|
|
191
|
-
});
|
|
192
|
-
});
|
|
192
|
+
const refined = refineOversizedSegment(segment, max);
|
|
193
|
+
expanded.push(...refined);
|
|
193
194
|
}
|
|
194
195
|
// Phase 2: merge undersized segments into previous (Opus-style breadcrumb check)
|
|
195
196
|
if (!min || expanded.length <= 1) {
|
|
@@ -210,6 +211,96 @@ function applySizeRules(segments, strategy) {
|
|
|
210
211
|
}
|
|
211
212
|
return merged;
|
|
212
213
|
}
|
|
214
|
+
/**
|
|
215
|
+
* Recursively refine an oversized segment by splitting at progressively finer
|
|
216
|
+
* heading levels (headingLevel+1, +2, ... up to h6). Falls back to AST node
|
|
217
|
+
* boundary splitting when no sub-headings exist.
|
|
218
|
+
*/
|
|
219
|
+
function refineOversizedSegment(segment, max) {
|
|
220
|
+
const nextLevel = segment.headingLevel + 1;
|
|
221
|
+
if (nextLevel > 6) {
|
|
222
|
+
return splitByNodeSizeSegments(segment, max);
|
|
223
|
+
}
|
|
224
|
+
// Find sub-heading boundaries at nextLevel within this segment's nodes
|
|
225
|
+
const subBoundaries = [];
|
|
226
|
+
const slugCounts = new Map();
|
|
227
|
+
for (let i = 0; i < segment.nodes.length; i += 1) {
|
|
228
|
+
const node = segment.nodes[i];
|
|
229
|
+
if (node.type === "heading" && node.depth === nextLevel) {
|
|
230
|
+
const heading = toString(node).trim() || "section";
|
|
231
|
+
const baseSlug = slugify(heading) || "section";
|
|
232
|
+
const count = (slugCounts.get(baseSlug) ?? 0) + 1;
|
|
233
|
+
slugCounts.set(baseSlug, count);
|
|
234
|
+
const slug = count === 1 ? baseSlug : `${baseSlug}-${count}`;
|
|
235
|
+
subBoundaries.push({ nodeIndex: i, heading, slug });
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (subBoundaries.length === 0) {
|
|
239
|
+
// No sub-headings at this level — try the next level down
|
|
240
|
+
const deeper = { ...segment, headingLevel: nextLevel };
|
|
241
|
+
return refineOversizedSegment(deeper, max);
|
|
242
|
+
}
|
|
243
|
+
const subSegments = [];
|
|
244
|
+
// Preamble: nodes before the first sub-heading (inherits parent heading)
|
|
245
|
+
if (subBoundaries[0].nodeIndex > 0) {
|
|
246
|
+
const preambleNodes = segment.nodes.slice(0, subBoundaries[0].nodeIndex);
|
|
247
|
+
const preambleContent = rawMarkdown(preambleNodes, segment.fullMarkdown);
|
|
248
|
+
if (preambleContent.trim()) {
|
|
249
|
+
subSegments.push({
|
|
250
|
+
...segment,
|
|
251
|
+
nodes: preambleNodes,
|
|
252
|
+
part: 1
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
// Create sub-segments for each sub-heading
|
|
257
|
+
for (let i = 0; i < subBoundaries.length; i += 1) {
|
|
258
|
+
const boundary = subBoundaries[i];
|
|
259
|
+
const next = subBoundaries[i + 1];
|
|
260
|
+
const startIdx = boundary.nodeIndex;
|
|
261
|
+
const endIdx = next ? next.nodeIndex : segment.nodes.length;
|
|
262
|
+
const sectionNodes = segment.nodes.slice(startIdx, endIdx);
|
|
263
|
+
const content = rawMarkdown(sectionNodes, segment.fullMarkdown);
|
|
264
|
+
if (!content.trim()) {
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
subSegments.push({
|
|
268
|
+
kind: "heading",
|
|
269
|
+
heading: boundary.heading,
|
|
270
|
+
headingLevel: nextLevel,
|
|
271
|
+
ancestorTexts: [...segment.ancestorTexts, ...(segment.heading ? [segment.heading] : [])],
|
|
272
|
+
ancestorSlugs: [...segment.ancestorSlugs, ...(segment.slug ? [segment.slug] : [])],
|
|
273
|
+
slug: boundary.slug,
|
|
274
|
+
nodes: sectionNodes,
|
|
275
|
+
fullMarkdown: segment.fullMarkdown,
|
|
276
|
+
part: 1
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
// Recursively refine any sub-segments that are still oversized
|
|
280
|
+
const result = [];
|
|
281
|
+
for (const sub of subSegments) {
|
|
282
|
+
const subLength = rawMarkdown(sub.nodes, sub.fullMarkdown).length;
|
|
283
|
+
if (subLength <= max) {
|
|
284
|
+
result.push(sub);
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
result.push(...refineOversizedSegment(sub, max));
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return result;
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Fallback: split an oversized segment at AST node boundaries, producing
|
|
294
|
+
* multi-part segments with the same heading metadata.
|
|
295
|
+
*/
|
|
296
|
+
function splitByNodeSizeSegments(segment, max) {
|
|
297
|
+
const nodeGroups = splitByNodeSize(segment.nodes, segment.fullMarkdown, max);
|
|
298
|
+
return nodeGroups.map((groupNodes, partIndex) => ({
|
|
299
|
+
...segment,
|
|
300
|
+
nodes: groupNodes,
|
|
301
|
+
part: partIndex + 1
|
|
302
|
+
}));
|
|
303
|
+
}
|
|
213
304
|
/**
|
|
214
305
|
* AST-safe max-size splitting (from Gemini approach).
|
|
215
306
|
*
|
package/dist/chunking.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AA4B5C,MAAM,eAAe,GAAkE;IACrF,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;CACN,CAAC;AAEF,oEAAoE;AAEpE,MAAM,UAAU,WAAW,CAAC,KAAuB;IACjD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1C,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACvC,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,cAAc,CAC7B;YACE;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,EAAE;gBACR,KAAK,EAAE,YAAY;gBACnB,YAAY,EAAE,KAAK,CAAC,QAAQ;gBAC5B,IAAI,EAAE,CAAC;aACR;SACF,EACD,KAAK,CAAC,QAAQ,CACf,CAAC;QACF,OAAO,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1D,OAAO,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,oEAAoE;AAEpE,SAAS,mBAAmB,CAAC,GAAS,EAAE,QAAgB,EAAE,WAAmB;IAC3E,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,KAAK,GAAsD,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAC;IAElE,mDAAmD;IACnD,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAEtE,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAE,QAAQ,IAAI,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,GAAG,eAAe,CAAC,QAAQ,CAAE,CAAC;QACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC;QAE/C,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,OAAO,EAAE,CAAC;gBACZ,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QACnE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC5C,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAClE,KAAK,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACd,UAAU,EAAE,QAAQ;YACpB,OAAO;YACP,YAAY,EAAE,IAAI,CAAC,KAAK;YACxB,aAAa;YACb,aAAa;YACb,IAAI;SACL,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL;gBACE,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,eAAe;gBACtB,YAAY,EAAE,QAAQ;gBACtB,IAAI,EAAE,CAAC;aACR;SACF,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,wDAAwD;IACxD,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,CAAC;IAC1E,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAG,WAAW,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAC7D,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,aAAa;gBACpB,YAAY,EAAE,QAAQ;gBACtB,IAAI,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAE,CAAC;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/D,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,YAAY;YACnB,YAAY,EAAE,QAAQ;YACtB,IAAI,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC;QACxB,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC;YACE;gBACE,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,eAAe;gBACtB,YAAY,EAAE,QAAQ;gBACtB,IAAI,EAAE,CAAC;aACR;SACF,CAAC;AACR,CAAC;AAED,oEAAoE;AAEpE,SAAS,UAAU,CACjB,aAAuB,EACvB,QAAgB,EAChB,kBAAoD;IAEpD,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC;IACzD,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;IACtF,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACrD,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEnC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,oEAAoE;AAEpE,SAAS,cAAc,CAAC,QAAmB,EAAE,QAA0B;IACrE,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC;IACpC,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC;IAEpC,8DAA8D;IAC9D,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAE9E,IAAI,CAAC,GAAG,IAAI,aAAa,IAAI,GAAG,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QAC7E,UAAU,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;YAC3C,QAAQ,CAAC,IAAI,CAAC;gBACZ,GAAG,OAAO;gBACV,KAAK,EAAE,UAAU;gBACjB,IAAI,EAAE,SAAS,GAAG,CAAC;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAErF,IACE,QAAQ;YACR,oBAAoB,GAAG,GAAG;YAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,EAC3B,CAAC;YACD,uBAAuB;YACvB,QAAQ,CAAC,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,KAAoB,EAAE,YAAoB,EAAE,OAAe;IAClF,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,OAAO,GAAkB,EAAE,CAAC;IAChC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC;QAE1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,OAAO,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC;YACjB,WAAW,GAAG,QAAQ,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,WAAW,IAAI,QAAQ,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,QAAQ,CAAC,QAAiB,EAAE,OAAgB;IACnD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,kBAAkB,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAW,EAAE,CAAW;IAClD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClB,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oEAAoE;AAEpE,SAAS,iBAAiB,CACxB,QAAgB,EAChB,QAAmB,EACnB,QAAgC;IAEhC,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,eAAe,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,QAAQ;YACR,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,aAAa,EAAE,OAAO,CAAC,YAAY;YACnC,OAAO;YACP,YAAY,EAAE,WAAW;YACzB,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC;YACvC,WAAW,EAAE,KAAK;YAClB,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB,EAAE,OAAgB;IACxD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,UAAU,OAAO,CAAC,IAAI,EAAE,CAAC;IAC7E,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,IAAI,KAAK,CAAC;YACvB,CAAC,CAAC,GAAG,QAAQ,YAAY;YACzB,CAAC,CAAC,GAAG,QAAQ,mBAAmB,OAAO,CAAC,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,MAAM,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,OAAO,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC;AACnC,CAAC;AAED,oEAAoE;AAEpE;;;;GAIG;AACH,SAAS,WAAW,CAAC,KAAoB,EAAE,WAAmB;IAC5D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IAEtC,IAAI,KAAK,CAAC,QAAQ,EAAE,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChD,OAAO,WAAW,CAAC,KAAK,CACtB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAC3B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CACzB,CAAC;IACJ,CAAC;IAED,WAAW;IACX,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACpD,CAAC"}
|
|
1
|
+
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AA4B5C,MAAM,eAAe,GAAkE;IACrF,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;CACN,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAE7C,oEAAoE;AAEpE,MAAM,UAAU,WAAW,CAAC,KAAuB;IACjD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1C,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACvC,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,cAAc,CAC7B;YACE;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,EAAE;gBACR,KAAK,EAAE,YAAY;gBACnB,YAAY,EAAE,KAAK,CAAC,QAAQ;gBAC5B,IAAI,EAAE,CAAC;aACR;SACF,EACD,KAAK,CAAC,QAAQ,CACf,CAAC;QACF,OAAO,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1D,OAAO,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,oEAAoE;AAEpE,SAAS,mBAAmB,CAAC,GAAS,EAAE,QAAgB,EAAE,WAAmB;IAC3E,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,KAAK,GAAsD,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9F,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA+B,CAAC;IAElE,mDAAmD;IACnD,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAEtE,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAE,QAAQ,IAAI,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,GAAG,eAAe,CAAC,QAAQ,CAAE,CAAC;QACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC;QAE/C,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,OAAO,EAAE,CAAC;gBACZ,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QACnE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC5C,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAClE,KAAK,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACd,UAAU,EAAE,QAAQ;YACpB,OAAO;YACP,YAAY,EAAE,IAAI,CAAC,KAAK;YACxB,aAAa;YACb,aAAa;YACb,IAAI;SACL,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL;gBACE,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,eAAe;gBACtB,YAAY,EAAE,QAAQ;gBACtB,IAAI,EAAE,CAAC;aACR;SACF,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,wDAAwD;IACxD,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,CAAC;IAC1E,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAG,WAAW,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAC7D,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,aAAa;gBACpB,YAAY,EAAE,QAAQ;gBACtB,IAAI,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAE,CAAC;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/D,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,YAAY;YACnB,YAAY,EAAE,QAAQ;YACtB,IAAI,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC;QACxB,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC;YACE;gBACE,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,EAAE;gBACjB,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,eAAe;gBACtB,YAAY,EAAE,QAAQ;gBACtB,IAAI,EAAE,CAAC;aACR;SACF,CAAC;AACR,CAAC;AAED,oEAAoE;AAEpE,SAAS,UAAU,CACjB,aAAuB,EACvB,QAAgB,EAChB,kBAAoD;IAEpD,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC;IACzD,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;IACtF,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACrD,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEnC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,oEAAoE;AAEpE,SAAS,cAAc,CAAC,QAAmB,EAAE,QAA0B;IACrE,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,IAAI,sBAAsB,CAAC;IAC9D,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC;IAEpC,8EAA8E;IAC9E,iDAAiD;IACjD,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAE9E,IAAI,aAAa,IAAI,GAAG,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,iFAAiF;IACjF,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAErF,IACE,QAAQ;YACR,oBAAoB,GAAG,GAAG;YAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,EAC3B,CAAC;YACD,uBAAuB;YACvB,QAAQ,CAAC,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,OAAgB,EAAE,GAAW;IAC3D,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,GAAG,CAAC,CAAC;IAC3C,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC/C,CAAC;IAED,uEAAuE;IACvE,MAAM,aAAa,GAAgE,EAAE,CAAC;IACtF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC;YAC/C,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAClD,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAChC,MAAM,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7D,aAAa,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,0DAA0D;QAC1D,MAAM,MAAM,GAAY,EAAE,GAAG,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC;QAChE,OAAO,sBAAsB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,WAAW,GAAc,EAAE,CAAC;IAElC,yEAAyE;IACzE,IAAI,aAAa,CAAC,CAAC,CAAE,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC;QAC1E,MAAM,eAAe,GAAG,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QACzE,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,WAAW,CAAC,IAAI,CAAC;gBACf,GAAG,OAAO;gBACV,KAAK,EAAE,aAAa;gBACpB,IAAI,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAE,CAAC;QACnC,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY,EAAE,SAAS;YACvB,aAAa,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxF,aAAa,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClF,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,YAAY;YACnB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,IAAI,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAED,+DAA+D;IAC/D,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAClE,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,OAAgB,EAAE,GAAW;IAC5D,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IAC7E,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QAChD,GAAG,OAAO;QACV,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,SAAS,GAAG,CAAC;KACpB,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,KAAoB,EAAE,YAAoB,EAAE,OAAe;IAClF,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,OAAO,GAAkB,EAAE,CAAC;IAChC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC;QAE1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,OAAO,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC;YACjB,WAAW,GAAG,QAAQ,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,WAAW,IAAI,QAAQ,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,QAAQ,CAAC,QAAiB,EAAE,OAAgB;IACnD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,kBAAkB,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAW,EAAE,CAAW;IAClD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClB,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oEAAoE;AAEpE,SAAS,iBAAiB,CACxB,QAAgB,EAChB,QAAmB,EACnB,QAAgC;IAEhC,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,eAAe,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,QAAQ;YACR,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,aAAa,EAAE,OAAO,CAAC,YAAY;YACnC,OAAO;YACP,YAAY,EAAE,WAAW;YACzB,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC;YACvC,WAAW,EAAE,KAAK;YAClB,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB,EAAE,OAAgB;IACxD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,UAAU,OAAO,CAAC,IAAI,EAAE,CAAC;IAC7E,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,IAAI,KAAK,CAAC;YACvB,CAAC,CAAC,GAAG,QAAQ,YAAY;YACzB,CAAC,CAAC,GAAG,QAAQ,mBAAmB,OAAO,CAAC,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,MAAM,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,OAAO,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC;AACnC,CAAC;AAED,oEAAoE;AAEpE;;;;GAIG;AACH,SAAS,WAAW,CAAC,KAAoB,EAAE,WAAmB;IAC5D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IAEtC,IAAI,KAAK,CAAC,QAAQ,EAAE,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChD,OAAO,WAAW,CAAC,KAAK,CACtB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAC3B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CACzB,CAAC;IACJ,CAAC;IAED,WAAW;IACX,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACpD,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Chunk, EmbeddingConfig } from "./types.js";
|
|
1
|
+
import type { Chunk, EmbedIncrementalOptions, EmbeddingConfig } from "./types.js";
|
|
2
2
|
/**
|
|
3
3
|
* Bumped only if toEmbeddingInput() changes in a way that isn't captured
|
|
4
4
|
* by the input text itself (e.g., normalization, encoding).
|
|
@@ -38,7 +38,7 @@ export declare function saveCache(baseDir: string, cache: EmbeddingCache, config
|
|
|
38
38
|
*/
|
|
39
39
|
export declare function embedChunksIncremental(chunks: Chunk[], provider: EmbeddingConfig & {
|
|
40
40
|
embed(texts: string[]): Promise<number[][]>;
|
|
41
|
-
}, cache: EmbeddingCache | null): Promise<{
|
|
41
|
+
}, cache: EmbeddingCache | null, options?: EmbedIncrementalOptions): Promise<{
|
|
42
42
|
vectorsByChunkId: Map<string, number[]>;
|
|
43
43
|
updatedCache: EmbeddingCache;
|
|
44
44
|
stats: EmbedIncrementalStats;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-cache.d.ts","sourceRoot":"","sources":["../src/embedding-cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"embedding-cache.d.ts","sourceRoot":"","sources":["../src/embedding-cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElF;;;GAGG;AACH,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAe5C,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,GAAG,MAAM,CAOhF;AAoBD;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,eAAe,GAC7B,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAiFhC;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC,CA2Cf;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,KAAK,EAAE,EACf,QAAQ,EAAE,eAAe,GAAG;IAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;CAAE,EAC3E,KAAK,EAAE,cAAc,GAAG,IAAI,EAC5B,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC;IACT,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACxC,YAAY,EAAE,cAAc,CAAC;IAC7B,KAAK,EAAE,qBAAqB,CAAC;CAC9B,CAAC,CAuFD"}
|
package/dist/embedding-cache.js
CHANGED
|
@@ -171,7 +171,7 @@ export async function saveCache(baseDir, cache, config) {
|
|
|
171
171
|
* Embed chunks incrementally: reuse cached vectors for unchanged chunks,
|
|
172
172
|
* only call the provider for misses.
|
|
173
173
|
*/
|
|
174
|
-
export async function embedChunksIncremental(chunks, provider, cache) {
|
|
174
|
+
export async function embedChunksIncremental(chunks, provider, cache, options) {
|
|
175
175
|
const config = {
|
|
176
176
|
provider: provider.provider,
|
|
177
177
|
model: provider.model,
|
|
@@ -193,7 +193,31 @@ export async function embedChunksIncremental(chunks, provider, cache) {
|
|
|
193
193
|
}
|
|
194
194
|
// Embed only misses
|
|
195
195
|
const missTexts = missIndices.map((i) => toEmbeddingInput(chunks[i]));
|
|
196
|
-
const missVectors =
|
|
196
|
+
const missVectors = [];
|
|
197
|
+
const batchSize = options?.batchSize;
|
|
198
|
+
const batchApiThreshold = options?.batchApiThreshold;
|
|
199
|
+
if (missTexts.length > 0 && batchApiThreshold && missTexts.length >= batchApiThreshold) {
|
|
200
|
+
// Send all at once so provider can use Batch API
|
|
201
|
+
missVectors.push(...await provider.embed(missTexts));
|
|
202
|
+
}
|
|
203
|
+
else if (missTexts.length > 0 && batchSize && batchSize > 0) {
|
|
204
|
+
let embeddedSoFar = 0;
|
|
205
|
+
for (let offset = 0; offset < missTexts.length; offset += batchSize) {
|
|
206
|
+
const batch = missTexts.slice(offset, offset + batchSize);
|
|
207
|
+
const batchVectors = await provider.embed(batch);
|
|
208
|
+
missVectors.push(...batchVectors);
|
|
209
|
+
embeddedSoFar += batch.length;
|
|
210
|
+
options.onProgress?.({
|
|
211
|
+
phase: "embedding",
|
|
212
|
+
completed: hitIndices.length + embeddedSoFar,
|
|
213
|
+
total: chunks.length,
|
|
214
|
+
cached: hitIndices.length,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
else if (missTexts.length > 0) {
|
|
219
|
+
missVectors.push(...await provider.embed(missTexts));
|
|
220
|
+
}
|
|
197
221
|
if (missVectors.length !== missTexts.length) {
|
|
198
222
|
throw new Error(`Embedding provider returned ${missVectors.length} vectors for ${missTexts.length} chunks`);
|
|
199
223
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-cache.js","sourceRoot":"","sources":["../src/embedding-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG7D;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAE5C,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAC1C,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,eAAe,GAAG,iBAAiB,CAAC;AAC1C,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAwBtC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAY,EAAE,MAAuB;IACtE,MAAM,KAAK,GAAG;QACZ,wBAAwB;QACxB,MAAM,CAAC,iBAAiB;QACxB,gBAAgB,CAAC,KAAK,CAAC;KACxB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;AAC5C,CAAC;AACD,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;AAChD,CAAC;AACD,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,uBAAuB,CAAC,OAAe;IACpD,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,aAA8B;IAE9B,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEvC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAEtD,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,0FAA0F;QAC1F,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;YACtG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,mEAAmE;QACrE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC3E,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/G,OAAO,CAAC,IAAI,CAAC,kEAAkE,IAAI,CAAC,aAAa,cAAc,aAAa,GAAG,CAAC,CAAC;QACjI,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,cAAc,KAAK,wBAAwB,EAAE,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,mEAAmE,IAAI,CAAC,cAAc,cAAc,wBAAwB,GAAG,CAAC,CAAC;QAC9I,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,kBAAkB,KAAK,aAAa,CAAC,iBAAiB,EAAE,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;QACzG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC3C,6EAA6E;gBAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAChC,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;YAE9C,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,GAAG,CAAC,WAAqB,CAAC;gBAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAkB,CAAC;gBACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAiC,CAAC,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC9E,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,KAAqB,EACrB,MAAuB;IAEvB,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAExC,sCAAsC;IACtC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,wBAAwB;IACxB,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,IAAI,GAAc;QACtB,aAAa,EAAE,aAAa;QAC5B,cAAc,EAAE,wBAAwB;QACxC,kBAAkB,EAAE,MAAM,CAAC,iBAAiB;KAC7C,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEnF,4BAA4B;IAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9D,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAClF,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IACD,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAe,EACf,QAA2E,EAC3E,KAA4B;
|
|
1
|
+
{"version":3,"file":"embedding-cache.js","sourceRoot":"","sources":["../src/embedding-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG7D;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAE5C,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAC1C,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,eAAe,GAAG,iBAAiB,CAAC;AAC1C,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAwBtC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAY,EAAE,MAAuB;IACtE,MAAM,KAAK,GAAG;QACZ,wBAAwB;QACxB,MAAM,CAAC,iBAAiB;QACxB,gBAAgB,CAAC,KAAK,CAAC;KACxB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;AAC5C,CAAC;AACD,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;AAChD,CAAC;AACD,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,uBAAuB,CAAC,OAAe;IACpD,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,aAA8B;IAE9B,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEvC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAEtD,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,0FAA0F;QAC1F,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;YACtG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,mEAAmE;QACrE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC3E,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/G,OAAO,CAAC,IAAI,CAAC,kEAAkE,IAAI,CAAC,aAAa,cAAc,aAAa,GAAG,CAAC,CAAC;QACjI,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,cAAc,KAAK,wBAAwB,EAAE,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,mEAAmE,IAAI,CAAC,cAAc,cAAc,wBAAwB,GAAG,CAAC,CAAC;QAC9I,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,kBAAkB,KAAK,aAAa,CAAC,iBAAiB,EAAE,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;QACzG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC3C,6EAA6E;gBAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAChC,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;YAE9C,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,GAAG,CAAC,WAAqB,CAAC;gBAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAkB,CAAC;gBACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAiC,CAAC,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC9E,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,KAAqB,EACrB,MAAuB;IAEvB,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAExC,sCAAsC;IACtC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,wBAAwB;IACxB,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,IAAI,GAAc;QACtB,aAAa,EAAE,aAAa;QAC5B,cAAc,EAAE,wBAAwB;QACxC,kBAAkB,EAAE,MAAM,CAAC,iBAAiB;KAC7C,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEnF,4BAA4B;IAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9D,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAClF,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IACD,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAe,EACf,QAA2E,EAC3E,KAA4B,EAC5B,OAAiC;IAMjC,MAAM,MAAM,GAAoB;QAC9B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;KAC9C,CAAC;IAEF,wCAAwC;IACxC,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;QAC5B,IAAI,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;IACvE,MAAM,WAAW,GAAe,EAAE,CAAC;IAEnC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,iBAAiB,GAAG,OAAO,EAAE,iBAAiB,CAAC;IACrD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,IAAI,SAAS,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;QACvF,iDAAiD;QACjD,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IACvD,CAAC;SAAM,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9D,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YACpE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;YAC1D,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjD,WAAW,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAClC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;YAC9B,OAAO,CAAC,UAAU,EAAE,CAAC;gBACnB,KAAK,EAAE,WAAW;gBAClB,SAAS,EAAE,UAAU,CAAC,MAAM,GAAG,aAAa;gBAC5C,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,+BAA+B,WAAW,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,SAAS,CAC3F,CAAC;IACJ,CAAC;IAED,gBAAgB;IAChB,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAoB,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEjD,aAAa;IACb,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,KAAK,GAAG,KAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACtC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACnD,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,kCAAkC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO;QACL,gBAAgB;QAChB,YAAY,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE;QACrC,KAAK,EAAE;YACL,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,IAAI,EAAE,UAAU,CAAC,MAAM;YACvB,MAAM,EAAE,WAAW,CAAC,MAAM;SAC3B;KACF,CAAC;AACJ,CAAC"}
|
package/dist/embedding.d.ts
CHANGED
|
@@ -5,21 +5,40 @@ import type { Chunk, EmbeddingProvider } from "./types.js";
|
|
|
5
5
|
*/
|
|
6
6
|
export declare function toEmbeddingInput(chunk: Chunk): string;
|
|
7
7
|
export declare function sha256hex(input: string): string;
|
|
8
|
+
export interface BatchProgressEvent {
|
|
9
|
+
phase: "batch-uploading" | "batch-polling" | "batch-downloading";
|
|
10
|
+
message: string;
|
|
11
|
+
/** Structured polling data — present when phase is "batch-polling" and counts are available. */
|
|
12
|
+
counts?: {
|
|
13
|
+
completed: number;
|
|
14
|
+
total: number;
|
|
15
|
+
failed: number;
|
|
16
|
+
};
|
|
17
|
+
elapsedSec?: number;
|
|
18
|
+
pollRemainingSec?: number;
|
|
19
|
+
/** Estimated seconds remaining. Undefined until enough data points. */
|
|
20
|
+
etaSec?: number;
|
|
21
|
+
}
|
|
8
22
|
export interface OpenAIEmbeddingProviderOptions {
|
|
9
23
|
apiKey: string;
|
|
10
24
|
model?: string;
|
|
11
25
|
dimensions?: number;
|
|
12
26
|
batchSize?: number;
|
|
27
|
+
batchApiThreshold?: number;
|
|
28
|
+
/** Name stored in batch metadata for identification and resume scoping. */
|
|
29
|
+
batchName?: string;
|
|
13
30
|
baseUrl?: string;
|
|
14
31
|
concurrency?: number;
|
|
15
32
|
maxRetries?: number;
|
|
16
33
|
retryBaseDelayMs?: number;
|
|
17
34
|
retryMaxDelayMs?: number;
|
|
35
|
+
onBatchProgress?: (event: BatchProgressEvent) => void;
|
|
18
36
|
}
|
|
19
37
|
export declare class NoopEmbeddingProvider implements EmbeddingProvider {
|
|
20
38
|
readonly name = "none";
|
|
21
39
|
readonly model = "none";
|
|
22
40
|
readonly dimensions = 0;
|
|
41
|
+
readonly costPerMillionTokens = 0;
|
|
23
42
|
readonly configFingerprint: string;
|
|
24
43
|
constructor();
|
|
25
44
|
embed(texts: string[]): Promise<number[][]>;
|
|
@@ -28,6 +47,7 @@ export declare class HashEmbeddingProvider implements EmbeddingProvider {
|
|
|
28
47
|
readonly name = "hash";
|
|
29
48
|
readonly model: string;
|
|
30
49
|
readonly dimensions: number;
|
|
50
|
+
readonly costPerMillionTokens = 0;
|
|
31
51
|
readonly configFingerprint: string;
|
|
32
52
|
constructor(options?: {
|
|
33
53
|
dimensions?: number;
|
|
@@ -39,17 +59,33 @@ export declare class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
|
|
39
59
|
readonly name = "openai";
|
|
40
60
|
readonly model: string;
|
|
41
61
|
readonly dimensions: number;
|
|
62
|
+
readonly costPerMillionTokens: number;
|
|
42
63
|
readonly configFingerprint: string;
|
|
64
|
+
readonly batchApiThreshold: number;
|
|
43
65
|
private readonly apiKey;
|
|
44
|
-
|
|
66
|
+
readonly batchSize: number;
|
|
45
67
|
private readonly baseUrl;
|
|
46
68
|
private readonly concurrency;
|
|
47
69
|
private readonly maxRetries;
|
|
48
70
|
private readonly retryBaseDelayMs;
|
|
49
71
|
private readonly retryMaxDelayMs;
|
|
72
|
+
private readonly batchName;
|
|
73
|
+
private readonly onBatchProgress;
|
|
50
74
|
constructor(options: OpenAIEmbeddingProviderOptions);
|
|
51
75
|
embed(texts: string[]): Promise<number[][]>;
|
|
52
76
|
private embedBatchWithRetry;
|
|
77
|
+
private buildBatchJsonl;
|
|
78
|
+
private uploadBatchFile;
|
|
79
|
+
private createBatch;
|
|
80
|
+
/**
|
|
81
|
+
* Search recent OpenAI batches for one whose metadata.content_sha matches,
|
|
82
|
+
* indicating the same embedding inputs were already submitted.
|
|
83
|
+
* Returns the batch if found in a usable state; null otherwise.
|
|
84
|
+
*/
|
|
85
|
+
private findExistingBatch;
|
|
86
|
+
private pollBatchUntilComplete;
|
|
87
|
+
private downloadBatchResults;
|
|
88
|
+
private embedViaBatchApi;
|
|
53
89
|
}
|
|
54
90
|
export declare function createEmbeddingProvider(input: {
|
|
55
91
|
provider: "none" | "hash" | "openai";
|
|
@@ -58,9 +94,13 @@ export declare function createEmbeddingProvider(input: {
|
|
|
58
94
|
model?: string;
|
|
59
95
|
baseUrl?: string;
|
|
60
96
|
batchSize?: number;
|
|
97
|
+
batchApiThreshold?: number;
|
|
98
|
+
batchName?: string;
|
|
61
99
|
concurrency?: number;
|
|
62
100
|
maxRetries?: number;
|
|
63
101
|
retryBaseDelayMs?: number;
|
|
64
102
|
retryMaxDelayMs?: number;
|
|
103
|
+
onBatchProgress?: (event: BatchProgressEvent) => void;
|
|
65
104
|
}): EmbeddingProvider;
|
|
105
|
+
export declare function formatDuration(seconds: number): string;
|
|
66
106
|
//# sourceMappingURL=embedding.d.ts.map
|
package/dist/embedding.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../src/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE3D;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAGrD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AASD,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../src/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE3D;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAGrD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AASD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,iBAAiB,GAAG,eAAe,GAAG,mBAAmB,CAAC;IACjE,OAAO,EAAE,MAAM,CAAC;IAChB,gGAAgG;IAChG,MAAM,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;CACvD;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,KAAK,UAAU;IACxB,QAAQ,CAAC,UAAU,KAAK;IACxB,QAAQ,CAAC,oBAAoB,KAAK;IAClC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;;IAU7B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGlD;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,oBAAoB,KAAK;IAClC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;gBAEvB,OAAO,GAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO;IAU3D,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGlD;AAgBD,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,QAAQ,CAAC,IAAI,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAEnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAoD;gBAExE,OAAO,EAAE,8BAA8B;IA0B7C,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAgCnC,mBAAmB;IAoEjC,OAAO,CAAC,eAAe;YAqBT,eAAe;YAqBf,WAAW;IA2BzB;;;;OAIG;YACW,iBAAiB;YA6CjB,sBAAsB;YAkFtB,oBAAoB;YAuCpB,gBAAgB;CAmC/B;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;CACvD,GAAG,iBAAiB,CAqEpB;AA6DD,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAKtD"}
|