@magda/semantic-indexer-framework 5.2.0 → 6.0.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunker.js +1 -1
- package/dist/chunker.js.map +1 -1
- package/dist/onRecordFoundStorageObject.js +50 -5
- package/dist/onRecordFoundStorageObject.js.map +1 -1
- package/dist/test/onRecordFoundStorageObject.spec.js +46 -1
- package/dist/test/onRecordFoundStorageObject.spec.js.map +1 -1
- package/dist/textSplitters.d.ts +87 -0
- package/dist/textSplitters.js +751 -0
- package/dist/textSplitters.js.map +1 -0
- package/package.json +4 -5
package/dist/chunker.js
CHANGED
package/dist/chunker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8BAA8B,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8BAA8B,EAAE,MAAM,oBAAoB,CAAC;AAepE,MAAM,OAAO,OAAO;IACR,QAAQ,CAAgB;IAChC,YAAY,QAAuB;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;CACJ;AAED,MAAM,OAAO,wBAAwB;IACzB,aAAa,CAAoB;IAEzC,YAAY,aAAgC;QACxC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IACvB,SAAS,CAAC;IACV,OAAO,CAAC;IAEhB,YAAY,SAAiB,EAAE,OAAe;QAC1C,IAAI,OAAO,IAAI,SAAS;YACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,EAAE,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChC,OAAO;gBACH;oBACI,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,CAAC;oBACX,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,OAAO,EAAE,CAAC;iBACb;aACJ,CAAC;QACN,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,8BAA8B,CAAC;YAChD,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,YAAY,EAAE,IAAI,CAAC,OAAO;YAC1B,aAAa,EAAE,IAAI;YACnB,eAAe,EAAE,KAAK;SACzB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC1D,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzD,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACpB,CAAC,EAAE,CAAC;YACR,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAED,MAAM,OAAO,wBAAwB;IACzB,SAAS,CAAS;IAClB,OAAO,CAAS;IAExB,YAAY,SAAiB,EAAE,OAAe;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,MAAM,GAAkB,EAAE,CAAC;QACjC,KACI,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,EAC9B,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,EACpC,CAAC;YACC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,OAAO,EAAE,IAAI,CAAC,OAAO;aACxB,CAAC,CAAC;QACP,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ"}
|
|
@@ -8,8 +8,11 @@ import { v4 as uuidv4 } from "uuid";
|
|
|
8
8
|
import { SkipError } from "./SkipError.js";
|
|
9
9
|
import urijs from "urijs";
|
|
10
10
|
import { pipeline } from "stream/promises";
|
|
11
|
+
import { createBrotliDecompress, createGunzip, createInflate } from "zlib";
|
|
11
12
|
import ServerError from "@magda/typescript-common/dist/ServerError.js";
|
|
12
13
|
import { deleteTempFile } from "./helpers.js";
|
|
14
|
+
class RetryableDownloadError extends Error {}
|
|
15
|
+
|
|
13
16
|
// The onRecordFound function passed to minion sdk to handle storage object records
|
|
14
17
|
export const onRecordFoundStorageObject = (options, chunker, embeddingApiClient, opensearchApiClient, minioClient, registryReadonlyClient) => {
|
|
15
18
|
return async (dist, registry) => {
|
|
@@ -86,7 +89,7 @@ export const onRecordFoundStorageObject = (options, chunker, embeddingApiClient,
|
|
|
86
89
|
};
|
|
87
90
|
};
|
|
88
91
|
async function downloadFileWithRetry(url, minioClient) {
|
|
89
|
-
return retry(() => downloadFile(url, minioClient), 1, 5, (err, retries) => {});
|
|
92
|
+
return retry(() => downloadFile(url, minioClient), 1, 5, (err, retries) => {}, (err) => err instanceof RetryableDownloadError);
|
|
90
93
|
}
|
|
91
94
|
async function downloadFile(url, minioClient) {
|
|
92
95
|
const uri = urijs(url);
|
|
@@ -100,12 +103,20 @@ async function downloadFile(url, minioClient) {
|
|
|
100
103
|
}
|
|
101
104
|
let response;
|
|
102
105
|
try {
|
|
103
|
-
response = await fetch(url
|
|
106
|
+
response = await fetch(url, {
|
|
107
|
+
compress: false,
|
|
108
|
+
headers: {
|
|
109
|
+
"accept-encoding": "gzip, deflate, br"
|
|
110
|
+
}
|
|
111
|
+
});
|
|
104
112
|
}
|
|
105
113
|
catch (err) {
|
|
106
|
-
throw new
|
|
114
|
+
throw new RetryableDownloadError(`Failed to download file because network error`);
|
|
107
115
|
}
|
|
108
116
|
if (!response.ok) {
|
|
117
|
+
if (response.status >= 500 || response.status === 429) {
|
|
118
|
+
throw new RetryableDownloadError(`Failed to download file because HTTP error ${response.status}`);
|
|
119
|
+
}
|
|
109
120
|
throw new SkipError(`Failed to download file because HTTP error ${response.status}`);
|
|
110
121
|
}
|
|
111
122
|
if (!response.body) {
|
|
@@ -115,15 +126,49 @@ async function downloadFile(url, minioClient) {
|
|
|
115
126
|
const tempFileName = `${uuidv4()}`;
|
|
116
127
|
const suffix = new urijs(url).suffix();
|
|
117
128
|
const tempFilePath = join(tempDir, `${tempFileName}.${suffix}`);
|
|
129
|
+
const responseStream = getDecodedResponseStream(response);
|
|
118
130
|
try {
|
|
119
131
|
const writeStream = fs.createWriteStream(tempFilePath);
|
|
120
|
-
await pipeline(
|
|
132
|
+
await pipeline(responseStream, writeStream);
|
|
121
133
|
return tempFilePath;
|
|
122
134
|
}
|
|
123
135
|
catch (err) {
|
|
124
136
|
await deleteTempFile(tempFilePath);
|
|
125
|
-
throw new SkipError(`Failed to write file`);
|
|
137
|
+
throw new SkipError(`Failed to write file: ${err.message}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function getDecodedResponseStream(response) {
|
|
141
|
+
if (!response.body) {
|
|
142
|
+
throw new SkipError("No response body to write to file");
|
|
143
|
+
}
|
|
144
|
+
let stream = response.body;
|
|
145
|
+
const contentEncodingHeader = response.headers.get("content-encoding")?.toLowerCase() || "";
|
|
146
|
+
const encodings = contentEncodingHeader.
|
|
147
|
+
split(",").
|
|
148
|
+
map((encoding) => encoding.trim()).
|
|
149
|
+
filter(Boolean);
|
|
150
|
+
if (encodings.length === 0 || encodings[0] === "identity") {
|
|
151
|
+
return stream;
|
|
152
|
+
}
|
|
153
|
+
// Content-Encoding is applied in order and must be decoded in reverse.
|
|
154
|
+
for (let i = encodings.length - 1; i >= 0; i--) {
|
|
155
|
+
const encoding = encodings[i];
|
|
156
|
+
switch (encoding) {
|
|
157
|
+
case "gzip":
|
|
158
|
+
case "x-gzip":
|
|
159
|
+
stream = stream.pipe(createGunzip());
|
|
160
|
+
break;
|
|
161
|
+
case "deflate":
|
|
162
|
+
stream = stream.pipe(createInflate());
|
|
163
|
+
break;
|
|
164
|
+
case "br":
|
|
165
|
+
stream = stream.pipe(createBrotliDecompress());
|
|
166
|
+
break;
|
|
167
|
+
default:
|
|
168
|
+
throw new SkipError(`Unsupported content encoding: ${encoding}`);
|
|
169
|
+
}
|
|
126
170
|
}
|
|
171
|
+
return stream;
|
|
127
172
|
}
|
|
128
173
|
export async function getParentRecordId(distributionId, registry) {
|
|
129
174
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"onRecordFoundStorageObject.js","sourceRoot":"","sources":["../src/onRecordFoundStorageObject.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAG7D,OAAO,KAAK,MAAM,sCAAsC,CAAC;AACzD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"onRecordFoundStorageObject.js","sourceRoot":"","sources":["../src/onRecordFoundStorageObject.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAG7D,OAAO,KAAK,MAAM,sCAAsC,CAAC;AACzD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAE3E,OAAO,WAAW,MAAM,4CAA4C,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,sBAAuB,SAAQ,KAAK;CAAG;AAE7C,mFAAmF;AACnF,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACtC,OAA+B,EAC/B,OAAgB,EAChB,kBAAsC,EACtC,mBAAwC,EACxC,WAAwB,EACxB,sBAAgC,EACf,EAAE;IACnB,OAAO,KAAK,EAAE,IAAY,EAAE,QAAQ,EAAE,EAAE;QACpC,IAAI,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,2BAA2B,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC;YAChE,MAAM,eAAe,GAAG,WAAW,IAAI,SAAS,CAAC;YACjD,IAAI,MAAM,GAAG,aAAa,IAAI,UAAU,CAAC;YACzC,IAAI,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;gBAC7B,MAAM,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/D,CAAC;YAED,gBAAgB;YAChB,IACI,CAAC,MAAM;gBACP,CAAC,eAAe;gBAChB,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7B,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CACjD,EACH,CAAC;gBACC,OAAO;YACX,CAAC;YAED,IAAI,aAA4B,CAAC;YACjC,IAAI,QAAQ,GAAkB,IAAI,CAAC;YAEnC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAC1C,IAAI,CAAC,EAAE,EACP,sBAAsB,CACzB,CAAC;YACF,IAAI,CAAC;gBACD,IAAI,CAAC;oBACD,IACI,OAAO,CAAC,gBAAgB,KAAK,SAAS;wBACtC,OAAO,CAAC,gBAAgB,EAC1B,CAAC;wBACC,QAAQ,GAAG,MAAM,qBAAqB,CAClC,eAAe,EACf,WAAW,CACd,CAAC;oBACN,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,MAAM,IAAI,SAAS,CACf,8BAA+B,GAAa,CAAC,OAAO,EAAE,CACzD,CAAC;gBACN,CAAC;gBACD,IAAI,CAAC;oBACD,aAAa,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC;wBAC9C,MAAM,EAAE,IAAI;wBACZ,MAAM,EAAE,MAAM;wBACd,QAAQ;wBACR,GAAG,EAAE,eAAe;wBACpB,gBAAgB,EAAE,sBAAsB;qBAC3C,CAAC,CAAC;oBAEH,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;wBACnD,MAAM,IAAI,SAAS,CACf,2EAA2E,CAC9E,CAAC;oBACN,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,MAAM,IAAI,SAAS,CACf,wDACK,GAAa,CAAC,OACnB,EAAE,CACL,CAAC;gBACN,CAAC;YACL,CAAC;oBAAS,CAAC;gBACP,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;YAED,MAAM,kBAAkB,CAAC;gBACrB,OAAO;gBACP,OAAO;gBACP,kBAAkB;gBAClB,mBAAmB;gBACnB,aAAa;gBACb,QAAQ,EAAE;oBACN,QAAQ,EAAE,IAAI,CAAC,EAAE;oBACjB,cAAc,EAAE,cAAc;oBAC9B,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,EAAE;oBAC5C,UAAU,EAAE,MAAM;iBACrB;aACJ,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;gBAC3B,OAAO,CAAC,IAAI,CACR,yBAAyB,IAAI,CAAC,EAAE,WAAW,EAC3C,GAAG,CAAC,OAAO,CACd,CAAC;gBACF,OAAO;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,KAAK,UAAU,qBAAqB,CAChC,GAAW,EACX,WAAwB;IAExB,OAAO,KAAK,CACR,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC,EACpC,CAAC,EACD,CAAC,EACD,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,GAAE,CAAC,EACpB,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,YAAY,sBAAsB,CACjD,CAAC;AACN,CAAC;AAED,KAAK,UAAU,YAAY,CACvB,GAAW,EACX,WAAwB;IAExB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,aAAa,EAAE,CAAC;QACjE,IAAI,CAAC;YACD,OAAO,MAAM,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,SAAS,CACf,uCAAwC,GAAa,CAAC,OAAO,EAAE,CAClE,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACD,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACxB,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE;gBACL,iBAAiB,EAAE,mBAAmB;aACzC;SACJ,CAAC,CAAC;IACP,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,sBAAsB,CAC5B,+CAA+C,CAClD,CAAC;IACN,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACf,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACpD,MAAM,IAAI,sBAAsB,CAC5B,8CAA8C,QAAQ,CAAC,MAAM,EAAE,CAClE,CAAC;QACN,CAAC;QACD,MAAM,IAAI,SAAS,CACf,8CAA8C,QAAQ,CAAC,MAAM,EAAE,CAClE,CAAC;IACN,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;IACzB,MAAM,YAAY,GAAG,GAAG,MAAM,EAAE,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,IAAI,MAAM,EAAE,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IAE1D,IAAI,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,cAAc,CAAC,YAAY,CAAC,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,yBAA0B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,CAAC;AACL,CAAC;AAED,SAAS,wBAAwB,CAAC,QAGjC;IACG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,MAAM,GAA0B,QAAQ,CAAC,IAAI,CAAC;IAClD,MAAM,qBAAqB,GACvB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAClE,MAAM,SAAS,GAAG,qBAAqB;SAClC,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAC;IAErB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;QACxD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,uEAAuE;IACvE,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,QAAQ;gBACT,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;gBACrC,MAAM;YACV,KAAK,SAAS;gBACV,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;gBACtC,MAAM;YACV,KAAK,IAAI;gBACL,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;gBAC/C,MAAM;YACV;gBACI,MAAM,IAAI,SAAS,CACf,iCAAiC,QAAQ,EAAE,CAC9C,CAAC;QACV,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACnC,cAAsB,EACtB,QAAkB;IAElB,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CACpC,CAAC,uBAAuB,CAAC,EACzB,SAAS,EACT,SAAS,EACT,IAAI,EACJ,SAAS,EACT,CAAC,wCAAwC,GAAG,cAAc,CAAC,CAC9D,CAAC;QAEF,IAAI,MAAM,YAAY,WAAW,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,mCAAmC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAChD,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}
|
|
@@ -7,6 +7,7 @@ import fs from "fs";
|
|
|
7
7
|
import nock from "nock";
|
|
8
8
|
import { tmpdir } from "os";
|
|
9
9
|
import { join } from "path";
|
|
10
|
+
import { gzipSync } from "zlib";
|
|
10
11
|
describe("onRecordFoundStorageObject", () => {
|
|
11
12
|
let testEnv;
|
|
12
13
|
beforeEach(() => {
|
|
@@ -79,7 +80,13 @@ describe("onRecordFoundStorageObject", () => {
|
|
|
79
80
|
});
|
|
80
81
|
it("should handle and index storage object with expected formatType, with autoDownloadFile enabled", async () => {
|
|
81
82
|
const fileContent = "test1";
|
|
82
|
-
nock("http://test.com")
|
|
83
|
+
nock("http://test.com")
|
|
84
|
+
.matchHeader("accept-encoding", /gzip,\s*deflate,\s*br/i)
|
|
85
|
+
.get("/file.csv")
|
|
86
|
+
.reply(200, gzipSync(Buffer.from(fileContent)), {
|
|
87
|
+
"content-type": "application/octet-stream",
|
|
88
|
+
"content-encoding": "gzip"
|
|
89
|
+
});
|
|
83
90
|
const record = createRecord({
|
|
84
91
|
id: "id1",
|
|
85
92
|
aspects: {
|
|
@@ -219,6 +226,44 @@ describe("onRecordFoundStorageObject", () => {
|
|
|
219
226
|
}
|
|
220
227
|
nock.cleanAll();
|
|
221
228
|
});
|
|
229
|
+
it("should skip record (without throwing) when server returns malformed compressed response", async () => {
|
|
230
|
+
const malformedCompressedBody = Buffer.concat([
|
|
231
|
+
gzipSync(Buffer.from("test1")),
|
|
232
|
+
Buffer.from("JUNK_TRAILING_BYTES")
|
|
233
|
+
]);
|
|
234
|
+
nock("http://test.com")
|
|
235
|
+
.matchHeader("accept-encoding", /gzip,\s*deflate,\s*br/i)
|
|
236
|
+
.get("/file.csv")
|
|
237
|
+
.reply(200, malformedCompressedBody, {
|
|
238
|
+
"content-type": "application/octet-stream",
|
|
239
|
+
"content-encoding": "gzip"
|
|
240
|
+
});
|
|
241
|
+
const record = createRecord({
|
|
242
|
+
id: "id1",
|
|
243
|
+
aspects: {
|
|
244
|
+
"dataset-format": { format: "csv" },
|
|
245
|
+
"dcat-distribution-strings": {
|
|
246
|
+
format: "csv",
|
|
247
|
+
downloadURL: "http://test.com/file.csv"
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
const userConfig = testEnv.updateUserConfig({
|
|
252
|
+
itemType: "storageObject",
|
|
253
|
+
formatTypes: ["csv"],
|
|
254
|
+
autoDownloadFile: true
|
|
255
|
+
});
|
|
256
|
+
const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
|
|
257
|
+
await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
|
|
258
|
+
testEnv.expectSuccessCalls({
|
|
259
|
+
createEmbeddingTextCallCount: 0,
|
|
260
|
+
chunkCallCount: 0,
|
|
261
|
+
embeddingApiCallCount: 0,
|
|
262
|
+
bulkIndexCallCount: 0,
|
|
263
|
+
deleteByQueryCallCount: 0
|
|
264
|
+
});
|
|
265
|
+
nock.cleanAll();
|
|
266
|
+
});
|
|
222
267
|
it("should filter out records with unexpected formatType", async () => {
|
|
223
268
|
testEnv.createEmbeddingTextStub.resolves({
|
|
224
269
|
text: "embedding text: test1"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"onRecordFoundStorageObject.spec.js","sourceRoot":"","sources":["../../src/test/onRecordFoundStorageObject.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAC9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IACxC,IAAI,OAAgC,CAAC;IAErC,UAAU,CAAC,GAAG,EAAE;QACZ,OAAO,GAAG,IAAI,uBAAuB,CAAC;YAClC,mBAAmB,EAAE,IAAI;SAC5B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACX,OAAO,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iGAAiG,EAAE,KAAK,IAAI,EAAE;QAC7G,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,EAAE;YACzD,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gGAAgG,EAAE,KAAK,IAAI,EAAE;QAC5G,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAEjE,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,uBAAuB,CAAC,SAAS,CACrC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,YAAY,GAAG,QAAQ,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;QAC7C,CAAC,CACJ,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAExC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,MAAM,WAAW,GACb,wDAAwD,CAAC;QAC7D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC/C,OAAO,CAAC,WAAW,CAAC,YAAY;aAC3B,QAAQ,CAAC,WAAW,CAAC;aACrB,SAAS,CAAC,KAAK,IAAI,EAAE;YAClB,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;YAC5C,OAAO,YAAY,CAAC;QACxB,CAAC,CAAC,CAAC;QAEP,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,WAAW;iBAC3B;aACJ;SACJ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,uBAAuB,CAAC,SAAS,CACrC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,YAAY,GAAG,QAAQ,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;QAC7C,CAAC,CACJ,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAExC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QAClE,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;gBACpC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,2BAA2B;iBAC3C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;QACxF,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;YAC5B,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,OAAO,CAAC,uBAAuB;aAC1B,QAAQ,CAAC;YACN,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CAAC;aACD,QAAQ,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,aAAa,GAAG;YAClB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,mBAAmB,CAAC,iBAAiB;aACxC,QAAQ,CAAC,MAAM,EAAE,IAAW,CAAC;aAC7B,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,mBAAmB,EAAE,KAAK;iBACrB,IAAI,EAAE;iBACN,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,EAAE;SACd,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE;gBACpB,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CACnE;YACI,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CACJ,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACtD,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,CAAC,EAAE;YACnE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,qBAAqB,CAAC,aAAa;YAC1D,IAAI,EAAE;gBACF,KAAK,EAAE;oBACH,IAAI,EAAE;wBACF,MAAM,EAAE;4BACJ,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE;4BACtC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;4BAC7B;gCACI,KAAK,EAAE;oCACH,UAAU,EAAE;wCACR,EAAE,EAAE,OAAO,CAAC,oBAAoB,EAAE;qCACrC;iCACJ;6BACJ;yBACJ;qBACJ;iBACJ;aACJ;YACD,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,UAAU,CAAC,OAAO;SAC9B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"onRecordFoundStorageObject.spec.js","sourceRoot":"","sources":["../../src/test/onRecordFoundStorageObject.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAC9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAEhC,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IACxC,IAAI,OAAgC,CAAC;IAErC,UAAU,CAAC,GAAG,EAAE;QACZ,OAAO,GAAG,IAAI,uBAAuB,CAAC;YAClC,mBAAmB,EAAE,IAAI;SAC5B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACX,OAAO,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iGAAiG,EAAE,KAAK,IAAI,EAAE;QAC7G,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,EAAE;YACzD,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gGAAgG,EAAE,KAAK,IAAI,EAAE;QAC5G,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,iBAAiB,CAAC;aAClB,WAAW,CAAC,iBAAiB,EAAE,wBAAwB,CAAC;aACxD,GAAG,CAAC,WAAW,CAAC;aAChB,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE;YAC5C,cAAc,EAAE,0BAA0B;YAC1C,kBAAkB,EAAE,MAAM;SAC7B,CAAC,CAAC;QAEP,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,uBAAuB,CAAC,SAAS,CACrC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,YAAY,GAAG,QAAQ,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;QAC7C,CAAC,CACJ,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAExC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,MAAM,WAAW,GACb,wDAAwD,CAAC;QAC7D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC/C,OAAO,CAAC,WAAW,CAAC,YAAY;aAC3B,QAAQ,CAAC,WAAW,CAAC;aACrB,SAAS,CAAC,KAAK,IAAI,EAAE;YAClB,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;YAC5C,OAAO,YAAY,CAAC;QACxB,CAAC,CAAC,CAAC;QAEP,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,WAAW;iBAC3B;aACJ;SACJ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,uBAAuB,CAAC,SAAS,CACrC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,YAAY,GAAG,QAAQ,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;QAC7C,CAAC,CACJ,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAExC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yFAAyF,EAAE,KAAK,IAAI,EAAE;QACrG,MAAM,uBAAuB,GAAG,MAAM,CAAC,MAAM,CAAC;YAC1C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,CAAC;aAClB,WAAW,CAAC,iBAAiB,EAAE,wBAAwB,CAAC;aACxD,GAAG,CAAC,WAAW,CAAC;aAChB,KAAK,CAAC,GAAG,EAAE,uBAAuB,EAAE;YACjC,cAAc,EAAE,0BAA0B;YAC1C,kBAAkB,EAAE,MAAM;SAC7B,CAAC,CAAC;QAEP,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QAClE,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;gBACpC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,2BAA2B;iBAC3C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;QACxF,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;YAC5B,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,OAAO,CAAC,uBAAuB;aAC1B,QAAQ,CAAC;YACN,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CAAC;aACD,QAAQ,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,aAAa,GAAG;YAClB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,mBAAmB,CAAC,iBAAiB;aACxC,QAAQ,CAAC,MAAM,EAAE,IAAW,CAAC;aAC7B,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,mBAAmB,EAAE,KAAK;iBACrB,IAAI,EAAE;iBACN,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,EAAE;SACd,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE;gBACpB,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CACnE;YACI,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CACJ,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACtD,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,CAAC,EAAE;YACnE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,qBAAqB,CAAC,aAAa;YAC1D,IAAI,EAAE;gBACF,KAAK,EAAE;oBACH,IAAI,EAAE;wBACF,MAAM,EAAE;4BACJ,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE;4BACtC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;4BAC7B;gCACI,KAAK,EAAE;oCACH,UAAU,EAAE;wCACR,EAAE,EAAE,OAAO,CAAC,oBAAoB,EAAE;qCACrC;iCACJ;6BACJ;yBACJ;qBACJ;iBACJ;aACJ;YACD,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,UAAU,CAAC,OAAO;SAC9B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type * as tiktoken from "js-tiktoken";
|
|
2
|
+
import { Document, BaseDocumentTransformer } from "@langchain/core/documents";
|
|
3
|
+
export interface TextSplitterParams {
|
|
4
|
+
chunkSize: number;
|
|
5
|
+
chunkOverlap: number;
|
|
6
|
+
keepSeparator: boolean;
|
|
7
|
+
stripWhitespace: boolean;
|
|
8
|
+
lengthFunction?: ((text: string) => number) | ((text: string) => Promise<number>);
|
|
9
|
+
}
|
|
10
|
+
interface ChunkWithMetadata {
|
|
11
|
+
text: string;
|
|
12
|
+
position: number;
|
|
13
|
+
overlap: number;
|
|
14
|
+
length: number;
|
|
15
|
+
}
|
|
16
|
+
export type TextSplitterChunkHeaderOptions = {
|
|
17
|
+
chunkHeader?: string;
|
|
18
|
+
chunkOverlapHeader?: string;
|
|
19
|
+
appendChunkOverlapHeader?: boolean;
|
|
20
|
+
};
|
|
21
|
+
export declare abstract class TextSplitter extends BaseDocumentTransformer implements TextSplitterParams {
|
|
22
|
+
lc_namespace: string[];
|
|
23
|
+
chunkSize: number;
|
|
24
|
+
chunkOverlap: number;
|
|
25
|
+
keepSeparator: boolean;
|
|
26
|
+
stripWhitespace: boolean;
|
|
27
|
+
lengthFunction: ((text: string) => number) | ((text: string) => Promise<number>);
|
|
28
|
+
constructor(fields?: Partial<TextSplitterParams>);
|
|
29
|
+
transformDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
|
|
30
|
+
abstract splitText(text: string): Promise<string[]>;
|
|
31
|
+
protected splitOnSeparator(text: string, separator: string, offset?: number): Promise<ChunkWithMetadata[]>;
|
|
32
|
+
createDocuments(texts: string[], metadatas?: Record<string, any>[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
|
|
33
|
+
private numberOfNewLines;
|
|
34
|
+
splitDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
|
|
35
|
+
private joinDocs;
|
|
36
|
+
mergeSplits(splits: ChunkWithMetadata[], separator: string): Promise<ChunkWithMetadata[]>;
|
|
37
|
+
}
|
|
38
|
+
export interface CharacterTextSplitterParams extends TextSplitterParams {
|
|
39
|
+
separator: string;
|
|
40
|
+
}
|
|
41
|
+
export declare class CharacterTextSplitter extends TextSplitter implements CharacterTextSplitterParams {
|
|
42
|
+
static lc_name(): string;
|
|
43
|
+
separator: string;
|
|
44
|
+
constructor(fields?: Partial<CharacterTextSplitterParams>);
|
|
45
|
+
splitText(text: string): Promise<string[]>;
|
|
46
|
+
}
|
|
47
|
+
export interface RecursiveCharacterTextSplitterParams extends TextSplitterParams {
|
|
48
|
+
separators: string[];
|
|
49
|
+
}
|
|
50
|
+
export declare const SupportedTextSplitterLanguages: readonly ["cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol"];
|
|
51
|
+
export type SupportedTextSplitterLanguage = typeof SupportedTextSplitterLanguages[number];
|
|
52
|
+
export declare class RecursiveCharacterTextSplitter extends TextSplitter implements RecursiveCharacterTextSplitterParams {
|
|
53
|
+
static lc_name(): string;
|
|
54
|
+
separators: string[];
|
|
55
|
+
constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>);
|
|
56
|
+
private _splitText;
|
|
57
|
+
splitText(text: string): Promise<string[]>;
|
|
58
|
+
splitTextWithMetadata(text: string): Promise<ChunkWithMetadata[]>;
|
|
59
|
+
static fromLanguage(language: SupportedTextSplitterLanguage, options?: Partial<RecursiveCharacterTextSplitterParams>): RecursiveCharacterTextSplitter;
|
|
60
|
+
static getSeparatorsForLanguage(language: SupportedTextSplitterLanguage): string[];
|
|
61
|
+
}
|
|
62
|
+
export interface TokenTextSplitterParams extends TextSplitterParams {
|
|
63
|
+
encodingName: tiktoken.TiktokenEncoding;
|
|
64
|
+
allowedSpecial: "all" | Array<string>;
|
|
65
|
+
disallowedSpecial: "all" | Array<string>;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Implementation of splitter which looks at tokens.
|
|
69
|
+
*/
|
|
70
|
+
export declare class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {
|
|
71
|
+
static lc_name(): string;
|
|
72
|
+
encodingName: tiktoken.TiktokenEncoding;
|
|
73
|
+
allowedSpecial: "all" | Array<string>;
|
|
74
|
+
disallowedSpecial: "all" | Array<string>;
|
|
75
|
+
private tokenizer;
|
|
76
|
+
constructor(fields?: Partial<TokenTextSplitterParams>);
|
|
77
|
+
splitText(text: string): Promise<string[]>;
|
|
78
|
+
}
|
|
79
|
+
export type MarkdownTextSplitterParams = TextSplitterParams;
|
|
80
|
+
export declare class MarkdownTextSplitter extends RecursiveCharacterTextSplitter implements MarkdownTextSplitterParams {
|
|
81
|
+
constructor(fields?: Partial<MarkdownTextSplitterParams>);
|
|
82
|
+
}
|
|
83
|
+
export type LatexTextSplitterParams = TextSplitterParams;
|
|
84
|
+
export declare class LatexTextSplitter extends RecursiveCharacterTextSplitter implements LatexTextSplitterParams {
|
|
85
|
+
constructor(fields?: Partial<LatexTextSplitterParams>);
|
|
86
|
+
}
|
|
87
|
+
export {};
|
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
// The MIT License
|
|
2
|
+
//
|
|
3
|
+
// Copyright (c) 2023 LangChain
|
|
4
|
+
//
|
|
5
|
+
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
// of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
// in the Software without restriction, including without limitation the rights
|
|
8
|
+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
// copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
// furnished to do so, subject to the following conditions:
|
|
11
|
+
//
|
|
12
|
+
// The above copyright notice and this permission notice shall be included in
|
|
13
|
+
// all copies or substantial portions of the Software.
|
|
14
|
+
//
|
|
15
|
+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
// THE SOFTWARE.
|
|
22
|
+
//
|
|
23
|
+
// Modifications: metadata-aware splitting (position/length/overlap),
|
|
24
|
+
// stripWhitespace option, and splitTextWithMetadata support.
|
|
25
|
+
import { Document, BaseDocumentTransformer } from "@langchain/core/documents";
|
|
26
|
+
import { getEncoding } from "@langchain/core/utils/tiktoken";
|
|
27
|
+
export class TextSplitter extends BaseDocumentTransformer {
|
|
28
|
+
lc_namespace = ["langchain", "document_transformers", "text_splitters"];
|
|
29
|
+
chunkSize = 1000;
|
|
30
|
+
chunkOverlap = 200;
|
|
31
|
+
keepSeparator = false;
|
|
32
|
+
stripWhitespace = true;
|
|
33
|
+
lengthFunction;
|
|
34
|
+
constructor(fields) {
|
|
35
|
+
super(fields);
|
|
36
|
+
this.chunkSize = fields?.chunkSize ?? this.chunkSize;
|
|
37
|
+
this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;
|
|
38
|
+
this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;
|
|
39
|
+
this.stripWhitespace = fields?.stripWhitespace ?? this.stripWhitespace;
|
|
40
|
+
this.lengthFunction =
|
|
41
|
+
fields?.lengthFunction ?? ((text) => text.length);
|
|
42
|
+
if (this.chunkOverlap >= this.chunkSize) {
|
|
43
|
+
throw new Error("Cannot have chunkOverlap >= chunkSize");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async transformDocuments(documents, chunkHeaderOptions = {}) {
|
|
47
|
+
return this.splitDocuments(documents, chunkHeaderOptions);
|
|
48
|
+
}
|
|
49
|
+
async splitOnSeparator(text, separator, offset = 0) {
|
|
50
|
+
const chunks = [];
|
|
51
|
+
if (!separator) {
|
|
52
|
+
for (let i = 0; i < text.length; i++) {
|
|
53
|
+
chunks.push({
|
|
54
|
+
text: text[i],
|
|
55
|
+
position: offset + i,
|
|
56
|
+
overlap: 0,
|
|
57
|
+
length: 1
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
return chunks;
|
|
61
|
+
}
|
|
62
|
+
const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, "\\$&");
|
|
63
|
+
const regex = new RegExp(regexEscapedSeparator, "g");
|
|
64
|
+
let lastIndex = 0;
|
|
65
|
+
let match;
|
|
66
|
+
while ((match = regex.exec(text)) !== null) {
|
|
67
|
+
const chunkText = text.slice(lastIndex, match.index);
|
|
68
|
+
if (chunkText !== "" || lastIndex === 0) {
|
|
69
|
+
chunks.push({
|
|
70
|
+
text: chunkText,
|
|
71
|
+
position: offset + lastIndex,
|
|
72
|
+
overlap: 0,
|
|
73
|
+
length: await this.lengthFunction(chunkText)
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
if (this.keepSeparator) {
|
|
77
|
+
lastIndex = match.index;
|
|
78
|
+
} else
|
|
79
|
+
{
|
|
80
|
+
lastIndex = match.index + match[0].length;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const chunkText = text.slice(lastIndex);
|
|
84
|
+
if (lastIndex < text.length) {
|
|
85
|
+
chunks.push({
|
|
86
|
+
text: chunkText,
|
|
87
|
+
position: offset + lastIndex,
|
|
88
|
+
overlap: 0,
|
|
89
|
+
length: await this.lengthFunction(chunkText)
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
return chunks;
|
|
93
|
+
}
|
|
94
|
+
async createDocuments(texts,
|
|
95
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
96
|
+
metadatas = [], chunkHeaderOptions = {}) {
|
|
97
|
+
// if no metadata is provided, we create an empty one for each text
|
|
98
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
99
|
+
const _metadatas = metadatas.length > 0 ?
|
|
100
|
+
metadatas :
|
|
101
|
+
[...Array(texts.length)].map(() => ({}));
|
|
102
|
+
const { chunkHeader = "", chunkOverlapHeader = "(cont'd) ", appendChunkOverlapHeader = false } = chunkHeaderOptions;
|
|
103
|
+
const documents = new Array();
|
|
104
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
105
|
+
const text = texts[i];
|
|
106
|
+
let lineCounterIndex = 1;
|
|
107
|
+
let prevChunk = null;
|
|
108
|
+
let indexPrevChunk = -1;
|
|
109
|
+
for (const chunk of await this.splitText(text)) {
|
|
110
|
+
let pageContent = chunkHeader;
|
|
111
|
+
// we need to count the \n that are in the text before getting removed by the splitting
|
|
112
|
+
const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);
|
|
113
|
+
if (prevChunk === null) {
|
|
114
|
+
const newLinesBeforeFirstChunk = this.numberOfNewLines(text, 0, indexChunk);
|
|
115
|
+
lineCounterIndex += newLinesBeforeFirstChunk;
|
|
116
|
+
} else
|
|
117
|
+
{
|
|
118
|
+
const indexEndPrevChunk = indexPrevChunk + (await this.lengthFunction(prevChunk));
|
|
119
|
+
if (indexEndPrevChunk < indexChunk) {
|
|
120
|
+
const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexEndPrevChunk, indexChunk);
|
|
121
|
+
lineCounterIndex += numberOfIntermediateNewLines;
|
|
122
|
+
} else
|
|
123
|
+
if (indexEndPrevChunk > indexChunk) {
|
|
124
|
+
const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexChunk, indexEndPrevChunk);
|
|
125
|
+
lineCounterIndex -= numberOfIntermediateNewLines;
|
|
126
|
+
}
|
|
127
|
+
if (appendChunkOverlapHeader) {
|
|
128
|
+
pageContent += chunkOverlapHeader;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
const newLinesCount = this.numberOfNewLines(chunk);
|
|
132
|
+
const loc = _metadatas[i].loc && typeof _metadatas[i].loc === "object" ?
|
|
133
|
+
{ ..._metadatas[i].loc } :
|
|
134
|
+
{};
|
|
135
|
+
loc.lines = {
|
|
136
|
+
from: lineCounterIndex,
|
|
137
|
+
to: lineCounterIndex + newLinesCount
|
|
138
|
+
};
|
|
139
|
+
const metadataWithLinesNumber = {
|
|
140
|
+
..._metadatas[i],
|
|
141
|
+
loc
|
|
142
|
+
};
|
|
143
|
+
pageContent += chunk;
|
|
144
|
+
documents.push(new Document({
|
|
145
|
+
pageContent,
|
|
146
|
+
metadata: metadataWithLinesNumber
|
|
147
|
+
}));
|
|
148
|
+
lineCounterIndex += newLinesCount;
|
|
149
|
+
prevChunk = chunk;
|
|
150
|
+
indexPrevChunk = indexChunk;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return documents;
|
|
154
|
+
}
|
|
155
|
+
numberOfNewLines(text, start, end) {
|
|
156
|
+
const textSection = text.slice(start, end);
|
|
157
|
+
return (textSection.match(/\n/g) || []).length;
|
|
158
|
+
}
|
|
159
|
+
async splitDocuments(documents, chunkHeaderOptions = {}) {
|
|
160
|
+
const selectedDocuments = documents.filter((doc) => doc.pageContent !== undefined);
|
|
161
|
+
const texts = selectedDocuments.map((doc) => doc.pageContent);
|
|
162
|
+
const metadatas = selectedDocuments.map((doc) => doc.metadata);
|
|
163
|
+
return this.createDocuments(texts, metadatas, chunkHeaderOptions);
|
|
164
|
+
}
|
|
165
|
+
async joinDocs(docs, separator) {
|
|
166
|
+
const texts = docs.map((doc) => doc.text);
|
|
167
|
+
let text = texts.join(separator);
|
|
168
|
+
if (this.stripWhitespace) {
|
|
169
|
+
text = text.trim();
|
|
170
|
+
}
|
|
171
|
+
const length = await this.lengthFunction(text);
|
|
172
|
+
const chunkWithMetadata = {
|
|
173
|
+
text,
|
|
174
|
+
position: docs[0].position,
|
|
175
|
+
overlap: 0,
|
|
176
|
+
length
|
|
177
|
+
};
|
|
178
|
+
return text === "" ? null : chunkWithMetadata;
|
|
179
|
+
}
|
|
180
|
+
async mergeSplits(splits, separator) {
|
|
181
|
+
const docs = [];
|
|
182
|
+
const currentDoc = [];
|
|
183
|
+
let lastMergedDoc = [];
|
|
184
|
+
const mergingDocs = [];
|
|
185
|
+
let total = 0;
|
|
186
|
+
for (let i = 0; i < splits.length; i++) {
|
|
187
|
+
const d = splits[i];
|
|
188
|
+
const _len = d.length;
|
|
189
|
+
if (total + _len + currentDoc.length * separator.length >
|
|
190
|
+
this.chunkSize) {
|
|
191
|
+
if (total > this.chunkSize) {
|
|
192
|
+
console.warn(`Created a chunk of size ${total}, +
|
|
193
|
+
which is longer than the specified ${this.chunkSize}`);
|
|
194
|
+
}
|
|
195
|
+
if (currentDoc.length > 0) {
|
|
196
|
+
const doc = await this.joinDocs(currentDoc, separator);
|
|
197
|
+
if (doc !== null) {
|
|
198
|
+
const overlapChunks = lastMergedDoc.filter((index) => mergingDocs.includes(index));
|
|
199
|
+
const overlapLength = overlapChunks.reduce((acc, index) => acc + splits[index].length, 0);
|
|
200
|
+
doc.overlap = overlapLength;
|
|
201
|
+
docs.push(doc);
|
|
202
|
+
lastMergedDoc = [...mergingDocs];
|
|
203
|
+
}
|
|
204
|
+
// Keep on popping if:
|
|
205
|
+
// - we have a larger chunk than in the chunk overlap
|
|
206
|
+
// - or if we still have any chunks and the length is long
|
|
207
|
+
while (total > this.chunkOverlap ||
|
|
208
|
+
total + _len + currentDoc.length * separator.length >
|
|
209
|
+
this.chunkSize &&
|
|
210
|
+
total > 0) {
|
|
211
|
+
total -= currentDoc[0].length;
|
|
212
|
+
currentDoc.shift();
|
|
213
|
+
mergingDocs.shift();
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
currentDoc.push(d);
|
|
218
|
+
mergingDocs.push(i);
|
|
219
|
+
total += _len;
|
|
220
|
+
}
|
|
221
|
+
const doc = await this.joinDocs(currentDoc, separator);
|
|
222
|
+
if (doc !== null) {
|
|
223
|
+
const overlapChunks = lastMergedDoc.filter((index) => mergingDocs.includes(index));
|
|
224
|
+
const overlapLength = overlapChunks.reduce((acc, index) => acc + splits[index].length, 0);
|
|
225
|
+
doc.overlap = overlapLength;
|
|
226
|
+
docs.push(doc);
|
|
227
|
+
}
|
|
228
|
+
return docs;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
export class CharacterTextSplitter extends TextSplitter {
|
|
232
|
+
static lc_name() {
|
|
233
|
+
return "CharacterTextSplitter";
|
|
234
|
+
}
|
|
235
|
+
separator = "\n\n";
|
|
236
|
+
constructor(fields) {
|
|
237
|
+
super(fields);
|
|
238
|
+
this.separator = fields?.separator ?? this.separator;
|
|
239
|
+
}
|
|
240
|
+
async splitText(text) {
|
|
241
|
+
// First we naively split the large input into a bunch of smaller ones.
|
|
242
|
+
const splits = await this.splitOnSeparator(text, this.separator);
|
|
243
|
+
return (await this.mergeSplits(splits, this.keepSeparator ? "" : this.separator)).map((chunk) => chunk.text);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
export const SupportedTextSplitterLanguages = [
|
|
247
|
+
"cpp",
|
|
248
|
+
"go",
|
|
249
|
+
"java",
|
|
250
|
+
"js",
|
|
251
|
+
"php",
|
|
252
|
+
"proto",
|
|
253
|
+
"python",
|
|
254
|
+
"rst",
|
|
255
|
+
"ruby",
|
|
256
|
+
"rust",
|
|
257
|
+
"scala",
|
|
258
|
+
"swift",
|
|
259
|
+
"markdown",
|
|
260
|
+
"latex",
|
|
261
|
+
"html",
|
|
262
|
+
"sol"];
|
|
263
|
+
|
|
264
|
+
export class RecursiveCharacterTextSplitter extends TextSplitter {
|
|
265
|
+
static lc_name() {
|
|
266
|
+
return "RecursiveCharacterTextSplitter";
|
|
267
|
+
}
|
|
268
|
+
separators = ["\n\n", "\n", " ", ""];
|
|
269
|
+
constructor(fields) {
|
|
270
|
+
super(fields);
|
|
271
|
+
this.separators = fields?.separators ?? this.separators;
|
|
272
|
+
this.keepSeparator = fields?.keepSeparator ?? true;
|
|
273
|
+
}
|
|
274
|
+
async _splitText(chunk, separators, offset = 0) {
|
|
275
|
+
const finalChunks = [];
|
|
276
|
+
// Get appropriate separator to use
|
|
277
|
+
let separator = separators[separators.length - 1];
|
|
278
|
+
let newSeparators;
|
|
279
|
+
for (let i = 0; i < separators.length; i += 1) {
|
|
280
|
+
const s = separators[i];
|
|
281
|
+
if (s === "") {
|
|
282
|
+
separator = s;
|
|
283
|
+
break;
|
|
284
|
+
}
|
|
285
|
+
if (chunk.text.includes(s)) {
|
|
286
|
+
separator = s;
|
|
287
|
+
newSeparators = separators.slice(i + 1);
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
// Now that we have the separator, split the text
|
|
292
|
+
const splits = await this.splitOnSeparator(chunk.text, separator, offset);
|
|
293
|
+
// Now go merging things, recursively splitting longer texts.
|
|
294
|
+
let goodSplits = [];
|
|
295
|
+
const _separator = this.keepSeparator ? "" : separator;
|
|
296
|
+
for (const s of splits) {
|
|
297
|
+
if (s.length < this.chunkSize) {
|
|
298
|
+
goodSplits.push(s);
|
|
299
|
+
} else
|
|
300
|
+
{
|
|
301
|
+
if (goodSplits.length) {
|
|
302
|
+
const mergedText = await this.mergeSplits(goodSplits, _separator);
|
|
303
|
+
finalChunks.push(...mergedText);
|
|
304
|
+
goodSplits = [];
|
|
305
|
+
}
|
|
306
|
+
if (!newSeparators) {
|
|
307
|
+
finalChunks.push(s);
|
|
308
|
+
} else
|
|
309
|
+
{
|
|
310
|
+
const otherInfo = await this._splitText(s, newSeparators, s.position);
|
|
311
|
+
finalChunks.push(...otherInfo);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (goodSplits.length) {
|
|
316
|
+
const mergedText = await this.mergeSplits(goodSplits, _separator);
|
|
317
|
+
finalChunks.push(...mergedText);
|
|
318
|
+
}
|
|
319
|
+
return finalChunks;
|
|
320
|
+
}
|
|
321
|
+
async splitText(text) {
|
|
322
|
+
const splits = (await this._splitText({ text, position: 0, overlap: 0, length: text.length }, this.separators)).map((chunk) => chunk.text);
|
|
323
|
+
return splits;
|
|
324
|
+
}
|
|
325
|
+
async splitTextWithMetadata(text) {
|
|
326
|
+
if (this.stripWhitespace === true || this.keepSeparator === false) {
|
|
327
|
+
console.warn("Might not produce correct results when stripWhitespace = true and keepSeparator = false");
|
|
328
|
+
}
|
|
329
|
+
const splits = await this._splitText({ text, position: 0, overlap: 0, length: text.length }, this.separators);
|
|
330
|
+
return splits;
|
|
331
|
+
}
|
|
332
|
+
static fromLanguage(language, options) {
|
|
333
|
+
return new RecursiveCharacterTextSplitter({
|
|
334
|
+
...options,
|
|
335
|
+
separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language)
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
static getSeparatorsForLanguage(language) {
|
|
339
|
+
if (language === "cpp") {
|
|
340
|
+
return [
|
|
341
|
+
// Split along class definitions
|
|
342
|
+
"\nclass ",
|
|
343
|
+
// Split along function definitions
|
|
344
|
+
"\nvoid ",
|
|
345
|
+
"\nint ",
|
|
346
|
+
"\nfloat ",
|
|
347
|
+
"\ndouble ",
|
|
348
|
+
// Split along control flow statements
|
|
349
|
+
"\nif ",
|
|
350
|
+
"\nfor ",
|
|
351
|
+
"\nwhile ",
|
|
352
|
+
"\nswitch ",
|
|
353
|
+
"\ncase ",
|
|
354
|
+
// Split by the normal type of lines
|
|
355
|
+
"\n\n",
|
|
356
|
+
"\n",
|
|
357
|
+
" ",
|
|
358
|
+
""];
|
|
359
|
+
|
|
360
|
+
} else
|
|
361
|
+
if (language === "go") {
|
|
362
|
+
return [
|
|
363
|
+
// Split along function definitions
|
|
364
|
+
"\nfunc ",
|
|
365
|
+
"\nvar ",
|
|
366
|
+
"\nconst ",
|
|
367
|
+
"\ntype ",
|
|
368
|
+
// Split along control flow statements
|
|
369
|
+
"\nif ",
|
|
370
|
+
"\nfor ",
|
|
371
|
+
"\nswitch ",
|
|
372
|
+
"\ncase ",
|
|
373
|
+
// Split by the normal type of lines
|
|
374
|
+
"\n\n",
|
|
375
|
+
"\n",
|
|
376
|
+
" ",
|
|
377
|
+
""];
|
|
378
|
+
|
|
379
|
+
} else
|
|
380
|
+
if (language === "java") {
|
|
381
|
+
return [
|
|
382
|
+
// Split along class definitions
|
|
383
|
+
"\nclass ",
|
|
384
|
+
// Split along method definitions
|
|
385
|
+
"\npublic ",
|
|
386
|
+
"\nprotected ",
|
|
387
|
+
"\nprivate ",
|
|
388
|
+
"\nstatic ",
|
|
389
|
+
// Split along control flow statements
|
|
390
|
+
"\nif ",
|
|
391
|
+
"\nfor ",
|
|
392
|
+
"\nwhile ",
|
|
393
|
+
"\nswitch ",
|
|
394
|
+
"\ncase ",
|
|
395
|
+
// Split by the normal type of lines
|
|
396
|
+
"\n\n",
|
|
397
|
+
"\n",
|
|
398
|
+
" ",
|
|
399
|
+
""];
|
|
400
|
+
|
|
401
|
+
} else
|
|
402
|
+
if (language === "js") {
|
|
403
|
+
return [
|
|
404
|
+
// Split along function definitions
|
|
405
|
+
"\nfunction ",
|
|
406
|
+
"\nconst ",
|
|
407
|
+
"\nlet ",
|
|
408
|
+
"\nvar ",
|
|
409
|
+
"\nclass ",
|
|
410
|
+
// Split along control flow statements
|
|
411
|
+
"\nif ",
|
|
412
|
+
"\nfor ",
|
|
413
|
+
"\nwhile ",
|
|
414
|
+
"\nswitch ",
|
|
415
|
+
"\ncase ",
|
|
416
|
+
"\ndefault ",
|
|
417
|
+
// Split by the normal type of lines
|
|
418
|
+
"\n\n",
|
|
419
|
+
"\n",
|
|
420
|
+
" ",
|
|
421
|
+
""];
|
|
422
|
+
|
|
423
|
+
} else
|
|
424
|
+
if (language === "php") {
|
|
425
|
+
return [
|
|
426
|
+
// Split along function definitions
|
|
427
|
+
"\nfunction ",
|
|
428
|
+
// Split along class definitions
|
|
429
|
+
"\nclass ",
|
|
430
|
+
// Split along control flow statements
|
|
431
|
+
"\nif ",
|
|
432
|
+
"\nforeach ",
|
|
433
|
+
"\nwhile ",
|
|
434
|
+
"\ndo ",
|
|
435
|
+
"\nswitch ",
|
|
436
|
+
"\ncase ",
|
|
437
|
+
// Split by the normal type of lines
|
|
438
|
+
"\n\n",
|
|
439
|
+
"\n",
|
|
440
|
+
" ",
|
|
441
|
+
""];
|
|
442
|
+
|
|
443
|
+
} else
|
|
444
|
+
if (language === "proto") {
|
|
445
|
+
return [
|
|
446
|
+
// Split along message definitions
|
|
447
|
+
"\nmessage ",
|
|
448
|
+
// Split along service definitions
|
|
449
|
+
"\nservice ",
|
|
450
|
+
// Split along enum definitions
|
|
451
|
+
"\nenum ",
|
|
452
|
+
// Split along option definitions
|
|
453
|
+
"\noption ",
|
|
454
|
+
// Split along import statements
|
|
455
|
+
"\nimport ",
|
|
456
|
+
// Split along syntax declarations
|
|
457
|
+
"\nsyntax ",
|
|
458
|
+
// Split by the normal type of lines
|
|
459
|
+
"\n\n",
|
|
460
|
+
"\n",
|
|
461
|
+
" ",
|
|
462
|
+
""];
|
|
463
|
+
|
|
464
|
+
} else
|
|
465
|
+
if (language === "python") {
|
|
466
|
+
return [
|
|
467
|
+
// First, try to split along class definitions
|
|
468
|
+
"\nclass ",
|
|
469
|
+
"\ndef ",
|
|
470
|
+
"\n\tdef ",
|
|
471
|
+
// Now split by the normal type of lines
|
|
472
|
+
"\n\n",
|
|
473
|
+
"\n",
|
|
474
|
+
" ",
|
|
475
|
+
""];
|
|
476
|
+
|
|
477
|
+
} else
|
|
478
|
+
if (language === "rst") {
|
|
479
|
+
return [
|
|
480
|
+
// Split along section titles
|
|
481
|
+
"\n===\n",
|
|
482
|
+
"\n---\n",
|
|
483
|
+
"\n***\n",
|
|
484
|
+
// Split along directive markers
|
|
485
|
+
"\n.. ",
|
|
486
|
+
// Split by the normal type of lines
|
|
487
|
+
"\n\n",
|
|
488
|
+
"\n",
|
|
489
|
+
" ",
|
|
490
|
+
""];
|
|
491
|
+
|
|
492
|
+
} else
|
|
493
|
+
if (language === "ruby") {
|
|
494
|
+
return [
|
|
495
|
+
// Split along method definitions
|
|
496
|
+
"\ndef ",
|
|
497
|
+
"\nclass ",
|
|
498
|
+
// Split along control flow statements
|
|
499
|
+
"\nif ",
|
|
500
|
+
"\nunless ",
|
|
501
|
+
"\nwhile ",
|
|
502
|
+
"\nfor ",
|
|
503
|
+
"\ndo ",
|
|
504
|
+
"\nbegin ",
|
|
505
|
+
"\nrescue ",
|
|
506
|
+
// Split by the normal type of lines
|
|
507
|
+
"\n\n",
|
|
508
|
+
"\n",
|
|
509
|
+
" ",
|
|
510
|
+
""];
|
|
511
|
+
|
|
512
|
+
} else
|
|
513
|
+
if (language === "rust") {
|
|
514
|
+
return [
|
|
515
|
+
// Split along function definitions
|
|
516
|
+
"\nfn ",
|
|
517
|
+
"\nconst ",
|
|
518
|
+
"\nlet ",
|
|
519
|
+
// Split along control flow statements
|
|
520
|
+
"\nif ",
|
|
521
|
+
"\nwhile ",
|
|
522
|
+
"\nfor ",
|
|
523
|
+
"\nloop ",
|
|
524
|
+
"\nmatch ",
|
|
525
|
+
"\nconst ",
|
|
526
|
+
// Split by the normal type of lines
|
|
527
|
+
"\n\n",
|
|
528
|
+
"\n",
|
|
529
|
+
" ",
|
|
530
|
+
""];
|
|
531
|
+
|
|
532
|
+
} else
|
|
533
|
+
if (language === "scala") {
|
|
534
|
+
return [
|
|
535
|
+
// Split along class definitions
|
|
536
|
+
"\nclass ",
|
|
537
|
+
"\nobject ",
|
|
538
|
+
// Split along method definitions
|
|
539
|
+
"\ndef ",
|
|
540
|
+
"\nval ",
|
|
541
|
+
"\nvar ",
|
|
542
|
+
// Split along control flow statements
|
|
543
|
+
"\nif ",
|
|
544
|
+
"\nfor ",
|
|
545
|
+
"\nwhile ",
|
|
546
|
+
"\nmatch ",
|
|
547
|
+
"\ncase ",
|
|
548
|
+
// Split by the normal type of lines
|
|
549
|
+
"\n\n",
|
|
550
|
+
"\n",
|
|
551
|
+
" ",
|
|
552
|
+
""];
|
|
553
|
+
|
|
554
|
+
} else
|
|
555
|
+
if (language === "swift") {
|
|
556
|
+
return [
|
|
557
|
+
// Split along function definitions
|
|
558
|
+
"\nfunc ",
|
|
559
|
+
// Split along class definitions
|
|
560
|
+
"\nclass ",
|
|
561
|
+
"\nstruct ",
|
|
562
|
+
"\nenum ",
|
|
563
|
+
// Split along control flow statements
|
|
564
|
+
"\nif ",
|
|
565
|
+
"\nfor ",
|
|
566
|
+
"\nwhile ",
|
|
567
|
+
"\ndo ",
|
|
568
|
+
"\nswitch ",
|
|
569
|
+
"\ncase ",
|
|
570
|
+
// Split by the normal type of lines
|
|
571
|
+
"\n\n",
|
|
572
|
+
"\n",
|
|
573
|
+
" ",
|
|
574
|
+
""];
|
|
575
|
+
|
|
576
|
+
} else
|
|
577
|
+
if (language === "markdown") {
|
|
578
|
+
return [
|
|
579
|
+
// First, try to split along Markdown headings (starting with level 2)
|
|
580
|
+
"\n## ",
|
|
581
|
+
"\n### ",
|
|
582
|
+
"\n#### ",
|
|
583
|
+
"\n##### ",
|
|
584
|
+
"\n###### ",
|
|
585
|
+
// Note the alternative syntax for headings (below) is not handled here
|
|
586
|
+
// Heading level 2
|
|
587
|
+
// ---------------
|
|
588
|
+
// End of code block
|
|
589
|
+
"```\n\n",
|
|
590
|
+
// Horizontal lines
|
|
591
|
+
"\n\n***\n\n",
|
|
592
|
+
"\n\n---\n\n",
|
|
593
|
+
"\n\n___\n\n",
|
|
594
|
+
// Note that this splitter doesn't handle horizontal lines defined
|
|
595
|
+
// by *three or more* of ***, ---, or ___, but this is not handled
|
|
596
|
+
"\n\n",
|
|
597
|
+
"\n",
|
|
598
|
+
" ",
|
|
599
|
+
""];
|
|
600
|
+
|
|
601
|
+
} else
|
|
602
|
+
if (language === "latex") {
|
|
603
|
+
return [
|
|
604
|
+
// First, try to split along Latex sections
|
|
605
|
+
"\n\\chapter{",
|
|
606
|
+
"\n\\section{",
|
|
607
|
+
"\n\\subsection{",
|
|
608
|
+
"\n\\subsubsection{",
|
|
609
|
+
// Now split by environments
|
|
610
|
+
"\n\\begin{enumerate}",
|
|
611
|
+
"\n\\begin{itemize}",
|
|
612
|
+
"\n\\begin{description}",
|
|
613
|
+
"\n\\begin{list}",
|
|
614
|
+
"\n\\begin{quote}",
|
|
615
|
+
"\n\\begin{quotation}",
|
|
616
|
+
"\n\\begin{verse}",
|
|
617
|
+
"\n\\begin{verbatim}",
|
|
618
|
+
// Now split by math environments
|
|
619
|
+
"\n\\begin{align}",
|
|
620
|
+
"$$",
|
|
621
|
+
"$",
|
|
622
|
+
// Now split by the normal type of lines
|
|
623
|
+
"\n\n",
|
|
624
|
+
"\n",
|
|
625
|
+
" ",
|
|
626
|
+
""];
|
|
627
|
+
|
|
628
|
+
} else
|
|
629
|
+
if (language === "html") {
|
|
630
|
+
return [
|
|
631
|
+
// First, try to split along HTML tags
|
|
632
|
+
"<body>",
|
|
633
|
+
"<div>",
|
|
634
|
+
"<p>",
|
|
635
|
+
"<br>",
|
|
636
|
+
"<li>",
|
|
637
|
+
"<h1>",
|
|
638
|
+
"<h2>",
|
|
639
|
+
"<h3>",
|
|
640
|
+
"<h4>",
|
|
641
|
+
"<h5>",
|
|
642
|
+
"<h6>",
|
|
643
|
+
"<span>",
|
|
644
|
+
"<table>",
|
|
645
|
+
"<tr>",
|
|
646
|
+
"<td>",
|
|
647
|
+
"<th>",
|
|
648
|
+
"<ul>",
|
|
649
|
+
"<ol>",
|
|
650
|
+
"<header>",
|
|
651
|
+
"<footer>",
|
|
652
|
+
"<nav>",
|
|
653
|
+
// Head
|
|
654
|
+
"<head>",
|
|
655
|
+
"<style>",
|
|
656
|
+
"<script>",
|
|
657
|
+
"<meta>",
|
|
658
|
+
"<title>",
|
|
659
|
+
// Normal type of lines
|
|
660
|
+
" ",
|
|
661
|
+
""];
|
|
662
|
+
|
|
663
|
+
} else
|
|
664
|
+
if (language === "sol") {
|
|
665
|
+
return [
|
|
666
|
+
// Split along compiler informations definitions
|
|
667
|
+
"\npragma ",
|
|
668
|
+
"\nusing ",
|
|
669
|
+
// Split along contract definitions
|
|
670
|
+
"\ncontract ",
|
|
671
|
+
"\ninterface ",
|
|
672
|
+
"\nlibrary ",
|
|
673
|
+
// Split along method definitions
|
|
674
|
+
"\nconstructor ",
|
|
675
|
+
"\ntype ",
|
|
676
|
+
"\nfunction ",
|
|
677
|
+
"\nevent ",
|
|
678
|
+
"\nmodifier ",
|
|
679
|
+
"\nerror ",
|
|
680
|
+
"\nstruct ",
|
|
681
|
+
"\nenum ",
|
|
682
|
+
// Split along control flow statements
|
|
683
|
+
"\nif ",
|
|
684
|
+
"\nfor ",
|
|
685
|
+
"\nwhile ",
|
|
686
|
+
"\ndo while ",
|
|
687
|
+
"\nassembly ",
|
|
688
|
+
// Split by the normal type of lines
|
|
689
|
+
"\n\n",
|
|
690
|
+
"\n",
|
|
691
|
+
" ",
|
|
692
|
+
""];
|
|
693
|
+
|
|
694
|
+
} else
|
|
695
|
+
{
|
|
696
|
+
throw new Error(`Language ${language} is not supported.`);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* Implementation of splitter which looks at tokens.
|
|
702
|
+
*/
|
|
703
|
+
export class TokenTextSplitter extends TextSplitter {
|
|
704
|
+
static lc_name() {
|
|
705
|
+
return "TokenTextSplitter";
|
|
706
|
+
}
|
|
707
|
+
encodingName;
|
|
708
|
+
allowedSpecial;
|
|
709
|
+
disallowedSpecial;
|
|
710
|
+
tokenizer;
|
|
711
|
+
constructor(fields) {
|
|
712
|
+
super(fields);
|
|
713
|
+
this.encodingName = fields?.encodingName ?? "gpt2";
|
|
714
|
+
this.allowedSpecial = fields?.allowedSpecial ?? [];
|
|
715
|
+
this.disallowedSpecial = fields?.disallowedSpecial ?? "all";
|
|
716
|
+
}
|
|
717
|
+
async splitText(text) {
|
|
718
|
+
if (!this.tokenizer) {
|
|
719
|
+
this.tokenizer = await getEncoding(this.encodingName);
|
|
720
|
+
}
|
|
721
|
+
const splits = [];
|
|
722
|
+
const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
|
|
723
|
+
let start_idx = 0;
|
|
724
|
+
while (start_idx < input_ids.length) {
|
|
725
|
+
if (start_idx > 0) {
|
|
726
|
+
start_idx -= this.chunkOverlap;
|
|
727
|
+
}
|
|
728
|
+
const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
|
|
729
|
+
const chunk_ids = input_ids.slice(start_idx, end_idx);
|
|
730
|
+
splits.push(this.tokenizer.decode(chunk_ids));
|
|
731
|
+
start_idx = end_idx;
|
|
732
|
+
}
|
|
733
|
+
return splits;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
export class MarkdownTextSplitter extends RecursiveCharacterTextSplitter {
|
|
737
|
+
constructor(fields) {
|
|
738
|
+
super({
|
|
739
|
+
...fields,
|
|
740
|
+
separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage("markdown")
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
export class LatexTextSplitter extends RecursiveCharacterTextSplitter {
|
|
745
|
+
constructor(fields) {
|
|
746
|
+
super({
|
|
747
|
+
...fields,
|
|
748
|
+
separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage("latex")
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"textSplitters.js","sourceRoot":"","sources":["../src/textSplitters.ts"],"names":[],"mappings":"AAAA,kBAAkB;AAClB,EAAE;AACF,+BAA+B;AAC/B,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAC/E,4EAA4E;AAC5E,wEAAwE;AACxE,2DAA2D;AAC3D,EAAE;AACF,6EAA6E;AAC7E,sDAAsD;AACtD,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,yEAAyE;AACzE,gFAAgF;AAChF,4EAA4E;AAC5E,gBAAgB;AAChB,EAAE;AACF,qEAAqE;AACrE,6DAA6D;AAG7D,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAyB7D,MAAM,OAAgB,YAAa,SAAQ,uBAAuB;IAE9D,YAAY,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;IAExE,SAAS,GAAG,IAAI,CAAC;IAEjB,YAAY,GAAG,GAAG,CAAC;IAEnB,aAAa,GAAG,KAAK,CAAC;IAEtB,eAAe,GAAG,IAAI,CAAC;IAEvB,cAAc,CAE4B;IAE1C,YAAY,MAAoC;QAC5C,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC;QAC9D,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,EAAE,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC;QACvE,IAAI,CAAC,cAAc;YACf,MAAM,EAAE,cAAc,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CACpB,SAAqB,EACrB,qBAAqD,EAAE;QAEvD,OAAO,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAC9D,CAAC;IAIS,KAAK,CAAC,gBAAgB,CAC5B,IAAY,EACZ,SAAiB,EACjB,SAAiB,CAAC;QAElB,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;oBACb,QAAQ,EAAE,MAAM,GAAG,CAAC;oBACpB,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,CAAC;iBACZ,CAAC,CAAC;YACP,CAAC;YACD,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,MAAM,qBAAqB,GAAG,SAAS,CAAC,OAAO,CAC3C,wBAAwB,EACxB,MAAM,CACT,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;QACrD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAA6B,CAAC;QAElC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACrD,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,SAAS;oBACf,QAAQ,EAAE,MAAM,GAAG,SAAS;oBAC5B,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC;iBAC/C,CAAC,CAAC;YACP,CAAC;YAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACrB,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACJ,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC9C,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,MAAM,GAAG,SAAS;gBAC5B,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC;aAC/C,CAAC,CAAC;QACP,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,eAAe,CACjB,KAAe;IACf,8DAA8D;IAC9D,YAAmC,EAAE,EACrC,qBAAqD,EAAE;QAEvD,mEAAmE;QACnE,8DAA8D;QAC9D,MAAM,UAAU,GACZ,SAAS,CAAC,MAAM,GAAG,CAAC;YAChB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,MAAM,EACF,WAAW,GAAG,EAAE,EAChB,kBAAkB,GAAG,WAAW,EAChC,wBAAwB,GAAG,KAAK,EACnC,GAAG,kBAAkB,CAAC;QACvB,MAAM,SAAS,GAAG,IAAI,KAAK,EAAY,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,SAAS,GAAG,IAAI,CAAC;YACrB,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,IAAI,WAAW,GAAG,WAAW,CAAC;gBAE9B,uFAAuF;gBACvF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;gBAC3D,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;oBACrB,MAAM,wBAAwB,GAAG,IAAI,CAAC,gBAAgB,CAClD,IAAI,EACJ,CAAC,EACD,UAAU,CACb,CAAC;oBACF,gBAAgB,IAAI,wBAAwB,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACJ,MAAM,iBAAiB,GACnB,cAAc,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;oBAC5D,IAAI,iBAAiB,GAAG,UAAU,EAAE,CAAC;wBACjC,MAAM,4BAA4B,GAAG,IAAI,CAAC,gBAAgB,CACtD,IAAI,EACJ,iBAAiB,EACjB,UAAU,CACb,CAAC;wBACF,gBAAgB,IAAI,4BAA4B,CAAC;oBACrD,CAAC;yBAAM,IAAI,iBAAiB,GAAG,UAAU,EAAE,CAAC;wBACxC,MAAM,4BAA4B,GAAG,IAAI,CAAC,gBAAgB,CACtD,IAAI,EACJ,UAAU,EACV,iBAAiB,CACpB,CAAC;wBACF,gBAAgB,IAAI,4BAA4B,CAAC;oBACrD,CAAC;oBACD,IAAI,wBAAwB,EAAE,CAAC;wBAC3B,WAAW,IAAI,kBAAkB,CAAC;oBACtC,CAAC;gBACL,CAAC;gBACD,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;gBAEnD,MAAM,GAAG,GACL,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ;oBACtD,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE;oBAC1B,CAAC,CAAC,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,GAAG;oBACR,IAAI,EAAE,gBAAgB;oBACtB,EAAE,EAAE,gBAAgB,GAAG,aAAa;iBACvC,CAAC;gBACF,MAAM,uBAAuB,GAAG;oBAC5B,GAAG,UAAU,CAAC,CAAC,CAAC;oBAChB,GAAG;iBACN,CAAC;gBAEF,WAAW,IAAI,KAAK,CAAC;gBACrB,SAAS,CAAC,IAAI,CACV,IAAI,QAAQ,CAAC;oBACT,WAAW;oBACX,QAAQ,EAAE,uBAAuB;iBACpC,CAAC,CACL,CAAC;gBACF,gBAAgB,IAAI,aAAa,CAAC;gBAClC,SAAS,GAAG,KAAK,CAAC;gBAClB,cAAc,GAAG,UAAU,CAAC;YAChC,CAAC;QACL,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,gBAAgB,CAAC,IAAY,EAAE,KAAc,EAAE,GAAY;QAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,cAAc,CAChB,SAAqB,EACrB,qBAAqD,EAAE;QAEvD,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,SAAS,CACzC,CAAC;QACF,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,kBAAkB,CAAC,CAAC;IACtE,CAAC;IAEO,KAAK,CAAC,QAAQ,CAClB,IAAyB,EACzB,SAAiB;QAEjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,iBAAiB,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ;YAC1B,OAAO,EAAE,CAAC;YACV,MAAM;SACT,CAAC;QACF,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,WAAW,CACb,MAA2B,EAC3B,SAAiB;QAEjB,MAAM,IAAI,GAAwB,EAAE,CAAC;QACrC,MAAM,UAAU,GAAwB,EAAE,CAAC;QAC3C,IAAI,aAAa,GAAa,EAAE,CAAC;QACjC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;YACtB,IACI,KAAK,GAAG,IAAI,GAAG,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;gBACnD,IAAI,CAAC,SAAS,EAChB,CAAC;gBACC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO,CAAC,IAAI,CACR,2BAA2B,KAAK;qCACnB,IAAI,CAAC,SAAS,EAAE,CAChC,CAAC;gBACN,CAAC;gBACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBACvD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;wBACf,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CACjD,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC9B,CAAC;wBACF,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAC1C,CAAC,CACJ,CAAC;wBACF,GAAG,CAAC,OAAO,GAAG,aAAa,CAAC;wBAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACf,aAAa,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;oBACrC,CAAC;oBACD,sBAAsB;oBACtB,qDAAqD;oBACrD,0DAA0D;oBAC1D,OACI,KAAK,GAAG,IAAI,CAAC,YAAY;wBACzB,CAAC,KAAK,GAAG,IAAI,GAAG,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;4BAChD,IAAI,CAAC,SAAS;4BACd,KAAK,GAAG,CAAC,CAAC,EAChB,CAAC;wBACC,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC9B,UAAU,CAAC,KAAK,EAAE,CAAC;wBACnB,WAAW,CAAC,KAAK,EAAE,CAAC;oBACxB,CAAC;gBACL,CAAC;YACL,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,KAAK,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACvD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACf,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CACjD,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC9B,CAAC;YACF,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAC1C,CAAC,CACJ,CAAC;YACF,GAAG,CAAC,OAAO,GAAG,aAAa,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AAMD,MAAM,OAAO,qBAAsB,SAAQ,YAAY;IAEnD,MAAM,CAAC,OAAO;QACV,OAAO,uBAAuB,CAAC;IACnC,CAAC;IAED,SAAS,GAAG,MAAM,CAAC;IAEnB,YAAY,MAA6C;QACrD,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QACxB,uEAAuE;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACjE,OAAO,CACH,MAAM,IAAI,CAAC,WAAW,CAClB,MAAM,EACN,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAC3C,CACJ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;CACJ;AAOD,MAAM,CAAC,MAAM,8BAA8B,GAAG;IAC1C,KAAK;IACL,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,KAAK;IACL,OAAO;IACP,QAAQ;IACR,KAAK;IACL,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,UAAU;IACV,OAAO;IACP,MAAM;IACN,KAAK;CACC,CAAC;AAIX,MAAM,OAAO,8BAA+B,SAAQ,YAAY;IAE5D,MAAM,CAAC,OAAO;QACV,OAAO,gCAAgC,CAAC;IAC5C,CAAC;IAED,UAAU,GAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAE/C,YAAY,MAAsD;QAC9D,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC;QACxD,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,IAAI,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,KAAwB,EACxB,UAAoB,EACpB,SAAiB,CAAC;QAElB,MAAM,WAAW,GAAwB,EAAE,CAAC;QAE5C,mCAAmC;QACnC,IAAI,SAAS,GAAW,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,IAAI,aAAa,CAAC;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;gBACX,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;YACV,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,SAAS,GAAG,CAAC,CAAC;gBACd,aAAa,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxC,MAAM;YACV,CAAC;QACL,CAAC;QAED,iDAAiD;QACjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACtC,KAAK,CAAC,IAAI,EACV,SAAS,EACT,MAAM,CACT,CAAC;QAEF,6DAA6D;QAC7D,IAAI,UAAU,GAAwB,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACvD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC5B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACpB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CACrC,UAAU,EACV,UAAU,CACb,CAAC;oBACF,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;oBAChC,UAAU,GAAG,EAAE,CAAC;gBACpB,CAAC;gBACD,IAAI,CAAC,aAAa,EAAE,CAAC;oBACjB,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACJ,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CACnC,CAAC,EACD,aAAa,EACb,CAAC,CAAC,QAAQ,CACb,CAAC;oBACF,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;QACL,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YAClE,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,WAAW,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QACxB,MAAM,MAAM,GAAG,CACX,MAAM,IAAI,CAAC,UAAU,CACjB,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EACtD,IAAI,CAAC,UAAU,CAClB,CACJ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,IAAY;QACpC,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;YAChE,OAAO,CAAC,IAAI,CACR,yFAAyF,CAC5F,CAAC;QACN,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAChC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EACtD,IAAI,CAAC,UAAU,CAClB,CAAC;QACF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,MAAM,CAAC,YAAY,CACf,QAAuC,EACvC,OAAuD;QAEvD,OAAO,IAAI,8BAA8B,CAAC;YACtC,GAAG,OAAO;YACV,UAAU,EAAE,8BAA8B,CAAC,wBAAwB,CAC/D,QAAQ,CACX;SACJ,CAAC,CAAC;IACP,CAAC;IAED,MAAM,CAAC,wBAAwB,CAAC,QAAuC;QACnE,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YACrB,OAAO;gBACH,gCAAgC;gBAChC,UAAU;gBACV,mCAAmC;gBACnC,SAAS;gBACT,QAAQ;gBACR,UAAU;gBACV,WAAW;gBACX,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,UAAU;gBACV,WAAW;gBACX,SAAS;gBACT,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC3B,OAAO;gBACH,mCAAmC;gBACnC,SAAS;gBACT,QAAQ;gBACR,UAAU;gBACV,SAAS;gBACT,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,WAAW;gBACX,SAAS;gBACT,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO;gBACH,gCAAgC;gBAChC,UAAU;gBACV,iCAAiC;gBACjC,WAAW;gBACX,cAAc;gBACd,YAAY;gBACZ,WAAW;gBACX,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,UAAU;gBACV,WAAW;gBACX,SAAS;gBACT,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC3B,OAAO;gBACH,mCAAmC;gBACnC,aAAa;gBACb,UAAU;gBACV,QAAQ;gBACR,QAAQ;gBACR,UAAU;gBACV,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,UAAU;gBACV,WAAW;gBACX,SAAS;gBACT,YAAY;gBACZ,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO;gBACH,mCAAmC;gBACnC,aAAa;gBACb,gCAAgC;gBAChC,UAAU;gBACV,sCAAsC;gBACtC,OAAO;gBACP,YAAY;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW;gBACX,SAAS;gBACT,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACH,kCAAkC;gBAClC,YAAY;gBACZ,kCAAkC;gBAClC,YAAY;gBACZ,+BAA+B;gBAC/B,SAAS;gBACT,iCAAiC;gBACjC,WAAW;gBACX,gCAAgC;gBAChC,WAAW;gBACX,kCAAkC;gBAClC,WAAW;gBACX,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO;gBACH,8CAA8C;gBAC9C,UAAU;gBACV,QAAQ;gBACR,UAAU;gBACV,wCAAwC;gBACxC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO;gBACH,6BAA6B;gBAC7B,SAAS;gBACT,SAAS;gBACT,SAAS;gBACT,gCAAgC;gBAChC,OAAO;gBACP,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO;gBACH,iCAAiC;gBACjC,QAAQ;gBACR,UAAU;gBACV,sCAAsC;gBACtC,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,QAAQ;gBACR,OAAO;gBACP,UAAU;gBACV,WAAW;gBACX,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO;gBACH,mCAAmC;gBACnC,OAAO;gBACP,UAAU;gBACV,QAAQ;gBACR,sCAAsC;gBACtC,OAAO;gBACP,UAAU;gBACV,QAAQ;gBACR,SAAS;gBACT,UAAU;gBACV,UAAU;gBACV,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACH,gCAAgC;gBAChC,UAAU;gBACV,WAAW;gBACX,iCAAiC;gBACjC,QAAQ;gBACR,QAAQ;gBACR,QAAQ;gBACR,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,UAAU;gBACV,UAAU;gBACV,SAAS;gBACT,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACH,mCAAmC;gBACnC,SAAS;gBACT,gCAAgC;gBAChC,UAAU;gBACV,WAAW;gBACX,SAAS;gBACT,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,UAAU;gBACV,OAAO;gBACP,WAAW;gBACX,SAAS;gBACT,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;YACjC,OAAO;gBACH,sEAAsE;gBACtE,OAAO;gBACP,QAAQ;gBACR,SAAS;gBACT,UAAU;gBACV,WAAW;gBACX,uEAAuE;gBACvE,kBAAkB;gBAClB,kBAAkB;gBAClB,oBAAoB;gBACpB,SAAS;gBACT,mBAAmB;gBACnB,aAAa;gBACb,aAAa;gBACb,aAAa;gBACb,kEAAkE;gBAClE,kEAAkE;gBAClE,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACH,2CAA2C;gBAC3C,cAAc;gBACd,cAAc;gBACd,iBAAiB;gBACjB,oBAAoB;gBAEpB,4BAA4B;gBAC5B,sBAAsB;gBACtB,oBAAoB;gBACpB,wBAAwB;gBACxB,iBAAiB;gBACjB,kBAAkB;gBAClB,sBAAsB;gBACtB,kBAAkB;gBAClB,qBAAqB;gBAErB,iCAAiC;gBACjC,kBAAkB;gBAClB,IAAI;gBACJ,GAAG;gBAEH,wCAAwC;gBACxC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO;gBACH,sCAAsC;gBACtC,QAAQ;gBACR,OAAO;gBACP,KAAK;gBACL,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,SAAS;gBACT,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,MAAM;gBACN,UAAU;gBACV,UAAU;gBACV,OAAO;gBACP,OAAO;gBACP,QAAQ;gBACR,SAAS;gBACT,UAAU;gBACV,QAAQ;gBACR,SAAS;gBACT,uBAAuB;gBACvB,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO;gBACH,gDAAgD;gBAChD,WAAW;gBACX,UAAU;gBACV,mCAAmC;gBACnC,aAAa;gBACb,cAAc;gBACd,YAAY;gBACZ,iCAAiC;gBACjC,gBAAgB;gBAChB,SAAS;gBACT,aAAa;gBACb,UAAU;gBACV,aAAa;gBACb,UAAU;gBACV,WAAW;gBACX,SAAS;gBACT,sCAAsC;gBACtC,OAAO;gBACP,QAAQ;gBACR,UAAU;gBACV,aAAa;gBACb,aAAa;gBACb,oCAAoC;gBACpC,MAAM;gBACN,IAAI;gBACJ,GAAG;gBACH,EAAE;aACL,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,oBAAoB,CAAC,CAAC;QAC9D,CAAC;IACL,CAAC;CACJ;AAQD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IAE/C,MAAM,CAAC,OAAO;QACV,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,YAAY,CAA4B;IAExC,cAAc,CAAwB;IAEtC,iBAAiB,CAAwB;IAEjC,SAAS,CAAoB;IAErC,YAAY,MAAyC;QACjD,KAAK,CAAC,MAAM,CAAC,CAAC;QAEd,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,YAAY,IAAI,MAAM,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,EAAE,CAAC;QACnD,IAAI,CAAC,iBAAiB,GAAG,MAAM,EAAE,iBAAiB,IAAI,KAAK,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAClB,IAAI,CAAC,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CACnC,IAAI,EACJ,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,iBAAiB,CACzB,CAAC;QAEF,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAChB,SAAS,IAAI,IAAI,CAAC,YAAY,CAAC;YACnC,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACpB,SAAS,GAAG,IAAI,CAAC,SAAS,EAC1B,SAAS,CAAC,MAAM,CACnB,CAAC;YACF,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YAC9C,SAAS,GAAG,OAAO,CAAC;QACxB,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAID,MAAM,OAAO,oBAAqB,SAAQ,8BAA8B;IAEpE,YAAY,MAA4C;QACpD,KAAK,CAAC;YACF,GAAG,MAAM;YACT,UAAU,EAAE,8BAA8B,CAAC,wBAAwB,CAC/D,UAAU,CACb;SACJ,CAAC,CAAC;IACP,CAAC;CACJ;AAID,MAAM,OAAO,iBAAkB,SAAQ,8BAA8B;IAEjE,YAAY,MAAyC;QACjD,KAAK,CAAC;YACF,GAAG,MAAM;YACT,UAAU,EAAE,8BAA8B,CAAC,wBAAwB,CAC/D,OAAO,CACV;SACJ,CAAC,CAAC;IACP,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@magda/semantic-indexer-framework",
|
|
3
3
|
"description": "MAGDA Semantic Indexer Framework",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "6.0.0-alpha.12",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
7
7
|
".": "./dist/index.js",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"author": "",
|
|
23
23
|
"license": "Apache-2.0",
|
|
24
24
|
"devDependencies": {
|
|
25
|
-
"@magda/scripts": "^
|
|
25
|
+
"@magda/scripts": "^6.0.0-alpha.12",
|
|
26
26
|
"@types/chai": "^4.3.11",
|
|
27
27
|
"@types/mocha": "^10.0.6",
|
|
28
28
|
"@types/uuid": "^8.0.0",
|
|
@@ -33,10 +33,9 @@
|
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@langchain/core": "^0.3.57",
|
|
36
|
-
"@langchain/textsplitters": "https://gitpkg.vercel.app/JHZhang2736/langchainjs/libs/langchain-textsplitters?textsplitters-build",
|
|
37
36
|
"@magda/esm-utils": "^1.0.1",
|
|
38
|
-
"@magda/minion-framework": "^
|
|
39
|
-
"@magda/typescript-common": "^
|
|
37
|
+
"@magda/minion-framework": "^6.0.0-alpha.12",
|
|
38
|
+
"@magda/typescript-common": "^6.0.0-alpha.12",
|
|
40
39
|
"@types/lodash": "^4.17.17",
|
|
41
40
|
"cross-fetch": "^4.0.0",
|
|
42
41
|
"minio": "^8.0.5",
|