@cogitator-ai/rag 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/chunkers/semantic-chunker.js +1 -1
- package/dist/chunkers/semantic-chunker.js.map +1 -1
- package/dist/loaders/json-loader.js +1 -1
- package/dist/loaders/json-loader.js.map +1 -1
- package/dist/loaders/pdf-loader.d.ts +3 -3
- package/dist/loaders/pdf-loader.d.ts.map +1 -1
- package/dist/loaders/pdf-loader.js +45 -42
- package/dist/loaders/pdf-loader.js.map +1 -1
- package/package.json +14 -13
package/README.md
CHANGED
|
@@ -341,7 +341,7 @@ const [ragSearch, ragIngest] = ragTools(pipeline, { allowedRoots: ['./knowledge-
|
|
|
341
341
|
|
|
342
342
|
const agent = new Agent({
|
|
343
343
|
name: 'docs-assistant',
|
|
344
|
-
model: 'openai/gpt-
|
|
344
|
+
model: 'openai/gpt-6.1-sol',
|
|
345
345
|
instructions: 'Use rag_search to find information before answering.',
|
|
346
346
|
tools: [tool(ragSearch), tool(ragIngest)],
|
|
347
347
|
});
|
|
@@ -188,7 +188,7 @@ export class SemanticChunker {
|
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
190
|
if (currentSentences.length > 0) {
|
|
191
|
-
chunks.push(this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order
|
|
191
|
+
chunks.push(this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order));
|
|
192
192
|
}
|
|
193
193
|
return chunks;
|
|
194
194
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,CAClF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -19,7 +19,7 @@ export class JSONLoader {
|
|
|
19
19
|
}
|
|
20
20
|
catch (err) {
|
|
21
21
|
const message = err instanceof Error ? err.message : String(err);
|
|
22
|
-
throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}
|
|
22
|
+
throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}`, { cause: err });
|
|
23
23
|
}
|
|
24
24
|
const items = Array.isArray(data) ? data : [data];
|
|
25
25
|
return items.map((item) => this.itemToDocument(item, filePath));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -7,9 +7,9 @@ export declare class PDFLoader implements DocumentLoader {
|
|
|
7
7
|
private readonly splitPages;
|
|
8
8
|
constructor(options?: PDFLoaderOptions);
|
|
9
9
|
load(source: string): Promise<RAGDocument[]>;
|
|
10
|
-
private
|
|
11
|
-
private
|
|
12
|
-
private
|
|
10
|
+
private parse;
|
|
11
|
+
private buildSingleDoc;
|
|
12
|
+
private buildPageDocs;
|
|
13
13
|
private buildPageDoc;
|
|
14
14
|
}
|
|
15
15
|
//# sourceMappingURL=pdf-loader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA0BvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAapC,KAAK;IAwBnB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,YAAY;CAkBrB"}
|
|
@@ -4,12 +4,18 @@ import { nanoid } from 'nanoid';
|
|
|
4
4
|
async function loadPdfParse() {
|
|
5
5
|
try {
|
|
6
6
|
const mod = await import('pdf-parse');
|
|
7
|
-
return mod.
|
|
7
|
+
return mod.PDFParse;
|
|
8
8
|
}
|
|
9
9
|
catch {
|
|
10
10
|
throw new Error('pdf-parse is required for PDFLoader. Install it: pnpm add pdf-parse');
|
|
11
11
|
}
|
|
12
12
|
}
|
|
13
|
+
function extractTitle(info) {
|
|
14
|
+
if (typeof info !== 'object' || info === null || !('Title' in info))
|
|
15
|
+
return undefined;
|
|
16
|
+
const title = info.Title;
|
|
17
|
+
return typeof title === 'string' && title.length > 0 ? title : undefined;
|
|
18
|
+
}
|
|
13
19
|
export class PDFLoader {
|
|
14
20
|
supportedTypes = ['pdf'];
|
|
15
21
|
splitPages;
|
|
@@ -17,57 +23,54 @@ export class PDFLoader {
|
|
|
17
23
|
this.splitPages = options?.splitPages ?? false;
|
|
18
24
|
}
|
|
19
25
|
async load(source) {
|
|
20
|
-
const
|
|
26
|
+
const PDFParse = await loadPdfParse();
|
|
21
27
|
const filePath = resolve(source);
|
|
22
28
|
const buffer = await readFile(filePath);
|
|
29
|
+
const parsed = await this.parse(PDFParse, buffer, filePath);
|
|
23
30
|
if (this.splitPages) {
|
|
24
|
-
return this.
|
|
31
|
+
return this.buildPageDocs(parsed, filePath);
|
|
25
32
|
}
|
|
26
|
-
return this.
|
|
33
|
+
return [this.buildSingleDoc(parsed, filePath)];
|
|
27
34
|
}
|
|
28
|
-
async
|
|
29
|
-
|
|
30
|
-
const title = result.info?.Title;
|
|
31
|
-
const metadata = { pages: result.numpages };
|
|
32
|
-
if (title)
|
|
33
|
-
metadata.title = title;
|
|
34
|
-
return [
|
|
35
|
-
{
|
|
36
|
-
id: nanoid(),
|
|
37
|
-
content: result.text,
|
|
38
|
-
source,
|
|
39
|
-
sourceType: 'pdf',
|
|
40
|
-
metadata,
|
|
41
|
-
},
|
|
42
|
-
];
|
|
43
|
-
}
|
|
44
|
-
async loadSplitPages(pdfParse, buffer, source) {
|
|
45
|
-
const pageTexts = [];
|
|
46
|
-
const result = await this.parseWithErrorContext(pdfParse, buffer, source, {
|
|
47
|
-
pagerender: async (pageData) => {
|
|
48
|
-
if (typeof pageData.getTextContent !== 'function')
|
|
49
|
-
return '';
|
|
50
|
-
const textContent = await pageData.getTextContent();
|
|
51
|
-
const text = textContent.items.map((item) => item.str).join('');
|
|
52
|
-
pageTexts.push(text);
|
|
53
|
-
return text;
|
|
54
|
-
},
|
|
55
|
-
});
|
|
56
|
-
const title = result.info?.Title;
|
|
57
|
-
const pages = pageTexts.filter((p) => p.trim().length > 0);
|
|
58
|
-
if (pages.length === 0) {
|
|
59
|
-
return [this.buildPageDoc(result.text, source, 1, result.numpages, title)];
|
|
60
|
-
}
|
|
61
|
-
return pages.map((text, i) => this.buildPageDoc(text.trim(), source, i + 1, result.numpages, title));
|
|
62
|
-
}
|
|
63
|
-
async parseWithErrorContext(pdfParse, buffer, source, options) {
|
|
35
|
+
async parse(PDFParse, buffer, source) {
|
|
36
|
+
let parser;
|
|
64
37
|
try {
|
|
65
|
-
|
|
38
|
+
parser = new PDFParse({ data: new Uint8Array(buffer) });
|
|
39
|
+
const info = await parser.getInfo();
|
|
40
|
+
const result = await parser.getText({ pageJoiner: '' });
|
|
41
|
+
return {
|
|
42
|
+
text: result.text,
|
|
43
|
+
totalPages: result.total,
|
|
44
|
+
pages: result.pages.map(({ num, text }) => ({ num, text })),
|
|
45
|
+
title: extractTitle(info.info),
|
|
46
|
+
};
|
|
66
47
|
}
|
|
67
48
|
catch (err) {
|
|
68
49
|
const message = err instanceof Error ? err.message : String(err);
|
|
69
|
-
throw new Error(`PDFLoader: failed to parse "${source}": ${message}
|
|
50
|
+
throw new Error(`PDFLoader: failed to parse "${source}": ${message}`, { cause: err });
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
await parser?.destroy();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
buildSingleDoc(parsed, source) {
|
|
57
|
+
const metadata = { pages: parsed.totalPages };
|
|
58
|
+
if (parsed.title)
|
|
59
|
+
metadata.title = parsed.title;
|
|
60
|
+
return {
|
|
61
|
+
id: nanoid(),
|
|
62
|
+
content: parsed.text,
|
|
63
|
+
source,
|
|
64
|
+
sourceType: 'pdf',
|
|
65
|
+
metadata,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
buildPageDocs(parsed, source) {
|
|
69
|
+
const pages = parsed.pages.filter((page) => page.text.trim().length > 0);
|
|
70
|
+
if (pages.length === 0) {
|
|
71
|
+
return [this.buildPageDoc(parsed.text, source, 1, parsed.totalPages, parsed.title)];
|
|
70
72
|
}
|
|
73
|
+
return pages.map((page) => this.buildPageDoc(page.text.trim(), source, page.num, parsed.totalPages, parsed.title));
|
|
71
74
|
}
|
|
72
75
|
buildPageDoc(content, source, pageNumber, totalPages, title) {
|
|
73
76
|
const metadata = { pageNumber, totalPages };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAahC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAa;IACjC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACtF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3E,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,QAA6B,EAC7B,MAAc,EACd,MAAc;QAEd,IAAI,MAAiC,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YACxD,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,KAAK;gBACxB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3D,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;aAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,MAAM,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACxF,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,EAAE,OAAO,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,MAAiB,EAAE,MAAc;QACtD,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;QACvE,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAEhD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO,EAAE,MAAM,CAAC,IAAI;YACpB,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;IAEO,aAAa,CAAC,MAAiB,EAAE,MAAc;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CACvF,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAyB;QAEzB,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/rag",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Retrieval Augmented Generation for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -14,13 +14,16 @@
|
|
|
14
14
|
"files": [
|
|
15
15
|
"dist"
|
|
16
16
|
],
|
|
17
|
+
"engines": {
|
|
18
|
+
"node": ">=22.12.0"
|
|
19
|
+
},
|
|
17
20
|
"dependencies": {
|
|
18
|
-
"
|
|
19
|
-
"
|
|
20
|
-
"
|
|
21
|
+
"@cogitator-ai/types": "0.25.0",
|
|
22
|
+
"nanoid": "^6.0.1",
|
|
23
|
+
"zod": "^4.6.5"
|
|
21
24
|
},
|
|
22
25
|
"peerDependencies": {
|
|
23
|
-
"@cogitator-ai/memory": ">=0.
|
|
26
|
+
"@cogitator-ai/memory": ">=0.8.1"
|
|
24
27
|
},
|
|
25
28
|
"peerDependenciesMeta": {
|
|
26
29
|
"@cogitator-ai/memory": {
|
|
@@ -29,15 +32,14 @@
|
|
|
29
32
|
},
|
|
30
33
|
"optionalDependencies": {
|
|
31
34
|
"cheerio": "^1.0.0",
|
|
32
|
-
"papaparse": "^5.
|
|
33
|
-
"pdf-parse": "^
|
|
35
|
+
"papaparse": "^5.7.0",
|
|
36
|
+
"pdf-parse": "^2.4.5"
|
|
34
37
|
},
|
|
35
38
|
"devDependencies": {
|
|
39
|
+
"@cogitator-ai/memory": "0.8.1",
|
|
36
40
|
"@types/papaparse": "^5.3.15",
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"vitest": "^4.0.18",
|
|
40
|
-
"@cogitator-ai/memory": "0.7.0"
|
|
41
|
+
"typescript": "^6.0.3",
|
|
42
|
+
"vitest": "^5.0.3"
|
|
41
43
|
},
|
|
42
44
|
"repository": {
|
|
43
45
|
"type": "git",
|
|
@@ -45,8 +47,7 @@
|
|
|
45
47
|
"directory": "packages/rag"
|
|
46
48
|
},
|
|
47
49
|
"publishConfig": {
|
|
48
|
-
"access": "public"
|
|
49
|
-
"registry": "https://npm.pkg.github.com"
|
|
50
|
+
"access": "public"
|
|
50
51
|
},
|
|
51
52
|
"license": "MIT",
|
|
52
53
|
"scripts": {
|