@equationalapplications/core-llm-wiki 5.0.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -2
- package/dist/{chunk-W53E3I44.mjs → chunk-DM7BKDCV.mjs} +148 -144
- package/dist/chunk-DM7BKDCV.mjs.map +1 -0
- package/dist/index.d.mts +62 -1
- package/dist/index.d.ts +62 -1
- package/dist/index.js +11 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/testing.js +7 -3
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-W53E3I44.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -969,6 +969,10 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
969
969
|
}
|
|
970
970
|
};
|
|
971
971
|
|
|
972
|
+
// src/utils/chunkingDefaults.ts
|
|
973
|
+
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
974
|
+
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
975
|
+
|
|
972
976
|
// src/services/IngestionService.ts
|
|
973
977
|
var IngestionService = class {
|
|
974
978
|
constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
@@ -987,10 +991,10 @@ var IngestionService = class {
|
|
|
987
991
|
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
988
992
|
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
989
993
|
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
990
|
-
const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ??
|
|
991
|
-
const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ??
|
|
994
|
+
const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? DEFAULT_MAX_CHUNK_LENGTH;
|
|
995
|
+
const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
|
|
992
996
|
const chunkOverlap = Math.min(
|
|
993
|
-
Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) :
|
|
997
|
+
Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : DEFAULT_CHUNK_OVERLAP,
|
|
994
998
|
maxChunkLength - 1
|
|
995
999
|
);
|
|
996
1000
|
const rawConcurrency = params.chunkConcurrency ?? this.options.config?.chunkConcurrency ?? 1;
|