@equationalapplications/core-llm-wiki 5.0.0 → 5.1.1

package/dist/index.d.mts

@@ -3,6 +3,51 @@ export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as Ext
 import { OkfFile } from '@equationalapplications/core-okf';
 import 'minisearch';
 
+/**
+ * Slices a string like `String.prototype.slice`, but clamps out-of-range
+ * indices instead of relying on JS's implicit clamping, normalizes a
+ * start-after-end range by swapping the two bounds, and nudges either bound
+ * off a UTF-16 surrogate pair boundary so slicing never splits one code
+ * point in half.
+ *
+ * @param value - The source string to slice.
+ * @param start - Start index; negative counts from the end, out-of-range
+ *   values are clamped to `[0, value.length]`.
+ * @param end - End index (exclusive); defaults to `value.length` when
+ *   omitted. Same clamping/negative-index rules as `start`.
+ * @returns The sliced substring, never splitting a surrogate pair.
+ */
+declare function safeSlice(value: string, start: number, end?: number): string;
+/**
+ * Splits `input` into chunks of at most `maxChunkLength` characters,
+ * preferring to split on a paragraph break, then a sentence terminator,
+ * then whitespace, falling back to a hard cut only when none of those are
+ * found within the window. Consecutive chunks overlap by up to `overlap`
+ * characters so context isn't lost at a chunk boundary. This is the exact
+ * chunking algorithm `IngestionService.ingestDocument` uses before
+ * embedding, so callers can reproduce ingest-time chunk boundaries exactly
+ * given the same `maxChunkLength`/`overlap` (see `DEFAULT_MAX_CHUNK_LENGTH`
+ * and `DEFAULT_CHUNK_OVERLAP` for the defaults ingest uses).
+ *
+ * @param input - The text to chunk; leading/trailing whitespace is trimmed
+ *   before chunking. Empty/whitespace-only input returns the empty result
+ *   below without validating `maxChunkLength`/`overlap`.
+ * @param maxChunkLength - For non-empty input, maximum characters per chunk;
+ *   must be an integer >= 2.
+ * @param overlap - For non-empty input, maximum number of characters each
+ *   chunk repeats from the end of the previous chunk; must be a
+ *   non-negative integer less than `maxChunkLength`. A chunk repeats fewer
+ *   characters when the previous chunk was shorter than `overlap`.
+ * @returns `chunks` — the resulting chunk strings (empty array for
+ *   empty/whitespace-only input); `truncated` — `true` if any split had to
+ *   fall back to a hard cut (no paragraph/sentence/whitespace boundary
+ *   found in the window).
+ */
+declare function chunkText(input: string, maxChunkLength: number, overlap: number): {
+    chunks: string[];
+    truncated: boolean;
+};
+
 declare function formatContext(bundle: MemoryBundle, options?: FormatContextOptions): string;
 
 /**
@@ -38,6 +83,22 @@ type GetRandomValues = (bytes: Uint8Array) => void;
  */
 declare function configureRandomSource(fn: GetRandomValues | null): void;
 
+/**
+ * Default maximum characters per chunk that `ingestDocument` uses when a
+ * caller doesn't override `maxChunkLength` (directly or via
+ * `WikiOptions.config.maxChunkLength`).
+ */
+declare const DEFAULT_MAX_CHUNK_LENGTH = 12000;
+/**
+ * Default/fallback character overlap between consecutive chunks.
+ * `ingestDocument` uses this when a caller doesn't override `chunkOverlap`
+ * (directly or via `WikiOptions.config.chunkOverlap`), and clamps the
+ * effective overlap (including this default) to `maxChunkLength - 1` when
+ * needed. The clamp is a no-op for the shipped defaults; it can also apply
+ * when a custom `maxChunkLength` makes the resolved overlap too large.
+ */
+declare const DEFAULT_CHUNK_OVERLAP = 400;
+
 interface LibrarianOptions {
     /** If provided, replaces the default Librarian system instructions. */
     systemPrompt?: string;
@@ -64,4 +125,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
 
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate, validateManifest };
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };

package/dist/index.d.ts

@@ -3,6 +3,51 @@ export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as Ext
 import { OkfFile } from '@equationalapplications/core-okf';
 import 'minisearch';
 
+/**
+ * Slices a string like `String.prototype.slice`, but clamps out-of-range
+ * indices instead of relying on JS's implicit clamping, normalizes a
+ * start-after-end range by swapping the two bounds, and nudges either bound
+ * off a UTF-16 surrogate pair boundary so slicing never splits one code
+ * point in half.
+ *
+ * @param value - The source string to slice.
+ * @param start - Start index; negative counts from the end, out-of-range
+ *   values are clamped to `[0, value.length]`.
+ * @param end - End index (exclusive); defaults to `value.length` when
+ *   omitted. Same clamping/negative-index rules as `start`.
+ * @returns The sliced substring, never splitting a surrogate pair.
+ */
+declare function safeSlice(value: string, start: number, end?: number): string;
+/**
+ * Splits `input` into chunks of at most `maxChunkLength` characters,
+ * preferring to split on a paragraph break, then a sentence terminator,
+ * then whitespace, falling back to a hard cut only when none of those are
+ * found within the window. Consecutive chunks overlap by up to `overlap`
+ * characters so context isn't lost at a chunk boundary. This is the exact
+ * chunking algorithm `IngestionService.ingestDocument` uses before
+ * embedding, so callers can reproduce ingest-time chunk boundaries exactly
+ * given the same `maxChunkLength`/`overlap` (see `DEFAULT_MAX_CHUNK_LENGTH`
+ * and `DEFAULT_CHUNK_OVERLAP` for the defaults ingest uses).
+ *
+ * @param input - The text to chunk; leading/trailing whitespace is trimmed
+ *   before chunking. Empty/whitespace-only input returns the empty result
+ *   below without validating `maxChunkLength`/`overlap`.
+ * @param maxChunkLength - For non-empty input, maximum characters per chunk;
+ *   must be an integer >= 2.
+ * @param overlap - For non-empty input, maximum number of characters each
+ *   chunk repeats from the end of the previous chunk; must be a
+ *   non-negative integer less than `maxChunkLength`. A chunk repeats fewer
+ *   characters when the previous chunk was shorter than `overlap`.
+ * @returns `chunks` — the resulting chunk strings (empty array for
+ *   empty/whitespace-only input); `truncated` — `true` if any split had to
+ *   fall back to a hard cut (no paragraph/sentence/whitespace boundary
+ *   found in the window).
+ */
+declare function chunkText(input: string, maxChunkLength: number, overlap: number): {
+    chunks: string[];
+    truncated: boolean;
+};
+
 declare function formatContext(bundle: MemoryBundle, options?: FormatContextOptions): string;
 
 /**
@@ -38,6 +83,22 @@ type GetRandomValues = (bytes: Uint8Array) => void;
  */
 declare function configureRandomSource(fn: GetRandomValues | null): void;
 
+/**
+ * Default maximum characters per chunk that `ingestDocument` uses when a
+ * caller doesn't override `maxChunkLength` (directly or via
+ * `WikiOptions.config.maxChunkLength`).
+ */
+declare const DEFAULT_MAX_CHUNK_LENGTH = 12000;
+/**
+ * Default/fallback character overlap between consecutive chunks.
+ * `ingestDocument` uses this when a caller doesn't override `chunkOverlap`
+ * (directly or via `WikiOptions.config.chunkOverlap`), and clamps the
+ * effective overlap (including this default) to `maxChunkLength - 1` when
+ * needed. The clamp is a no-op for the shipped defaults; it can also apply
+ * when a custom `maxChunkLength` makes the resolved overlap too large.
+ */
+declare const DEFAULT_CHUNK_OVERLAP = 400;
+
 interface LibrarianOptions {
     /** If provided, replaces the default Librarian system instructions. */
     systemPrompt?: string;
@@ -64,4 +125,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
 
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate, validateManifest };
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };

package/dist/index.js

@@ -3063,6 +3063,10 @@ ${JSON.stringify(facts, null, 2)}`
   }
 };
 
+// src/utils/chunkingDefaults.ts
+var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
+var DEFAULT_CHUNK_OVERLAP = 400;
+
 // src/services/IngestionService.ts
 var IngestionService = class {
   constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
@@ -3081,10 +3085,10 @@ var IngestionService = class {
     if (!sourceRef) throw new Error("Invalid sourceRef");
     const sourceHash = normalizeSourceHash(params.sourceHash);
     if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
-    const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? 12e3;
-    const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? 400;
+    const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? DEFAULT_MAX_CHUNK_LENGTH;
+    const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
     const chunkOverlap = Math.min(
-      Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : 400,
+      Number.isFinite(rawOverlap) && rawOverlap >= 0 ? Math.floor(rawOverlap) : DEFAULT_CHUNK_OVERLAP,
       maxChunkLength - 1
     );
     const rawConcurrency = params.chunkConcurrency ?? this.options.config?.chunkConcurrency ?? 1;
@@ -3791,10 +3795,10 @@ var MaintenanceService = class {
     const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
     const insertedFacts = [];
     const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
-    const healFactsForDedupe = (await this.entryRepo.findInferredTitlesByEntityId(entityId)).filter((f) => !safeDeletedSet.has(f.id));
     await this.db.withTransactionAsync(async (tx) => {
       await this.entryRepo.downgradeByIds(safeDowngraded, entityId, tx);
       await this.entryRepo.softDeleteByIds(safeDeleted, entityId, tx);
+      const healFactsForDedupe = await this.entryRepo.findInferredTitlesByEntityId(entityId, tx);
       for (const fact of validNewFacts) {
         const newTokens = titleTokens(fact.title);
         let skip = false;
@@ -6526,7 +6530,9 @@ function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 
+exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
 exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
+exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
 exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;
 exports.HEAL_RECHECK_MS = HEAL_RECHECK_MS;
 exports.HOOK_TIMEOUT_MARKER = HOOK_TIMEOUT_MARKER;
@@ -6539,6 +6545,7 @@ exports.PrunePartialFailureError = PrunePartialFailureError;
 exports.WikiBusyError = WikiBusyError;
 exports.WikiMemory = WikiMemory;
 exports.WikiTransactionError = WikiTransactionError;
+exports.chunkText = chunkText;
 exports.configureRandomSource = configureRandomSource;
 exports.createWiki = createWiki;
 exports.formatContext = formatContext;
@@ -6549,6 +6556,7 @@ exports.hydrateLibrarianPrompt = hydrateLibrarianPrompt;
 exports.mapLibrarianOptionsToReadOptions = mapLibrarianOptionsToReadOptions;
 exports.parseEmbedding = parseEmbedding;
 exports.parseOkfBundle = parseOkfBundle;
+exports.safeSlice = safeSlice;
 exports.validateLibrarianPromptTemplate = validateLibrarianPromptTemplate;
 exports.validateManifest = validateManifest;
 //# sourceMappingURL=index.js.map