@equationalapplications/core-llm-wiki 5.5.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -31
- package/dist/{chunk-QVY5DFJV.mjs → chunk-HREQ6F6Z.mjs} +241 -45
- package/dist/chunk-HREQ6F6Z.mjs.map +1 -0
- package/dist/index.d.mts +18 -3
- package/dist/index.d.ts +18 -3
- package/dist/index.js +241 -42
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-D0RnZjyW.d.mts → testing-DgqVB29I.d.mts} +72 -6
- package/dist/{testing-D0RnZjyW.d.ts → testing-DgqVB29I.d.ts} +72 -6
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +238 -42
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-QVY5DFJV.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DgqVB29I.mjs';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiMemoryTestAccess, Z as WikiOutboxEvent, _ as WikiParseError, $ as WikiSourceRefHashCollision, a0 as WikiStrictOntologyViolation, a1 as WikiTask, a2 as WikiTransactionError } from './testing-DgqVB29I.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -48,6 +48,21 @@ declare function chunkText(input: string, maxChunkLength: number, overlap: numbe
|
|
|
48
48
|
truncated: boolean;
|
|
49
49
|
};
|
|
50
50
|
|
|
51
|
+
/** Per-candidate anchor ratio used at L0: `min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE)`. */
|
|
52
|
+
declare const HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
53
|
+
/**
|
|
54
|
+
* Default body truncation cap applied at L3 only. A fact whose body is at or
|
|
55
|
+
* below this cap passes through unchanged; a fact above it is sliced to this
|
|
56
|
+
* length with a `...[truncated at N chars, original was M]` marker.
|
|
57
|
+
*/
|
|
58
|
+
declare const HEAL_MAX_FACT_BODY_CHARS_L3 = 4000;
|
|
59
|
+
/**
|
|
60
|
+
* Hard ceiling on the `allTasks` slice fed to the heal prompt at L0. Tasks
|
|
61
|
+
* are entity-global (not batch-local), so this is a constant rather than a
|
|
62
|
+
* function of batch length.
|
|
63
|
+
*/
|
|
64
|
+
declare const HEAL_MAX_TASKS = 50;
|
|
65
|
+
|
|
51
66
|
declare function formatContext(bundle: MemoryBundle, options?: FormatContextOptions): string;
|
|
52
67
|
|
|
53
68
|
/**
|
|
@@ -130,4 +145,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
130
145
|
|
|
131
146
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
132
147
|
|
|
133
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
148
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DgqVB29I.js';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiMemoryTestAccess, Z as WikiOutboxEvent, _ as WikiParseError, $ as WikiSourceRefHashCollision, a0 as WikiStrictOntologyViolation, a1 as WikiTask, a2 as WikiTransactionError } from './testing-DgqVB29I.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -48,6 +48,21 @@ declare function chunkText(input: string, maxChunkLength: number, overlap: numbe
|
|
|
48
48
|
truncated: boolean;
|
|
49
49
|
};
|
|
50
50
|
|
|
51
|
+
/** Per-candidate anchor ratio used at L0: `min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE)`. */
|
|
52
|
+
declare const HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
53
|
+
/**
|
|
54
|
+
* Default body truncation cap applied at L3 only. A fact whose body is at or
|
|
55
|
+
* below this cap passes through unchanged; a fact above it is sliced to this
|
|
56
|
+
* length with a `...[truncated at N chars, original was M]` marker.
|
|
57
|
+
*/
|
|
58
|
+
declare const HEAL_MAX_FACT_BODY_CHARS_L3 = 4000;
|
|
59
|
+
/**
|
|
60
|
+
* Hard ceiling on the `allTasks` slice fed to the heal prompt at L0. Tasks
|
|
61
|
+
* are entity-global (not batch-local), so this is a constant rather than a
|
|
62
|
+
* function of batch length.
|
|
63
|
+
*/
|
|
64
|
+
declare const HEAL_MAX_TASKS = 50;
|
|
65
|
+
|
|
51
66
|
declare function formatContext(bundle: MemoryBundle, options?: FormatContextOptions): string;
|
|
52
67
|
|
|
53
68
|
/**
|
|
@@ -130,4 +145,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
130
145
|
|
|
131
146
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
132
147
|
|
|
133
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
148
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.js
CHANGED
|
@@ -3692,7 +3692,12 @@ function extractParsePosition(err) {
|
|
|
3692
3692
|
return match ? Number(match[1]) : null;
|
|
3693
3693
|
}
|
|
3694
3694
|
function safeErrorToString(e) {
|
|
3695
|
-
|
|
3695
|
+
let isErrorLike = false;
|
|
3696
|
+
try {
|
|
3697
|
+
isErrorLike = e instanceof Error;
|
|
3698
|
+
} catch {
|
|
3699
|
+
}
|
|
3700
|
+
if (isErrorLike) {
|
|
3696
3701
|
const msg = readErrorField(e, "message");
|
|
3697
3702
|
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
3698
3703
|
const name = readErrorField(e, "name");
|
|
@@ -3717,16 +3722,54 @@ function readErrorField(e, key) {
|
|
|
3717
3722
|
}
|
|
3718
3723
|
}
|
|
3719
3724
|
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
3725
|
+
let isErrorLike = false;
|
|
3726
|
+
try {
|
|
3727
|
+
isErrorLike = err instanceof Error;
|
|
3728
|
+
} catch {
|
|
3729
|
+
}
|
|
3720
3730
|
if (sanitizeRankerErrors === false) {
|
|
3721
|
-
return
|
|
3731
|
+
return isErrorLike ? err : new Error(safeErrorToString(err));
|
|
3732
|
+
}
|
|
3733
|
+
let errLike = null;
|
|
3734
|
+
if (isErrorLike) errLike = err;
|
|
3735
|
+
let typeName;
|
|
3736
|
+
try {
|
|
3737
|
+
if (errLike) {
|
|
3738
|
+
const rawName = errLike.constructor?.name;
|
|
3739
|
+
typeName = typeof rawName === "string" ? rawName : "Error";
|
|
3740
|
+
} else {
|
|
3741
|
+
typeName = typeof err;
|
|
3742
|
+
}
|
|
3743
|
+
} catch {
|
|
3744
|
+
typeName = isErrorLike ? "Error" : typeof err;
|
|
3745
|
+
}
|
|
3746
|
+
let innerCause;
|
|
3747
|
+
if (errLike) {
|
|
3748
|
+
let cause;
|
|
3749
|
+
try {
|
|
3750
|
+
cause = errLike.cause;
|
|
3751
|
+
} catch {
|
|
3752
|
+
cause = void 0;
|
|
3753
|
+
}
|
|
3754
|
+
if (cause !== void 0) {
|
|
3755
|
+
let causeName;
|
|
3756
|
+
try {
|
|
3757
|
+
const rawCauseName = cause?.constructor?.name;
|
|
3758
|
+
causeName = typeof rawCauseName === "string" ? rawCauseName : typeof cause;
|
|
3759
|
+
} catch {
|
|
3760
|
+
causeName = typeof cause;
|
|
3761
|
+
}
|
|
3762
|
+
innerCause = new Error(`Caused by: ${causeName}`);
|
|
3763
|
+
}
|
|
3722
3764
|
}
|
|
3723
|
-
const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
|
|
3724
|
-
const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
|
|
3725
3765
|
const sanitized = new Error(
|
|
3726
3766
|
`VectorRanker ${typeName} (message scrubbed for security)`,
|
|
3727
3767
|
innerCause ? { cause: innerCause } : void 0
|
|
3728
3768
|
);
|
|
3729
|
-
|
|
3769
|
+
try {
|
|
3770
|
+
sanitized.name = typeName;
|
|
3771
|
+
} catch {
|
|
3772
|
+
}
|
|
3730
3773
|
return sanitized;
|
|
3731
3774
|
}
|
|
3732
3775
|
function safeSlice(value, start, end) {
|
|
@@ -3910,6 +3953,12 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
3910
3953
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
3911
3954
|
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
3912
3955
|
|
|
3956
|
+
// src/utils/healConstants.ts
|
|
3957
|
+
var HEAL_MAX_ANCHORS = 50;
|
|
3958
|
+
var HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
3959
|
+
var HEAL_MAX_FACT_BODY_CHARS_L3 = 4e3;
|
|
3960
|
+
var HEAL_MAX_TASKS = 50;
|
|
3961
|
+
|
|
3913
3962
|
// src/services/PromptService.ts
|
|
3914
3963
|
var PromptService = class {
|
|
3915
3964
|
constructor(globalOverrides) {
|
|
@@ -3977,25 +4026,67 @@ Current Facts:
|
|
|
3977
4026
|
${JSON.stringify(currentFacts, null, 2)}`
|
|
3978
4027
|
};
|
|
3979
4028
|
}
|
|
3980
|
-
|
|
4029
|
+
/**
|
|
4030
|
+
* Heal-prompt level interpretation for the `attemptLevel` ladder.
|
|
4031
|
+
*
|
|
4032
|
+
* Caller contract: `documentAnchors` may be a slice sized for `batch.length`
|
|
4033
|
+
* or a larger set (e.g. a cache hit from `_selectHealAnchors`). This function
|
|
4034
|
+
* applies the prompt-side anchor cap `min(HEAL_MAX_ANCHORS=50, batch.length
|
|
4035
|
+
* * HEAL_ANCHORS_PER_CANDIDATE=4)` so the rendered prompt is bounded
|
|
4036
|
+
* regardless of caller input. `HEAL_MAX_ANCHORS` and
|
|
4037
|
+
* `HEAL_ANCHORS_PER_CANDIDATE` live here too — keeping the formula
|
|
4038
|
+
* co-located with its application avoids a "MaintenanceService policy"
|
|
4039
|
+
* import cycle (`PromptService` is constructed before `MaintenanceService`
|
|
4040
|
+
* exists) and makes the cap testable without a `MaintenanceService`
|
|
4041
|
+
* instance. Task 3 exports the same two constants from `MaintenanceService`
|
|
4042
|
+
* for caller-side overfetch sizing; the values must match.
|
|
4043
|
+
*
|
|
4044
|
+
* Level semantics:
|
|
4045
|
+
* - L0: allTasks + recentEvents + full candidate bodies; anchors re-capped
|
|
4046
|
+
* - L1: drop allTasks; recentEvents present; candidate bodies full
|
|
4047
|
+
* - L2: drop allTasks and recentEvents; candidate bodies full
|
|
4048
|
+
* - L3: drop allTasks and recentEvents; truncate each candidate body to
|
|
4049
|
+
* `bodyTruncationChars` and emit a `degraded` record per truncated fact
|
|
4050
|
+
*/
|
|
4051
|
+
buildHealPrompt(healCandidates, documentAnchors, allTasks, recentEvents, runtimeOverride, attemptLevel, bodyTruncationChars = HEAL_MAX_FACT_BODY_CHARS_L3) {
|
|
4052
|
+
const effectiveTasks = attemptLevel >= 1 ? [] : allTasks;
|
|
4053
|
+
const effectiveEvents = attemptLevel >= 2 ? [] : recentEvents;
|
|
4054
|
+
const maxAnchors = Math.max(1, Math.min(HEAL_MAX_ANCHORS, healCandidates.length * HEAL_ANCHORS_PER_CANDIDATE));
|
|
4055
|
+
const effectiveAnchors = documentAnchors.slice(0, maxAnchors);
|
|
4056
|
+
const { shapedCandidates, degraded } = applyBodyTruncation(
|
|
4057
|
+
healCandidates,
|
|
4058
|
+
attemptLevel,
|
|
4059
|
+
bodyTruncationChars
|
|
4060
|
+
);
|
|
3981
4061
|
const template = runtimeOverride ?? this.globalOverrides?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT;
|
|
3982
4062
|
if (/\{\{\s*healCandidates\s*\}\}/.test(template) || /\{\{\s*documentAnchors\s*\}\}/.test(template) || /\{\{\s*allTasks\s*\}\}/.test(template) || /\{\{\s*recentEvents\s*\}\}/.test(template)) {
|
|
3983
4063
|
return {
|
|
3984
|
-
|
|
3985
|
-
|
|
4064
|
+
prompts: {
|
|
4065
|
+
systemPrompt: this.hydrate(template, {
|
|
4066
|
+
healCandidates: shapedCandidates,
|
|
4067
|
+
documentAnchors: effectiveAnchors,
|
|
4068
|
+
allTasks: effectiveTasks,
|
|
4069
|
+
recentEvents: effectiveEvents
|
|
4070
|
+
}),
|
|
4071
|
+
userPrompt: "Please heal the memory graph."
|
|
4072
|
+
},
|
|
4073
|
+
degraded
|
|
3986
4074
|
};
|
|
3987
4075
|
}
|
|
3988
4076
|
return {
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
4077
|
+
prompts: {
|
|
4078
|
+
systemPrompt: template,
|
|
4079
|
+
userPrompt: `Heal Candidates:
|
|
4080
|
+
${JSON.stringify(shapedCandidates, null, 2)}
|
|
3992
4081
|
Document Anchors (DO NOT MODIFY OR DELETE):
|
|
3993
|
-
${JSON.stringify(
|
|
4082
|
+
${JSON.stringify(effectiveAnchors, null, 2)}
|
|
3994
4083
|
All Tasks:
|
|
3995
|
-
${JSON.stringify(
|
|
4084
|
+
${JSON.stringify(effectiveTasks, null, 2)}
|
|
3996
4085
|
Recent Events:
|
|
3997
|
-
${JSON.stringify(
|
|
4086
|
+
${JSON.stringify(effectiveEvents, null, 2)}
|
|
3998
4087
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
4088
|
+
},
|
|
4089
|
+
degraded
|
|
3999
4090
|
};
|
|
4000
4091
|
}
|
|
4001
4092
|
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
@@ -4015,6 +4106,36 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
4015
4106
|
};
|
|
4016
4107
|
}
|
|
4017
4108
|
};
|
|
4109
|
+
function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
4110
|
+
if (attemptLevel < 3) {
|
|
4111
|
+
return { shapedCandidates: candidates, degraded: [] };
|
|
4112
|
+
}
|
|
4113
|
+
const shapedCandidates = [];
|
|
4114
|
+
const degraded = [];
|
|
4115
|
+
for (const c of candidates) {
|
|
4116
|
+
if (typeof c !== "object" || c === null) {
|
|
4117
|
+
shapedCandidates.push(c);
|
|
4118
|
+
continue;
|
|
4119
|
+
}
|
|
4120
|
+
const fact = c;
|
|
4121
|
+
const body = typeof fact.body === "string" ? fact.body : "";
|
|
4122
|
+
if (body.length <= bodyTruncationChars) {
|
|
4123
|
+
shapedCandidates.push(c);
|
|
4124
|
+
continue;
|
|
4125
|
+
}
|
|
4126
|
+
if (typeof fact.id !== "string") {
|
|
4127
|
+
shapedCandidates.push(c);
|
|
4128
|
+
continue;
|
|
4129
|
+
}
|
|
4130
|
+
const originalBodyChars = body.length;
|
|
4131
|
+
const prefix = safeSlice(body, 0, bodyTruncationChars);
|
|
4132
|
+
const truncatedBodyChars = prefix.length;
|
|
4133
|
+
const truncated = `${prefix}\u2026[truncated at ${truncatedBodyChars} chars, original was ${originalBodyChars}]`;
|
|
4134
|
+
shapedCandidates.push({ ...fact, body: truncated });
|
|
4135
|
+
degraded.push({ id: fact.id, originalBodyChars, truncatedBodyChars });
|
|
4136
|
+
}
|
|
4137
|
+
return { shapedCandidates, degraded };
|
|
4138
|
+
}
|
|
4018
4139
|
|
|
4019
4140
|
// src/utils/chunkingDefaults.ts
|
|
4020
4141
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
@@ -4497,11 +4618,24 @@ var TRUNCATION_PATTERNS = [
|
|
|
4497
4618
|
/finish[_ ]?reason/i
|
|
4498
4619
|
];
|
|
4499
4620
|
var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
|
|
4621
|
+
function getErrorMessage(err) {
|
|
4622
|
+
try {
|
|
4623
|
+
return err instanceof Error ? err.message : String(err ?? "");
|
|
4624
|
+
} catch {
|
|
4625
|
+
return void 0;
|
|
4626
|
+
}
|
|
4627
|
+
}
|
|
4500
4628
|
function isTruncationError(err) {
|
|
4501
|
-
const message =
|
|
4629
|
+
const message = getErrorMessage(err);
|
|
4630
|
+
if (message === void 0) return false;
|
|
4502
4631
|
if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
|
|
4503
4632
|
return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
|
|
4504
4633
|
}
|
|
4634
|
+
function isConfigError(err) {
|
|
4635
|
+
const message = getErrorMessage(err);
|
|
4636
|
+
if (message === void 0) return false;
|
|
4637
|
+
return EXCEEDS_LIMIT_PATTERN.test(message);
|
|
4638
|
+
}
|
|
4505
4639
|
function initialBatchSize(maxOutputTokens) {
|
|
4506
4640
|
if (!maxOutputTokens || !Number.isFinite(maxOutputTokens) || maxOutputTokens <= 0) {
|
|
4507
4641
|
return DEFAULT_BATCH_SIZE;
|
|
@@ -4543,14 +4677,18 @@ async function runBatched(args) {
|
|
|
4543
4677
|
const single = candidate.slice(0, 1);
|
|
4544
4678
|
return { batch: single, prompts: await buildPrompt(single) };
|
|
4545
4679
|
};
|
|
4546
|
-
const onFailure = async (batch, err) => {
|
|
4547
|
-
if (batch.length
|
|
4548
|
-
if (
|
|
4549
|
-
|
|
4680
|
+
const onFailure = async (batch, err, attemptLevel, fromCall) => {
|
|
4681
|
+
if (batch.length === 1) {
|
|
4682
|
+
if (fromCall && isTruncationError(err) && attemptLevel < 3) {
|
|
4683
|
+
await attempt(batch, void 0, attemptLevel + 1);
|
|
4684
|
+
} else {
|
|
4685
|
+
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
4686
|
+
skipped.push({ item: batch[0], reason });
|
|
4550
4687
|
onSkip?.(batch[0], err);
|
|
4551
4688
|
}
|
|
4552
4689
|
return;
|
|
4553
4690
|
}
|
|
4691
|
+
if (fromCall && !isTruncationError(err)) throw err;
|
|
4554
4692
|
const mid = Math.ceil(batch.length / 2);
|
|
4555
4693
|
if (mid < batchSize) batchSize = mid;
|
|
4556
4694
|
let i = 0;
|
|
@@ -4561,23 +4699,22 @@ async function runBatched(args) {
|
|
|
4561
4699
|
i += trimmed.batch.length;
|
|
4562
4700
|
}
|
|
4563
4701
|
};
|
|
4564
|
-
const attempt = async (batch, prebuilt) => {
|
|
4702
|
+
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
4565
4703
|
if (batch.length === 0) return;
|
|
4566
|
-
const prompts = prebuilt
|
|
4704
|
+
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
4567
4705
|
batches++;
|
|
4568
4706
|
let responseText;
|
|
4569
4707
|
try {
|
|
4570
4708
|
responseText = await call(prompts);
|
|
4571
4709
|
} catch (err) {
|
|
4572
|
-
|
|
4573
|
-
await onFailure(batch, err);
|
|
4710
|
+
await onFailure(batch, err, attemptLevel, true);
|
|
4574
4711
|
return;
|
|
4575
4712
|
}
|
|
4576
4713
|
let result;
|
|
4577
4714
|
try {
|
|
4578
4715
|
result = parse(responseText, batch);
|
|
4579
4716
|
} catch (err) {
|
|
4580
|
-
await onFailure(batch, err);
|
|
4717
|
+
await onFailure(batch, err, attemptLevel, false);
|
|
4581
4718
|
return;
|
|
4582
4719
|
}
|
|
4583
4720
|
results.push(result);
|
|
@@ -4586,7 +4723,7 @@ async function runBatched(args) {
|
|
|
4586
4723
|
while (index < items.length) {
|
|
4587
4724
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
4588
4725
|
index += batch.length;
|
|
4589
|
-
await attempt(batch, prompts);
|
|
4726
|
+
await attempt(batch, prompts, 0);
|
|
4590
4727
|
}
|
|
4591
4728
|
return { results, skipped, batches };
|
|
4592
4729
|
}
|
|
@@ -4597,7 +4734,6 @@ var MIN_TOKENS_TO_QUALIFY = 3;
|
|
|
4597
4734
|
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
4598
4735
|
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
4599
4736
|
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
4600
|
-
var HEAL_MAX_ANCHORS = 50;
|
|
4601
4737
|
var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
4602
4738
|
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
4603
4739
|
var HEAL_BATCH_SIZE = 25;
|
|
@@ -4605,7 +4741,12 @@ var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
|
4605
4741
|
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
4606
4742
|
var formatSkipError = (err) => {
|
|
4607
4743
|
let base;
|
|
4608
|
-
|
|
4744
|
+
let isErrorLike = false;
|
|
4745
|
+
try {
|
|
4746
|
+
isErrorLike = err instanceof Error;
|
|
4747
|
+
} catch {
|
|
4748
|
+
}
|
|
4749
|
+
if (isErrorLike || typeof err !== "object" && typeof err !== "function") {
|
|
4609
4750
|
base = safeErrorToString(err);
|
|
4610
4751
|
} else {
|
|
4611
4752
|
try {
|
|
@@ -5023,6 +5164,10 @@ var MaintenanceService = class {
|
|
|
5023
5164
|
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
5024
5165
|
throw new Error("Invalid batchSize: must be an integer >= 1");
|
|
5025
5166
|
}
|
|
5167
|
+
const bodyTruncationChars = options?.bodyTruncationChars ?? HEAL_MAX_FACT_BODY_CHARS_L3;
|
|
5168
|
+
if (!Number.isInteger(bodyTruncationChars) || bodyTruncationChars < 1) {
|
|
5169
|
+
throw new Error("Invalid bodyTruncationChars: must be an integer >= 1");
|
|
5170
|
+
}
|
|
5026
5171
|
const now = Date.now();
|
|
5027
5172
|
const recheckCutoff = now - HEAL_RECHECK_MS;
|
|
5028
5173
|
const orphanAfterDays = this.options.config?.orphanAfterDays !== void 0 ? this.options.config?.orphanAfterDays : 30;
|
|
@@ -5063,29 +5208,44 @@ var MaintenanceService = class {
|
|
|
5063
5208
|
downgraded: staleDowngradedIds.length,
|
|
5064
5209
|
deleted: orphanedIds.length,
|
|
5065
5210
|
newFactsCreated: 0,
|
|
5066
|
-
skipped:
|
|
5211
|
+
skipped: [],
|
|
5212
|
+
degraded: [],
|
|
5067
5213
|
remaining: counts2.eligible,
|
|
5068
5214
|
deferred: counts2.deferred
|
|
5069
5215
|
};
|
|
5070
5216
|
}
|
|
5071
|
-
const allTasks = await this.taskRepo.findAllPending([entityId]);
|
|
5217
|
+
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
5072
5218
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
5073
5219
|
const toPromptShape = (f) => {
|
|
5074
5220
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
5075
5221
|
return { ...rest, tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags };
|
|
5076
5222
|
};
|
|
5077
5223
|
const anchorCache = /* @__PURE__ */ new Map();
|
|
5224
|
+
const degraded = [];
|
|
5078
5225
|
const outcome = await runBatched({
|
|
5079
5226
|
items: healCandidates,
|
|
5080
|
-
buildPrompt: async (batch) => {
|
|
5081
|
-
const documentAnchors = await this._selectHealAnchors(
|
|
5082
|
-
|
|
5227
|
+
buildPrompt: async (batch, attemptLevel = 0) => {
|
|
5228
|
+
const documentAnchors = await this._selectHealAnchors(
|
|
5229
|
+
entityId,
|
|
5230
|
+
batch,
|
|
5231
|
+
// Per-batch anchor cap: `batch.length * HEAL_ANCHORS_PER_CANDIDATE` (capped at
|
|
5232
|
+
// HEAL_MAX_ANCHORS) right-sizes the anchor lookup so a 1-fact batch does
|
|
5233
|
+
// not overfetch the same 200 keyword hits a 25-fact batch once did.
|
|
5234
|
+
// buildHealPrompt applies the matching cap on its side — values must match.
|
|
5235
|
+
Math.min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE),
|
|
5236
|
+
anchorCache
|
|
5237
|
+
);
|
|
5238
|
+
const { prompts, degraded: batchDegraded } = await this.promptService.buildHealPrompt(
|
|
5083
5239
|
batch.map(toPromptShape),
|
|
5084
5240
|
documentAnchors,
|
|
5085
5241
|
allTasks,
|
|
5086
5242
|
recentEvents,
|
|
5087
|
-
promptOverride
|
|
5243
|
+
promptOverride,
|
|
5244
|
+
attemptLevel,
|
|
5245
|
+
bodyTruncationChars
|
|
5088
5246
|
);
|
|
5247
|
+
degraded.push(...batchDegraded);
|
|
5248
|
+
return prompts;
|
|
5089
5249
|
},
|
|
5090
5250
|
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
5091
5251
|
parse: (responseText, batch) => {
|
|
@@ -5159,8 +5319,14 @@ var MaintenanceService = class {
|
|
|
5159
5319
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
5160
5320
|
healFactsForDedupe.push({ id, title: fact.title });
|
|
5161
5321
|
}
|
|
5322
|
+
const callErrorIds = new Set(
|
|
5323
|
+
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
5324
|
+
);
|
|
5162
5325
|
await this.entryRepo.markHealChecked(
|
|
5163
|
-
[
|
|
5326
|
+
[
|
|
5327
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
5328
|
+
...insertedFacts.map((f) => f.id)
|
|
5329
|
+
],
|
|
5164
5330
|
entityId,
|
|
5165
5331
|
now,
|
|
5166
5332
|
tx
|
|
@@ -5178,6 +5344,13 @@ var MaintenanceService = class {
|
|
|
5178
5344
|
await this.embeddingService.embedFact(fact);
|
|
5179
5345
|
}
|
|
5180
5346
|
this.searchService.evictCache(entityId);
|
|
5347
|
+
const skippedIds = new Set(outcome.skipped.map(({ item }) => item.id));
|
|
5348
|
+
const healedDegraded = degraded.filter((d) => !skippedIds.has(d.id));
|
|
5349
|
+
for (const d of healedDegraded) {
|
|
5350
|
+
console.warn(
|
|
5351
|
+
`[WikiMemory] heal healed under degraded context ${entityId}/${d.id}: body truncated from ${d.originalBodyChars} to ${d.truncatedBodyChars} chars`
|
|
5352
|
+
);
|
|
5353
|
+
}
|
|
5181
5354
|
let scanned = outcome.skipped.length;
|
|
5182
5355
|
for (const batchResult of outcome.results) scanned += batchResult.batch.length;
|
|
5183
5356
|
const allDowngraded = /* @__PURE__ */ new Set([...staleDowngradedIds, ...safeDowngraded]);
|
|
@@ -5188,7 +5361,8 @@ var MaintenanceService = class {
|
|
|
5188
5361
|
downgraded: allDowngraded.size,
|
|
5189
5362
|
deleted: allDeleted.size,
|
|
5190
5363
|
newFactsCreated: insertedFacts.length,
|
|
5191
|
-
skipped: outcome.skipped.
|
|
5364
|
+
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
5365
|
+
degraded: healedDegraded,
|
|
5192
5366
|
remaining: counts.eligible,
|
|
5193
5367
|
deferred: counts.deferred
|
|
5194
5368
|
};
|
|
@@ -5272,7 +5446,15 @@ var MaintenanceService = class {
|
|
|
5272
5446
|
};
|
|
5273
5447
|
}
|
|
5274
5448
|
if (outcome.skipped.length > 0) {
|
|
5275
|
-
|
|
5449
|
+
const callErrorIds = new Set(
|
|
5450
|
+
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
5451
|
+
);
|
|
5452
|
+
await this.entryRepo.markOntologyChecked(
|
|
5453
|
+
outcome.skipped.filter(({ item }) => !callErrorIds.has(item.id)).map(({ item }) => item.id),
|
|
5454
|
+
entityId,
|
|
5455
|
+
now,
|
|
5456
|
+
this.db
|
|
5457
|
+
);
|
|
5276
5458
|
}
|
|
5277
5459
|
this.searchService.evictCache(entityId);
|
|
5278
5460
|
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
@@ -5388,7 +5570,7 @@ var MaintenanceService = class {
|
|
|
5388
5570
|
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
5389
5571
|
* per-pass — see the call site in doRunHeal.
|
|
5390
5572
|
*/
|
|
5391
|
-
async _selectHealAnchors(entityId, batch, cache) {
|
|
5573
|
+
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache) {
|
|
5392
5574
|
const query = batch.map((f) => f.title).join(" ").trim();
|
|
5393
5575
|
if (!query) return [];
|
|
5394
5576
|
const cached = cache?.get(query);
|
|
@@ -5396,7 +5578,7 @@ var MaintenanceService = class {
|
|
|
5396
5578
|
const hits = this.searchService.searchKeyword(
|
|
5397
5579
|
query,
|
|
5398
5580
|
[entityId],
|
|
5399
|
-
|
|
5581
|
+
cap * HEAL_ANCHOR_SEARCH_OVERFETCH
|
|
5400
5582
|
);
|
|
5401
5583
|
const hitIds = hits.map((h) => h.id);
|
|
5402
5584
|
const anchors = [];
|
|
@@ -5407,7 +5589,7 @@ var MaintenanceService = class {
|
|
|
5407
5589
|
const row = byId.get(id);
|
|
5408
5590
|
if (!row) continue;
|
|
5409
5591
|
anchors.push(row);
|
|
5410
|
-
if (anchors.length >=
|
|
5592
|
+
if (anchors.length >= cap) break;
|
|
5411
5593
|
}
|
|
5412
5594
|
}
|
|
5413
5595
|
cache?.set(query, anchors);
|
|
@@ -6297,7 +6479,12 @@ var RetrievalService = class {
|
|
|
6297
6479
|
}
|
|
6298
6480
|
}
|
|
6299
6481
|
} catch (rankerErr) {
|
|
6300
|
-
|
|
6482
|
+
let isErrorLike = false;
|
|
6483
|
+
try {
|
|
6484
|
+
isErrorLike = rankerErr instanceof Error;
|
|
6485
|
+
} catch {
|
|
6486
|
+
}
|
|
6487
|
+
const rankerError = isErrorLike ? rankerErr : new Error(safeErrorToString(rankerErr));
|
|
6301
6488
|
const policy = this.options.vectorRankerFallback ?? "js-cosine";
|
|
6302
6489
|
this.options.onVectorRankerFallback?.({
|
|
6303
6490
|
error: this._sanitizeRankerError(rankerError),
|
|
@@ -6409,12 +6596,21 @@ var RetrievalService = class {
|
|
|
6409
6596
|
}
|
|
6410
6597
|
}
|
|
6411
6598
|
} catch (err) {
|
|
6412
|
-
|
|
6599
|
+
let isErrorLike = false;
|
|
6600
|
+
try {
|
|
6601
|
+
isErrorLike = err instanceof Error;
|
|
6602
|
+
} catch {
|
|
6603
|
+
}
|
|
6604
|
+
const error = isErrorLike ? err : new Error(safeErrorToString(err));
|
|
6413
6605
|
if (rankerShouldRethrow) {
|
|
6414
6606
|
throw error;
|
|
6415
6607
|
}
|
|
6416
6608
|
if (pendingRankerFallbackError) {
|
|
6417
|
-
|
|
6609
|
+
try {
|
|
6610
|
+
error.cause = pendingRankerFallbackError;
|
|
6611
|
+
} catch {
|
|
6612
|
+
this.options.onRetrievalFallback?.(pendingRankerFallbackError);
|
|
6613
|
+
}
|
|
6418
6614
|
pendingRankerFallbackError = void 0;
|
|
6419
6615
|
}
|
|
6420
6616
|
this.options.onRetrievalFallback?.(error);
|
|
@@ -8408,7 +8604,10 @@ function createWiki(db, options) {
|
|
|
8408
8604
|
exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
|
|
8409
8605
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
8410
8606
|
exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
|
|
8607
|
+
exports.HEAL_ANCHORS_PER_CANDIDATE = HEAL_ANCHORS_PER_CANDIDATE;
|
|
8411
8608
|
exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;
|
|
8609
|
+
exports.HEAL_MAX_FACT_BODY_CHARS_L3 = HEAL_MAX_FACT_BODY_CHARS_L3;
|
|
8610
|
+
exports.HEAL_MAX_TASKS = HEAL_MAX_TASKS;
|
|
8412
8611
|
exports.HEAL_RECHECK_MS = HEAL_RECHECK_MS;
|
|
8413
8612
|
exports.HOOK_TIMEOUT_MARKER = HOOK_TIMEOUT_MARKER;
|
|
8414
8613
|
exports.ONTOLOGY_BACKFILL_BATCH_SIZE = ONTOLOGY_BACKFILL_BATCH_SIZE;
|