@equationalapplications/core-llm-wiki 5.5.1 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -31
- package/dist/{chunk-JQRA6BYQ.mjs → chunk-HREQ6F6Z.mjs} +168 -39
- package/dist/chunk-HREQ6F6Z.mjs.map +1 -0
- package/dist/index.d.mts +18 -3
- package/dist/index.d.ts +18 -3
- package/dist/index.js +168 -36
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-D0RnZjyW.d.mts → testing-DgqVB29I.d.mts} +72 -6
- package/dist/{testing-D0RnZjyW.d.ts → testing-DgqVB29I.d.ts} +72 -6
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +165 -36
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-JQRA6BYQ.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DgqVB29I.mjs';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiMemoryTestAccess, Z as WikiOutboxEvent, _ as WikiParseError, $ as WikiSourceRefHashCollision, a0 as WikiStrictOntologyViolation, a1 as WikiTask, a2 as WikiTransactionError } from './testing-DgqVB29I.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -48,6 +48,21 @@ declare function chunkText(input: string, maxChunkLength: number, overlap: numbe
|
|
|
48
48
|
truncated: boolean;
|
|
49
49
|
};
|
|
50
50
|
|
|
51
|
+
/** Per-candidate anchor ratio used at L0: `min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE)`. */
|
|
52
|
+
declare const HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
53
|
+
/**
|
|
54
|
+
* Default body truncation cap applied at L3 only. A fact whose body is at or
|
|
55
|
+
* below this cap passes through unchanged; a fact above it is sliced to this
|
|
56
|
+
* length with a `...[truncated at N chars, original was M]` marker.
|
|
57
|
+
*/
|
|
58
|
+
declare const HEAL_MAX_FACT_BODY_CHARS_L3 = 4000;
|
|
59
|
+
/**
|
|
60
|
+
* Hard ceiling on the `allTasks` slice fed to the heal prompt at L0. Tasks
|
|
61
|
+
* are entity-global (not batch-local), so this is a constant rather than a
|
|
62
|
+
* function of batch length.
|
|
63
|
+
*/
|
|
64
|
+
declare const HEAL_MAX_TASKS = 50;
|
|
65
|
+
|
|
51
66
|
declare function formatContext(bundle: MemoryBundle, options?: FormatContextOptions): string;
|
|
52
67
|
|
|
53
68
|
/**
|
|
@@ -130,4 +145,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
130
145
|
|
|
131
146
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
132
147
|
|
|
133
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
148
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DgqVB29I.js';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiMemoryTestAccess, Z as WikiOutboxEvent, _ as WikiParseError, $ as WikiSourceRefHashCollision, a0 as WikiStrictOntologyViolation, a1 as WikiTask, a2 as WikiTransactionError } from './testing-DgqVB29I.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -48,6 +48,21 @@ declare function chunkText(input: string, maxChunkLength: number, overlap: numbe
|
|
|
48
48
|
truncated: boolean;
|
|
49
49
|
};
|
|
50
50
|
|
|
51
|
+
/** Per-candidate anchor ratio used at L0: `min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE)`. */
|
|
52
|
+
declare const HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
53
|
+
/**
|
|
54
|
+
* Default body truncation cap applied at L3 only. A fact whose body is at or
|
|
55
|
+
* below this cap passes through unchanged; a fact above it is sliced to this
|
|
56
|
+
* length with a `...[truncated at N chars, original was M]` marker.
|
|
57
|
+
*/
|
|
58
|
+
declare const HEAL_MAX_FACT_BODY_CHARS_L3 = 4000;
|
|
59
|
+
/**
|
|
60
|
+
* Hard ceiling on the `allTasks` slice fed to the heal prompt at L0. Tasks
|
|
61
|
+
* are entity-global (not batch-local), so this is a constant rather than a
|
|
62
|
+
* function of batch length.
|
|
63
|
+
*/
|
|
64
|
+
declare const HEAL_MAX_TASKS = 50;
|
|
65
|
+
|
|
51
66
|
declare function formatContext(bundle: MemoryBundle, options?: FormatContextOptions): string;
|
|
52
67
|
|
|
53
68
|
/**
|
|
@@ -130,4 +145,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
130
145
|
|
|
131
146
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
132
147
|
|
|
133
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
148
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.js
CHANGED
|
@@ -3953,6 +3953,12 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
3953
3953
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
3954
3954
|
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
3955
3955
|
|
|
3956
|
+
// src/utils/healConstants.ts
|
|
3957
|
+
var HEAL_MAX_ANCHORS = 50;
|
|
3958
|
+
var HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
3959
|
+
var HEAL_MAX_FACT_BODY_CHARS_L3 = 4e3;
|
|
3960
|
+
var HEAL_MAX_TASKS = 50;
|
|
3961
|
+
|
|
3956
3962
|
// src/services/PromptService.ts
|
|
3957
3963
|
var PromptService = class {
|
|
3958
3964
|
constructor(globalOverrides) {
|
|
@@ -4020,25 +4026,67 @@ Current Facts:
|
|
|
4020
4026
|
${JSON.stringify(currentFacts, null, 2)}`
|
|
4021
4027
|
};
|
|
4022
4028
|
}
|
|
4023
|
-
|
|
4029
|
+
/**
|
|
4030
|
+
* Heal-prompt level interpretation for the `attemptLevel` ladder.
|
|
4031
|
+
*
|
|
4032
|
+
* Caller contract: `documentAnchors` may be a slice sized for `batch.length`
|
|
4033
|
+
* or a larger set (e.g. a cache hit from `_selectHealAnchors`). This function
|
|
4034
|
+
* applies the prompt-side anchor cap `min(HEAL_MAX_ANCHORS=50, batch.length
|
|
4035
|
+
* * HEAL_ANCHORS_PER_CANDIDATE=4)` so the rendered prompt is bounded
|
|
4036
|
+
* regardless of caller input. `HEAL_MAX_ANCHORS` and
|
|
4037
|
+
* `HEAL_ANCHORS_PER_CANDIDATE` live here too — keeping the formula
|
|
4038
|
+
* co-located with its application avoids a "MaintenanceService policy"
|
|
4039
|
+
* import cycle (`PromptService` is constructed before `MaintenanceService`
|
|
4040
|
+
* exists) and makes the cap testable without a `MaintenanceService`
|
|
4041
|
+
* instance. Task 3 exports the same two constants from `MaintenanceService`
|
|
4042
|
+
* for caller-side overfetch sizing; the values must match.
|
|
4043
|
+
*
|
|
4044
|
+
* Level semantics:
|
|
4045
|
+
* - L0: allTasks + recentEvents + full candidate bodies; anchors re-capped
|
|
4046
|
+
* - L1: drop allTasks; recentEvents present; candidate bodies full
|
|
4047
|
+
* - L2: drop allTasks and recentEvents; candidate bodies full
|
|
4048
|
+
* - L3: drop allTasks and recentEvents; truncate each candidate body to
|
|
4049
|
+
* `bodyTruncationChars` and emit a `degraded` record per truncated fact
|
|
4050
|
+
*/
|
|
4051
|
+
buildHealPrompt(healCandidates, documentAnchors, allTasks, recentEvents, runtimeOverride, attemptLevel, bodyTruncationChars = HEAL_MAX_FACT_BODY_CHARS_L3) {
|
|
4052
|
+
const effectiveTasks = attemptLevel >= 1 ? [] : allTasks;
|
|
4053
|
+
const effectiveEvents = attemptLevel >= 2 ? [] : recentEvents;
|
|
4054
|
+
const maxAnchors = Math.max(1, Math.min(HEAL_MAX_ANCHORS, healCandidates.length * HEAL_ANCHORS_PER_CANDIDATE));
|
|
4055
|
+
const effectiveAnchors = documentAnchors.slice(0, maxAnchors);
|
|
4056
|
+
const { shapedCandidates, degraded } = applyBodyTruncation(
|
|
4057
|
+
healCandidates,
|
|
4058
|
+
attemptLevel,
|
|
4059
|
+
bodyTruncationChars
|
|
4060
|
+
);
|
|
4024
4061
|
const template = runtimeOverride ?? this.globalOverrides?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT;
|
|
4025
4062
|
if (/\{\{\s*healCandidates\s*\}\}/.test(template) || /\{\{\s*documentAnchors\s*\}\}/.test(template) || /\{\{\s*allTasks\s*\}\}/.test(template) || /\{\{\s*recentEvents\s*\}\}/.test(template)) {
|
|
4026
4063
|
return {
|
|
4027
|
-
|
|
4028
|
-
|
|
4064
|
+
prompts: {
|
|
4065
|
+
systemPrompt: this.hydrate(template, {
|
|
4066
|
+
healCandidates: shapedCandidates,
|
|
4067
|
+
documentAnchors: effectiveAnchors,
|
|
4068
|
+
allTasks: effectiveTasks,
|
|
4069
|
+
recentEvents: effectiveEvents
|
|
4070
|
+
}),
|
|
4071
|
+
userPrompt: "Please heal the memory graph."
|
|
4072
|
+
},
|
|
4073
|
+
degraded
|
|
4029
4074
|
};
|
|
4030
4075
|
}
|
|
4031
4076
|
return {
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4077
|
+
prompts: {
|
|
4078
|
+
systemPrompt: template,
|
|
4079
|
+
userPrompt: `Heal Candidates:
|
|
4080
|
+
${JSON.stringify(shapedCandidates, null, 2)}
|
|
4035
4081
|
Document Anchors (DO NOT MODIFY OR DELETE):
|
|
4036
|
-
${JSON.stringify(
|
|
4082
|
+
${JSON.stringify(effectiveAnchors, null, 2)}
|
|
4037
4083
|
All Tasks:
|
|
4038
|
-
${JSON.stringify(
|
|
4084
|
+
${JSON.stringify(effectiveTasks, null, 2)}
|
|
4039
4085
|
Recent Events:
|
|
4040
|
-
${JSON.stringify(
|
|
4086
|
+
${JSON.stringify(effectiveEvents, null, 2)}
|
|
4041
4087
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
4088
|
+
},
|
|
4089
|
+
degraded
|
|
4042
4090
|
};
|
|
4043
4091
|
}
|
|
4044
4092
|
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
@@ -4058,6 +4106,36 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
4058
4106
|
};
|
|
4059
4107
|
}
|
|
4060
4108
|
};
|
|
4109
|
+
function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
4110
|
+
if (attemptLevel < 3) {
|
|
4111
|
+
return { shapedCandidates: candidates, degraded: [] };
|
|
4112
|
+
}
|
|
4113
|
+
const shapedCandidates = [];
|
|
4114
|
+
const degraded = [];
|
|
4115
|
+
for (const c of candidates) {
|
|
4116
|
+
if (typeof c !== "object" || c === null) {
|
|
4117
|
+
shapedCandidates.push(c);
|
|
4118
|
+
continue;
|
|
4119
|
+
}
|
|
4120
|
+
const fact = c;
|
|
4121
|
+
const body = typeof fact.body === "string" ? fact.body : "";
|
|
4122
|
+
if (body.length <= bodyTruncationChars) {
|
|
4123
|
+
shapedCandidates.push(c);
|
|
4124
|
+
continue;
|
|
4125
|
+
}
|
|
4126
|
+
if (typeof fact.id !== "string") {
|
|
4127
|
+
shapedCandidates.push(c);
|
|
4128
|
+
continue;
|
|
4129
|
+
}
|
|
4130
|
+
const originalBodyChars = body.length;
|
|
4131
|
+
const prefix = safeSlice(body, 0, bodyTruncationChars);
|
|
4132
|
+
const truncatedBodyChars = prefix.length;
|
|
4133
|
+
const truncated = `${prefix}\u2026[truncated at ${truncatedBodyChars} chars, original was ${originalBodyChars}]`;
|
|
4134
|
+
shapedCandidates.push({ ...fact, body: truncated });
|
|
4135
|
+
degraded.push({ id: fact.id, originalBodyChars, truncatedBodyChars });
|
|
4136
|
+
}
|
|
4137
|
+
return { shapedCandidates, degraded };
|
|
4138
|
+
}
|
|
4061
4139
|
|
|
4062
4140
|
// src/utils/chunkingDefaults.ts
|
|
4063
4141
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
@@ -4540,16 +4618,24 @@ var TRUNCATION_PATTERNS = [
|
|
|
4540
4618
|
/finish[_ ]?reason/i
|
|
4541
4619
|
];
|
|
4542
4620
|
var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
|
|
4543
|
-
function
|
|
4544
|
-
let message;
|
|
4621
|
+
function getErrorMessage(err) {
|
|
4545
4622
|
try {
|
|
4546
|
-
|
|
4623
|
+
return err instanceof Error ? err.message : String(err ?? "");
|
|
4547
4624
|
} catch {
|
|
4548
|
-
return
|
|
4625
|
+
return void 0;
|
|
4549
4626
|
}
|
|
4627
|
+
}
|
|
4628
|
+
function isTruncationError(err) {
|
|
4629
|
+
const message = getErrorMessage(err);
|
|
4630
|
+
if (message === void 0) return false;
|
|
4550
4631
|
if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
|
|
4551
4632
|
return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
|
|
4552
4633
|
}
|
|
4634
|
+
function isConfigError(err) {
|
|
4635
|
+
const message = getErrorMessage(err);
|
|
4636
|
+
if (message === void 0) return false;
|
|
4637
|
+
return EXCEEDS_LIMIT_PATTERN.test(message);
|
|
4638
|
+
}
|
|
4553
4639
|
function initialBatchSize(maxOutputTokens) {
|
|
4554
4640
|
if (!maxOutputTokens || !Number.isFinite(maxOutputTokens) || maxOutputTokens <= 0) {
|
|
4555
4641
|
return DEFAULT_BATCH_SIZE;
|
|
@@ -4591,14 +4677,18 @@ async function runBatched(args) {
|
|
|
4591
4677
|
const single = candidate.slice(0, 1);
|
|
4592
4678
|
return { batch: single, prompts: await buildPrompt(single) };
|
|
4593
4679
|
};
|
|
4594
|
-
const onFailure = async (batch, err) => {
|
|
4595
|
-
if (batch.length
|
|
4596
|
-
if (
|
|
4597
|
-
|
|
4680
|
+
const onFailure = async (batch, err, attemptLevel, fromCall) => {
|
|
4681
|
+
if (batch.length === 1) {
|
|
4682
|
+
if (fromCall && isTruncationError(err) && attemptLevel < 3) {
|
|
4683
|
+
await attempt(batch, void 0, attemptLevel + 1);
|
|
4684
|
+
} else {
|
|
4685
|
+
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
4686
|
+
skipped.push({ item: batch[0], reason });
|
|
4598
4687
|
onSkip?.(batch[0], err);
|
|
4599
4688
|
}
|
|
4600
4689
|
return;
|
|
4601
4690
|
}
|
|
4691
|
+
if (fromCall && !isTruncationError(err)) throw err;
|
|
4602
4692
|
const mid = Math.ceil(batch.length / 2);
|
|
4603
4693
|
if (mid < batchSize) batchSize = mid;
|
|
4604
4694
|
let i = 0;
|
|
@@ -4609,23 +4699,22 @@ async function runBatched(args) {
|
|
|
4609
4699
|
i += trimmed.batch.length;
|
|
4610
4700
|
}
|
|
4611
4701
|
};
|
|
4612
|
-
const attempt = async (batch, prebuilt) => {
|
|
4702
|
+
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
4613
4703
|
if (batch.length === 0) return;
|
|
4614
|
-
const prompts = prebuilt
|
|
4704
|
+
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
4615
4705
|
batches++;
|
|
4616
4706
|
let responseText;
|
|
4617
4707
|
try {
|
|
4618
4708
|
responseText = await call(prompts);
|
|
4619
4709
|
} catch (err) {
|
|
4620
|
-
|
|
4621
|
-
await onFailure(batch, err);
|
|
4710
|
+
await onFailure(batch, err, attemptLevel, true);
|
|
4622
4711
|
return;
|
|
4623
4712
|
}
|
|
4624
4713
|
let result;
|
|
4625
4714
|
try {
|
|
4626
4715
|
result = parse(responseText, batch);
|
|
4627
4716
|
} catch (err) {
|
|
4628
|
-
await onFailure(batch, err);
|
|
4717
|
+
await onFailure(batch, err, attemptLevel, false);
|
|
4629
4718
|
return;
|
|
4630
4719
|
}
|
|
4631
4720
|
results.push(result);
|
|
@@ -4634,7 +4723,7 @@ async function runBatched(args) {
|
|
|
4634
4723
|
while (index < items.length) {
|
|
4635
4724
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
4636
4725
|
index += batch.length;
|
|
4637
|
-
await attempt(batch, prompts);
|
|
4726
|
+
await attempt(batch, prompts, 0);
|
|
4638
4727
|
}
|
|
4639
4728
|
return { results, skipped, batches };
|
|
4640
4729
|
}
|
|
@@ -4645,7 +4734,6 @@ var MIN_TOKENS_TO_QUALIFY = 3;
|
|
|
4645
4734
|
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
4646
4735
|
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
4647
4736
|
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
4648
|
-
var HEAL_MAX_ANCHORS = 50;
|
|
4649
4737
|
var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
4650
4738
|
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
4651
4739
|
var HEAL_BATCH_SIZE = 25;
|
|
@@ -5076,6 +5164,10 @@ var MaintenanceService = class {
|
|
|
5076
5164
|
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
5077
5165
|
throw new Error("Invalid batchSize: must be an integer >= 1");
|
|
5078
5166
|
}
|
|
5167
|
+
const bodyTruncationChars = options?.bodyTruncationChars ?? HEAL_MAX_FACT_BODY_CHARS_L3;
|
|
5168
|
+
if (!Number.isInteger(bodyTruncationChars) || bodyTruncationChars < 1) {
|
|
5169
|
+
throw new Error("Invalid bodyTruncationChars: must be an integer >= 1");
|
|
5170
|
+
}
|
|
5079
5171
|
const now = Date.now();
|
|
5080
5172
|
const recheckCutoff = now - HEAL_RECHECK_MS;
|
|
5081
5173
|
const orphanAfterDays = this.options.config?.orphanAfterDays !== void 0 ? this.options.config?.orphanAfterDays : 30;
|
|
@@ -5116,29 +5208,44 @@ var MaintenanceService = class {
|
|
|
5116
5208
|
downgraded: staleDowngradedIds.length,
|
|
5117
5209
|
deleted: orphanedIds.length,
|
|
5118
5210
|
newFactsCreated: 0,
|
|
5119
|
-
skipped:
|
|
5211
|
+
skipped: [],
|
|
5212
|
+
degraded: [],
|
|
5120
5213
|
remaining: counts2.eligible,
|
|
5121
5214
|
deferred: counts2.deferred
|
|
5122
5215
|
};
|
|
5123
5216
|
}
|
|
5124
|
-
const allTasks = await this.taskRepo.findAllPending([entityId]);
|
|
5217
|
+
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
5125
5218
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
5126
5219
|
const toPromptShape = (f) => {
|
|
5127
5220
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
5128
5221
|
return { ...rest, tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags };
|
|
5129
5222
|
};
|
|
5130
5223
|
const anchorCache = /* @__PURE__ */ new Map();
|
|
5224
|
+
const degraded = [];
|
|
5131
5225
|
const outcome = await runBatched({
|
|
5132
5226
|
items: healCandidates,
|
|
5133
|
-
buildPrompt: async (batch) => {
|
|
5134
|
-
const documentAnchors = await this._selectHealAnchors(
|
|
5135
|
-
|
|
5227
|
+
buildPrompt: async (batch, attemptLevel = 0) => {
|
|
5228
|
+
const documentAnchors = await this._selectHealAnchors(
|
|
5229
|
+
entityId,
|
|
5230
|
+
batch,
|
|
5231
|
+
// Per-batch anchor cap: `batch.length * HEAL_ANCHORS_PER_CANDIDATE` (capped at
|
|
5232
|
+
// HEAL_MAX_ANCHORS) right-sizes the anchor lookup so a 1-fact batch does
|
|
5233
|
+
// not overfetch the same 200 keyword hits a 25-fact batch once did.
|
|
5234
|
+
// buildHealPrompt applies the matching cap on its side — values must match.
|
|
5235
|
+
Math.min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE),
|
|
5236
|
+
anchorCache
|
|
5237
|
+
);
|
|
5238
|
+
const { prompts, degraded: batchDegraded } = await this.promptService.buildHealPrompt(
|
|
5136
5239
|
batch.map(toPromptShape),
|
|
5137
5240
|
documentAnchors,
|
|
5138
5241
|
allTasks,
|
|
5139
5242
|
recentEvents,
|
|
5140
|
-
promptOverride
|
|
5243
|
+
promptOverride,
|
|
5244
|
+
attemptLevel,
|
|
5245
|
+
bodyTruncationChars
|
|
5141
5246
|
);
|
|
5247
|
+
degraded.push(...batchDegraded);
|
|
5248
|
+
return prompts;
|
|
5142
5249
|
},
|
|
5143
5250
|
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
5144
5251
|
parse: (responseText, batch) => {
|
|
@@ -5212,8 +5319,14 @@ var MaintenanceService = class {
|
|
|
5212
5319
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
5213
5320
|
healFactsForDedupe.push({ id, title: fact.title });
|
|
5214
5321
|
}
|
|
5322
|
+
const callErrorIds = new Set(
|
|
5323
|
+
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
5324
|
+
);
|
|
5215
5325
|
await this.entryRepo.markHealChecked(
|
|
5216
|
-
[
|
|
5326
|
+
[
|
|
5327
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
5328
|
+
...insertedFacts.map((f) => f.id)
|
|
5329
|
+
],
|
|
5217
5330
|
entityId,
|
|
5218
5331
|
now,
|
|
5219
5332
|
tx
|
|
@@ -5231,6 +5344,13 @@ var MaintenanceService = class {
|
|
|
5231
5344
|
await this.embeddingService.embedFact(fact);
|
|
5232
5345
|
}
|
|
5233
5346
|
this.searchService.evictCache(entityId);
|
|
5347
|
+
const skippedIds = new Set(outcome.skipped.map(({ item }) => item.id));
|
|
5348
|
+
const healedDegraded = degraded.filter((d) => !skippedIds.has(d.id));
|
|
5349
|
+
for (const d of healedDegraded) {
|
|
5350
|
+
console.warn(
|
|
5351
|
+
`[WikiMemory] heal healed under degraded context ${entityId}/${d.id}: body truncated from ${d.originalBodyChars} to ${d.truncatedBodyChars} chars`
|
|
5352
|
+
);
|
|
5353
|
+
}
|
|
5234
5354
|
let scanned = outcome.skipped.length;
|
|
5235
5355
|
for (const batchResult of outcome.results) scanned += batchResult.batch.length;
|
|
5236
5356
|
const allDowngraded = /* @__PURE__ */ new Set([...staleDowngradedIds, ...safeDowngraded]);
|
|
@@ -5241,7 +5361,8 @@ var MaintenanceService = class {
|
|
|
5241
5361
|
downgraded: allDowngraded.size,
|
|
5242
5362
|
deleted: allDeleted.size,
|
|
5243
5363
|
newFactsCreated: insertedFacts.length,
|
|
5244
|
-
skipped: outcome.skipped.
|
|
5364
|
+
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
5365
|
+
degraded: healedDegraded,
|
|
5245
5366
|
remaining: counts.eligible,
|
|
5246
5367
|
deferred: counts.deferred
|
|
5247
5368
|
};
|
|
@@ -5325,7 +5446,15 @@ var MaintenanceService = class {
|
|
|
5325
5446
|
};
|
|
5326
5447
|
}
|
|
5327
5448
|
if (outcome.skipped.length > 0) {
|
|
5328
|
-
|
|
5449
|
+
const callErrorIds = new Set(
|
|
5450
|
+
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
5451
|
+
);
|
|
5452
|
+
await this.entryRepo.markOntologyChecked(
|
|
5453
|
+
outcome.skipped.filter(({ item }) => !callErrorIds.has(item.id)).map(({ item }) => item.id),
|
|
5454
|
+
entityId,
|
|
5455
|
+
now,
|
|
5456
|
+
this.db
|
|
5457
|
+
);
|
|
5329
5458
|
}
|
|
5330
5459
|
this.searchService.evictCache(entityId);
|
|
5331
5460
|
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
@@ -5441,7 +5570,7 @@ var MaintenanceService = class {
|
|
|
5441
5570
|
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
5442
5571
|
* per-pass — see the call site in doRunHeal.
|
|
5443
5572
|
*/
|
|
5444
|
-
async _selectHealAnchors(entityId, batch, cache) {
|
|
5573
|
+
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache) {
|
|
5445
5574
|
const query = batch.map((f) => f.title).join(" ").trim();
|
|
5446
5575
|
if (!query) return [];
|
|
5447
5576
|
const cached = cache?.get(query);
|
|
@@ -5449,7 +5578,7 @@ var MaintenanceService = class {
|
|
|
5449
5578
|
const hits = this.searchService.searchKeyword(
|
|
5450
5579
|
query,
|
|
5451
5580
|
[entityId],
|
|
5452
|
-
|
|
5581
|
+
cap * HEAL_ANCHOR_SEARCH_OVERFETCH
|
|
5453
5582
|
);
|
|
5454
5583
|
const hitIds = hits.map((h) => h.id);
|
|
5455
5584
|
const anchors = [];
|
|
@@ -5460,7 +5589,7 @@ var MaintenanceService = class {
|
|
|
5460
5589
|
const row = byId.get(id);
|
|
5461
5590
|
if (!row) continue;
|
|
5462
5591
|
anchors.push(row);
|
|
5463
|
-
if (anchors.length >=
|
|
5592
|
+
if (anchors.length >= cap) break;
|
|
5464
5593
|
}
|
|
5465
5594
|
}
|
|
5466
5595
|
cache?.set(query, anchors);
|
|
@@ -8475,7 +8604,10 @@ function createWiki(db, options) {
|
|
|
8475
8604
|
exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
|
|
8476
8605
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
8477
8606
|
exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
|
|
8607
|
+
exports.HEAL_ANCHORS_PER_CANDIDATE = HEAL_ANCHORS_PER_CANDIDATE;
|
|
8478
8608
|
exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;
|
|
8609
|
+
exports.HEAL_MAX_FACT_BODY_CHARS_L3 = HEAL_MAX_FACT_BODY_CHARS_L3;
|
|
8610
|
+
exports.HEAL_MAX_TASKS = HEAL_MAX_TASKS;
|
|
8479
8611
|
exports.HEAL_RECHECK_MS = HEAL_RECHECK_MS;
|
|
8480
8612
|
exports.HOOK_TIMEOUT_MARKER = HOOK_TIMEOUT_MARKER;
|
|
8481
8613
|
exports.ONTOLOGY_BACKFILL_BATCH_SIZE = ONTOLOGY_BACKFILL_BATCH_SIZE;
|