@remnic/core 9.3.735 → 9.3.736
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-boundary.d.ts +5 -5
- package/dist/access-cli.js +1 -1
- package/dist/access-http.d.ts +5 -5
- package/dist/access-mcp.d.ts +5 -5
- package/dist/access-operations.d.ts +5 -5
- package/dist/access-schema.d.ts +4 -4
- package/dist/{access-service-CVx7mmvK.d.ts → access-service-CdC0q4pr.d.ts} +3 -3
- package/dist/access-service.d.ts +5 -5
- package/dist/access-surface-catalog.d.ts +5 -5
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +4 -4
- package/dist/briefing.d.ts +1 -1
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +1 -1
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-BVhDEUhT.d.ts → catalog-D2NLDg4S.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-FLCYY5JX.js → chunk-AE2HPUP7.js} +208 -185
- package/dist/chunk-AE2HPUP7.js.map +1 -0
- package/dist/{cli-Cx5Mxwk7.d.ts → cli-BHy6Bo6u.d.ts} +3 -3
- package/dist/cli.d.ts +6 -6
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/contradiction/index.d.ts +1 -1
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +4 -4
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/identity-continuity.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +1 -1
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/mcp-memory-inspector-app.d.ts +5 -5
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +2 -2
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +2 -2
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/orchestration/compression-guideline-coordinator.d.ts +1 -1
- package/dist/orchestration/maintenance.d.ts +2 -2
- package/dist/{orchestrator-oF0cceBX.d.ts → orchestrator-C-xp_rtZ.d.ts} +64 -5
- package/dist/orchestrator.d.ts +4 -4
- package/dist/orchestrator.js +1 -3
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray.d.ts +1 -1
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/retrieval-agents.d.ts +1 -1
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/schemas.d.ts +24 -24
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-DKGbufYO.d.ts → semantic-consolidation-COdwU-me.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +3 -3
- package/dist/signal.d.ts +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/types.d.ts +42 -42
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-C7UCxpuO.d.ts → types-w3Y232yi.d.ts} +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/package.json +2 -2
- package/src/orchestration/entity-synthesis-coordinator.ts +319 -0
- package/src/orchestrator.ts +12 -235
- package/dist/chunk-FLCYY5JX.js.map +0 -1
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Entity synthesis coordinator — extracted from the orchestrator (issue #1526).
|
|
3
|
+
*
|
|
4
|
+
* Owns the entity-synthesis lifecycle: processing the queued entities whose
|
|
5
|
+
* evidence has changed since their last synthesis, gathering timeline +
|
|
6
|
+
* structured-section evidence, deduplicating it, batch-feeding it through the
|
|
7
|
+
* fast-tier LLM to produce a compact "current truth" synthesis, and persisting
|
|
8
|
+
* the result with drift/bookkeeping metadata.
|
|
9
|
+
*
|
|
10
|
+
* Moved here (behavior-preserving):
|
|
11
|
+
* - module-level helpers `dedupeEntitySynthesisEvidenceEntries`,
|
|
12
|
+
* `flattenStructuredSectionEvidence`, `fingerprintEntitySynthesisEvidence`
|
|
13
|
+
* - the orchestrator method `processEntitySynthesisQueue`
|
|
14
|
+
*
|
|
15
|
+
* The orchestrator constructs one instance and delegates the public entrypoint
|
|
16
|
+
* to it. Storage is resolved per-call via the injected accessor so namespace-
|
|
17
|
+
* scoped and default-namespace calls share one coordinator.
|
|
18
|
+
*
|
|
19
|
+
* Behavior-preserving move from orchestrator.ts. No logic changes — the
|
|
20
|
+
* orchestrator keeps a thin delegating `processEntitySynthesisQueue` so
|
|
21
|
+
* existing call sites (consolidation pass, access-service) and tests that
|
|
22
|
+
* exercise the public API continue to work.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { createHash } from "node:crypto";
|
|
26
|
+
|
|
27
|
+
import type { StorageManager } from "../index.js";
|
|
28
|
+
import {
|
|
29
|
+
compareEntityTimestamps,
|
|
30
|
+
fingerprintEntityStructuredFacts,
|
|
31
|
+
parseEntityFile,
|
|
32
|
+
} from "../storage.js";
|
|
33
|
+
import type {
|
|
34
|
+
EntityStructuredSection,
|
|
35
|
+
EntityTimelineEntry,
|
|
36
|
+
PluginConfig,
|
|
37
|
+
} from "../types.js";
|
|
38
|
+
import { log } from "../logger.js";
|
|
39
|
+
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
// Evidence helpers (moved verbatim from orchestrator.ts module scope)
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Deduplicate synthesis-evidence entries by normalized text, keeping the
|
|
46
|
+
* newest and oldest timestamp per unique fact when they differ.
|
|
47
|
+
*/
|
|
48
|
+
export function dedupeEntitySynthesisEvidenceEntries(
|
|
49
|
+
entries: EntityTimelineEntry[],
|
|
50
|
+
): EntityTimelineEntry[] {
|
|
51
|
+
const dedupedEvidenceEntries: EntityTimelineEntry[] = [];
|
|
52
|
+
const evidenceByFact = new Map<string, {
|
|
53
|
+
newest: EntityTimelineEntry;
|
|
54
|
+
oldest: EntityTimelineEntry;
|
|
55
|
+
}>();
|
|
56
|
+
|
|
57
|
+
for (const entry of entries) {
|
|
58
|
+
const normalizedFact = entry.text.trim();
|
|
59
|
+
if (!normalizedFact) continue;
|
|
60
|
+
const existing = evidenceByFact.get(normalizedFact);
|
|
61
|
+
if (!existing) {
|
|
62
|
+
evidenceByFact.set(normalizedFact, { newest: entry, oldest: entry });
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (compareEntityTimestamps(entry.timestamp, existing.newest.timestamp) > 0) {
|
|
66
|
+
existing.newest = entry;
|
|
67
|
+
}
|
|
68
|
+
if (compareEntityTimestamps(entry.timestamp, existing.oldest.timestamp) < 0) {
|
|
69
|
+
existing.oldest = entry;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
for (const { newest, oldest } of evidenceByFact.values()) {
|
|
74
|
+
dedupedEvidenceEntries.push(newest);
|
|
75
|
+
const newestKey = [
|
|
76
|
+
newest.timestamp,
|
|
77
|
+
newest.source ?? "",
|
|
78
|
+
newest.sessionKey ?? "",
|
|
79
|
+
newest.principal ?? "",
|
|
80
|
+
newest.text,
|
|
81
|
+
].join("\u0000");
|
|
82
|
+
const oldestKey = [
|
|
83
|
+
oldest.timestamp,
|
|
84
|
+
oldest.source ?? "",
|
|
85
|
+
oldest.sessionKey ?? "",
|
|
86
|
+
oldest.principal ?? "",
|
|
87
|
+
oldest.text,
|
|
88
|
+
].join("\u0000");
|
|
89
|
+
if (oldestKey !== newestKey) {
|
|
90
|
+
dedupedEvidenceEntries.push(oldest);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return dedupedEvidenceEntries;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function flattenStructuredSectionEvidence(
|
|
98
|
+
sections: EntityStructuredSection[] | undefined,
|
|
99
|
+
): EntityTimelineEntry[] {
|
|
100
|
+
return (sections ?? []).flatMap((section) =>
|
|
101
|
+
section.facts
|
|
102
|
+
.map((fact) => fact.trim())
|
|
103
|
+
.filter((fact) => fact.length > 0)
|
|
104
|
+
.map((fact) => ({
|
|
105
|
+
timestamp: "",
|
|
106
|
+
text: fact,
|
|
107
|
+
source: `section:${section.title}`,
|
|
108
|
+
})),
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function fingerprintEntitySynthesisEvidence(entity: {
|
|
113
|
+
timeline: EntityTimelineEntry[];
|
|
114
|
+
structuredSections?: EntityStructuredSection[];
|
|
115
|
+
}): string {
|
|
116
|
+
const fingerprint = createHash("sha256");
|
|
117
|
+
const timelineEntries = entity.timeline
|
|
118
|
+
.map((entry) => [
|
|
119
|
+
entry.timestamp,
|
|
120
|
+
entry.source ?? "",
|
|
121
|
+
entry.sessionKey ?? "",
|
|
122
|
+
entry.principal ?? "",
|
|
123
|
+
entry.text,
|
|
124
|
+
].join("\u0000"))
|
|
125
|
+
.sort();
|
|
126
|
+
const timelineEntrySeparator = String.fromCharCode(1);
|
|
127
|
+
const structuredFactsSeparator = String.fromCharCode(2);
|
|
128
|
+
fingerprint.update(timelineEntries.join(timelineEntrySeparator));
|
|
129
|
+
fingerprint.update(structuredFactsSeparator);
|
|
130
|
+
fingerprint.update(fingerprintEntityStructuredFacts(entity) ?? "");
|
|
131
|
+
return fingerprint.digest("hex");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// Coordinator
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
/** Dependencies injected by the orchestrator. */
|
|
139
|
+
export interface EntitySynthesisCoordinatorDeps {
|
|
140
|
+
config: PluginConfig;
|
|
141
|
+
/** Resolve the storage manager for a namespace (or the default). */
|
|
142
|
+
getStorage: (namespace?: string) => Promise<StorageManager>;
|
|
143
|
+
/** Fast-tier LLM completion (same signature as Orchestrator.fastChatCompletion). */
|
|
144
|
+
fastChatCompletion: (
|
|
145
|
+
messages: Array<{ role: string; content: string }>,
|
|
146
|
+
options: {
|
|
147
|
+
temperature?: number;
|
|
148
|
+
maxTokens?: number;
|
|
149
|
+
timeoutMs?: number;
|
|
150
|
+
operation?: string;
|
|
151
|
+
priority?: "background" | "recall-critical";
|
|
152
|
+
},
|
|
153
|
+
) => Promise<{ content: string } | null>;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Coordinates entity synthesis refresh for queued entities. Owns the
|
|
158
|
+
* evidence-gathering, dedup, batched-LLM-synthesis, and persistence logic
|
|
159
|
+
* that previously lived inline in the orchestrator.
|
|
160
|
+
*/
|
|
161
|
+
export class EntitySynthesisCoordinator {
|
|
162
|
+
constructor(private readonly deps: EntitySynthesisCoordinatorDeps) {}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Process up to `maxEntities` entities from the synthesis queue for the
|
|
166
|
+
* given namespace. Returns the number of entities whose synthesis was
|
|
167
|
+
* successfully refreshed.
|
|
168
|
+
*/
|
|
169
|
+
async processQueue(
|
|
170
|
+
namespace?: string,
|
|
171
|
+
maxEntities: number = 5,
|
|
172
|
+
): Promise<number> {
|
|
173
|
+
const { config } = this.deps;
|
|
174
|
+
if (
|
|
175
|
+
!config.entitySummaryEnabled
|
|
176
|
+
|| maxEntities <= 0
|
|
177
|
+
|| config.entitySynthesisMaxTokens <= 0
|
|
178
|
+
) return 0;
|
|
179
|
+
const storage = await this.deps.getStorage(namespace);
|
|
180
|
+
const queued = await storage.refreshEntitySynthesisQueue();
|
|
181
|
+
let processed = 0;
|
|
182
|
+
let attempted = 0;
|
|
183
|
+
|
|
184
|
+
for (const entityName of queued) {
|
|
185
|
+
if (attempted >= maxEntities) break;
|
|
186
|
+
attempted += 1;
|
|
187
|
+
try {
|
|
188
|
+
const raw = await storage.readEntity(entityName);
|
|
189
|
+
if (!raw) continue;
|
|
190
|
+
const entity = parseEntityFile(raw, config.entitySchemas);
|
|
191
|
+
const previousSynthesis = entity.synthesis || entity.summary || "";
|
|
192
|
+
const sortedTimelineEntries = entity.timeline
|
|
193
|
+
.slice()
|
|
194
|
+
.sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
195
|
+
const newerTimelineEntries = sortedTimelineEntries.filter(
|
|
196
|
+
(entry) =>
|
|
197
|
+
!entity.synthesisUpdatedAt
|
|
198
|
+
|| compareEntityTimestamps(entry.timestamp, entity.synthesisUpdatedAt) > 0,
|
|
199
|
+
);
|
|
200
|
+
const appendedTimelineEntries = entity.synthesisTimelineCount === undefined
|
|
201
|
+
? []
|
|
202
|
+
: entity.timeline.slice(Math.max(0, entity.synthesisTimelineCount));
|
|
203
|
+
const structuredEvidenceEntries = flattenStructuredSectionEvidence(entity.structuredSections);
|
|
204
|
+
const structuredEvidenceCount = structuredEvidenceEntries.length;
|
|
205
|
+
const structuredEvidenceDigest = fingerprintEntityStructuredFacts(entity);
|
|
206
|
+
const structuredEvidenceDrifted = structuredEvidenceDigest !== (entity.synthesisStructuredFactDigest?.trim() || undefined);
|
|
207
|
+
const appendedStructuredEvidenceEntries = entity.synthesisStructuredFactCount === undefined
|
|
208
|
+
|| structuredEvidenceDrifted
|
|
209
|
+
? structuredEvidenceEntries
|
|
210
|
+
: structuredEvidenceEntries.slice(Math.max(0, entity.synthesisStructuredFactCount));
|
|
211
|
+
const candidateEvidenceEntries = [
|
|
212
|
+
...newerTimelineEntries,
|
|
213
|
+
...appendedTimelineEntries,
|
|
214
|
+
...appendedStructuredEvidenceEntries,
|
|
215
|
+
]
|
|
216
|
+
.slice()
|
|
217
|
+
.sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
218
|
+
const dedupedEvidenceEntries = dedupeEntitySynthesisEvidenceEntries(
|
|
219
|
+
candidateEvidenceEntries.length > 0
|
|
220
|
+
? candidateEvidenceEntries
|
|
221
|
+
: [...sortedTimelineEntries, ...structuredEvidenceEntries],
|
|
222
|
+
);
|
|
223
|
+
const chronologicalEvidenceEntries = dedupedEvidenceEntries
|
|
224
|
+
.slice()
|
|
225
|
+
.sort((left, right) => compareEntityTimestamps(left.timestamp, right.timestamp));
|
|
226
|
+
if (chronologicalEvidenceEntries.length === 0) continue;
|
|
227
|
+
const latestEvidenceTimestamp = chronologicalEvidenceEntries
|
|
228
|
+
.slice()
|
|
229
|
+
.reverse()
|
|
230
|
+
.map((entry) => entry.timestamp?.trim() || undefined)
|
|
231
|
+
.find((timestamp) => Boolean(timestamp));
|
|
232
|
+
const previousSynthesisUpdatedAt = entity.synthesisUpdatedAt?.trim() || undefined;
|
|
233
|
+
const nextSynthesisUpdatedAt = compareEntityTimestamps(
|
|
234
|
+
latestEvidenceTimestamp,
|
|
235
|
+
previousSynthesisUpdatedAt,
|
|
236
|
+
) >= 0
|
|
237
|
+
? latestEvidenceTimestamp
|
|
238
|
+
: previousSynthesisUpdatedAt;
|
|
239
|
+
const evidenceBatches: typeof chronologicalEvidenceEntries[] = [];
|
|
240
|
+
for (let index = 0; index < chronologicalEvidenceEntries.length; index += 8) {
|
|
241
|
+
evidenceBatches.push(chronologicalEvidenceEntries.slice(index, index + 8));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
let nextSynthesis = previousSynthesis;
|
|
245
|
+
let batchFailed = false;
|
|
246
|
+
for (const evidenceEntries of evidenceBatches) {
|
|
247
|
+
const evidenceText = evidenceEntries
|
|
248
|
+
.map((entry) => {
|
|
249
|
+
const sectionTitle = entry.source?.startsWith("section:")
|
|
250
|
+
? entry.source.slice("section:".length)
|
|
251
|
+
: "";
|
|
252
|
+
const metadata = [
|
|
253
|
+
`timestamp=${entry.timestamp}`,
|
|
254
|
+
sectionTitle ? `section=${sectionTitle}` : entry.source ? `source=${entry.source}` : "",
|
|
255
|
+
entry.sessionKey ? `session=${entry.sessionKey}` : "",
|
|
256
|
+
entry.principal ? `principal=${entry.principal}` : "",
|
|
257
|
+
]
|
|
258
|
+
.filter(Boolean)
|
|
259
|
+
.join(", ");
|
|
260
|
+
return `- ${metadata}: ${entry.text}`;
|
|
261
|
+
})
|
|
262
|
+
.join("\n");
|
|
263
|
+
const response = await this.deps.fastChatCompletion(
|
|
264
|
+
[
|
|
265
|
+
{
|
|
266
|
+
role: "system",
|
|
267
|
+
content:
|
|
268
|
+
"Rewrite the entity synthesis as compact current truth. Preserve uncertainty when evidence conflicts. Return plain text only.",
|
|
269
|
+
},
|
|
270
|
+
{
|
|
271
|
+
role: "user",
|
|
272
|
+
content: [
|
|
273
|
+
`Entity: ${entity.name} (${entity.type})`,
|
|
274
|
+
nextSynthesis ? `Previous synthesis:\n${nextSynthesis}` : "Previous synthesis: none",
|
|
275
|
+
`New evidence:\n${evidenceText}`,
|
|
276
|
+
].join("\n\n"),
|
|
277
|
+
},
|
|
278
|
+
],
|
|
279
|
+
{
|
|
280
|
+
temperature: 0.2,
|
|
281
|
+
maxTokens: config.entitySynthesisMaxTokens,
|
|
282
|
+
operation: "entity_summary",
|
|
283
|
+
priority: "background",
|
|
284
|
+
},
|
|
285
|
+
);
|
|
286
|
+
const synthesis = response?.content?.trim().replace(/^["']|["']$/g, "");
|
|
287
|
+
const maxSynthesisChars = Math.max(2_000, config.entitySynthesisMaxTokens * 8);
|
|
288
|
+
if (!synthesis || synthesis.length < 10 || synthesis.length > maxSynthesisChars) {
|
|
289
|
+
batchFailed = true;
|
|
290
|
+
break;
|
|
291
|
+
}
|
|
292
|
+
nextSynthesis = synthesis;
|
|
293
|
+
}
|
|
294
|
+
if (batchFailed || nextSynthesis.length === 0) continue;
|
|
295
|
+
const latestRaw = await storage.readEntity(entityName);
|
|
296
|
+
if (!latestRaw) continue;
|
|
297
|
+
const latestEntity = parseEntityFile(latestRaw, config.entitySchemas);
|
|
298
|
+
if (
|
|
299
|
+
fingerprintEntitySynthesisEvidence(latestEntity)
|
|
300
|
+
!== fingerprintEntitySynthesisEvidence(entity)
|
|
301
|
+
) {
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
await storage.updateEntitySynthesis(entityName, nextSynthesis, {
|
|
305
|
+
entityUpdatedAt: new Date().toISOString(),
|
|
306
|
+
synthesisStructuredFactDigest: structuredEvidenceDigest,
|
|
307
|
+
synthesisStructuredFactCount: structuredEvidenceCount,
|
|
308
|
+
synthesisTimelineCount: entity.timeline.length,
|
|
309
|
+
updatedAt: nextSynthesisUpdatedAt,
|
|
310
|
+
});
|
|
311
|
+
processed += 1;
|
|
312
|
+
} catch (err) {
|
|
313
|
+
log.debug(`entity synthesis refresh failed for ${entityName}: ${err}`);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
return processed;
|
|
318
|
+
}
|
|
319
|
+
}
|
package/src/orchestrator.ts
CHANGED
|
@@ -79,12 +79,9 @@ import {
|
|
|
79
79
|
} from "./search/factory.js";
|
|
80
80
|
import { NoopSearchBackend } from "./search/noop-backend.js";
|
|
81
81
|
import {
|
|
82
|
-
compareEntityTimestamps,
|
|
83
82
|
StorageManager,
|
|
84
83
|
ContentHashIndex,
|
|
85
|
-
fingerprintEntityStructuredFacts,
|
|
86
84
|
normalizeAttributePairs,
|
|
87
|
-
parseEntityFile,
|
|
88
85
|
} from "./storage.js";
|
|
89
86
|
import { sanitizeMemoryContent } from "./sanitize.js";
|
|
90
87
|
import { ThreadingManager } from "./threading.js";
|
|
@@ -103,6 +100,7 @@ import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coo
|
|
|
103
100
|
import { CompressionGuidelineCoordinator } from "./orchestration/compression-guideline-coordinator.js";
|
|
104
101
|
import { SemanticConsolidationCoordinator } from "./orchestration/semantic-consolidation-coordinator.js";
|
|
105
102
|
import { LifecyclePolicyCoordinator } from "./orchestration/lifecycle-policy-coordinator.js";
|
|
103
|
+
import { EntitySynthesisCoordinator } from "./orchestration/entity-synthesis-coordinator.js";
|
|
106
104
|
import {
|
|
107
105
|
runLiveConnectorsOnce,
|
|
108
106
|
type LiveConnectorsRunSummary,
|
|
@@ -451,8 +449,6 @@ import type {
|
|
|
451
449
|
RecallPlanMode,
|
|
452
450
|
RecallSectionConfig,
|
|
453
451
|
RecallTierExplain,
|
|
454
|
-
EntityStructuredSection,
|
|
455
|
-
EntityTimelineEntry,
|
|
456
452
|
} from "./types.js";
|
|
457
453
|
import { disposeDefaultArchiveScoring, getDefaultArchiveScoring, memoryFileToScoreItem } from "./recall/archive-scoring.js";
|
|
458
454
|
|
|
@@ -492,92 +488,6 @@ interface ExtractionRunResult {
|
|
|
492
488
|
postPersistMetadataFailed?: boolean;
|
|
493
489
|
}
|
|
494
490
|
|
|
495
|
-
export function dedupeEntitySynthesisEvidenceEntries(
|
|
496
|
-
entries: EntityTimelineEntry[],
|
|
497
|
-
): EntityTimelineEntry[] {
|
|
498
|
-
const dedupedEvidenceEntries: EntityTimelineEntry[] = [];
|
|
499
|
-
const evidenceByFact = new Map<string, {
|
|
500
|
-
newest: EntityTimelineEntry;
|
|
501
|
-
oldest: EntityTimelineEntry;
|
|
502
|
-
}>();
|
|
503
|
-
|
|
504
|
-
for (const entry of entries) {
|
|
505
|
-
const normalizedFact = entry.text.trim();
|
|
506
|
-
if (!normalizedFact) continue;
|
|
507
|
-
const existing = evidenceByFact.get(normalizedFact);
|
|
508
|
-
if (!existing) {
|
|
509
|
-
evidenceByFact.set(normalizedFact, { newest: entry, oldest: entry });
|
|
510
|
-
continue;
|
|
511
|
-
}
|
|
512
|
-
if (compareEntityTimestamps(entry.timestamp, existing.newest.timestamp) > 0) {
|
|
513
|
-
existing.newest = entry;
|
|
514
|
-
}
|
|
515
|
-
if (compareEntityTimestamps(entry.timestamp, existing.oldest.timestamp) < 0) {
|
|
516
|
-
existing.oldest = entry;
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
for (const { newest, oldest } of evidenceByFact.values()) {
|
|
521
|
-
dedupedEvidenceEntries.push(newest);
|
|
522
|
-
const newestKey = [
|
|
523
|
-
newest.timestamp,
|
|
524
|
-
newest.source ?? "",
|
|
525
|
-
newest.sessionKey ?? "",
|
|
526
|
-
newest.principal ?? "",
|
|
527
|
-
newest.text,
|
|
528
|
-
].join("\u0000");
|
|
529
|
-
const oldestKey = [
|
|
530
|
-
oldest.timestamp,
|
|
531
|
-
oldest.source ?? "",
|
|
532
|
-
oldest.sessionKey ?? "",
|
|
533
|
-
oldest.principal ?? "",
|
|
534
|
-
oldest.text,
|
|
535
|
-
].join("\u0000");
|
|
536
|
-
if (oldestKey !== newestKey) {
|
|
537
|
-
dedupedEvidenceEntries.push(oldest);
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
return dedupedEvidenceEntries;
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
function flattenStructuredSectionEvidence(
|
|
545
|
-
sections: EntityStructuredSection[] | undefined,
|
|
546
|
-
): EntityTimelineEntry[] {
|
|
547
|
-
return (sections ?? []).flatMap((section) =>
|
|
548
|
-
section.facts
|
|
549
|
-
.map((fact) => fact.trim())
|
|
550
|
-
.filter((fact) => fact.length > 0)
|
|
551
|
-
.map((fact) => ({
|
|
552
|
-
timestamp: "",
|
|
553
|
-
text: fact,
|
|
554
|
-
source: `section:${section.title}`,
|
|
555
|
-
})),
|
|
556
|
-
);
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
function fingerprintEntitySynthesisEvidence(entity: {
|
|
560
|
-
timeline: EntityTimelineEntry[];
|
|
561
|
-
structuredSections?: EntityStructuredSection[];
|
|
562
|
-
}): string {
|
|
563
|
-
const fingerprint = createHash("sha256");
|
|
564
|
-
const timelineEntries = entity.timeline
|
|
565
|
-
.map((entry) => [
|
|
566
|
-
entry.timestamp,
|
|
567
|
-
entry.source ?? "",
|
|
568
|
-
entry.sessionKey ?? "",
|
|
569
|
-
entry.principal ?? "",
|
|
570
|
-
entry.text,
|
|
571
|
-
].join("\u0000"))
|
|
572
|
-
.sort();
|
|
573
|
-
const timelineEntrySeparator = String.fromCharCode(1);
|
|
574
|
-
const structuredFactsSeparator = String.fromCharCode(2);
|
|
575
|
-
fingerprint.update(timelineEntries.join(timelineEntrySeparator));
|
|
576
|
-
fingerprint.update(structuredFactsSeparator);
|
|
577
|
-
fingerprint.update(fingerprintEntityStructuredFacts(entity) ?? "");
|
|
578
|
-
return fingerprint.digest("hex");
|
|
579
|
-
}
|
|
580
|
-
|
|
581
491
|
export interface GraphRecallSnapshot {
|
|
582
492
|
recordedAt: string;
|
|
583
493
|
mode: RecallPlanMode | string;
|
|
@@ -1904,6 +1814,7 @@ export class Orchestrator {
|
|
|
1904
1814
|
readonly compressionGuidelineCoordinator: CompressionGuidelineCoordinator;
|
|
1905
1815
|
readonly semanticConsolidationCoordinator: SemanticConsolidationCoordinator;
|
|
1906
1816
|
readonly lifecyclePolicyCoordinator: LifecyclePolicyCoordinator;
|
|
1817
|
+
readonly entitySynthesisCoordinator: EntitySynthesisCoordinator;
|
|
1907
1818
|
/**
|
|
1908
1819
|
* In-memory X-ray snapshot from the most recent `recall()` call that
|
|
1909
1820
|
* was invoked with `xrayCapture: true` (issue #570 PR 1). Scope is
|
|
@@ -3027,6 +2938,12 @@ export class Orchestrator {
|
|
|
3027
2938
|
fastChatCompletion: (messages, options) =>
|
|
3028
2939
|
this.fastChatCompletion(messages, options),
|
|
3029
2940
|
});
|
|
2941
|
+
this.entitySynthesisCoordinator = new EntitySynthesisCoordinator({
|
|
2942
|
+
config,
|
|
2943
|
+
getStorage: (namespace) => this.getStorage(namespace),
|
|
2944
|
+
fastChatCompletion: (messages, options) =>
|
|
2945
|
+
this.fastChatCompletion(messages, options),
|
|
2946
|
+
});
|
|
3030
2947
|
|
|
3031
2948
|
this.sessionObserver = new SessionObserverState({
|
|
3032
2949
|
memoryDir: config.memoryDir,
|
|
@@ -4444,150 +4361,10 @@ export class Orchestrator {
|
|
|
4444
4361
|
namespace?: string,
|
|
4445
4362
|
maxEntities: number = 5,
|
|
4446
4363
|
): Promise<number> {
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
) return 0;
|
|
4452
|
-
const storage = await this.getStorage(namespace);
|
|
4453
|
-
const queued = await storage.refreshEntitySynthesisQueue();
|
|
4454
|
-
let processed = 0;
|
|
4455
|
-
let attempted = 0;
|
|
4456
|
-
|
|
4457
|
-
for (const entityName of queued) {
|
|
4458
|
-
if (attempted >= maxEntities) break;
|
|
4459
|
-
attempted += 1;
|
|
4460
|
-
try {
|
|
4461
|
-
const raw = await storage.readEntity(entityName);
|
|
4462
|
-
if (!raw) continue;
|
|
4463
|
-
const entity = parseEntityFile(raw, this.config.entitySchemas);
|
|
4464
|
-
const previousSynthesis = entity.synthesis || entity.summary || "";
|
|
4465
|
-
const sortedTimelineEntries = entity.timeline
|
|
4466
|
-
.slice()
|
|
4467
|
-
.sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
4468
|
-
const newerTimelineEntries = sortedTimelineEntries.filter(
|
|
4469
|
-
(entry) =>
|
|
4470
|
-
!entity.synthesisUpdatedAt
|
|
4471
|
-
|| compareEntityTimestamps(entry.timestamp, entity.synthesisUpdatedAt) > 0,
|
|
4472
|
-
);
|
|
4473
|
-
const appendedTimelineEntries = entity.synthesisTimelineCount === undefined
|
|
4474
|
-
? []
|
|
4475
|
-
: entity.timeline.slice(Math.max(0, entity.synthesisTimelineCount));
|
|
4476
|
-
const structuredEvidenceEntries = flattenStructuredSectionEvidence(entity.structuredSections);
|
|
4477
|
-
const structuredEvidenceCount = structuredEvidenceEntries.length;
|
|
4478
|
-
const structuredEvidenceDigest = fingerprintEntityStructuredFacts(entity);
|
|
4479
|
-
const structuredEvidenceDrifted = structuredEvidenceDigest !== (entity.synthesisStructuredFactDigest?.trim() || undefined);
|
|
4480
|
-
const appendedStructuredEvidenceEntries = entity.synthesisStructuredFactCount === undefined
|
|
4481
|
-
|| structuredEvidenceDrifted
|
|
4482
|
-
? structuredEvidenceEntries
|
|
4483
|
-
: structuredEvidenceEntries.slice(Math.max(0, entity.synthesisStructuredFactCount));
|
|
4484
|
-
const candidateEvidenceEntries = [
|
|
4485
|
-
...newerTimelineEntries,
|
|
4486
|
-
...appendedTimelineEntries,
|
|
4487
|
-
...appendedStructuredEvidenceEntries,
|
|
4488
|
-
]
|
|
4489
|
-
.slice()
|
|
4490
|
-
.sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
4491
|
-
const dedupedEvidenceEntries = dedupeEntitySynthesisEvidenceEntries(
|
|
4492
|
-
candidateEvidenceEntries.length > 0
|
|
4493
|
-
? candidateEvidenceEntries
|
|
4494
|
-
: [...sortedTimelineEntries, ...structuredEvidenceEntries],
|
|
4495
|
-
);
|
|
4496
|
-
const chronologicalEvidenceEntries = dedupedEvidenceEntries
|
|
4497
|
-
.slice()
|
|
4498
|
-
.sort((left, right) => compareEntityTimestamps(left.timestamp, right.timestamp));
|
|
4499
|
-
if (chronologicalEvidenceEntries.length === 0) continue;
|
|
4500
|
-
const latestEvidenceTimestamp = chronologicalEvidenceEntries
|
|
4501
|
-
.slice()
|
|
4502
|
-
.reverse()
|
|
4503
|
-
.map((entry) => entry.timestamp?.trim() || undefined)
|
|
4504
|
-
.find((timestamp) => Boolean(timestamp));
|
|
4505
|
-
const previousSynthesisUpdatedAt = entity.synthesisUpdatedAt?.trim() || undefined;
|
|
4506
|
-
const nextSynthesisUpdatedAt = compareEntityTimestamps(
|
|
4507
|
-
latestEvidenceTimestamp,
|
|
4508
|
-
previousSynthesisUpdatedAt,
|
|
4509
|
-
) >= 0
|
|
4510
|
-
? latestEvidenceTimestamp
|
|
4511
|
-
: previousSynthesisUpdatedAt;
|
|
4512
|
-
const evidenceBatches: typeof chronologicalEvidenceEntries[] = [];
|
|
4513
|
-
for (let index = 0; index < chronologicalEvidenceEntries.length; index += 8) {
|
|
4514
|
-
evidenceBatches.push(chronologicalEvidenceEntries.slice(index, index + 8));
|
|
4515
|
-
}
|
|
4516
|
-
|
|
4517
|
-
let nextSynthesis = previousSynthesis;
|
|
4518
|
-
let batchFailed = false;
|
|
4519
|
-
for (const evidenceEntries of evidenceBatches) {
|
|
4520
|
-
const evidenceText = evidenceEntries
|
|
4521
|
-
.map((entry) => {
|
|
4522
|
-
const sectionTitle = entry.source?.startsWith("section:")
|
|
4523
|
-
? entry.source.slice("section:".length)
|
|
4524
|
-
: "";
|
|
4525
|
-
const metadata = [
|
|
4526
|
-
`timestamp=${entry.timestamp}`,
|
|
4527
|
-
sectionTitle ? `section=${sectionTitle}` : entry.source ? `source=${entry.source}` : "",
|
|
4528
|
-
entry.sessionKey ? `session=${entry.sessionKey}` : "",
|
|
4529
|
-
entry.principal ? `principal=${entry.principal}` : "",
|
|
4530
|
-
]
|
|
4531
|
-
.filter(Boolean)
|
|
4532
|
-
.join(", ");
|
|
4533
|
-
return `- ${metadata}: ${entry.text}`;
|
|
4534
|
-
})
|
|
4535
|
-
.join("\n");
|
|
4536
|
-
const response = await this.fastChatCompletion(
|
|
4537
|
-
[
|
|
4538
|
-
{
|
|
4539
|
-
role: "system",
|
|
4540
|
-
content:
|
|
4541
|
-
"Rewrite the entity synthesis as compact current truth. Preserve uncertainty when evidence conflicts. Return plain text only.",
|
|
4542
|
-
},
|
|
4543
|
-
{
|
|
4544
|
-
role: "user",
|
|
4545
|
-
content: [
|
|
4546
|
-
`Entity: ${entity.name} (${entity.type})`,
|
|
4547
|
-
nextSynthesis ? `Previous synthesis:\n${nextSynthesis}` : "Previous synthesis: none",
|
|
4548
|
-
`New evidence:\n${evidenceText}`,
|
|
4549
|
-
].join("\n\n"),
|
|
4550
|
-
},
|
|
4551
|
-
],
|
|
4552
|
-
{
|
|
4553
|
-
temperature: 0.2,
|
|
4554
|
-
maxTokens: this.config.entitySynthesisMaxTokens,
|
|
4555
|
-
operation: "entity_summary",
|
|
4556
|
-
priority: "background",
|
|
4557
|
-
},
|
|
4558
|
-
);
|
|
4559
|
-
const synthesis = response?.content?.trim().replace(/^["']|["']$/g, "");
|
|
4560
|
-
const maxSynthesisChars = Math.max(2_000, this.config.entitySynthesisMaxTokens * 8);
|
|
4561
|
-
if (!synthesis || synthesis.length < 10 || synthesis.length > maxSynthesisChars) {
|
|
4562
|
-
batchFailed = true;
|
|
4563
|
-
break;
|
|
4564
|
-
}
|
|
4565
|
-
nextSynthesis = synthesis;
|
|
4566
|
-
}
|
|
4567
|
-
if (batchFailed || nextSynthesis.length === 0) continue;
|
|
4568
|
-
const latestRaw = await storage.readEntity(entityName);
|
|
4569
|
-
if (!latestRaw) continue;
|
|
4570
|
-
const latestEntity = parseEntityFile(latestRaw, this.config.entitySchemas);
|
|
4571
|
-
if (
|
|
4572
|
-
fingerprintEntitySynthesisEvidence(latestEntity)
|
|
4573
|
-
!== fingerprintEntitySynthesisEvidence(entity)
|
|
4574
|
-
) {
|
|
4575
|
-
continue;
|
|
4576
|
-
}
|
|
4577
|
-
await storage.updateEntitySynthesis(entityName, nextSynthesis, {
|
|
4578
|
-
entityUpdatedAt: new Date().toISOString(),
|
|
4579
|
-
synthesisStructuredFactDigest: structuredEvidenceDigest,
|
|
4580
|
-
synthesisStructuredFactCount: structuredEvidenceCount,
|
|
4581
|
-
synthesisTimelineCount: entity.timeline.length,
|
|
4582
|
-
updatedAt: nextSynthesisUpdatedAt,
|
|
4583
|
-
});
|
|
4584
|
-
processed += 1;
|
|
4585
|
-
} catch (err) {
|
|
4586
|
-
log.debug(`entity synthesis refresh failed for ${entityName}: ${err}`);
|
|
4587
|
-
}
|
|
4588
|
-
}
|
|
4589
|
-
|
|
4590
|
-
return processed;
|
|
4364
|
+
// Issue #1526: entity synthesis lifecycle moved to EntitySynthesisCoordinator.
|
|
4365
|
+
// Thin delegation keeps all call sites (consolidation pass, access-service)
|
|
4366
|
+
// and tests that exercise the public API working unchanged.
|
|
4367
|
+
return this.entitySynthesisCoordinator.processQueue(namespace, maxEntities);
|
|
4591
4368
|
}
|
|
4592
4369
|
|
|
4593
4370
|
async generateDaySummary(
|