@remnic/core 9.3.738 → 9.3.739
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-boundary.d.ts +6 -6
- package/dist/access-cli.js +1 -1
- package/dist/access-http.d.ts +6 -6
- package/dist/access-mcp.d.ts +6 -6
- package/dist/access-operations.d.ts +6 -6
- package/dist/{access-service-DwQ6BIQw.d.ts → access-service-Cbk-Kv_A.d.ts} +1 -1
- package/dist/access-service.d.ts +6 -6
- package/dist/access-surface-catalog.d.ts +6 -6
- package/dist/bootstrap.d.ts +5 -5
- package/dist/{chunk-EKQ3LWBM.js → chunk-IEWUPZ75.js} +583 -511
- package/dist/chunk-IEWUPZ75.js.map +1 -0
- package/dist/{cli-BlEIWDQw.d.ts → cli-BQW2xKdb.d.ts} +2 -2
- package/dist/cli.d.ts +7 -7
- package/dist/explicit-capture.d.ts +5 -5
- package/dist/index.d.ts +8 -8
- package/dist/index.js +1 -1
- package/dist/mcp-memory-inspector-app.d.ts +6 -6
- package/dist/{orchestrator-CJZ21ZiB.d.ts → orchestrator-BOAFN_Yr.d.ts} +186 -56
- package/dist/orchestrator.d.ts +4 -4
- package/dist/orchestrator.js +1 -1
- package/dist/schemas.d.ts +22 -22
- package/dist/transfer/types.d.ts +12 -12
- package/package.json +2 -2
- package/src/orchestration/conversation-index-coordinator.ts +344 -0
- package/src/orchestration/recall-rerank-coordinator.ts +425 -0
- package/src/orchestrator.ts +39 -552
- package/dist/chunk-EKQ3LWBM.js.map +0 -1
package/src/orchestrator.ts
CHANGED
|
@@ -102,6 +102,8 @@ import { SemanticConsolidationCoordinator } from "./orchestration/semantic-conso
|
|
|
102
102
|
import { LifecyclePolicyCoordinator } from "./orchestration/lifecycle-policy-coordinator.js";
|
|
103
103
|
import { EntitySynthesisCoordinator } from "./orchestration/entity-synthesis-coordinator.js";
|
|
104
104
|
import { RecallResultFormatter } from "./orchestration/recall-result-formatter.js";
|
|
105
|
+
import { ConversationIndexCoordinator } from "./orchestration/conversation-index-coordinator.js";
|
|
106
|
+
import { RecallRerankCoordinator } from "./orchestration/recall-rerank-coordinator.js";
|
|
105
107
|
export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
|
|
106
108
|
import {
|
|
107
109
|
runLiveConnectorsOnce,
|
|
@@ -121,16 +123,8 @@ import {
|
|
|
121
123
|
type ParallelSearchResult,
|
|
122
124
|
} from "./retrieval-agents.js";
|
|
123
125
|
import { RerankCache, rerankLocalOrNoop } from "./rerank.js";
|
|
124
|
-
import {
|
|
125
|
-
applyMemoryWorthFilter,
|
|
126
|
-
buildMemoryWorthCounterMap,
|
|
127
|
-
type MemoryWorthCounters,
|
|
128
|
-
} from "./memory-worth-filter.js";
|
|
129
|
-
import { applyTrustScoreStage, buildTrustSignalsForRerank, projectTrustForXray } from "./trust-score-stage.js";
|
|
126
|
+
import { projectTrustForXray } from "./trust-score-stage.js";
|
|
130
127
|
import type { TrustStageResultItem } from "./trust-score-stage.js";
|
|
131
|
-
import { renderEpistemicHedge, type TrustSignals } from "./trust-score.js";
|
|
132
|
-
import { reorderRecallResultsWithMmr } from "./recall-mmr.js";
|
|
133
|
-
import { applyReasoningTraceBoost } from "./reasoning-trace-recall.js";
|
|
134
128
|
import { buildRetrievedMemoryProvenance } from "./memory-provenance.js";
|
|
135
129
|
import {
|
|
136
130
|
applyTemporalSupersession,
|
|
@@ -330,9 +324,6 @@ import {
|
|
|
330
324
|
type SemanticConsolidationLlmOperator,
|
|
331
325
|
type SemanticConsolidationResult,
|
|
332
326
|
} from "./semantic-consolidation.js";
|
|
333
|
-
import { chunkTranscriptEntries } from "./conversation-index/chunker.js";
|
|
334
|
-
import { writeConversationChunks } from "./conversation-index/indexer.js";
|
|
335
|
-
import { cleanupConversationChunks } from "./conversation-index/cleanup.js";
|
|
336
327
|
import {
|
|
337
328
|
type ConversationIndexBackend,
|
|
338
329
|
type ConversationIndexBackendInspection,
|
|
@@ -1817,31 +1808,6 @@ export class Orchestrator {
|
|
|
1817
1808
|
readonly lcmEngine: LcmEngine | null = null;
|
|
1818
1809
|
private readonly rerankCache = new RerankCache();
|
|
1819
1810
|
|
|
1820
|
-
/**
|
|
1821
|
-
* Short-TTL cache for Memory Worth counter lookups so interactive recall
|
|
1822
|
-
* doesn't trigger a full `readAllMemories` scan per query. Keyed by
|
|
1823
|
-
* namespace; the filter unions across namespaces at query time. The TTL
|
|
1824
|
-
* is intentionally short (seconds, not minutes) because counters are
|
|
1825
|
-
* mutated by `recordMemoryOutcome` asynchronously and we'd rather serve
|
|
1826
|
-
* a 30-second-stale worth score than a stable-but-wrong one.
|
|
1827
|
-
*/
|
|
1828
|
-
private readonly memoryWorthCounterCache = new Map<
|
|
1829
|
-
string,
|
|
1830
|
-
{ at: number; counters: ReadonlyMap<string, MemoryWorthCounters> }
|
|
1831
|
-
>();
|
|
1832
|
-
private static readonly MEMORY_WORTH_CACHE_TTL_MS = 30_000;
|
|
1833
|
-
/**
|
|
1834
|
-
* Issue #1577 — per-namespace TrustScore signal map cache. Same TTL/shape
|
|
1835
|
-
* discipline as {@link memoryWorthCounterCache}: seconds-scale, evicted on
|
|
1836
|
-
* every call, keyed by namespace. Holds the frontmatter-derived signals
|
|
1837
|
-
* (worth, provenance, faithfulness, corroboration, recency) so the trust
|
|
1838
|
-
* stage doesn't trigger a fresh `readAllMemories` scan per query.
|
|
1839
|
-
*/
|
|
1840
|
-
private readonly trustSignalCache = new Map<
|
|
1841
|
-
string,
|
|
1842
|
-
{ at: number; signals: ReadonlyMap<string, TrustSignals> }
|
|
1843
|
-
>();
|
|
1844
|
-
private static readonly TRUST_SIGNAL_CACHE_TTL_MS = 30_000;
|
|
1845
1811
|
/**
|
|
1846
1812
|
* Per-session workspace selections keyed by sessionKey.
|
|
1847
1813
|
* Set by the before_agent_start hook so recall() uses the correct
|
|
@@ -1909,6 +1875,12 @@ export class Orchestrator {
|
|
|
1909
1875
|
* to RecallResultFormatter.
|
|
1910
1876
|
*/
|
|
1911
1877
|
readonly recallResultFormatter: RecallResultFormatter;
|
|
1878
|
+
/**
|
|
1879
|
+
* Issue #1526: conversation-index subsystem moved to
|
|
1880
|
+
* ConversationIndexCoordinator.
|
|
1881
|
+
*/
|
|
1882
|
+
readonly conversationIndexCoordinator: ConversationIndexCoordinator;
|
|
1883
|
+
readonly recallRerankCoordinator: RecallRerankCoordinator;
|
|
1912
1884
|
private heartbeatObserverChains = new Map<string, Promise<void>>();
|
|
1913
1885
|
private recentExtractionFingerprints = new Map<string, number>();
|
|
1914
1886
|
private readonly consolidationObservers = new Set<
|
|
@@ -1917,7 +1889,6 @@ export class Orchestrator {
|
|
|
1917
1889
|
private wearablesServiceInstance: WearablesService | null = null;
|
|
1918
1890
|
private wearablesAutoSyncHandle: { stop(): Promise<void> } | null = null;
|
|
1919
1891
|
private lastQmdReprobeAtMs = 0;
|
|
1920
|
-
private readonly conversationIndexLastUpdateAtMs = new Map<string, number>();
|
|
1921
1892
|
private lastFileHygieneRunAtMs = 0;
|
|
1922
1893
|
// Pattern-reinforcement cadence gate (issue #687 PR 2/4). Tracks the
|
|
1923
1894
|
// last successful run so `runPatternReinforcement` can short-circuit
|
|
@@ -2898,6 +2869,18 @@ export class Orchestrator {
|
|
|
2898
2869
|
"conversation-index",
|
|
2899
2870
|
"chunks",
|
|
2900
2871
|
);
|
|
2872
|
+
this.conversationIndexCoordinator = new ConversationIndexCoordinator({
|
|
2873
|
+
config,
|
|
2874
|
+
getTranscript: () => this.transcript,
|
|
2875
|
+
getBackend: () => this.conversationIndexBackend,
|
|
2876
|
+
indexDir: this.conversationIndexDir,
|
|
2877
|
+
});
|
|
2878
|
+
this.recallRerankCoordinator = new RecallRerankCoordinator({
|
|
2879
|
+
getConfig: () => this.config,
|
|
2880
|
+
getStorage: (namespace) => this.getStorage(namespace),
|
|
2881
|
+
readQmdResultMemory: (resultPath, fallbackStorage, recallNamespaces) =>
|
|
2882
|
+
this.readQmdResultMemory(resultPath, fallbackStorage, recallNamespaces),
|
|
2883
|
+
});
|
|
2901
2884
|
this.modelRegistry = new ModelRegistry(config.memoryDir);
|
|
2902
2885
|
this.relevance = new RelevanceStore(config.memoryDir);
|
|
2903
2886
|
this.negatives = new NegativeExampleStore(config.memoryDir);
|
|
@@ -5006,63 +4989,21 @@ export class Orchestrator {
|
|
|
5006
4989
|
retrievalQuery: string,
|
|
5007
4990
|
topK: number,
|
|
5008
4991
|
): Promise<Array<{ path: string; snippet: string; score: number }>> {
|
|
5009
|
-
|
|
5010
|
-
return this.conversationIndexBackend.search(retrievalQuery, topK);
|
|
5011
|
-
}
|
|
5012
|
-
return [];
|
|
4992
|
+
return this.conversationIndexCoordinator.search(retrievalQuery, topK);
|
|
5013
4993
|
}
|
|
5014
4994
|
|
|
5015
4995
|
private formatConversationRecallSection(
|
|
5016
4996
|
results: Array<{ path: string; snippet: string; score: number }>,
|
|
5017
4997
|
maxChars: number,
|
|
5018
4998
|
): string | null {
|
|
5019
|
-
|
|
5020
|
-
|
|
5021
|
-
|
|
5022
|
-
|
|
5023
|
-
if (!r?.snippet) continue;
|
|
5024
|
-
const chunk =
|
|
5025
|
-
`### ${r.path}\n` +
|
|
5026
|
-
`Score: ${r.score.toFixed(3)}\n\n` +
|
|
5027
|
-
`${r.snippet.trim()}\n`;
|
|
5028
|
-
if (used + chunk.length > maxChars) break;
|
|
5029
|
-
lines.push(chunk);
|
|
5030
|
-
used += chunk.length;
|
|
5031
|
-
}
|
|
5032
|
-
return used > 0 ? lines.join("\n") : null;
|
|
5033
|
-
}
|
|
5034
|
-
|
|
5035
|
-
private async countConversationChunkDocs(dir: string): Promise<number> {
|
|
5036
|
-
try {
|
|
5037
|
-
const entries = await readdir(dir, { withFileTypes: true });
|
|
5038
|
-
let total = 0;
|
|
5039
|
-
for (const entry of entries) {
|
|
5040
|
-
const fullPath = path.join(dir, entry.name);
|
|
5041
|
-
if (entry.isDirectory()) {
|
|
5042
|
-
total += await this.countConversationChunkDocs(fullPath);
|
|
5043
|
-
continue;
|
|
5044
|
-
}
|
|
5045
|
-
if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
5046
|
-
total += 1;
|
|
5047
|
-
}
|
|
5048
|
-
}
|
|
5049
|
-
return total;
|
|
5050
|
-
} catch {
|
|
5051
|
-
return 0;
|
|
5052
|
-
}
|
|
4999
|
+
return this.conversationIndexCoordinator.formatRecallSection(
|
|
5000
|
+
results,
|
|
5001
|
+
maxChars,
|
|
5002
|
+
);
|
|
5053
5003
|
}
|
|
5054
5004
|
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
hours: number = 24,
|
|
5058
|
-
): Promise<ReturnType<typeof chunkTranscriptEntries>> {
|
|
5059
|
-
const entries = await this.transcript.readRecent(hours, sessionKey);
|
|
5060
|
-
const effectiveSessionKey = sessionKey ?? "all-sessions";
|
|
5061
|
-
return chunkTranscriptEntries(effectiveSessionKey, entries, {
|
|
5062
|
-
maxChars: this.config.conversationRecallMaxChars * 2,
|
|
5063
|
-
maxTurns: Math.max(10, this.config.hourlySummariesMaxTurnsPerRun),
|
|
5064
|
-
});
|
|
5065
|
-
}
|
|
5005
|
+
// Issue #1526: countConversationChunkDocs / buildConversationIndexChunks moved
|
|
5006
|
+
// to ConversationIndexCoordinator (internal helpers, no orchestrator callers).
|
|
5066
5007
|
|
|
5067
5008
|
async getConversationIndexHealth(): Promise<{
|
|
5068
5009
|
enabled: boolean;
|
|
@@ -5087,37 +5028,7 @@ export class Orchestrator {
|
|
|
5087
5028
|
};
|
|
5088
5029
|
};
|
|
5089
5030
|
}> {
|
|
5090
|
-
|
|
5091
|
-
this.conversationIndexDir,
|
|
5092
|
-
);
|
|
5093
|
-
const lastUpdateAtMs = Math.max(
|
|
5094
|
-
0,
|
|
5095
|
-
...this.conversationIndexLastUpdateAtMs.values(),
|
|
5096
|
-
);
|
|
5097
|
-
const lastUpdateAt =
|
|
5098
|
-
lastUpdateAtMs > 0 ? new Date(lastUpdateAtMs).toISOString() : null;
|
|
5099
|
-
|
|
5100
|
-
if (!resolveIndexingCapabilities(this.config).conversationIndex) {
|
|
5101
|
-
return {
|
|
5102
|
-
enabled: false,
|
|
5103
|
-
backend: this.config.conversationIndexBackend,
|
|
5104
|
-
status: "disabled",
|
|
5105
|
-
chunkDocCount,
|
|
5106
|
-
lastUpdateAt,
|
|
5107
|
-
};
|
|
5108
|
-
}
|
|
5109
|
-
const backendHealth = this.conversationIndexBackend
|
|
5110
|
-
? await this.conversationIndexBackend.health()
|
|
5111
|
-
: {
|
|
5112
|
-
backend: this.config.conversationIndexBackend,
|
|
5113
|
-
status: "degraded" as const,
|
|
5114
|
-
};
|
|
5115
|
-
return {
|
|
5116
|
-
enabled: true,
|
|
5117
|
-
chunkDocCount,
|
|
5118
|
-
lastUpdateAt,
|
|
5119
|
-
...backendHealth,
|
|
5120
|
-
};
|
|
5031
|
+
return this.conversationIndexCoordinator.getHealth();
|
|
5121
5032
|
}
|
|
5122
5033
|
|
|
5123
5034
|
async inspectConversationIndex(): Promise<
|
|
@@ -5127,53 +5038,7 @@ export class Orchestrator {
|
|
|
5127
5038
|
lastUpdateAt: string | null;
|
|
5128
5039
|
}
|
|
5129
5040
|
> {
|
|
5130
|
-
|
|
5131
|
-
this.conversationIndexDir,
|
|
5132
|
-
);
|
|
5133
|
-
const lastUpdateAtMs = Math.max(
|
|
5134
|
-
0,
|
|
5135
|
-
...this.conversationIndexLastUpdateAtMs.values(),
|
|
5136
|
-
);
|
|
5137
|
-
const lastUpdateAt =
|
|
5138
|
-
lastUpdateAtMs > 0 ? new Date(lastUpdateAtMs).toISOString() : null;
|
|
5139
|
-
|
|
5140
|
-
if (!resolveIndexingCapabilities(this.config).conversationIndex) {
|
|
5141
|
-
return {
|
|
5142
|
-
enabled: false,
|
|
5143
|
-
backend: this.config.conversationIndexBackend,
|
|
5144
|
-
status: "disabled",
|
|
5145
|
-
available: false,
|
|
5146
|
-
indexPath: this.conversationIndexDir,
|
|
5147
|
-
supportsIncrementalUpdate: true,
|
|
5148
|
-
message: "Conversation index disabled by config",
|
|
5149
|
-
metadata: {
|
|
5150
|
-
chunkCount: chunkDocCount,
|
|
5151
|
-
},
|
|
5152
|
-
chunkDocCount,
|
|
5153
|
-
lastUpdateAt,
|
|
5154
|
-
};
|
|
5155
|
-
}
|
|
5156
|
-
|
|
5157
|
-
const inspection = this.conversationIndexBackend
|
|
5158
|
-
? await this.conversationIndexBackend.inspect()
|
|
5159
|
-
: {
|
|
5160
|
-
backend: this.config.conversationIndexBackend,
|
|
5161
|
-
status: "degraded" as const,
|
|
5162
|
-
available: false,
|
|
5163
|
-
indexPath: this.conversationIndexDir,
|
|
5164
|
-
supportsIncrementalUpdate: true,
|
|
5165
|
-
message: "Conversation index backend unavailable",
|
|
5166
|
-
metadata: {
|
|
5167
|
-
chunkCount: chunkDocCount,
|
|
5168
|
-
},
|
|
5169
|
-
};
|
|
5170
|
-
|
|
5171
|
-
return {
|
|
5172
|
-
enabled: true,
|
|
5173
|
-
chunkDocCount,
|
|
5174
|
-
lastUpdateAt,
|
|
5175
|
-
...inspection,
|
|
5176
|
-
};
|
|
5041
|
+
return this.conversationIndexCoordinator.inspect();
|
|
5177
5042
|
}
|
|
5178
5043
|
|
|
5179
5044
|
async getRecoverySummary(sessionKey?: string): Promise<{
|
|
@@ -5199,54 +5064,7 @@ export class Orchestrator {
|
|
|
5199
5064
|
retryAfterMs?: number;
|
|
5200
5065
|
embedded?: boolean;
|
|
5201
5066
|
}> {
|
|
5202
|
-
|
|
5203
|
-
return { chunks: 0, skipped: true, reason: "disabled", embedded: false };
|
|
5204
|
-
}
|
|
5205
|
-
const enforceMinInterval = opts?.enforceMinInterval !== false;
|
|
5206
|
-
if (enforceMinInterval) {
|
|
5207
|
-
const minIntervalMs = Math.max(
|
|
5208
|
-
0,
|
|
5209
|
-
this.config.conversationIndexMinUpdateIntervalMs,
|
|
5210
|
-
);
|
|
5211
|
-
const now = Date.now();
|
|
5212
|
-
const last = this.conversationIndexLastUpdateAtMs.get(sessionKey) ?? 0;
|
|
5213
|
-
const elapsed = now - last;
|
|
5214
|
-
if (minIntervalMs > 0 && elapsed < minIntervalMs) {
|
|
5215
|
-
return {
|
|
5216
|
-
chunks: 0,
|
|
5217
|
-
skipped: true,
|
|
5218
|
-
reason: "min_interval",
|
|
5219
|
-
retryAfterMs: minIntervalMs - elapsed,
|
|
5220
|
-
embedded: false,
|
|
5221
|
-
};
|
|
5222
|
-
}
|
|
5223
|
-
}
|
|
5224
|
-
const chunks = await this.buildConversationIndexChunks(sessionKey, hours);
|
|
5225
|
-
await writeConversationChunks(this.conversationIndexDir, chunks);
|
|
5226
|
-
const retentionCutoffMs =
|
|
5227
|
-
Number.isFinite(this.config.conversationIndexRetentionDays) &&
|
|
5228
|
-
this.config.conversationIndexRetentionDays > 0
|
|
5229
|
-
? Date.now() -
|
|
5230
|
-
this.config.conversationIndexRetentionDays * 24 * 60 * 60 * 1000
|
|
5231
|
-
: undefined;
|
|
5232
|
-
await cleanupConversationChunks(
|
|
5233
|
-
this.conversationIndexDir,
|
|
5234
|
-
this.config.conversationIndexRetentionDays,
|
|
5235
|
-
);
|
|
5236
|
-
const shouldEmbed =
|
|
5237
|
-
opts?.embed ?? this.config.conversationIndexEmbedOnUpdate;
|
|
5238
|
-
let embedded = false;
|
|
5239
|
-
|
|
5240
|
-
if (this.conversationIndexBackend) {
|
|
5241
|
-
const result = await this.conversationIndexBackend.update(chunks, {
|
|
5242
|
-
embed: shouldEmbed,
|
|
5243
|
-
...(retentionCutoffMs !== undefined ? { retentionCutoffMs } : {}),
|
|
5244
|
-
});
|
|
5245
|
-
embedded = result.embedded;
|
|
5246
|
-
}
|
|
5247
|
-
|
|
5248
|
-
this.conversationIndexLastUpdateAtMs.set(sessionKey, Date.now());
|
|
5249
|
-
return { chunks: chunks.length, skipped: false, embedded };
|
|
5067
|
+
return this.conversationIndexCoordinator.update(sessionKey, hours, opts);
|
|
5250
5068
|
}
|
|
5251
5069
|
|
|
5252
5070
|
async rebuildConversationIndex(
|
|
@@ -5260,42 +5078,7 @@ export class Orchestrator {
|
|
|
5260
5078
|
embedded?: boolean;
|
|
5261
5079
|
rebuilt?: boolean;
|
|
5262
5080
|
}> {
|
|
5263
|
-
|
|
5264
|
-
return {
|
|
5265
|
-
chunks: 0,
|
|
5266
|
-
skipped: true,
|
|
5267
|
-
reason: "disabled",
|
|
5268
|
-
embedded: false,
|
|
5269
|
-
rebuilt: false,
|
|
5270
|
-
};
|
|
5271
|
-
}
|
|
5272
|
-
|
|
5273
|
-
const chunks = await this.buildConversationIndexChunks(sessionKey, hours);
|
|
5274
|
-
await writeConversationChunks(this.conversationIndexDir, chunks);
|
|
5275
|
-
await cleanupConversationChunks(
|
|
5276
|
-
this.conversationIndexDir,
|
|
5277
|
-
this.config.conversationIndexRetentionDays,
|
|
5278
|
-
);
|
|
5279
|
-
|
|
5280
|
-
const shouldEmbed =
|
|
5281
|
-
opts?.embed ?? this.config.conversationIndexEmbedOnUpdate;
|
|
5282
|
-
let embedded = false;
|
|
5283
|
-
let rebuilt = false;
|
|
5284
|
-
if (this.conversationIndexBackend) {
|
|
5285
|
-
const result = await this.conversationIndexBackend.rebuild(chunks, {
|
|
5286
|
-
embed: shouldEmbed,
|
|
5287
|
-
});
|
|
5288
|
-
embedded = result.embedded;
|
|
5289
|
-
rebuilt = result.rebuilt;
|
|
5290
|
-
}
|
|
5291
|
-
|
|
5292
|
-
const stamp = Date.now();
|
|
5293
|
-
if (sessionKey) {
|
|
5294
|
-
this.conversationIndexLastUpdateAtMs.set(sessionKey, stamp);
|
|
5295
|
-
} else {
|
|
5296
|
-
this.conversationIndexLastUpdateAtMs.set("__rebuild__", stamp);
|
|
5297
|
-
}
|
|
5298
|
-
return { chunks: chunks.length, skipped: false, embedded, rebuilt };
|
|
5081
|
+
return this.conversationIndexCoordinator.rebuild(sessionKey, hours, opts);
|
|
5299
5082
|
}
|
|
5300
5083
|
|
|
5301
5084
|
/**
|
|
@@ -17365,156 +17148,15 @@ export class Orchestrator {
|
|
|
17365
17148
|
return matches[0] ?? null;
|
|
17366
17149
|
}
|
|
17367
17150
|
|
|
17151
|
+
// Issue #1526: recall-rerank methods moved to RecallRerankCoordinator.
|
|
17152
|
+
// Thin delegation keeps the private API stable for callers + tests.
|
|
17368
17153
|
private async applyMemoryWorthRerank(
|
|
17369
17154
|
results: QmdSearchResult[],
|
|
17370
17155
|
namespaces: string[],
|
|
17371
17156
|
): Promise<QmdSearchResult[]> {
|
|
17372
|
-
|
|
17373
|
-
// namespace the recall spans — the recall path itself already
|
|
17374
|
-
// aggregates candidates from multiple namespaces, so we must do the
|
|
17375
|
-
// same when looking up counters. Per-namespace results are cached with
|
|
17376
|
-
// a short TTL so interactive recall doesn't trigger a full
|
|
17377
|
-
// `readAllMemories` scan per query (addresses codex P2 on PR 4).
|
|
17378
|
-
const counters = new Map<string, MemoryWorthCounters>();
|
|
17379
|
-
const seenNamespaces = new Set<string>();
|
|
17380
|
-
const nowMs = Date.now();
|
|
17381
|
-
|
|
17382
|
-
// Evict all expired entries on every call so long-running processes
|
|
17383
|
-
// touching a high-cardinality namespace set (coding/project overlays,
|
|
17384
|
-
// per-branch) don't grow the cache unboundedly. Without this, an entry
|
|
17385
|
-
// for a namespace that's never looked up again would pin its full
|
|
17386
|
-
// counter map forever.
|
|
17387
|
-
for (const [key, entry] of this.memoryWorthCounterCache) {
|
|
17388
|
-
if (nowMs - entry.at >= Orchestrator.MEMORY_WORTH_CACHE_TTL_MS) {
|
|
17389
|
-
this.memoryWorthCounterCache.delete(key);
|
|
17390
|
-
}
|
|
17391
|
-
}
|
|
17392
|
-
|
|
17393
|
-
for (const ns of namespaces) {
|
|
17394
|
-
if (seenNamespaces.has(ns)) continue;
|
|
17395
|
-
seenNamespaces.add(ns);
|
|
17396
|
-
try {
|
|
17397
|
-
const cached = this.memoryWorthCounterCache.get(ns);
|
|
17398
|
-
let nsMap: ReadonlyMap<string, MemoryWorthCounters> | undefined;
|
|
17399
|
-
if (
|
|
17400
|
-
cached &&
|
|
17401
|
-
nowMs - cached.at < Orchestrator.MEMORY_WORTH_CACHE_TTL_MS
|
|
17402
|
-
) {
|
|
17403
|
-
nsMap = cached.counters;
|
|
17404
|
-
} else {
|
|
17405
|
-
const storage = await this.getStorage(ns);
|
|
17406
|
-
const memories = await storage.readAllMemories();
|
|
17407
|
-
nsMap = buildMemoryWorthCounterMap(memories);
|
|
17408
|
-
this.memoryWorthCounterCache.set(ns, { at: nowMs, counters: nsMap });
|
|
17409
|
-
}
|
|
17410
|
-
for (const [path, c] of nsMap) counters.set(path, c);
|
|
17411
|
-
} catch (err) {
|
|
17412
|
-
log.debug("memory-worth: failed to read namespace, skipping", {
|
|
17413
|
-
namespace: ns,
|
|
17414
|
-
error: (err as Error).message,
|
|
17415
|
-
});
|
|
17416
|
-
}
|
|
17417
|
-
}
|
|
17418
|
-
|
|
17419
|
-
// For candidates whose path didn't show up in any hot-tier namespace
|
|
17420
|
-
// scan (typical of cold-tier / archive fallback), try a direct
|
|
17421
|
-
// per-path read. Without this, cold-tier candidates always stay at
|
|
17422
|
-
// multiplier 1.0 even when they have outcome history. Errors are
|
|
17423
|
-
// swallowed so a single unreadable archive entry can't break the
|
|
17424
|
-
// whole recall.
|
|
17425
|
-
const missing = results.filter((r) => !counters.has(r.path));
|
|
17426
|
-
if (missing.length > 0) {
|
|
17427
|
-
// Use the first-seen namespace's storage as the reader — all
|
|
17428
|
-
// StorageManagers share the same on-disk format, and
|
|
17429
|
-
// `readMemoryByPath` takes an absolute path so the baseDir doesn't
|
|
17430
|
-
// have to match.
|
|
17431
|
-
let reader: StorageManager | null = null;
|
|
17432
|
-
for (const ns of namespaces) {
|
|
17433
|
-
try {
|
|
17434
|
-
reader = await this.getStorage(ns);
|
|
17435
|
-
break;
|
|
17436
|
-
} catch {
|
|
17437
|
-
// try next namespace
|
|
17438
|
-
}
|
|
17439
|
-
}
|
|
17440
|
-
if (reader) {
|
|
17441
|
-
for (const r of missing) {
|
|
17442
|
-
try {
|
|
17443
|
-
const memory = await this.readQmdResultMemory(r.path, reader, namespaces);
|
|
17444
|
-
if (!memory) continue;
|
|
17445
|
-
const fm = memory.frontmatter;
|
|
17446
|
-
if (fm.mw_success === undefined && fm.mw_fail === undefined) continue;
|
|
17447
|
-
counters.set(r.path, {
|
|
17448
|
-
mw_success: fm.mw_success,
|
|
17449
|
-
mw_fail: fm.mw_fail,
|
|
17450
|
-
lastAccessed: fm.lastAccessed,
|
|
17451
|
-
});
|
|
17452
|
-
} catch (err) {
|
|
17453
|
-
log.debug("memory-worth: direct path lookup failed", {
|
|
17454
|
-
path: r.path,
|
|
17455
|
-
error: (err as Error).message,
|
|
17456
|
-
});
|
|
17457
|
-
}
|
|
17458
|
-
}
|
|
17459
|
-
}
|
|
17460
|
-
}
|
|
17461
|
-
|
|
17462
|
-
// If no memory in the candidate set has any counter data, the filter
|
|
17463
|
-
// would be a no-op — skip the reorder to avoid spurious log spam.
|
|
17464
|
-
if (counters.size === 0) return results;
|
|
17465
|
-
|
|
17466
|
-
// Preserve upstream ordering (reranker, specialized tiers, etc.) for
|
|
17467
|
-
// neutral candidates. The upstream stages set `memoryResults` in their
|
|
17468
|
-
// intended order but often leave `r.score` as the raw QMD score. If we
|
|
17469
|
-
// sorted by `r.score * multiplier` directly, neutral candidates
|
|
17470
|
-
// (multiplier 1.0) would snap back to raw-QMD order and silently undo
|
|
17471
|
-
// the reranker. Feed the filter a synthetic monotone-decreasing rank
|
|
17472
|
-
// score so it uses input position as the baseline, then applies the
|
|
17473
|
-
// multiplier on top. Ties fall back to the stable secondary key in
|
|
17474
|
-
// `applyMemoryWorthFilter`.
|
|
17475
|
-
const rankedInputs = results.map((r, i) => ({
|
|
17476
|
-
path: r.path,
|
|
17477
|
-
// Large positive rank score so multiplier math stays well-scaled and
|
|
17478
|
-
// we never hit zero; descending so earlier items rank higher.
|
|
17479
|
-
score: results.length - i,
|
|
17480
|
-
}));
|
|
17481
|
-
const filtered = applyMemoryWorthFilter(rankedInputs, {
|
|
17482
|
-
counters,
|
|
17483
|
-
now: new Date(),
|
|
17484
|
-
halfLifeMs:
|
|
17485
|
-
this.config.recallMemoryWorthHalfLifeMs > 0
|
|
17486
|
-
? this.config.recallMemoryWorthHalfLifeMs
|
|
17487
|
-
: undefined,
|
|
17488
|
-
});
|
|
17489
|
-
|
|
17490
|
-
// Reconstruct the QmdSearchResult list in the new order. `.score` is
|
|
17491
|
-
// preserved from the upstream pipeline (rerank, tier scoring, etc.) —
|
|
17492
|
-
// we only reorder. Writing the synthetic rank-weighted score back
|
|
17493
|
-
// would contaminate downstream logic (telemetry, confidence gates)
|
|
17494
|
-
// that expects the original QMD/rerank score semantics.
|
|
17495
|
-
const byPath = new Map(results.map((r) => [r.path, r]));
|
|
17496
|
-
const reordered: QmdSearchResult[] = [];
|
|
17497
|
-
for (const item of filtered) {
|
|
17498
|
-
const original = byPath.get(item.path);
|
|
17499
|
-
if (original) reordered.push(original);
|
|
17500
|
-
}
|
|
17501
|
-
return reordered;
|
|
17157
|
+
return this.recallRerankCoordinator.applyMemoryWorthRerank(results, namespaces);
|
|
17502
17158
|
}
|
|
17503
17159
|
|
|
17504
|
-
/**
|
|
17505
|
-
* Issue #1577 — unified TrustScore recall stage. Thin wiring over the pure
|
|
17506
|
-
* {@link applyTrustScoreStage} scorer + the {@link buildTrustSignalsForRerank}
|
|
17507
|
-
* signal builder. The stage subsumes the Memory Worth multiplier — the
|
|
17508
|
-
* orchestrator runs exactly one of the two (mutual exclusion, rule 39; the
|
|
17509
|
-
* double-multiplier test in trust-score-stage.test.ts pins it structurally).
|
|
17510
|
-
*
|
|
17511
|
-
* Returns the admitted results AND the per-path trust map (including
|
|
17512
|
-
* quarantined items) so the caller can: (a) render epistemic hedges, (b)
|
|
17513
|
-
* surface quarantined items in X-ray with a reason (rule 34), and (c) filter
|
|
17514
|
-
* quarantined paths from fallback recall branches. The trust map is a
|
|
17515
|
-
* per-recall local — never instance state — so concurrent recalls cannot
|
|
17516
|
-
* race on it (review: shared-trust-map concurrency).
|
|
17517
|
-
*/
|
|
17518
17160
|
private async applyTrustScoreRerank(
|
|
17519
17161
|
results: QmdSearchResult[],
|
|
17520
17162
|
namespaces: string[],
|
|
@@ -17522,74 +17164,9 @@ export class Orchestrator {
|
|
|
17522
17164
|
results: QmdSearchResult[];
|
|
17523
17165
|
trustByPath: Map<string, TrustStageResultItem> | null;
|
|
17524
17166
|
}> {
|
|
17525
|
-
|
|
17526
|
-
const now = new Date();
|
|
17527
|
-
const halfLifeDays =
|
|
17528
|
-
this.config.recallMemoryWorthHalfLifeMs > 0
|
|
17529
|
-
? this.config.recallMemoryWorthHalfLifeMs / (24 * 60 * 60 * 1000)
|
|
17530
|
-
: undefined;
|
|
17531
|
-
// Cold-tier direct-fallback reader: resolve once, reuse for every missing
|
|
17532
|
-
// candidate (mirrors the memory-worth filter's reader selection).
|
|
17533
|
-
let fallbackReader: StorageManager | null = null;
|
|
17534
|
-
const signals = await buildTrustSignalsForRerank(
|
|
17535
|
-
results.map((r) => r.path),
|
|
17536
|
-
namespaces,
|
|
17537
|
-
{
|
|
17538
|
-
readNamespaceMemories: async (ns) => (await this.getStorage(ns)).readAllMemories(),
|
|
17539
|
-
readMemoryFrontmatter: async (path) => {
|
|
17540
|
-
if (!fallbackReader) {
|
|
17541
|
-
for (const ns of namespaces) {
|
|
17542
|
-
try {
|
|
17543
|
-
fallbackReader = await this.getStorage(ns);
|
|
17544
|
-
break;
|
|
17545
|
-
} catch {
|
|
17546
|
-
// try next namespace
|
|
17547
|
-
}
|
|
17548
|
-
}
|
|
17549
|
-
}
|
|
17550
|
-
if (!fallbackReader) return null;
|
|
17551
|
-
const memory = await this.readQmdResultMemory(path, fallbackReader, namespaces);
|
|
17552
|
-
return memory ? memory.frontmatter : null;
|
|
17553
|
-
},
|
|
17554
|
-
},
|
|
17555
|
-
{ cache: this.trustSignalCache, ttlMs: Orchestrator.TRUST_SIGNAL_CACHE_TTL_MS },
|
|
17556
|
-
now,
|
|
17557
|
-
{
|
|
17558
|
-
recencyHalfLifeDays: halfLifeDays,
|
|
17559
|
-
logDebug: (message, context) => log.debug(message, context),
|
|
17560
|
-
},
|
|
17561
|
-
);
|
|
17562
|
-
if (signals.size === 0) {
|
|
17563
|
-
return { results, trustByPath: null };
|
|
17564
|
-
}
|
|
17565
|
-
// Synthetic monotone-decreasing rank so the multiplier rebias is applied
|
|
17566
|
-
// on top of upstream ordering, not raw QMD scores (see applyMemoryWorthRerank).
|
|
17567
|
-
const rankedInputs = results.map((r, i) => ({ path: r.path, score: results.length - i }));
|
|
17568
|
-
const stage = applyTrustScoreStage(rankedInputs, {
|
|
17569
|
-
signals,
|
|
17570
|
-
weights: this.config.trustScoreWeights,
|
|
17571
|
-
minMultiplier: this.config.trustScoreMinMultiplier,
|
|
17572
|
-
maxMultiplier: this.config.trustScoreMaxMultiplier,
|
|
17573
|
-
quarantine: this.config.trustScoreQuarantine,
|
|
17574
|
-
});
|
|
17575
|
-
const trustByPath = new Map(stage.all.map((item) => [item.path, item]));
|
|
17576
|
-
const byPath = new Map(results.map((r) => [r.path, r]));
|
|
17577
|
-
const admitted = stage.admitted
|
|
17578
|
-
.map((item) => byPath.get(item.path))
|
|
17579
|
-
.filter((r): r is QmdSearchResult => r !== undefined);
|
|
17580
|
-
return { results: admitted, trustByPath };
|
|
17167
|
+
return this.recallRerankCoordinator.applyTrustScoreRerank(results, namespaces);
|
|
17581
17168
|
}
|
|
17582
17169
|
|
|
17583
|
-
/**
|
|
17584
|
-
* Issue #1577 — apply the TrustScore stage (or, when trust is off, the Memory
|
|
17585
|
-
* Worth multiplier fallback) to ONE recall branch's results, returning the
|
|
17586
|
-
* scored results + the per-path trust map. Thin wiring over
|
|
17587
|
-
* {@link applyTrustScoreRerank} so every recall path — hot QMD, embedding
|
|
17588
|
-
* fallback, recent scan — applies the SAME multiplier gate (rule 41: a
|
|
17589
|
-
* feature gate must apply across ALL parallel recall paths). TrustScore
|
|
17590
|
-
* subsumes Memory Worth; exactly one runs (rule 39). Fail-open on lookup
|
|
17591
|
-
* errors so a storage hiccup never breaks a fallback path.
|
|
17592
|
-
*/
|
|
17593
17170
|
private async applyTrustScoreToBranch(
|
|
17594
17171
|
results: QmdSearchResult[],
|
|
17595
17172
|
namespaces: string[],
|
|
@@ -17599,25 +17176,7 @@ export class Orchestrator {
|
|
|
17599
17176
|
results: QmdSearchResult[];
|
|
17600
17177
|
trustByPath: Map<string, TrustStageResultItem> | null;
|
|
17601
17178
|
}> {
|
|
17602
|
-
|
|
17603
|
-
try {
|
|
17604
|
-
return await this.applyTrustScoreRerank(results, namespaces);
|
|
17605
|
-
} catch (err) {
|
|
17606
|
-
log.debug(`trust-score stage (${label}) failed open`, {
|
|
17607
|
-
error: (err as Error).message,
|
|
17608
|
-
});
|
|
17609
|
-
}
|
|
17610
|
-
} else if (caps.recallMemoryWorthFilter && results.length > 0) {
|
|
17611
|
-
try {
|
|
17612
|
-
const filtered = await this.applyMemoryWorthRerank(results, namespaces);
|
|
17613
|
-
return { results: filtered, trustByPath: null };
|
|
17614
|
-
} catch (err) {
|
|
17615
|
-
log.debug(`memory-worth filter (${label}) failed open`, {
|
|
17616
|
-
error: (err as Error).message,
|
|
17617
|
-
});
|
|
17618
|
-
}
|
|
17619
|
-
}
|
|
17620
|
-
return { results, trustByPath: null };
|
|
17179
|
+
return this.recallRerankCoordinator.applyTrustScoreToBranch(results, namespaces, caps, label);
|
|
17621
17180
|
}
|
|
17622
17181
|
|
|
17623
17182
|
private diversifyAndLimitRecallResults(
|
|
@@ -17625,89 +17184,17 @@ export class Orchestrator {
|
|
|
17625
17184
|
results: QmdSearchResult[],
|
|
17626
17185
|
limit: number,
|
|
17627
17186
|
retrievalQuery?: string,
|
|
17628
|
-
// `caps` is additive AND last (issue #1523) so the positional call shape
|
|
17629
|
-
// stays backward-compatible: the recall pipeline threads a resolved set,
|
|
17630
|
-
// but callers that omit it (e.g. direct unit-test invocations) get an
|
|
17631
|
-
// equivalent set derived from the same config — behavior-preserving.
|
|
17632
17187
|
caps: CapabilitySet = resolveCapabilities(this.config),
|
|
17633
17188
|
): QmdSearchResult[] {
|
|
17634
|
-
|
|
17635
|
-
typeof limit === "number" && Number.isFinite(limit)
|
|
17636
|
-
? Math.max(0, Math.floor(limit))
|
|
17637
|
-
: 0;
|
|
17638
|
-
if (!Array.isArray(results) || results.length === 0) return [];
|
|
17639
|
-
// `recallResultLimit === 0` is a true zero limit (e.g. when
|
|
17640
|
-
// `memoriesSectionEnabled` is false) and must return an empty array so
|
|
17641
|
-
// the memories section is genuinely skipped. This mirrors the
|
|
17642
|
-
// `slice(0, 0)` semantics of every call site this helper replaced.
|
|
17643
|
-
if (safeLimit === 0) return [];
|
|
17644
|
-
// Issue #564 PR 3: when the feature flag is on, boost reasoning_trace
|
|
17645
|
-
// memories for problem-solving asks so they bubble up ahead of ordinary
|
|
17646
|
-
// facts/decisions before MMR picks the final section. No-op when the
|
|
17647
|
-
// flag is off or the query is not a problem-solving ask.
|
|
17648
|
-
const boosted =
|
|
17649
|
-
caps.recallReasoningTraceBoost && typeof retrievalQuery === "string"
|
|
17650
|
-
? applyReasoningTraceBoost(results, {
|
|
17651
|
-
enabled: true,
|
|
17652
|
-
query: retrievalQuery,
|
|
17653
|
-
})
|
|
17654
|
-
: results;
|
|
17655
|
-
const diversified = this.applyMmrToQmdResults(sectionId, boosted, caps);
|
|
17656
|
-
return diversified.slice(0, safeLimit);
|
|
17189
|
+
return this.recallRerankCoordinator.diversifyAndLimitRecallResults(sectionId, results, limit, retrievalQuery, caps);
|
|
17657
17190
|
}
|
|
17658
17191
|
|
|
17659
|
-
/**
|
|
17660
|
-
* Apply Maximal Marginal Relevance to a section's ordered candidate list.
|
|
17661
|
-
*
|
|
17662
|
-
* Operates per-section so one redundant cluster cannot dominate a section,
|
|
17663
|
-
* and so one section's MMR pass cannot starve other sections. Returns the
|
|
17664
|
-
* input unchanged when disabled, when there are fewer than 2 candidates, or
|
|
17665
|
-
* when no budget information is available.
|
|
17666
|
-
*/
|
|
17667
17192
|
private applyMmrToQmdResults(
|
|
17668
17193
|
sectionId: string,
|
|
17669
17194
|
results: QmdSearchResult[],
|
|
17670
|
-
// Additive `caps` (issue #1523); defaults to a config-derived set so direct
|
|
17671
|
-
// callers that omit it behave identically to the threaded recall path.
|
|
17672
17195
|
caps: CapabilitySet = resolveCapabilities(this.config),
|
|
17673
17196
|
): QmdSearchResult[] {
|
|
17674
|
-
|
|
17675
|
-
if (!Array.isArray(results) || results.length < 2) return results;
|
|
17676
|
-
|
|
17677
|
-
// Config is runtime API (see AGENTS.md §4): preserve `0` as a true zero
|
|
17678
|
-
// limit rather than coercing it to a non-zero value. A configured topN of
|
|
17679
|
-
// 0 means "apply MMR over an empty window" — i.e. skip the reorder and
|
|
17680
|
-
// return the upstream candidates unchanged. This keeps read-time
|
|
17681
|
-
// behavior symmetric with the write-time semantics parseConfig exposes.
|
|
17682
|
-
const configuredTopN = this.config.recallMmrTopN;
|
|
17683
|
-
const topN =
|
|
17684
|
-
typeof configuredTopN === "number" && Number.isFinite(configuredTopN)
|
|
17685
|
-
? Math.max(0, Math.floor(configuredTopN))
|
|
17686
|
-
: 40;
|
|
17687
|
-
if (topN === 0) return results;
|
|
17688
|
-
const lambda = this.config.recallMmrLambda ?? 0.7;
|
|
17689
|
-
|
|
17690
|
-
// Delegate to the pure helper so candidate keying (path-first, index
|
|
17691
|
-
// suffixed for uniqueness) and the head-of-list diversity metric are
|
|
17692
|
-
// exercised by the same code path that the unit tests cover.
|
|
17693
|
-
const { reordered, diversity } = reorderRecallResultsWithMmr(results, {
|
|
17694
|
-
lambda,
|
|
17695
|
-
topN,
|
|
17696
|
-
});
|
|
17697
|
-
|
|
17698
|
-
try {
|
|
17699
|
-
log.info(
|
|
17700
|
-
`recall_mmr: section=${sectionId} kept=${diversity.kept}/${diversity.considered} ` +
|
|
17701
|
-
`headReorderCount=${diversity.headReorderCount} ` +
|
|
17702
|
-
`avgSimBefore=${diversity.avgPairwiseSimBefore.toFixed(3)} ` +
|
|
17703
|
-
`avgSimAfter=${diversity.avgPairwiseSimAfter.toFixed(3)} ` +
|
|
17704
|
-
`lambda=${lambda.toFixed(2)}`,
|
|
17705
|
-
);
|
|
17706
|
-
} catch {
|
|
17707
|
-
// Metrics must never break recall.
|
|
17708
|
-
}
|
|
17709
|
-
|
|
17710
|
-
return reordered;
|
|
17197
|
+
return this.recallRerankCoordinator.applyMmrToQmdResults(sectionId, results, caps);
|
|
17711
17198
|
}
|
|
17712
17199
|
|
|
17713
17200
|
private buildLastRecallBudgetSummary(options: {
|