@remnic/core 9.3.653 → 9.3.655
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +24 -24
- package/dist/access-http.d.ts +4 -4
- package/dist/access-http.js +17 -17
- package/dist/access-mcp.d.ts +4 -4
- package/dist/access-mcp.js +16 -16
- package/dist/access-schema.d.ts +12 -12
- package/dist/{access-service-CdJFd3_b.d.ts → access-service-BEJvriUt.d.ts} +11 -2
- package/dist/access-service.d.ts +4 -4
- package/dist/access-service.js +15 -15
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +1 -1
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +3 -3
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +3 -3
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +1 -1
- package/dist/calibration.d.ts +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +4 -4
- package/dist/{chunk-GI45G4BK.js → chunk-2RCGZ67B.js} +4 -4
- package/dist/{chunk-BEMWL2FZ.js → chunk-54LOUIBE.js} +2 -2
- package/dist/{chunk-E3J6O6N7.js → chunk-55ZMNKMQ.js} +20 -9
- package/dist/{chunk-E3J6O6N7.js.map → chunk-55ZMNKMQ.js.map} +1 -1
- package/dist/{chunk-7WEB3FLJ.js → chunk-5PLUC5OB.js} +2 -2
- package/dist/{chunk-SPMZZUEJ.js → chunk-5QD3QD76.js} +2684 -401
- package/dist/chunk-5QD3QD76.js.map +1 -0
- package/dist/{chunk-WLGE6KEO.js → chunk-67G4T7KI.js} +3 -3
- package/dist/{chunk-JX2RINDR.js → chunk-6G5JEN55.js} +2 -2
- package/dist/{chunk-R3PQUPQ4.js → chunk-6IMKOIZ6.js} +85 -3
- package/dist/chunk-6IMKOIZ6.js.map +1 -0
- package/dist/{chunk-KJDKZVF3.js → chunk-A3Y37UWI.js} +3 -3
- package/dist/{chunk-CFOCZPIQ.js → chunk-BGKXTVNG.js} +2 -2
- package/dist/{chunk-QQHIQ7JD.js → chunk-COVZLGMR.js} +87 -18
- package/dist/chunk-COVZLGMR.js.map +1 -0
- package/dist/{chunk-JVRPJ7D4.js → chunk-EKQMQQ3U.js} +48 -12
- package/dist/chunk-EKQMQQ3U.js.map +1 -0
- package/dist/{chunk-H3PHZLMF.js → chunk-GKKAXVAJ.js} +20 -11
- package/dist/chunk-GKKAXVAJ.js.map +1 -0
- package/dist/{chunk-JBHXMCYN.js → chunk-GRYAECRV.js} +2 -2
- package/dist/{chunk-EHQLDFSH.js → chunk-IQ53ZSXV.js} +2 -2
- package/dist/{chunk-C63WC454.js → chunk-KOI765XP.js} +125 -1
- package/dist/chunk-KOI765XP.js.map +1 -0
- package/dist/{chunk-IVYSVAC6.js → chunk-KZZ4YAEC.js} +2 -2
- package/dist/{chunk-2DGQLOOM.js → chunk-M3VYPE2H.js} +1 -1
- package/dist/{chunk-2DGQLOOM.js.map → chunk-M3VYPE2H.js.map} +1 -1
- package/dist/{chunk-JF7SFXTG.js → chunk-NCSJKK23.js} +2 -2
- package/dist/{chunk-XMN6MMTU.js → chunk-NRBGRZW4.js} +2 -2
- package/dist/{chunk-NOBL7OUP.js → chunk-OKW6F5S5.js} +12 -5
- package/dist/{chunk-NOBL7OUP.js.map → chunk-OKW6F5S5.js.map} +1 -1
- package/dist/{chunk-BNFRL6QW.js → chunk-PTMJ2FH2.js} +2 -2
- package/dist/{chunk-KWM33SPU.js → chunk-PVE7KSQP.js} +2 -2
- package/dist/{chunk-EW52H5EM.js → chunk-QDVQ4AN2.js} +12 -5
- package/dist/chunk-QDVQ4AN2.js.map +1 -0
- package/dist/{chunk-PYWNNF2I.js → chunk-QRSKPI62.js} +99 -66
- package/dist/chunk-QRSKPI62.js.map +1 -0
- package/dist/{chunk-YM3LR4LS.js → chunk-SSSXWIBP.js} +5 -5
- package/dist/{chunk-C43KEWEV.js → chunk-TDZSSJV4.js} +1 -1
- package/dist/chunk-TDZSSJV4.js.map +1 -0
- package/dist/{chunk-Y7NWBBHV.js → chunk-TEO46GMM.js} +2 -2
- package/dist/{chunk-AJE7FJVE.js → chunk-UCEABZZN.js} +2 -2
- package/dist/{chunk-IENGGY2C.js → chunk-UCEDY5M7.js} +2 -2
- package/dist/{chunk-PRQXUSQV.js → chunk-UYNFWZWG.js} +2 -2
- package/dist/{chunk-V4UDXYGG.js → chunk-WDTUYOLS.js} +2 -2
- package/dist/{chunk-RZOBQ23O.js → chunk-XOFXKASO.js} +2 -2
- package/dist/chunk-XRKQOQLY.js +212 -0
- package/dist/chunk-XRKQOQLY.js.map +1 -0
- package/dist/{chunk-WTI35CVJ.js → chunk-YYN3LIYA.js} +5 -5
- package/dist/{cli-DDo7Qgs-.d.ts → cli-BGahB_d3.d.ts} +3 -3
- package/dist/cli.d.ts +5 -5
- package/dist/cli.js +29 -29
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +3 -3
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +1 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +3 -3
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/contradiction/index.d.ts +19 -1
- package/dist/contradiction/index.js +1 -1
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +3 -3
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +3 -3
- package/dist/explicit-capture.js +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/identity-continuity.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/index.d.ts +8 -8
- package/dist/index.js +37 -35
- package/dist/index.js.map +1 -1
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +4 -4
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +1 -1
- package/dist/namespaces/migrate.js +11 -11
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +15 -4
- package/dist/namespaces/search.js +7 -7
- package/dist/namespaces/storage.d.ts +52 -3
- package/dist/namespaces/storage.js +9 -5
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +14 -14
- package/dist/{orchestrator-8fTZsa0y.d.ts → orchestrator-BgzZlWxH.d.ts} +500 -3
- package/dist/orchestrator.d.ts +3 -3
- package/dist/orchestrator.js +20 -20
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +5 -1
- package/dist/qmd.js +2 -2
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/{resolution-3SAP4SH2.js → resolution-IDTEBJFS.js} +2 -2
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +2 -2
- package/dist/retrieval-agents.d.ts +1 -1
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/schemas.d.ts +22 -22
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/factory.js +6 -6
- package/dist/search/index.d.ts +1 -1
- package/dist/search/index.js +6 -6
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/lancedb-backend.js +2 -2
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.js +2 -2
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/orama-backend.js +2 -2
- package/dist/search/port.d.ts +17 -1
- package/dist/search/port.js +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-DKdYzQOg.d.ts → semantic-consolidation-Z8d_uMq8.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +2 -2
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/types.d.ts +12 -12
- package/dist/{types-D8yUmSik.d.ts → types-2OPlQWJG.d.ts} +23 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +3 -3
- package/package.json +1 -1
- package/src/access-http.ts +7 -0
- package/src/access-mcp.ts +7 -0
- package/src/access-service.ts +12 -0
- package/src/cli.ts +104 -0
- package/src/config.test.ts +109 -0
- package/src/config.ts +164 -0
- package/src/contradiction/contradiction.test.ts +284 -0
- package/src/contradiction/resolution.ts +151 -4
- package/src/explicit-capture.ts +31 -10
- package/src/index.ts +10 -0
- package/src/maintenance/namespace-planner.test.ts +1120 -0
- package/src/maintenance/namespace-planner.ts +893 -0
- package/src/namespaces/catalog.test.ts +3356 -0
- package/src/namespaces/catalog.ts +2123 -0
- package/src/namespaces/search.test.ts +130 -2
- package/src/namespaces/search.ts +71 -10
- package/src/namespaces/storage.ts +210 -30
- package/src/orchestrator-flush.test.ts +720 -0
- package/src/orchestrator.ts +881 -239
- package/src/qmd-client.test.ts +59 -0
- package/src/qmd.ts +124 -84
- package/src/search/port.ts +16 -0
- package/src/types.ts +23 -0
- package/dist/chunk-C43KEWEV.js.map +0 -1
- package/dist/chunk-C63WC454.js.map +0 -1
- package/dist/chunk-EW52H5EM.js.map +0 -1
- package/dist/chunk-H3PHZLMF.js.map +0 -1
- package/dist/chunk-JVRPJ7D4.js.map +0 -1
- package/dist/chunk-ORGWWNJG.js +0 -131
- package/dist/chunk-ORGWWNJG.js.map +0 -1
- package/dist/chunk-PYWNNF2I.js.map +0 -1
- package/dist/chunk-QQHIQ7JD.js.map +0 -1
- package/dist/chunk-R3PQUPQ4.js.map +0 -1
- package/dist/chunk-SPMZZUEJ.js.map +0 -1
- /package/dist/{chunk-GI45G4BK.js.map → chunk-2RCGZ67B.js.map} +0 -0
- /package/dist/{chunk-BEMWL2FZ.js.map → chunk-54LOUIBE.js.map} +0 -0
- /package/dist/{chunk-7WEB3FLJ.js.map → chunk-5PLUC5OB.js.map} +0 -0
- /package/dist/{chunk-WLGE6KEO.js.map → chunk-67G4T7KI.js.map} +0 -0
- /package/dist/{chunk-JX2RINDR.js.map → chunk-6G5JEN55.js.map} +0 -0
- /package/dist/{chunk-KJDKZVF3.js.map → chunk-A3Y37UWI.js.map} +0 -0
- /package/dist/{chunk-CFOCZPIQ.js.map → chunk-BGKXTVNG.js.map} +0 -0
- /package/dist/{chunk-JBHXMCYN.js.map → chunk-GRYAECRV.js.map} +0 -0
- /package/dist/{chunk-EHQLDFSH.js.map → chunk-IQ53ZSXV.js.map} +0 -0
- /package/dist/{chunk-IVYSVAC6.js.map → chunk-KZZ4YAEC.js.map} +0 -0
- /package/dist/{chunk-JF7SFXTG.js.map → chunk-NCSJKK23.js.map} +0 -0
- /package/dist/{chunk-XMN6MMTU.js.map → chunk-NRBGRZW4.js.map} +0 -0
- /package/dist/{chunk-BNFRL6QW.js.map → chunk-PTMJ2FH2.js.map} +0 -0
- /package/dist/{chunk-KWM33SPU.js.map → chunk-PVE7KSQP.js.map} +0 -0
- /package/dist/{chunk-YM3LR4LS.js.map → chunk-SSSXWIBP.js.map} +0 -0
- /package/dist/{chunk-Y7NWBBHV.js.map → chunk-TEO46GMM.js.map} +0 -0
- /package/dist/{chunk-AJE7FJVE.js.map → chunk-UCEABZZN.js.map} +0 -0
- /package/dist/{chunk-IENGGY2C.js.map → chunk-UCEDY5M7.js.map} +0 -0
- /package/dist/{chunk-PRQXUSQV.js.map → chunk-UYNFWZWG.js.map} +0 -0
- /package/dist/{chunk-V4UDXYGG.js.map → chunk-WDTUYOLS.js.map} +0 -0
- /package/dist/{chunk-RZOBQ23O.js.map → chunk-XOFXKASO.js.map} +0 -0
- /package/dist/{chunk-WTI35CVJ.js.map → chunk-YYN3LIYA.js.map} +0 -0
- /package/dist/{resolution-3SAP4SH2.js.map → resolution-IDTEBJFS.js.map} +0 -0
package/src/orchestrator.ts
CHANGED
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import os from "node:os";
|
|
8
8
|
import { createHash, randomBytes } from "node:crypto";
|
|
9
|
-
import { existsSync } from "node:fs";
|
|
9
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
10
10
|
import {
|
|
11
11
|
mkdir,
|
|
12
12
|
readdir,
|
|
@@ -297,8 +297,23 @@ import {
|
|
|
297
297
|
type ConversationIndexBackendInspection,
|
|
298
298
|
type ConversationQmdRuntime,
|
|
299
299
|
} from "./conversation-index/backend.js";
|
|
300
|
-
import {
|
|
301
|
-
|
|
300
|
+
import {
|
|
301
|
+
NamespaceStorageRouter,
|
|
302
|
+
} from "./namespaces/storage.js";
|
|
303
|
+
import {
|
|
304
|
+
NamespaceCatalog,
|
|
305
|
+
} from "./namespaces/catalog.js";
|
|
306
|
+
import {
|
|
307
|
+
planNamespaceMaintenance,
|
|
308
|
+
runNamespaceMaintenanceBatchPlan,
|
|
309
|
+
type NamespaceMaintenancePlan,
|
|
310
|
+
type NamespaceMaintenanceSkipReason,
|
|
311
|
+
} from "./maintenance/namespace-planner.js";
|
|
312
|
+
import {
|
|
313
|
+
namespaceIdentityFromToken,
|
|
314
|
+
namespaceIdentityToken,
|
|
315
|
+
normalizeNamespaceIdentity,
|
|
316
|
+
} from "./namespaces/identity.js";
|
|
302
317
|
import {
|
|
303
318
|
canReadNamespace,
|
|
304
319
|
defaultNamespaceForPrincipal,
|
|
@@ -327,6 +342,7 @@ import { parseFlexibleIsoTimestamp } from "./utils/iso-timestamp.js";
|
|
|
327
342
|
import { TierMigrationExecutor } from "./tier-migration.js";
|
|
328
343
|
import { decideTierTransition, type MemoryTier } from "./tier-routing.js";
|
|
329
344
|
import {
|
|
345
|
+
isSafeRouteNamespace,
|
|
330
346
|
selectRouteRule,
|
|
331
347
|
type RouteRule,
|
|
332
348
|
type RoutingEngineOptions,
|
|
@@ -1754,9 +1770,20 @@ export function resolvePersistedMemoryRelativePath(options: {
|
|
|
1754
1770
|
return path.join(subtree, `${options.memoryId}.md`);
|
|
1755
1771
|
}
|
|
1756
1772
|
|
|
1773
|
+
function qmdMaintenanceSkipReasonForError(error: unknown): NamespaceMaintenanceSkipReason | null {
|
|
1774
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1775
|
+
return /^QMD (?:update|embed) skipped by .*min-interval gate$/.test(message)
|
|
1776
|
+
? "throttled"
|
|
1777
|
+
: null;
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1757
1780
|
export class Orchestrator {
|
|
1758
1781
|
readonly storage: StorageManager;
|
|
1759
1782
|
private readonly storageRouter: NamespaceStorageRouter;
|
|
1783
|
+
/** Rebuildable namespace catalog (issue #1499). Inert unless namespaces enabled. */
|
|
1784
|
+
readonly namespaceCatalog: NamespaceCatalog;
|
|
1785
|
+
private readonly namespaceStorageDirHints = new Map<string, Set<string>>();
|
|
1786
|
+
private namespaceStorageDirHintsLoaded = false;
|
|
1760
1787
|
private readonly namespaceSearchRouter: NamespaceSearchRouter;
|
|
1761
1788
|
qmd: SearchBackend;
|
|
1762
1789
|
private readonly conversationQmd?: ConversationQmdRuntime;
|
|
@@ -1897,6 +1924,7 @@ export class Orchestrator {
|
|
|
1897
1924
|
private qmdMaintenancePending = false;
|
|
1898
1925
|
private qmdMaintenanceInFlight = false;
|
|
1899
1926
|
private lastQmdEmbedAtMs = 0;
|
|
1927
|
+
private lastQmdEmbedAtMsByNamespace = new Map<string, number>();
|
|
1900
1928
|
private lastQmdReprobeAtMs = 0;
|
|
1901
1929
|
private tierMigrationInFlight = false;
|
|
1902
1930
|
private lastTierMigrationRunAtMs = 0;
|
|
@@ -2245,6 +2273,188 @@ export class Orchestrator {
|
|
|
2245
2273
|
);
|
|
2246
2274
|
}
|
|
2247
2275
|
|
|
2276
|
+
private rememberNamespaceStorageDirHint(namespace: string, storageDir?: string): void {
|
|
2277
|
+
if (!this.config.namespacesEnabled || !storageDir) return;
|
|
2278
|
+
const ns = normalizeNamespaceIdentity(namespace);
|
|
2279
|
+
if (!ns) return;
|
|
2280
|
+
const defaultNs = normalizeNamespaceIdentity(this.config.defaultNamespace);
|
|
2281
|
+
if (ns !== defaultNs && !isSafeRouteNamespace(ns)) return;
|
|
2282
|
+
|
|
2283
|
+
if (!this.storageDirMatchesNamespaceHint(ns, storageDir)) return;
|
|
2284
|
+
|
|
2285
|
+
const resolvedStorageDir = path.resolve(storageDir);
|
|
2286
|
+
let hints = this.namespaceStorageDirHints.get(resolvedStorageDir);
|
|
2287
|
+
if (!hints) {
|
|
2288
|
+
hints = new Set<string>();
|
|
2289
|
+
this.namespaceStorageDirHints.set(resolvedStorageDir, hints);
|
|
2290
|
+
}
|
|
2291
|
+
hints.add(ns);
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
private storageDirMatchesNamespaceHint(namespace: string, storageDir: string): boolean {
|
|
2295
|
+
const ns = normalizeNamespaceIdentity(namespace);
|
|
2296
|
+
if (!ns) return false;
|
|
2297
|
+
|
|
2298
|
+
const resolvedStorageDir = path.resolve(storageDir);
|
|
2299
|
+
const resolvedMemoryDir = path.resolve(this.config.memoryDir);
|
|
2300
|
+
const defaultNs = normalizeNamespaceIdentity(this.config.defaultNamespace);
|
|
2301
|
+
if (resolvedStorageDir === resolvedMemoryDir) return ns === defaultNs;
|
|
2302
|
+
|
|
2303
|
+
const resolvedNamespacesDir = path.join(resolvedMemoryDir, "namespaces");
|
|
2304
|
+
if (!isPathInsideStorageRoot(resolvedNamespacesDir, resolvedStorageDir)) return false;
|
|
2305
|
+
|
|
2306
|
+
const rawRoot = path.resolve(resolvedNamespacesDir, ns);
|
|
2307
|
+
const tokenRoot = path.resolve(resolvedNamespacesDir, namespaceIdentityToken(ns));
|
|
2308
|
+
return resolvedStorageDir === rawRoot || resolvedStorageDir === tokenRoot;
|
|
2309
|
+
}
|
|
2310
|
+
|
|
2311
|
+
private namespaceStorageDirHintOwnershipRank(
|
|
2312
|
+
record: { namespace: string },
|
|
2313
|
+
resolvedStorageDir: string,
|
|
2314
|
+
configured: Set<string>,
|
|
2315
|
+
): number {
|
|
2316
|
+
if (resolvedStorageDir === path.resolve(this.config.memoryDir)) {
|
|
2317
|
+
return record.namespace === normalizeNamespaceIdentity(this.config.defaultNamespace)
|
|
2318
|
+
? 0
|
|
2319
|
+
: 3;
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
const leaf = path.basename(resolvedStorageDir);
|
|
2323
|
+
const tokenOwnsRoot = namespaceIdentityToken(record.namespace) === leaf;
|
|
2324
|
+
if (tokenOwnsRoot && configured.has(record.namespace)) return 0;
|
|
2325
|
+
if (record.namespace === leaf) return 1;
|
|
2326
|
+
if (tokenOwnsRoot) return 2;
|
|
2327
|
+
return 3;
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
private preferNamespaceStorageDirHintOwner(
|
|
2331
|
+
current: { namespace: string; identityToken: string; storageDir: string },
|
|
2332
|
+
candidate: { namespace: string; identityToken: string; storageDir: string },
|
|
2333
|
+
resolvedStorageDir: string,
|
|
2334
|
+
configured: Set<string>,
|
|
2335
|
+
): { namespace: string; identityToken: string; storageDir: string } {
|
|
2336
|
+
const currentRank = this.namespaceStorageDirHintOwnershipRank(
|
|
2337
|
+
current,
|
|
2338
|
+
resolvedStorageDir,
|
|
2339
|
+
configured,
|
|
2340
|
+
);
|
|
2341
|
+
const candidateRank = this.namespaceStorageDirHintOwnershipRank(
|
|
2342
|
+
candidate,
|
|
2343
|
+
resolvedStorageDir,
|
|
2344
|
+
configured,
|
|
2345
|
+
);
|
|
2346
|
+
if (candidateRank < currentRank) return candidate;
|
|
2347
|
+
if (candidateRank > currentRank) return current;
|
|
2348
|
+
|
|
2349
|
+
const byName = candidate.namespace.localeCompare(current.namespace);
|
|
2350
|
+
if (byName < 0) return candidate;
|
|
2351
|
+
if (byName > 0) return current;
|
|
2352
|
+
return candidate.identityToken.localeCompare(current.identityToken) < 0
|
|
2353
|
+
? candidate
|
|
2354
|
+
: current;
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
private loadNamespaceStorageDirHintsFromCatalog(): void {
|
|
2358
|
+
if (this.namespaceStorageDirHintsLoaded || !this.namespaceCatalog.enabled) return;
|
|
2359
|
+
this.namespaceStorageDirHintsLoaded = true;
|
|
2360
|
+
const catalogPath = path.join(this.config.memoryDir, "state", "namespaces.jsonl");
|
|
2361
|
+
if (!existsSync(catalogPath)) return;
|
|
2362
|
+
|
|
2363
|
+
let body: string;
|
|
2364
|
+
try {
|
|
2365
|
+
body = readFileSync(catalogPath, "utf8");
|
|
2366
|
+
} catch {
|
|
2367
|
+
return;
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
const compactedByNamespace = new Map<
|
|
2371
|
+
string,
|
|
2372
|
+
{ namespace: string; identityToken: string; storageDir: string }
|
|
2373
|
+
>();
|
|
2374
|
+
for (const line of body.split(/\r?\n/)) {
|
|
2375
|
+
const trimmed = line.trim();
|
|
2376
|
+
if (!trimmed) continue;
|
|
2377
|
+
try {
|
|
2378
|
+
const parsed = JSON.parse(trimmed) as unknown;
|
|
2379
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) continue;
|
|
2380
|
+
const record = parsed as Record<string, unknown>;
|
|
2381
|
+
if (
|
|
2382
|
+
typeof record.namespace !== "string" ||
|
|
2383
|
+
typeof record.storageDir !== "string" ||
|
|
2384
|
+
typeof record.identityToken !== "string"
|
|
2385
|
+
) {
|
|
2386
|
+
continue;
|
|
2387
|
+
}
|
|
2388
|
+
const namespace = normalizeNamespaceIdentity(record.namespace);
|
|
2389
|
+
if (!namespace || record.identityToken !== namespaceIdentityToken(namespace)) continue;
|
|
2390
|
+
compactedByNamespace.set(namespace, {
|
|
2391
|
+
namespace,
|
|
2392
|
+
identityToken: record.identityToken,
|
|
2393
|
+
storageDir: record.storageDir,
|
|
2394
|
+
});
|
|
2395
|
+
} catch {
|
|
2396
|
+
// Catalog hints are best-effort. The catalog reader still owns full recovery.
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
const configured = new Set(
|
|
2401
|
+
this.configuredNamespaces().map((namespace) => normalizeNamespaceIdentity(namespace)),
|
|
2402
|
+
);
|
|
2403
|
+
const preferredByStorageDir = new Map<
|
|
2404
|
+
string,
|
|
2405
|
+
{ namespace: string; identityToken: string; storageDir: string }
|
|
2406
|
+
>();
|
|
2407
|
+
for (const record of compactedByNamespace.values()) {
|
|
2408
|
+
if (!this.storageDirMatchesNamespaceHint(record.namespace, record.storageDir)) {
|
|
2409
|
+
continue;
|
|
2410
|
+
}
|
|
2411
|
+
const resolvedStorageDir = path.resolve(record.storageDir);
|
|
2412
|
+
const current = preferredByStorageDir.get(resolvedStorageDir);
|
|
2413
|
+
preferredByStorageDir.set(
|
|
2414
|
+
resolvedStorageDir,
|
|
2415
|
+
current
|
|
2416
|
+
? this.preferNamespaceStorageDirHintOwner(
|
|
2417
|
+
current,
|
|
2418
|
+
record,
|
|
2419
|
+
resolvedStorageDir,
|
|
2420
|
+
configured,
|
|
2421
|
+
)
|
|
2422
|
+
: record,
|
|
2423
|
+
);
|
|
2424
|
+
}
|
|
2425
|
+
for (const record of preferredByStorageDir.values()) {
|
|
2426
|
+
this.rememberNamespaceStorageDirHint(record.namespace, record.storageDir);
|
|
2427
|
+
}
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
/**
|
|
2431
|
+
* Shared namespace maintenance planner (issue #1500). This extends the
|
|
2432
|
+
* #1499 catalog-union QMD helper into a reusable contract: configured
|
|
2433
|
+
* namespaces are always considered, dynamic catalog namespaces are admitted
|
|
2434
|
+
* only when their live router root still matches real memory data, and branch
|
|
2435
|
+
* namespaces are opt-in. Recurring jobs use the per-cycle budget; startup and
|
|
2436
|
+
* recovery discovery paths use the same safety filters without that cycle
|
|
2437
|
+
* budget so every live namespace is ensured/synced.
|
|
2438
|
+
*/
|
|
2439
|
+
private async namespaceMaintenancePlan(jobName: string): Promise<NamespaceMaintenancePlan> {
|
|
2440
|
+
return planNamespaceMaintenance(this.config, {
|
|
2441
|
+
jobName,
|
|
2442
|
+
catalog: this.namespaceCatalog,
|
|
2443
|
+
});
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
private async maintenanceNamespaces(
|
|
2447
|
+
jobName = "qmd",
|
|
2448
|
+
budgetMode: "cycle" | "unbounded" = "unbounded",
|
|
2449
|
+
): Promise<string[]> {
|
|
2450
|
+
const plan = await planNamespaceMaintenance(this.config, {
|
|
2451
|
+
jobName,
|
|
2452
|
+
catalog: this.namespaceCatalog,
|
|
2453
|
+
budgetMode,
|
|
2454
|
+
});
|
|
2455
|
+
return plan.namespaces.map((candidate) => candidate.namespace);
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2248
2458
|
private buildConfiguredQmdSearchOptions(
|
|
2249
2459
|
queryText: string,
|
|
2250
2460
|
): SearchQueryOptions | undefined {
|
|
@@ -2350,7 +2560,21 @@ export class Orchestrator {
|
|
|
2350
2560
|
storageDir: config.profilingStorageDir || path.join(config.memoryDir, "profiling"),
|
|
2351
2561
|
maxTraces: config.profilingMaxTraces,
|
|
2352
2562
|
});
|
|
2353
|
-
|
|
2563
|
+
// Namespace catalog (issue #1499): downstream, rebuildable metadata index.
|
|
2564
|
+
// Inert unless namespacesEnabled is true. Storage resolution registers
|
|
2565
|
+
// namespaces via the router's onResolve hook; the touch is best-effort and
|
|
2566
|
+
// a catalog write failure never affects storage resolution.
|
|
2567
|
+
this.namespaceCatalog = new NamespaceCatalog(config);
|
|
2568
|
+
this.storageRouter = new NamespaceStorageRouter(config, {
|
|
2569
|
+
// Return the registration promise (round 6, codex P2 — NEFoX) so the
|
|
2570
|
+
// router's resolve-hook dedup only marks a namespace notified when the
|
|
2571
|
+
// catalog actually APPENDED. A dropped append (rebuild-lock timeout) or a
|
|
2572
|
+
// failure resolves to `false`/rejects, so the next `storageFor` retries.
|
|
2573
|
+
onResolve: (namespace, storageDir) => {
|
|
2574
|
+
this.rememberNamespaceStorageDirHint(namespace, storageDir);
|
|
2575
|
+
return this.namespaceCatalog.registerResolved(namespace, storageDir);
|
|
2576
|
+
},
|
|
2577
|
+
});
|
|
2354
2578
|
this.namespaceSearchRouter = new NamespaceSearchRouter(
|
|
2355
2579
|
config,
|
|
2356
2580
|
this.storageRouter,
|
|
@@ -2839,6 +3063,14 @@ export class Orchestrator {
|
|
|
2839
3063
|
await sm.ensureDirectories();
|
|
2840
3064
|
await sm.loadAliases().catch(() => undefined);
|
|
2841
3065
|
}
|
|
3066
|
+
// Explicitly seed the catalog with all configured namespaces at startup
|
|
3067
|
+
// (round 6, cursor Medium — NBLlR). The storageFor loop above fires the
|
|
3068
|
+
// router's onResolve hook, but a warm router cache (reused instance
|
|
3069
|
+
// across stop/start) can skip onResolve, leaving policy namespaces absent
|
|
3070
|
+
// from the live catalog until an operator runs `rebuild --apply`. This
|
|
3071
|
+
// call is cheap, idempotent, and best-effort: a catalog failure must
|
|
3072
|
+
// never break initialization (rule #13, #40).
|
|
3073
|
+
await this.namespaceCatalog.registerConfiguredNamespaces().catch(() => undefined);
|
|
2842
3074
|
}
|
|
2843
3075
|
await this.relevance.load();
|
|
2844
3076
|
await this.negatives.load();
|
|
@@ -2907,8 +3139,15 @@ export class Orchestrator {
|
|
|
2907
3139
|
const available = await this.qmd.probe();
|
|
2908
3140
|
if (available) {
|
|
2909
3141
|
log.info(`Search backend: available ${this.qmd.debugStatus()}`);
|
|
3142
|
+
// Ensure collections at startup for the catalog-union namespace set, not
|
|
3143
|
+
// just the configured set (issue #1499 sweep, same class as NHZEV): a
|
|
3144
|
+
// dynamic namespace that exists only in the persisted catalog must have
|
|
3145
|
+
// its QMD collection checked/ensured on boot so recall against it works
|
|
3146
|
+
// after a restart. `registerConfiguredNamespaces()` already seeded the
|
|
3147
|
+
// catalog above, so `maintenanceNamespaces()` is readable here; it falls
|
|
3148
|
+
// back to the configured set on any catalog read failure.
|
|
2910
3149
|
const namespaces = this.config.namespacesEnabled
|
|
2911
|
-
? this.
|
|
3150
|
+
? await this.maintenanceNamespaces()
|
|
2912
3151
|
: [this.config.defaultNamespace];
|
|
2913
3152
|
const states = await Promise.all(
|
|
2914
3153
|
namespaces.map(async (namespace) => {
|
|
@@ -3031,8 +3270,12 @@ export class Orchestrator {
|
|
|
3031
3270
|
try {
|
|
3032
3271
|
log.info("QMD startup sync: updating index to match current disk state");
|
|
3033
3272
|
if (this.config.namespacesEnabled) {
|
|
3273
|
+
// Cover cataloged dynamic namespaces at startup too (NHZEV, codex P2):
|
|
3274
|
+
// a dynamic namespace written before a daemon restart must be synced on
|
|
3275
|
+
// boot, not only by the debounced runQmdMaintenance() path. Same union +
|
|
3276
|
+
// catalog-read-failure fallback as runQmdMaintenance.
|
|
3034
3277
|
await this.namespaceSearchRouter.updateNamespaces(
|
|
3035
|
-
this.
|
|
3278
|
+
await this.maintenanceNamespaces(),
|
|
3036
3279
|
{ signal },
|
|
3037
3280
|
);
|
|
3038
3281
|
} else {
|
|
@@ -3310,9 +3553,16 @@ export class Orchestrator {
|
|
|
3310
3553
|
this.namespaceSearchRouter.clearCache();
|
|
3311
3554
|
}
|
|
3312
3555
|
|
|
3313
|
-
// Ensure collections — namespace-aware when enabled
|
|
3556
|
+
// Ensure collections — namespace-aware when enabled.
|
|
3557
|
+
// Use the catalog-union namespace set (issue #1499 sweep, same class as
|
|
3558
|
+
// NHZEV): this is the QMD startup-recovery sync that ensures collections AND
|
|
3559
|
+
// runs `updateNamespaces(...)` below over the SAME `namespaces` set. A dynamic
|
|
3560
|
+
// namespace that exists only in the persisted catalog must be ensured and
|
|
3561
|
+
// re-synced here too, otherwise after a backend-was-unavailable-at-boot
|
|
3562
|
+
// recovery its collection stays stale. Falls back to the configured set on any
|
|
3563
|
+
// catalog read failure.
|
|
3314
3564
|
const namespaces = this.config.namespacesEnabled
|
|
3315
|
-
? this.
|
|
3565
|
+
? await this.maintenanceNamespaces()
|
|
3316
3566
|
: [this.config.defaultNamespace];
|
|
3317
3567
|
|
|
3318
3568
|
const states = await Promise.all(
|
|
@@ -3934,6 +4184,7 @@ export class Orchestrator {
|
|
|
3934
4184
|
}
|
|
3935
4185
|
|
|
3936
4186
|
for (const cluster of clusters) {
|
|
4187
|
+
let canonicalWriteCompleted = false;
|
|
3937
4188
|
try {
|
|
3938
4189
|
// Operator-aware prompt (issue #561 PR 3): ask the LLM to pick the
|
|
3939
4190
|
// SPLIT/MERGE/UPDATE operator alongside the canonical output. Falls
|
|
@@ -4057,6 +4308,7 @@ export class Orchestrator {
|
|
|
4057
4308
|
derivedVia: operator,
|
|
4058
4309
|
},
|
|
4059
4310
|
);
|
|
4311
|
+
canonicalWriteCompleted = true;
|
|
4060
4312
|
|
|
4061
4313
|
result.memoriesConsolidated++;
|
|
4062
4314
|
|
|
@@ -4090,17 +4342,27 @@ export class Orchestrator {
|
|
|
4090
4342
|
this.contentHashIndex.remove(m.content);
|
|
4091
4343
|
}
|
|
4092
4344
|
}
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
m.path
|
|
4102
|
-
m.frontmatter
|
|
4103
|
-
|
|
4345
|
+
// Best-effort index cleanup: a failure here (e.g. on-disk index save
|
|
4346
|
+
// under disk-full) must NOT abort the archival loop and thereby skip
|
|
4347
|
+
// the catalog write touch below for an already-durable canonical write
|
|
4348
|
+
// (kilo NV0mh).
|
|
4349
|
+
try {
|
|
4350
|
+
await this.embeddingFallback.removeFromIndex(m.frontmatter.id);
|
|
4351
|
+
if (
|
|
4352
|
+
this.config.queryAwareIndexingEnabled &&
|
|
4353
|
+
m.path &&
|
|
4354
|
+
m.frontmatter?.created
|
|
4355
|
+
) {
|
|
4356
|
+
deindexMemory(
|
|
4357
|
+
targetStorage.dir,
|
|
4358
|
+
m.path,
|
|
4359
|
+
m.frontmatter.created,
|
|
4360
|
+
m.frontmatter.tags ?? [],
|
|
4361
|
+
);
|
|
4362
|
+
}
|
|
4363
|
+
} catch (cleanupErr) {
|
|
4364
|
+
log.warn(
|
|
4365
|
+
`[semantic-consolidation] index cleanup failed (non-fatal): ${cleanupErr}`,
|
|
4104
4366
|
);
|
|
4105
4367
|
}
|
|
4106
4368
|
result.memoriesArchived++;
|
|
@@ -4115,6 +4377,21 @@ export class Orchestrator {
|
|
|
4115
4377
|
`[semantic-consolidation] cluster processing failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
4116
4378
|
);
|
|
4117
4379
|
result.errors++;
|
|
4380
|
+
} finally {
|
|
4381
|
+
if (canonicalWriteCompleted) {
|
|
4382
|
+
// Catalog write touch (issue #1499 sweep): record after the canonical
|
|
4383
|
+
// write and, on the happy path, after archival of superseded cluster
|
|
4384
|
+
// memories, so `lastWriteAt` reflects every durable mutation in this
|
|
4385
|
+
// consolidation (cursor NUtCK). The `finally` also covers partial
|
|
4386
|
+
// failures where the canonical memory was written but a later archive
|
|
4387
|
+
// step throws and the cluster catch continues (codex NY-dK).
|
|
4388
|
+
// Best-effort; namespace decoded from the storage dir since this path
|
|
4389
|
+
// has no routed namespace name.
|
|
4390
|
+
this.markCatalogWrite(
|
|
4391
|
+
this.namespaceFromStorageDir(targetStorage.dir),
|
|
4392
|
+
targetStorage.dir,
|
|
4393
|
+
);
|
|
4394
|
+
}
|
|
4118
4395
|
}
|
|
4119
4396
|
}
|
|
4120
4397
|
|
|
@@ -7194,6 +7471,22 @@ export class Orchestrator {
|
|
|
7194
7471
|
} else {
|
|
7195
7472
|
recallNamespaces = readableRecallNamespaces;
|
|
7196
7473
|
}
|
|
7474
|
+
// Catalog touch (issue #1499): record reads against the recalled namespaces
|
|
7475
|
+
// so the catalog reflects active read scopes. Best-effort, failure-tolerant.
|
|
7476
|
+
// Round 3 (codex P2): gate behind the no_recall guard — when the planner
|
|
7477
|
+
// selects `no_recall` retrieval is skipped entirely (see the early return at
|
|
7478
|
+
// `recallMode === "no_recall"` below), so marking every readable namespace as
|
|
7479
|
+
// read would falsely inflate `lastReadAt` / catalog recency.
|
|
7480
|
+
// Round 4 (codex P2): also skip when the effective memory result limit is
|
|
7481
|
+
// zero (`topK: 0`, a disabled/zero `memories` recall section, etc.). The QMD
|
|
7482
|
+
// path explicitly returns before searching when `recallResultLimit <= 0`, so
|
|
7483
|
+
// no namespace is actually read and the touch would be spurious.
|
|
7484
|
+
// NOTE: the catalog read touch is recorded LATER, immediately after the
|
|
7485
|
+
// Phase 1 `throwIfRecallAborted` gate (round 6, codex P2 / cursor Medium —
|
|
7486
|
+
// NDXHa/NDmle), so it fires only once retrieval is actually about to run.
|
|
7487
|
+
// Recording it here (recall entry) would set `lastReadAt` for recalls that
|
|
7488
|
+
// are aborted, error out, or short-circuit before any QMD/filesystem read.
|
|
7489
|
+
|
|
7197
7490
|
// Effective LCM read NAMESPACE SET (#1505 thread "Include coding fallback
|
|
7198
7491
|
// namespaces in LCM reads"). `observe` archives LCM / structured history
|
|
7199
7492
|
// under `${effectiveNamespace}:${sessionKey}` for whichever namespace was
|
|
@@ -7473,6 +7766,21 @@ export class Orchestrator {
|
|
|
7473
7766
|
// --- Phase 1: Launch ALL independent data fetches in parallel ---
|
|
7474
7767
|
throwIfRecallAborted(options.abortSignal);
|
|
7475
7768
|
|
|
7769
|
+
// Catalog read touch (issue #1499): record reads against the recalled
|
|
7770
|
+
// namespaces HERE — after the abort gate, immediately before retrieval
|
|
7771
|
+
// actually runs — so `lastReadAt` reflects a real read, not a recall that was
|
|
7772
|
+
// aborted/errored/short-circuited before reaching this point (round 3/4/6,
|
|
7773
|
+
// codex/cursor — no_recall, zero-limit, aborted, and pre-read-error cases).
|
|
7774
|
+
// `no_recall` already returned earlier, so it cannot reach here. Best-effort
|
|
7775
|
+
// and failure-tolerant.
|
|
7776
|
+
if (
|
|
7777
|
+
this.namespaceCatalog.enabled &&
|
|
7778
|
+
recallResultLimit > 0 &&
|
|
7779
|
+
!options.abortSignal?.aborted
|
|
7780
|
+
) {
|
|
7781
|
+
for (const ns of recallNamespaces) this.markCatalogRead(ns);
|
|
7782
|
+
}
|
|
7783
|
+
|
|
7476
7784
|
// 0. Shared context (v4.0, optional)
|
|
7477
7785
|
const sharedContextPromise = (async (): Promise<string | null> => {
|
|
7478
7786
|
if (
|
|
@@ -12523,6 +12831,9 @@ export class Orchestrator {
|
|
|
12523
12831
|
storage,
|
|
12524
12832
|
threadIdForExtraction,
|
|
12525
12833
|
{ sessionKey, principal, validAt: sourceValidAt },
|
|
12834
|
+
// Pass the KNOWN base namespace (NHIdx) so the catalog write touch records the
|
|
12835
|
+
// real namespace rather than a guess decoded from the storage dir.
|
|
12836
|
+
selfNamespace,
|
|
12526
12837
|
);
|
|
12527
12838
|
let postPersistMetadataFailed = false;
|
|
12528
12839
|
meta ??= await storage.loadMeta();
|
|
@@ -13004,25 +13315,80 @@ export class Orchestrator {
|
|
|
13004
13315
|
|
|
13005
13316
|
try {
|
|
13006
13317
|
if (this.config.namespacesEnabled) {
|
|
13007
|
-
|
|
13008
|
-
|
|
13318
|
+
// Include cataloged dynamic namespaces, not just the configured set
|
|
13319
|
+
// (NGnei), but run through the namespace-aware maintenance planner so
|
|
13320
|
+
// each namespace is budgeted, lock-protected, and status-recorded
|
|
13321
|
+
// independently (issue #1500).
|
|
13322
|
+
const plan = await this.namespaceMaintenancePlan("qmd");
|
|
13323
|
+
const now = Date.now();
|
|
13324
|
+
const lastEmbedAtByNamespace =
|
|
13325
|
+
this.lastQmdEmbedAtMsByNamespace ?? (this.lastQmdEmbedAtMsByNamespace = new Map());
|
|
13326
|
+
const dueEmbedNamespaces = (namespaces: string[]): string[] => {
|
|
13327
|
+
if (!this.config.qmdAutoEmbedEnabled) return [];
|
|
13328
|
+
return namespaces.filter(
|
|
13329
|
+
(namespace) =>
|
|
13330
|
+
now - (lastEmbedAtByNamespace.get(namespace) ?? 0) >= this.config.qmdEmbedMinIntervalMs,
|
|
13331
|
+
);
|
|
13332
|
+
};
|
|
13333
|
+
const markEmbedded = (namespaces: string[]): void => {
|
|
13334
|
+
if (namespaces.length === 0) return;
|
|
13335
|
+
for (const namespace of namespaces) {
|
|
13336
|
+
lastEmbedAtByNamespace.set(namespace, now);
|
|
13337
|
+
}
|
|
13338
|
+
this.lastQmdEmbedAtMs = now;
|
|
13339
|
+
};
|
|
13340
|
+
await runNamespaceMaintenanceBatchPlan(
|
|
13341
|
+
this.config,
|
|
13342
|
+
plan,
|
|
13343
|
+
async (candidates) => {
|
|
13344
|
+
const namespaces = candidates.map((candidate) => candidate.namespace);
|
|
13345
|
+
const embedNamespaces = dueEmbedNamespaces(namespaces);
|
|
13346
|
+
let result: Awaited<ReturnType<NamespaceSearchRouter["updateNamespacesDetailed"]>>;
|
|
13347
|
+
try {
|
|
13348
|
+
result = await this.namespaceSearchRouter.updateNamespacesDetailed(
|
|
13349
|
+
namespaces,
|
|
13350
|
+
undefined,
|
|
13351
|
+
{ strict: true },
|
|
13352
|
+
);
|
|
13353
|
+
} catch (error) {
|
|
13354
|
+
if (
|
|
13355
|
+
embedNamespaces.length > 0 &&
|
|
13356
|
+
qmdMaintenanceSkipReasonForError(error) === "throttled"
|
|
13357
|
+
) {
|
|
13358
|
+
await this.namespaceSearchRouter.embedNamespaces(embedNamespaces, { strict: true });
|
|
13359
|
+
markEmbedded(embedNamespaces);
|
|
13360
|
+
}
|
|
13361
|
+
throw error;
|
|
13362
|
+
}
|
|
13363
|
+
if (result.backendCount <= 0) {
|
|
13364
|
+
throw new Error("no eligible QMD backend for selected namespaces");
|
|
13365
|
+
}
|
|
13366
|
+
if (result.eligibleNamespaces.length !== namespaces.length) {
|
|
13367
|
+
const eligible = new Set(result.eligibleNamespaces);
|
|
13368
|
+
const missing = namespaces.filter((namespace) => !eligible.has(namespace));
|
|
13369
|
+
throw new Error(`QMD backend ineligible for selected namespaces (${missing.length})`);
|
|
13370
|
+
}
|
|
13371
|
+
if (embedNamespaces.length > 0) {
|
|
13372
|
+
await this.namespaceSearchRouter.embedNamespaces(embedNamespaces, { strict: true });
|
|
13373
|
+
markEmbedded(embedNamespaces);
|
|
13374
|
+
}
|
|
13375
|
+
return { itemCount: result.backendCount };
|
|
13376
|
+
},
|
|
13377
|
+
this.namespaceCatalog,
|
|
13378
|
+
{
|
|
13379
|
+
skipReasonForError: qmdMaintenanceSkipReasonForError,
|
|
13380
|
+
},
|
|
13009
13381
|
);
|
|
13010
13382
|
} else {
|
|
13011
13383
|
await this.qmd.update();
|
|
13012
|
-
|
|
13013
|
-
|
|
13014
|
-
|
|
13015
|
-
|
|
13016
|
-
|
|
13017
|
-
) {
|
|
13018
|
-
if (this.config.namespacesEnabled) {
|
|
13019
|
-
await this.namespaceSearchRouter.embedNamespaces(
|
|
13020
|
-
this.configuredNamespaces(),
|
|
13021
|
-
);
|
|
13022
|
-
} else {
|
|
13384
|
+
const now = Date.now();
|
|
13385
|
+
if (
|
|
13386
|
+
this.config.qmdAutoEmbedEnabled &&
|
|
13387
|
+
now - this.lastQmdEmbedAtMs >= this.config.qmdEmbedMinIntervalMs
|
|
13388
|
+
) {
|
|
13023
13389
|
await this.qmd.embed();
|
|
13390
|
+
this.lastQmdEmbedAtMs = now;
|
|
13024
13391
|
}
|
|
13025
|
-
this.lastQmdEmbedAtMs = now;
|
|
13026
13392
|
}
|
|
13027
13393
|
} finally {
|
|
13028
13394
|
this.qmdMaintenanceInFlight = false;
|
|
@@ -13037,6 +13403,7 @@ export class Orchestrator {
|
|
|
13037
13403
|
storage: StorageManager,
|
|
13038
13404
|
threadIdForExtraction?: string | null,
|
|
13039
13405
|
sourceContext?: { sessionKey?: string; principal?: string; validAt?: string },
|
|
13406
|
+
baseNamespace?: string,
|
|
13040
13407
|
): Promise<string[]> {
|
|
13041
13408
|
// Inline source attribution (issue #369). When enabled, every extracted
|
|
13042
13409
|
// fact is rewritten to carry a compact provenance tag inside its body so
|
|
@@ -13315,7 +13682,7 @@ export class Orchestrator {
|
|
|
13315
13682
|
// `createdAt` as the ordering anchor instead of the old fact's
|
|
13316
13683
|
// timestamp, ensuring supersession fires correctly even when
|
|
13317
13684
|
// the matching fact predates conflicting candidates.
|
|
13318
|
-
await applyTemporalSupersession({
|
|
13685
|
+
const hashDedupSupersession = await applyTemporalSupersession({
|
|
13319
13686
|
storage: sharedStorage,
|
|
13320
13687
|
newMemoryId: hashDedupMatchingFact.frontmatter.id,
|
|
13321
13688
|
entityRef: options.entityRef,
|
|
@@ -13324,6 +13691,19 @@ export class Orchestrator {
|
|
|
13324
13691
|
enabled: true,
|
|
13325
13692
|
useCallerTimestamp: true,
|
|
13326
13693
|
});
|
|
13694
|
+
// Catalog touch (issue #1499 — codex P2 NElSf): this dedup branch
|
|
13695
|
+
// returns WITHOUT reaching the post-write `markCatalogWrite` below,
|
|
13696
|
+
// but `applyTemporalSupersession` mutated the shared namespace
|
|
13697
|
+
// (it rewrote frontmatter to retire stale shared facts). When any
|
|
13698
|
+
// ids were actually superseded, the shared namespace changed, so we
|
|
13699
|
+
// must record the write — otherwise the shared record's
|
|
13700
|
+
// `lastWriteAt` stays stale and `writtenSince` maintenance / QMD
|
|
13701
|
+
// fanout skips the namespace after a supersession-only update.
|
|
13702
|
+
// Best-effort and failure-tolerant (markCatalogWrite swallows
|
|
13703
|
+
// errors); only touch when work happened to avoid spurious writes.
|
|
13704
|
+
if (hashDedupSupersession.supersededIds.length > 0) {
|
|
13705
|
+
this.markCatalogWrite(this.config.sharedNamespace, sharedStorage.dir);
|
|
13706
|
+
}
|
|
13327
13707
|
// Active matching fact exists — normal short-circuit is safe.
|
|
13328
13708
|
return;
|
|
13329
13709
|
}
|
|
@@ -13418,6 +13798,16 @@ export class Orchestrator {
|
|
|
13418
13798
|
);
|
|
13419
13799
|
}
|
|
13420
13800
|
}
|
|
13801
|
+
// Catalog touch (issue #1499, Issue B + ordering sweep): a shared-
|
|
13802
|
+
// namespace promotion is the ONLY write the shared namespace receives on
|
|
13803
|
+
// this path, so without this the shared record's lastWriteAt stays stale
|
|
13804
|
+
// and `writtenSince` filters / maintenance fanout skip it. Record AFTER
|
|
13805
|
+
// the promoted write and the shared temporal-supersession attempt so the
|
|
13806
|
+
// catalog timestamp never precedes a later durable frontmatter mutation in
|
|
13807
|
+
// the same promotion pass. The hot-path source-namespace touch uses a
|
|
13808
|
+
// different storage dir, so this does not double-count the source.
|
|
13809
|
+
// Best-effort and failure-tolerant — it must never crash the promotion.
|
|
13810
|
+
this.markCatalogWrite(this.config.sharedNamespace, sharedStorage.dir);
|
|
13421
13811
|
trackPersistedId(sharedStorage, promotedId, {
|
|
13422
13812
|
includeReturnedIds: false,
|
|
13423
13813
|
});
|
|
@@ -13778,6 +14168,19 @@ export class Orchestrator {
|
|
|
13778
14168
|
// affect both the dedup fingerprint and importance (issue #519 procedure routing).
|
|
13779
14169
|
let writeCategory = fact.category;
|
|
13780
14170
|
let targetStorage = storage;
|
|
14171
|
+
// Track the KNOWN target namespace NAME alongside targetStorage (round 6,
|
|
14172
|
+
// codex P2 — NCQI0). Re-deriving it from `targetStorage.dir` mangles a raw
|
|
14173
|
+
// namespace literally named like a canonical token (e.g. `ns-616c706861`
|
|
14174
|
+
// served from its legacy raw dir decodes to `alpha`). We seed it from the
|
|
14175
|
+
// EXPLICIT base namespace the caller used to obtain `storage` (NHIdx, codex
|
|
14176
|
+
// P2) — `selfNamespace`/`writeNamespaceOverride` — so the catalog write touch
|
|
14177
|
+
// records the real namespace, not a guess decoded from the directory. We only
|
|
14178
|
+
// fall back to decoding the dir when no base namespace was passed (legacy
|
|
14179
|
+
// callers). The EXPLICIT routed name (below) still overrides this verbatim.
|
|
14180
|
+
let targetNamespaceName =
|
|
14181
|
+
baseNamespace && baseNamespace.length > 0
|
|
14182
|
+
? baseNamespace
|
|
14183
|
+
: this.namespaceFromStorageDir(targetStorage.dir);
|
|
13781
14184
|
let routedRuleId: string | undefined;
|
|
13782
14185
|
let routedNamespaceExplicit = false;
|
|
13783
14186
|
if (routeRules.length > 0) {
|
|
@@ -13794,6 +14197,7 @@ export class Orchestrator {
|
|
|
13794
14197
|
targetStorage = await this.storageRouter.storageFor(
|
|
13795
14198
|
selected.target.namespace,
|
|
13796
14199
|
);
|
|
14200
|
+
targetNamespaceName = selected.target.namespace;
|
|
13797
14201
|
}
|
|
13798
14202
|
}
|
|
13799
14203
|
} catch (err) {
|
|
@@ -13823,6 +14227,7 @@ export class Orchestrator {
|
|
|
13823
14227
|
targetStorage = await this.storageRouter.storageFor(
|
|
13824
14228
|
this.config.sharedNamespace,
|
|
13825
14229
|
);
|
|
14230
|
+
targetNamespaceName = this.config.sharedNamespace;
|
|
13826
14231
|
log.debug(
|
|
13827
14232
|
`scope-routing: fact "${fact.content.slice(0, 60)}…" routed to shared namespace (scope=global)`,
|
|
13828
14233
|
);
|
|
@@ -14198,41 +14603,49 @@ export class Orchestrator {
|
|
|
14198
14603
|
contentHashSource: rawChunkedContent,
|
|
14199
14604
|
},
|
|
14200
14605
|
);
|
|
14606
|
+
try {
|
|
14607
|
+
// Write individual chunks with parent reference
|
|
14608
|
+
for (const chunk of chunkResult.chunks) {
|
|
14609
|
+
// Score each chunk's importance separately
|
|
14610
|
+
const chunkImportance = scoreImportance(
|
|
14611
|
+
chunk.content,
|
|
14612
|
+
writeCategory,
|
|
14613
|
+
fact.tags,
|
|
14614
|
+
);
|
|
14615
|
+
const chunkWriteSource =
|
|
14616
|
+
(fact as any).source === "proactive"
|
|
14617
|
+
? "chunking-proactive"
|
|
14618
|
+
: "chunking";
|
|
14201
14619
|
|
|
14202
|
-
|
|
14203
|
-
|
|
14204
|
-
|
|
14205
|
-
|
|
14206
|
-
|
|
14207
|
-
|
|
14208
|
-
|
|
14209
|
-
|
|
14210
|
-
|
|
14211
|
-
|
|
14212
|
-
|
|
14213
|
-
|
|
14214
|
-
|
|
14215
|
-
|
|
14216
|
-
|
|
14217
|
-
|
|
14218
|
-
|
|
14219
|
-
|
|
14220
|
-
|
|
14221
|
-
|
|
14222
|
-
|
|
14223
|
-
|
|
14224
|
-
|
|
14225
|
-
|
|
14226
|
-
|
|
14227
|
-
|
|
14228
|
-
|
|
14229
|
-
|
|
14230
|
-
|
|
14231
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14232
|
-
memoryKind,
|
|
14233
|
-
validAt: sourceContext?.validAt,
|
|
14234
|
-
},
|
|
14235
|
-
);
|
|
14620
|
+
await targetStorage.writeChunk(
|
|
14621
|
+
parentId,
|
|
14622
|
+
chunk.index,
|
|
14623
|
+
chunkResult.chunks.length,
|
|
14624
|
+
writeCategory,
|
|
14625
|
+
// Each chunk carries its own inline citation so provenance
|
|
14626
|
+
// survives when a single chunk is quoted in isolation.
|
|
14627
|
+
applyInlineCitation(chunk.content),
|
|
14628
|
+
{
|
|
14629
|
+
confidence: fact.confidence,
|
|
14630
|
+
tags: fact.tags,
|
|
14631
|
+
entityRef: fact.entityRef,
|
|
14632
|
+
source: chunkWriteSource,
|
|
14633
|
+
importance: chunkImportance,
|
|
14634
|
+
intentGoal: inferredIntent?.goal,
|
|
14635
|
+
intentActionType: inferredIntent?.actionType,
|
|
14636
|
+
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14637
|
+
memoryKind,
|
|
14638
|
+
validAt: sourceContext?.validAt,
|
|
14639
|
+
},
|
|
14640
|
+
);
|
|
14641
|
+
}
|
|
14642
|
+
} finally {
|
|
14643
|
+
// The parent memory is durable once writeMemory returns `parentId`.
|
|
14644
|
+
// Touch immediately around the chunk-write loop so a later chunk
|
|
14645
|
+
// failure still surfaces the partially durable parent/chunk files to
|
|
14646
|
+
// catalog-driven `writtenSince` maintenance. The final touch below
|
|
14647
|
+
// still refreshes `lastWriteAt` after later durable writes on success.
|
|
14648
|
+
this.markCatalogWrite(targetNamespaceName, targetStorage.dir);
|
|
14236
14649
|
}
|
|
14237
14650
|
|
|
14238
14651
|
if (routedRuleId) {
|
|
@@ -14308,62 +14721,71 @@ export class Orchestrator {
|
|
|
14308
14721
|
// directly for embedding-fallback sync of each chunk document.
|
|
14309
14722
|
await this.indexPersistedMemory(targetStorage, chunkId);
|
|
14310
14723
|
}
|
|
14311
|
-
|
|
14312
|
-
|
|
14313
|
-
|
|
14314
|
-
|
|
14315
|
-
|
|
14316
|
-
|
|
14317
|
-
|
|
14318
|
-
|
|
14319
|
-
|
|
14320
|
-
|
|
14321
|
-
|
|
14322
|
-
|
|
14323
|
-
|
|
14324
|
-
|
|
14325
|
-
|
|
14326
|
-
|
|
14327
|
-
}
|
|
14328
|
-
// v8.2: graph edge building for chunked memories
|
|
14329
|
-
if (this.config.multiGraphMemoryEnabled) {
|
|
14330
|
-
try {
|
|
14331
|
-
const graphContext = await ensureGraphContext(targetStorage);
|
|
14332
|
-
const entityRef =
|
|
14333
|
-
typeof (fact as any).entityRef === "string"
|
|
14334
|
-
? (fact as any).entityRef
|
|
14335
|
-
: undefined;
|
|
14336
|
-
const parentRelPath = resolvePersistedMemoryRelativePath({
|
|
14337
|
-
memoryId: parentId,
|
|
14338
|
-
pathById: graphContext.memoryPathById,
|
|
14339
|
-
category: writeCategory,
|
|
14340
|
-
});
|
|
14341
|
-
graphContext.memoryPathById.set(parentId, parentRelPath);
|
|
14342
|
-
appendMemoryToGraphContext({
|
|
14343
|
-
allMemsForGraph: graphContext.allMemsForGraph,
|
|
14344
|
-
storageDir: targetStorage.dir,
|
|
14345
|
-
memoryRelPath: parentRelPath,
|
|
14346
|
-
memoryId: parentId,
|
|
14347
|
-
category: writeCategory,
|
|
14348
|
-
content: fact.content ?? "",
|
|
14349
|
-
entityRef,
|
|
14724
|
+
try {
|
|
14725
|
+
if (
|
|
14726
|
+
this.config.verbatimArtifactsEnabled &&
|
|
14727
|
+
this.config.verbatimArtifactCategories.includes(writeCategory) &&
|
|
14728
|
+
fact.confidence >= this.config.verbatimArtifactsMinConfidence
|
|
14729
|
+
) {
|
|
14730
|
+
// Reuse citedChunkedContent so the artifact carries the same citation
|
|
14731
|
+
// timestamp as the parent memory write above (Fix #3 — duplicate-citation).
|
|
14732
|
+
await targetStorage.writeArtifact(citedChunkedContent, {
|
|
14733
|
+
confidence: fact.confidence,
|
|
14734
|
+
tags: [...fact.tags, "artifact", "chunked-parent"],
|
|
14735
|
+
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
14736
|
+
sourceMemoryId: parentId,
|
|
14737
|
+
intentGoal: inferredIntent?.goal,
|
|
14738
|
+
intentActionType: inferredIntent?.actionType,
|
|
14739
|
+
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14350
14740
|
});
|
|
14351
|
-
await this.buildGraphEdge(
|
|
14352
|
-
targetStorage,
|
|
14353
|
-
parentRelPath,
|
|
14354
|
-
entityRef,
|
|
14355
|
-
parentId,
|
|
14356
|
-
fact.content ?? "",
|
|
14357
|
-
graphContext.allMemsForGraph,
|
|
14358
|
-
graphContext.memoryPathById,
|
|
14359
|
-
threadIdForExtraction ?? undefined,
|
|
14360
|
-
threadEpisodeIdsForGraph,
|
|
14361
|
-
graphContext.previousPersistedRelPath,
|
|
14362
|
-
);
|
|
14363
|
-
graphContext.previousPersistedRelPath = parentRelPath;
|
|
14364
|
-
} catch {
|
|
14365
|
-
/* fail-open */
|
|
14366
14741
|
}
|
|
14742
|
+
// v8.2: graph edge building for chunked memories
|
|
14743
|
+
if (this.config.multiGraphMemoryEnabled) {
|
|
14744
|
+
try {
|
|
14745
|
+
const graphContext = await ensureGraphContext(targetStorage);
|
|
14746
|
+
const entityRef =
|
|
14747
|
+
typeof (fact as any).entityRef === "string"
|
|
14748
|
+
? (fact as any).entityRef
|
|
14749
|
+
: undefined;
|
|
14750
|
+
const parentRelPath = resolvePersistedMemoryRelativePath({
|
|
14751
|
+
memoryId: parentId,
|
|
14752
|
+
pathById: graphContext.memoryPathById,
|
|
14753
|
+
category: writeCategory,
|
|
14754
|
+
});
|
|
14755
|
+
graphContext.memoryPathById.set(parentId, parentRelPath);
|
|
14756
|
+
appendMemoryToGraphContext({
|
|
14757
|
+
allMemsForGraph: graphContext.allMemsForGraph,
|
|
14758
|
+
storageDir: targetStorage.dir,
|
|
14759
|
+
memoryRelPath: parentRelPath,
|
|
14760
|
+
memoryId: parentId,
|
|
14761
|
+
category: writeCategory,
|
|
14762
|
+
content: fact.content ?? "",
|
|
14763
|
+
entityRef,
|
|
14764
|
+
});
|
|
14765
|
+
await this.buildGraphEdge(
|
|
14766
|
+
targetStorage,
|
|
14767
|
+
parentRelPath,
|
|
14768
|
+
entityRef,
|
|
14769
|
+
parentId,
|
|
14770
|
+
fact.content ?? "",
|
|
14771
|
+
graphContext.allMemsForGraph,
|
|
14772
|
+
graphContext.memoryPathById,
|
|
14773
|
+
threadIdForExtraction ?? undefined,
|
|
14774
|
+
threadEpisodeIdsForGraph,
|
|
14775
|
+
graphContext.previousPersistedRelPath,
|
|
14776
|
+
);
|
|
14777
|
+
graphContext.previousPersistedRelPath = parentRelPath;
|
|
14778
|
+
} catch {
|
|
14779
|
+
/* fail-open */
|
|
14780
|
+
}
|
|
14781
|
+
}
|
|
14782
|
+
} finally {
|
|
14783
|
+
// Catalog touch (issue #1499): refresh AFTER later chunked
|
|
14784
|
+
// source-namespace durable mutations — temporal supersession, shared
|
|
14785
|
+
// promotion, optional artifact writes, and graph-edge writes — so
|
|
14786
|
+
// `lastWriteAt` cannot precede later file changes on successful
|
|
14787
|
+
// completion. Use the KNOWN routed name, not a dir-decoded guess.
|
|
14788
|
+
this.markCatalogWrite(targetNamespaceName, targetStorage.dir);
|
|
14367
14789
|
}
|
|
14368
14790
|
trackBehaviorSignals(
|
|
14369
14791
|
targetStorage,
|
|
@@ -14469,120 +14891,150 @@ export class Orchestrator {
|
|
|
14469
14891
|
} catch (err) {
|
|
14470
14892
|
log.warn(`temporal-supersession: unexpected error: ${err}`);
|
|
14471
14893
|
}
|
|
14472
|
-
|
|
14473
|
-
|
|
14474
|
-
|
|
14475
|
-
|
|
14894
|
+
try {
|
|
14895
|
+
trackBehaviorSignals(
|
|
14896
|
+
targetStorage,
|
|
14897
|
+
buildBehaviorSignalsForMemory({
|
|
14898
|
+
memoryId,
|
|
14899
|
+
category: writeCategory,
|
|
14900
|
+
content: fact.content,
|
|
14901
|
+
namespace: this.namespaceFromStorageDir(targetStorage.dir),
|
|
14902
|
+
confidence: fact.confidence,
|
|
14903
|
+
source: "extraction",
|
|
14904
|
+
}),
|
|
14905
|
+
);
|
|
14906
|
+
trackPersistedId(targetStorage, memoryId);
|
|
14907
|
+
if (
|
|
14908
|
+
threadEpisodeIdsForGraph &&
|
|
14909
|
+
!threadEpisodeIdsForGraph.includes(memoryId)
|
|
14910
|
+
) {
|
|
14911
|
+
threadEpisodeIdsForGraph.push(memoryId);
|
|
14912
|
+
}
|
|
14913
|
+
await this.indexPersistedMemory(targetStorage, memoryId);
|
|
14914
|
+
await promoteMemoryToShared({
|
|
14915
|
+
sourceStorage: targetStorage,
|
|
14476
14916
|
category: writeCategory,
|
|
14477
14917
|
content: fact.content,
|
|
14478
|
-
namespace: this.namespaceFromStorageDir(targetStorage.dir),
|
|
14479
14918
|
confidence: fact.confidence,
|
|
14480
|
-
|
|
14481
|
-
|
|
14482
|
-
);
|
|
14483
|
-
trackPersistedId(targetStorage, memoryId);
|
|
14484
|
-
if (
|
|
14485
|
-
threadEpisodeIdsForGraph &&
|
|
14486
|
-
!threadEpisodeIdsForGraph.includes(memoryId)
|
|
14487
|
-
) {
|
|
14488
|
-
threadEpisodeIdsForGraph.push(memoryId);
|
|
14489
|
-
}
|
|
14490
|
-
await this.indexPersistedMemory(targetStorage, memoryId);
|
|
14491
|
-
await promoteMemoryToShared({
|
|
14492
|
-
sourceStorage: targetStorage,
|
|
14493
|
-
category: writeCategory,
|
|
14494
|
-
content: fact.content,
|
|
14495
|
-
confidence: fact.confidence,
|
|
14496
|
-
tags: fact.tags,
|
|
14497
|
-
entityRef:
|
|
14498
|
-
typeof (fact as any).entityRef === "string"
|
|
14499
|
-
? (fact as any).entityRef
|
|
14500
|
-
: undefined,
|
|
14501
|
-
structuredAttributes: fact.structuredAttributes,
|
|
14502
|
-
sourceMemoryId: memoryId,
|
|
14503
|
-
importance,
|
|
14504
|
-
intentGoal: inferredIntent?.goal,
|
|
14505
|
-
intentActionType: inferredIntent?.actionType,
|
|
14506
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14507
|
-
memoryKind,
|
|
14508
|
-
validAt: sourceContext?.validAt,
|
|
14509
|
-
source: extractionWriteSource,
|
|
14510
|
-
});
|
|
14511
|
-
// v8.2: graph edge building (fail-open — errors caught inside GraphIndex)
|
|
14512
|
-
if (this.config.multiGraphMemoryEnabled) {
|
|
14513
|
-
try {
|
|
14514
|
-
const graphContext = await ensureGraphContext(targetStorage);
|
|
14515
|
-
const entityRef =
|
|
14919
|
+
tags: fact.tags,
|
|
14920
|
+
entityRef:
|
|
14516
14921
|
typeof (fact as any).entityRef === "string"
|
|
14517
14922
|
? (fact as any).entityRef
|
|
14518
|
-
: undefined
|
|
14519
|
-
|
|
14520
|
-
memoryId,
|
|
14521
|
-
pathById: graphContext.memoryPathById,
|
|
14522
|
-
category: writeCategory,
|
|
14523
|
-
});
|
|
14524
|
-
graphContext.memoryPathById.set(memoryId, memoryRelPath);
|
|
14525
|
-
appendMemoryToGraphContext({
|
|
14526
|
-
allMemsForGraph: graphContext.allMemsForGraph,
|
|
14527
|
-
storageDir: targetStorage.dir,
|
|
14528
|
-
memoryRelPath: memoryRelPath,
|
|
14529
|
-
memoryId,
|
|
14530
|
-
category: writeCategory,
|
|
14531
|
-
content: fact.content ?? "",
|
|
14532
|
-
entityRef,
|
|
14533
|
-
});
|
|
14534
|
-
await this.buildGraphEdge(
|
|
14535
|
-
targetStorage,
|
|
14536
|
-
memoryRelPath,
|
|
14537
|
-
entityRef,
|
|
14538
|
-
memoryId,
|
|
14539
|
-
fact.content ?? "",
|
|
14540
|
-
graphContext.allMemsForGraph,
|
|
14541
|
-
graphContext.memoryPathById,
|
|
14542
|
-
threadIdForExtraction ?? undefined,
|
|
14543
|
-
threadEpisodeIdsForGraph,
|
|
14544
|
-
graphContext.previousPersistedRelPath,
|
|
14545
|
-
);
|
|
14546
|
-
graphContext.previousPersistedRelPath = memoryRelPath;
|
|
14547
|
-
} catch {
|
|
14548
|
-
/* fail-open */
|
|
14549
|
-
}
|
|
14550
|
-
}
|
|
14551
|
-
if (
|
|
14552
|
-
this.config.verbatimArtifactsEnabled &&
|
|
14553
|
-
this.config.verbatimArtifactCategories.includes(writeCategory) &&
|
|
14554
|
-
fact.confidence >= this.config.verbatimArtifactsMinConfidence
|
|
14555
|
-
) {
|
|
14556
|
-
// Reuse citedFactContent so the artifact carries the same citation
|
|
14557
|
-
// timestamp as the memory write above (Fix #3 — duplicate-citation).
|
|
14558
|
-
await targetStorage.writeArtifact(citedFactContent, {
|
|
14559
|
-
confidence: fact.confidence,
|
|
14560
|
-
tags: [...fact.tags, "artifact"],
|
|
14561
|
-
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
14923
|
+
: undefined,
|
|
14924
|
+
structuredAttributes: fact.structuredAttributes,
|
|
14562
14925
|
sourceMemoryId: memoryId,
|
|
14926
|
+
importance,
|
|
14563
14927
|
intentGoal: inferredIntent?.goal,
|
|
14564
14928
|
intentActionType: inferredIntent?.actionType,
|
|
14565
14929
|
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14930
|
+
memoryKind,
|
|
14931
|
+
validAt: sourceContext?.validAt,
|
|
14932
|
+
source: extractionWriteSource,
|
|
14566
14933
|
});
|
|
14567
|
-
|
|
14568
|
-
|
|
14569
|
-
|
|
14570
|
-
|
|
14571
|
-
|
|
14572
|
-
|
|
14573
|
-
|
|
14574
|
-
|
|
14575
|
-
|
|
14576
|
-
|
|
14577
|
-
|
|
14578
|
-
|
|
14579
|
-
|
|
14580
|
-
|
|
14581
|
-
|
|
14582
|
-
|
|
14583
|
-
|
|
14584
|
-
|
|
14585
|
-
|
|
14934
|
+
// v8.2: graph edge building (fail-open — errors caught inside GraphIndex)
|
|
14935
|
+
if (this.config.multiGraphMemoryEnabled) {
|
|
14936
|
+
try {
|
|
14937
|
+
const graphContext = await ensureGraphContext(targetStorage);
|
|
14938
|
+
const entityRef =
|
|
14939
|
+
typeof (fact as any).entityRef === "string"
|
|
14940
|
+
? (fact as any).entityRef
|
|
14941
|
+
: undefined;
|
|
14942
|
+
const memoryRelPath = resolvePersistedMemoryRelativePath({
|
|
14943
|
+
memoryId,
|
|
14944
|
+
pathById: graphContext.memoryPathById,
|
|
14945
|
+
category: writeCategory,
|
|
14946
|
+
});
|
|
14947
|
+
graphContext.memoryPathById.set(memoryId, memoryRelPath);
|
|
14948
|
+
appendMemoryToGraphContext({
|
|
14949
|
+
allMemsForGraph: graphContext.allMemsForGraph,
|
|
14950
|
+
storageDir: targetStorage.dir,
|
|
14951
|
+
memoryRelPath: memoryRelPath,
|
|
14952
|
+
memoryId,
|
|
14953
|
+
category: writeCategory,
|
|
14954
|
+
content: fact.content ?? "",
|
|
14955
|
+
entityRef,
|
|
14956
|
+
});
|
|
14957
|
+
await this.buildGraphEdge(
|
|
14958
|
+
targetStorage,
|
|
14959
|
+
memoryRelPath,
|
|
14960
|
+
entityRef,
|
|
14961
|
+
memoryId,
|
|
14962
|
+
fact.content ?? "",
|
|
14963
|
+
graphContext.allMemsForGraph,
|
|
14964
|
+
graphContext.memoryPathById,
|
|
14965
|
+
threadIdForExtraction ?? undefined,
|
|
14966
|
+
threadEpisodeIdsForGraph,
|
|
14967
|
+
graphContext.previousPersistedRelPath,
|
|
14968
|
+
);
|
|
14969
|
+
graphContext.previousPersistedRelPath = memoryRelPath;
|
|
14970
|
+
} catch {
|
|
14971
|
+
/* fail-open */
|
|
14972
|
+
}
|
|
14973
|
+
}
|
|
14974
|
+
if (
|
|
14975
|
+
this.config.verbatimArtifactsEnabled &&
|
|
14976
|
+
this.config.verbatimArtifactCategories.includes(writeCategory) &&
|
|
14977
|
+
fact.confidence >= this.config.verbatimArtifactsMinConfidence
|
|
14978
|
+
) {
|
|
14979
|
+
// Reuse citedFactContent so the artifact carries the same citation
|
|
14980
|
+
// timestamp as the memory write above (Fix #3 — duplicate-citation).
|
|
14981
|
+
await targetStorage.writeArtifact(citedFactContent, {
|
|
14982
|
+
confidence: fact.confidence,
|
|
14983
|
+
tags: [...fact.tags, "artifact"],
|
|
14984
|
+
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
14985
|
+
sourceMemoryId: memoryId,
|
|
14986
|
+
intentGoal: inferredIntent?.goal,
|
|
14987
|
+
intentActionType: inferredIntent?.actionType,
|
|
14988
|
+
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14989
|
+
});
|
|
14990
|
+
}
|
|
14991
|
+
// Register in content-hash index after successful write.
|
|
14992
|
+
// Thread 3 fix: canonicalize by stripping any pre-existing citation so
|
|
14993
|
+
// the stored hash matches what the dedup check computes via
|
|
14994
|
+
// stripCitationForTemplate before calling contentHashIndex.has().
|
|
14995
|
+
if (this.contentHashIndex) {
|
|
14996
|
+
const canonicalFactContent =
|
|
14997
|
+
citationEnabled &&
|
|
14998
|
+
hasCitationForTemplate(fact.content, citationTemplate)
|
|
14999
|
+
? stripCitationForTemplate(fact.content, citationTemplate)
|
|
15000
|
+
: fact.content;
|
|
15001
|
+
const hashRegisterKey =
|
|
15002
|
+
writeCategory === "procedure"
|
|
15003
|
+
? buildProcedurePersistBody(fact.content, fact.procedureSteps)
|
|
15004
|
+
: canonicalFactContent;
|
|
15005
|
+
this.contentHashIndex.add(hashRegisterKey);
|
|
15006
|
+
}
|
|
15007
|
+
} finally {
|
|
15008
|
+
// Catalog touch (issue #1499): record AFTER every synchronous
|
|
15009
|
+
// source-namespace mutation in the non-chunked path: writeMemory,
|
|
15010
|
+
// temporal supersession, graph edges, and optional verbatim artifacts.
|
|
15011
|
+
// The `finally` preserves the write touch when post-write indexing or
|
|
15012
|
+
// promotion fails after the canonical memory is already durable. Use the
|
|
15013
|
+
// KNOWN routed name, not a dir-decoded guess (NCQI0).
|
|
15014
|
+
this.markCatalogWrite(targetNamespaceName, targetStorage.dir);
|
|
15015
|
+
}
|
|
15016
|
+
}
|
|
15017
|
+
|
|
15018
|
+
// Tracks whether THIS extraction persisted any durable, non-fact output to the
|
|
15019
|
+
// BASE namespace's storage (entity / relationship / profile / question). The
|
|
15020
|
+
// per-fact `markCatalogWrite` only fires inside the fact write loop, so a
|
|
15021
|
+
// fact-less extraction that still persists durable data must record exactly one
|
|
15022
|
+
// base-namespace catalog touch after all writes complete (NHZEZ, codex P2).
|
|
15023
|
+
let durableNonFactWritten = false;
|
|
15024
|
+
let durableNonFactTouchRecorded = false;
|
|
15025
|
+
const touchBaseNonFactNamespace = () => {
|
|
15026
|
+
const baseTouchNamespace =
|
|
15027
|
+
baseNamespace && baseNamespace.length > 0
|
|
15028
|
+
? baseNamespace
|
|
15029
|
+
: this.namespaceFromStorageDir(storage.dir);
|
|
15030
|
+
this.markCatalogWrite(baseTouchNamespace, storage.dir);
|
|
15031
|
+
};
|
|
15032
|
+
const recordDurableNonFactWrite = () => {
|
|
15033
|
+
durableNonFactWritten = true;
|
|
15034
|
+
if (durableNonFactTouchRecorded) return;
|
|
15035
|
+
durableNonFactTouchRecorded = true;
|
|
15036
|
+
touchBaseNonFactNamespace();
|
|
15037
|
+
};
|
|
14586
15038
|
for (const entity of entities) {
|
|
14587
15039
|
try {
|
|
14588
15040
|
const name = (entity as any)?.name;
|
|
@@ -14607,7 +15059,10 @@ export class Orchestrator {
|
|
|
14607
15059
|
? (entity as any).structuredSections
|
|
14608
15060
|
: undefined,
|
|
14609
15061
|
});
|
|
14610
|
-
if (id)
|
|
15062
|
+
if (id) {
|
|
15063
|
+
trackPersistedId(storage, id);
|
|
15064
|
+
recordDurableNonFactWrite();
|
|
15065
|
+
}
|
|
14611
15066
|
} catch (err) {
|
|
14612
15067
|
log.warn(`persistExtraction: entity write failed: ${err}`);
|
|
14613
15068
|
}
|
|
@@ -14626,10 +15081,12 @@ export class Orchestrator {
|
|
|
14626
15081
|
target: rel.target,
|
|
14627
15082
|
label: rel.label,
|
|
14628
15083
|
});
|
|
15084
|
+
recordDurableNonFactWrite();
|
|
14629
15085
|
await storage.addEntityRelationship(rel.target, {
|
|
14630
15086
|
target: rel.source,
|
|
14631
15087
|
label: `${rel.label} (reverse)`,
|
|
14632
15088
|
});
|
|
15089
|
+
recordDurableNonFactWrite();
|
|
14633
15090
|
} catch (err) {
|
|
14634
15091
|
log.debug(`relationship persist failed: ${err}`);
|
|
14635
15092
|
}
|
|
@@ -14658,23 +15115,49 @@ export class Orchestrator {
|
|
|
14658
15115
|
|
|
14659
15116
|
if (profileUpdates.length > 0) {
|
|
14660
15117
|
await storage.appendToProfile(profileUpdates);
|
|
15118
|
+
recordDurableNonFactWrite();
|
|
14661
15119
|
}
|
|
14662
15120
|
|
|
14663
15121
|
// Persist questions
|
|
14664
15122
|
for (const q of questions) {
|
|
14665
15123
|
const id = await storage.writeQuestion(q.question, q.context, q.priority);
|
|
14666
|
-
if (id)
|
|
15124
|
+
if (id) {
|
|
15125
|
+
trackPersistedId(storage, id);
|
|
15126
|
+
recordDurableNonFactWrite();
|
|
15127
|
+
}
|
|
14667
15128
|
}
|
|
14668
15129
|
|
|
14669
|
-
// Persist identity reflection
|
|
15130
|
+
// Persist identity reflection. This writes durable namespace-local state, so
|
|
15131
|
+
// an identity-ONLY extraction (no facts/entities/profile/questions) still
|
|
15132
|
+
// counts as a durable non-fact write for the catalog touch below (NIIly).
|
|
15133
|
+
// Only count it when the write actually succeeds (best-effort write); the
|
|
15134
|
+
// touch is recorded AFTER this so a rolled-back/failed write never touches.
|
|
14670
15135
|
if (this.config.identityEnabled && result.identityReflection) {
|
|
14671
15136
|
try {
|
|
14672
15137
|
await storage.appendIdentityReflection(result.identityReflection);
|
|
15138
|
+
recordDurableNonFactWrite();
|
|
14673
15139
|
} catch (err) {
|
|
14674
15140
|
log.debug(`identity reflection write failed: ${err}`);
|
|
14675
15141
|
}
|
|
14676
15142
|
}
|
|
14677
15143
|
|
|
15144
|
+
// Catalog touch for durable NON-FACT outputs (NHZEZ / NIIly, codex P2). The
|
|
15145
|
+
// per-fact `markCatalogWrite` above only fires inside the fact write loop, so
|
|
15146
|
+
// an extraction that persists ONLY entities, relationships, profile updates,
|
|
15147
|
+
// questions, or an identity reflection (no facts) would record durable data to
|
|
15148
|
+
// the BASE namespace's storage without ever touching the catalog — leaving that
|
|
15149
|
+
// namespace's `lastWriteAt` stale so `listNamespaces({writtenSince})` /
|
|
15150
|
+
// write-recency QMD maintenance miss the write. All of these are written to the
|
|
15151
|
+
// BASE `storage` (not the per-fact routed `targetStorage`), so we record ONE
|
|
15152
|
+
// base-namespace touch here, AFTER every non-fact write completes. Use the
|
|
15153
|
+
// KNOWN base namespace name, not a dir-decoded guess (NCQI0). One touch per
|
|
15154
|
+
// namespace per extraction — `markWrite` is idempotent, so if the fact path
|
|
15155
|
+
// already touched the base namespace this only refreshes `lastWriteAt`.
|
|
15156
|
+
// Best-effort and failure-tolerant (markCatalogWrite swallows errors).
|
|
15157
|
+
if (durableNonFactWritten) {
|
|
15158
|
+
touchBaseNonFactNamespace();
|
|
15159
|
+
}
|
|
15160
|
+
|
|
14678
15161
|
// Save content-hash index after batch
|
|
14679
15162
|
if (this.contentHashIndex) {
|
|
14680
15163
|
await this.contentHashIndex
|
|
@@ -14912,6 +15395,11 @@ export class Orchestrator {
|
|
|
14912
15395
|
log.info("running consolidation pass");
|
|
14913
15396
|
let merged = 0;
|
|
14914
15397
|
let invalidated = 0;
|
|
15398
|
+
// Tracks whether any consolidation memory-item action (UPDATE / MERGE /
|
|
15399
|
+
// INVALIDATE) durably rewrote memory state. A consolidation pass that only
|
|
15400
|
+
// mutates memory items (no profile/entity updates) still changes the default
|
|
15401
|
+
// namespace's data, so its catalog `lastWriteAt` must refresh too (NIBOi).
|
|
15402
|
+
let memoryItemMutated = false;
|
|
14915
15403
|
|
|
14916
15404
|
// Flush access tracking buffer first
|
|
14917
15405
|
if (this.accessTrackingBuffer.size > 0) {
|
|
@@ -14955,6 +15443,7 @@ export class Orchestrator {
|
|
|
14955
15443
|
: null;
|
|
14956
15444
|
if (await this.storage.invalidateMemory(item.existingId)) {
|
|
14957
15445
|
invalidated += 1;
|
|
15446
|
+
memoryItemMutated = true;
|
|
14958
15447
|
await this.embeddingFallback.removeFromIndex(item.existingId);
|
|
14959
15448
|
if (toInvalidate?.path && toInvalidate.frontmatter?.created) {
|
|
14960
15449
|
deindexMemory(
|
|
@@ -14976,6 +15465,7 @@ export class Orchestrator {
|
|
|
14976
15465
|
lineage: [item.existingId],
|
|
14977
15466
|
},
|
|
14978
15467
|
);
|
|
15468
|
+
memoryItemMutated = true;
|
|
14979
15469
|
await this.indexPersistedMemory(this.storage, item.existingId);
|
|
14980
15470
|
// updateMemory() only changes content/updated/lineage — path, created, and tags
|
|
14981
15471
|
// are preserved, so the temporal/tag index entry is already correct; no reindex needed.
|
|
@@ -14991,6 +15481,7 @@ export class Orchestrator {
|
|
|
14991
15481
|
lineage: [item.existingId, item.mergeWith],
|
|
14992
15482
|
},
|
|
14993
15483
|
);
|
|
15484
|
+
memoryItemMutated = true;
|
|
14994
15485
|
await this.indexPersistedMemory(this.storage, item.existingId);
|
|
14995
15486
|
// updateMemory() only changes content/updated/supersedes/lineage — path, created, and tags
|
|
14996
15487
|
// are preserved, so the temporal/tag index entry for the survivor is already correct.
|
|
@@ -15035,9 +15526,24 @@ export class Orchestrator {
|
|
|
15035
15526
|
});
|
|
15036
15527
|
}
|
|
15037
15528
|
|
|
15529
|
+
// Catalog write touch accounting (issue #1499 sweep): consolidation persists
|
|
15530
|
+
// durable mutations directly to the default-namespace `this.storage`, bypassing
|
|
15531
|
+
// the extraction write path. We do NOT touch here — later maintenance steps in
|
|
15532
|
+
// this same function (entity-file merges, expired-commitment / TTL cleanup,
|
|
15533
|
+
// fact archival) can ALSO mutate the namespace on a run with no LLM outputs
|
|
15534
|
+
// (NIjwl). So we accumulate every durable mutation into `memoryItemMutated` and
|
|
15535
|
+
// record ONE consolidated touch AFTER all mutation-producing steps complete,
|
|
15536
|
+
// just before returning (rule #25: touch after the write commits). LLM
|
|
15537
|
+
// profile/entity updates and memory-item actions (UPDATE / MERGE / INVALIDATE)
|
|
15538
|
+
// count here (NIBOi).
|
|
15539
|
+
if (result.profileUpdates.length > 0 || result.entityUpdates.length > 0) {
|
|
15540
|
+
memoryItemMutated = true;
|
|
15541
|
+
}
|
|
15542
|
+
|
|
15038
15543
|
// Merge fragmented entity files
|
|
15039
15544
|
const entitiesMerged = await this.storage.mergeFragmentedEntities();
|
|
15040
15545
|
if (entitiesMerged > 0) {
|
|
15546
|
+
memoryItemMutated = true;
|
|
15041
15547
|
log.info(`merged ${entitiesMerged} fragmented entity files`);
|
|
15042
15548
|
}
|
|
15043
15549
|
|
|
@@ -15048,6 +15554,10 @@ export class Orchestrator {
|
|
|
15048
15554
|
5,
|
|
15049
15555
|
);
|
|
15050
15556
|
if (synthesized > 0) {
|
|
15557
|
+
// Entity synthesis rewrites entity files — a durable namespace mutation,
|
|
15558
|
+
// so record it for the catalog touch even when it is the only change in
|
|
15559
|
+
// the pass (codex). Otherwise lastWriteAt goes stale.
|
|
15560
|
+
memoryItemMutated = true;
|
|
15051
15561
|
log.info(`refreshed ${synthesized} entity syntheses`);
|
|
15052
15562
|
}
|
|
15053
15563
|
} catch (err) {
|
|
@@ -15060,6 +15570,7 @@ export class Orchestrator {
|
|
|
15060
15570
|
this.config.commitmentDecayDays,
|
|
15061
15571
|
);
|
|
15062
15572
|
if (deletedCommitments.length > 0) {
|
|
15573
|
+
memoryItemMutated = true;
|
|
15063
15574
|
log.info(`cleaned ${deletedCommitments.length} expired commitments`);
|
|
15064
15575
|
if (this.config.queryAwareIndexingEnabled) {
|
|
15065
15576
|
for (const m of deletedCommitments) {
|
|
@@ -15089,6 +15600,7 @@ export class Orchestrator {
|
|
|
15089
15600
|
lifecycle.transitionedToExpired.length > 0 ||
|
|
15090
15601
|
lifecycle.deletedResolved.length > 0
|
|
15091
15602
|
) {
|
|
15603
|
+
memoryItemMutated = true;
|
|
15092
15604
|
log.info(
|
|
15093
15605
|
`commitment ledger lifecycle: expired ${lifecycle.transitionedToExpired.length}, cleaned ${lifecycle.deletedResolved.length}`,
|
|
15094
15606
|
);
|
|
@@ -15101,6 +15613,7 @@ export class Orchestrator {
|
|
|
15101
15613
|
// Clean memories past their TTL (speculative memories auto-expire)
|
|
15102
15614
|
const deletedTTL = await this.storage.cleanExpiredTTL();
|
|
15103
15615
|
if (deletedTTL.length > 0) {
|
|
15616
|
+
memoryItemMutated = true;
|
|
15104
15617
|
log.info(`cleaned ${deletedTTL.length} TTL-expired memories`);
|
|
15105
15618
|
if (this.config.queryAwareIndexingEnabled) {
|
|
15106
15619
|
for (const m of deletedTTL) {
|
|
@@ -15119,7 +15632,12 @@ export class Orchestrator {
|
|
|
15119
15632
|
try {
|
|
15120
15633
|
const lightSleepStartedAt = new Date().toISOString();
|
|
15121
15634
|
const lifecycleCorpus = await this.storage.readAllMemories();
|
|
15122
|
-
|
|
15635
|
+
// Lifecycle frontmatter writes count as durable mutations for the catalog
|
|
15636
|
+
// touch below (codex NR-tS), even when no other consolidation step set
|
|
15637
|
+
// memoryItemMutated.
|
|
15638
|
+
if ((await this.runLifecyclePolicyPass(lifecycleCorpus)) > 0) {
|
|
15639
|
+
memoryItemMutated = true;
|
|
15640
|
+
}
|
|
15123
15641
|
await this.recordScheduledDreamsPhaseRun(
|
|
15124
15642
|
"lightSleep",
|
|
15125
15643
|
lifecycleCorpus.length,
|
|
@@ -15139,13 +15657,17 @@ export class Orchestrator {
|
|
|
15139
15657
|
|
|
15140
15658
|
try {
|
|
15141
15659
|
const deepSleepStartedAt = new Date().toISOString();
|
|
15142
|
-
|
|
15660
|
+
// Tier migrations move/rewrite memory files; count them as durable
|
|
15661
|
+
// mutations for the catalog touch below (codex NThSW).
|
|
15662
|
+
const tierMigration = await this.runTierMigrationCycle(this.storage, "maintenance");
|
|
15663
|
+
if (tierMigration.migrated > 0) memoryItemMutated = true;
|
|
15143
15664
|
allMemories = await this.storage.readAllMemories();
|
|
15144
15665
|
|
|
15145
15666
|
// Fact archival pass (v6.0) — move old, low-importance, rarely-accessed facts to archive/
|
|
15146
15667
|
if (this.config.factArchivalEnabled) {
|
|
15147
15668
|
const archived = await this.runFactArchival(allMemories);
|
|
15148
15669
|
if (archived > 0) {
|
|
15670
|
+
memoryItemMutated = true;
|
|
15149
15671
|
log.info(`archived ${archived} old low-importance facts`);
|
|
15150
15672
|
}
|
|
15151
15673
|
}
|
|
@@ -15268,6 +15790,10 @@ export class Orchestrator {
|
|
|
15268
15790
|
);
|
|
15269
15791
|
if (profileResult) {
|
|
15270
15792
|
await this.storage.writeProfile(profileResult.consolidatedProfile);
|
|
15793
|
+
// Profile consolidation rewrites profile.md — a durable namespace
|
|
15794
|
+
// mutation; record it for the catalog touch even when it is the only
|
|
15795
|
+
// change in the pass (codex). Otherwise lastWriteAt goes stale.
|
|
15796
|
+
memoryItemMutated = true;
|
|
15271
15797
|
log.info(
|
|
15272
15798
|
`profile.md consolidated: removed ${profileResult.removedCount} items — ${profileResult.summary}`,
|
|
15273
15799
|
);
|
|
@@ -15352,6 +15878,21 @@ export class Orchestrator {
|
|
|
15352
15878
|
}
|
|
15353
15879
|
}
|
|
15354
15880
|
|
|
15881
|
+
// Consolidated catalog write touch (issue #1499 sweep; NIBOi + NIjwl). One
|
|
15882
|
+
// touch covering EVERY durable namespace mutation this pass made — LLM
|
|
15883
|
+
// profile/entity/memory-item actions AND cleanup-only maintenance (entity-file
|
|
15884
|
+
// merges, expired-commitment / ledger-lifecycle / TTL cleanup, fact archival).
|
|
15885
|
+
// Recorded here, after all mutation-producing steps, so a cleanup-only run that
|
|
15886
|
+
// rewrote the store still refreshes `lastWriteAt` (rule #25). The default
|
|
15887
|
+
// namespace is always configured/cataloged; `markWrite` is idempotent so this
|
|
15888
|
+
// only refreshes recency. Best-effort and failure-tolerant.
|
|
15889
|
+
if (memoryItemMutated) {
|
|
15890
|
+
this.markCatalogWrite(
|
|
15891
|
+
this.namespaceFromStorageDir(this.storage.dir),
|
|
15892
|
+
this.storage.dir,
|
|
15893
|
+
);
|
|
15894
|
+
}
|
|
15895
|
+
|
|
15355
15896
|
log.info("consolidation complete");
|
|
15356
15897
|
return { memoriesProcessed: allMemories.length, merged, invalidated };
|
|
15357
15898
|
}
|
|
@@ -15801,14 +16342,17 @@ export class Orchestrator {
|
|
|
15801
16342
|
|
|
15802
16343
|
async runLifecyclePolicyNow(storage: StorageManager = this.storage): Promise<{ memoriesAssessed: number }> {
|
|
15803
16344
|
const lifecycleCorpus = await storage.readAllMemories();
|
|
15804
|
-
|
|
16345
|
+
// Record the catalog write when the pass rewrote any frontmatter (codex NR-tS).
|
|
16346
|
+
if ((await this.runLifecyclePolicyPass(lifecycleCorpus, storage)) > 0) {
|
|
16347
|
+
this.markCatalogWrite(this.namespaceFromStorageDir(storage.dir), storage.dir);
|
|
16348
|
+
}
|
|
15805
16349
|
return { memoriesAssessed: lifecycleCorpus.length };
|
|
15806
16350
|
}
|
|
15807
16351
|
|
|
15808
16352
|
private async runLifecyclePolicyPass(
|
|
15809
16353
|
allMemories: MemoryFile[],
|
|
15810
16354
|
storage: StorageManager = this.storage,
|
|
15811
|
-
): Promise<
|
|
16355
|
+
): Promise<number> {
|
|
15812
16356
|
const now = new Date();
|
|
15813
16357
|
const nowIso = now.toISOString();
|
|
15814
16358
|
const countsByState: Record<LifecycleState, number> = {
|
|
@@ -15885,7 +16429,9 @@ export class Orchestrator {
|
|
|
15885
16429
|
if (wrote) updatedCount += 1;
|
|
15886
16430
|
}
|
|
15887
16431
|
|
|
15888
|
-
|
|
16432
|
+
// Report how many memories had frontmatter rewritten so callers can record a
|
|
16433
|
+
// catalog write touch for lifecycle-only passes (codex NR-tS).
|
|
16434
|
+
if (!this.config.lifecycleMetricsEnabled) return updatedCount;
|
|
15889
16435
|
|
|
15890
16436
|
const total = evaluatedCount;
|
|
15891
16437
|
const metrics = {
|
|
@@ -15910,6 +16456,7 @@ export class Orchestrator {
|
|
|
15910
16456
|
);
|
|
15911
16457
|
await mkdir(path.dirname(metricsPath), { recursive: true });
|
|
15912
16458
|
await writeFile(metricsPath, JSON.stringify(metrics, null, 2), "utf-8");
|
|
16459
|
+
return updatedCount;
|
|
15913
16460
|
}
|
|
15914
16461
|
|
|
15915
16462
|
/**
|
|
@@ -16030,9 +16577,10 @@ export class Orchestrator {
|
|
|
16030
16577
|
new Date(b.frontmatter.created).getTime(),
|
|
16031
16578
|
);
|
|
16032
16579
|
|
|
16033
|
-
// Keep recent memories
|
|
16034
|
-
|
|
16035
|
-
const
|
|
16580
|
+
// Keep recent memories, with explicit zero handling so `slice(-0)` does not
|
|
16581
|
+
// accidentally keep every memory out of the summarization candidate set.
|
|
16582
|
+
const recentToKeep = Math.max(0, this.config.summarizationRecentToKeep);
|
|
16583
|
+
const toSummarize = recentToKeep > 0 ? sorted.slice(0, -recentToKeep) : sorted;
|
|
16036
16584
|
|
|
16037
16585
|
// Filter candidates for summarization
|
|
16038
16586
|
const candidates = toSummarize.filter((m) => {
|
|
@@ -16093,6 +16641,15 @@ export class Orchestrator {
|
|
|
16093
16641
|
summary.id,
|
|
16094
16642
|
);
|
|
16095
16643
|
|
|
16644
|
+
// Catalog write touch (issue #1499 sweep): summarization writes a durable
|
|
16645
|
+
// summary and then rewrites source-memory archive status, bypassing the
|
|
16646
|
+
// extraction write path. Record the touch after both mutations complete so
|
|
16647
|
+
// `lastWriteAt` covers the final archived-state write.
|
|
16648
|
+
this.markCatalogWrite(
|
|
16649
|
+
this.namespaceFromStorageDir(this.storage.dir),
|
|
16650
|
+
this.storage.dir,
|
|
16651
|
+
);
|
|
16652
|
+
|
|
16096
16653
|
log.info(
|
|
16097
16654
|
`created summary ${summary.id} from ${batch.length} memories, archived ${archived}`,
|
|
16098
16655
|
);
|
|
@@ -16127,8 +16684,12 @@ export class Orchestrator {
|
|
|
16127
16684
|
private static readonly IDENTITY_CONSOLIDATE_THRESHOLD = 8_000;
|
|
16128
16685
|
|
|
16129
16686
|
private async autoConsolidateIdentity(): Promise<void> {
|
|
16687
|
+
// Fan out over the catalog-union namespace set (issue #1499 sweep): a dynamic
|
|
16688
|
+
// namespace that accumulated IDENTITY.md reflections must also be eligible for
|
|
16689
|
+
// auto-consolidation, otherwise its identity file grows unbounded and is never
|
|
16690
|
+
// consolidated. Falls back to the configured set on any catalog read failure.
|
|
16130
16691
|
const namespaces = this.config.namespacesEnabled
|
|
16131
|
-
? this.
|
|
16692
|
+
? await this.maintenanceNamespaces()
|
|
16132
16693
|
: [this.config.defaultNamespace];
|
|
16133
16694
|
|
|
16134
16695
|
for (const namespace of namespaces) {
|
|
@@ -16190,6 +16751,19 @@ export class Orchestrator {
|
|
|
16190
16751
|
identityNamespace,
|
|
16191
16752
|
);
|
|
16192
16753
|
await storage.writeIdentityReflections("");
|
|
16754
|
+
// NRcCL (codex P2): record a per-namespace catalog write for THIS namespace
|
|
16755
|
+
// after the identity files are updated. This fan-out can mutate a dynamic
|
|
16756
|
+
// namespace via `writeIdentity`/`writeIdentityReflections`, but the
|
|
16757
|
+
// consolidation pass's only consolidated touch covers `this.storage` (the
|
|
16758
|
+
// default) and only fires when `memoryItemMutated` was set by OTHER work — so
|
|
16759
|
+
// a namespace whose sole mutation in the pass is identity consolidation would
|
|
16760
|
+
// otherwise keep a stale `lastWriteAt`, making `listNamespaces({ writtenSince })`
|
|
16761
|
+
// and catalog-recency consumers miss the write. Best-effort and
|
|
16762
|
+
// failure-tolerant (`markCatalogWrite` swallows errors, never crashing the
|
|
16763
|
+
// consolidation; gotcha #13, rule #40). No double-count with the consolidated
|
|
16764
|
+
// touch above: that one is gated on `memoryItemMutated` (which identity
|
|
16765
|
+
// consolidation does not set), and `markWrite` is idempotent regardless.
|
|
16766
|
+
this.markCatalogWrite(namespace, storage.dir);
|
|
16193
16767
|
log.info(
|
|
16194
16768
|
`IDENTITY(${namespace}) consolidated: ${identityContent.length} → ${newContent.length} chars, ${result.learnedPatterns.length} patterns`,
|
|
16195
16769
|
);
|
|
@@ -18538,7 +19112,75 @@ export class Orchestrator {
|
|
|
18538
19112
|
return this.config.defaultNamespace;
|
|
18539
19113
|
const m = resolvedStorageDir.match(/[\\/]namespaces[\\/]([^\\/]+)$/);
|
|
18540
19114
|
if (!m?.[1]) return this.config.defaultNamespace;
|
|
18541
|
-
|
|
19115
|
+
const dirName = m[1];
|
|
19116
|
+
// Token-shaped raw names (round 6, codex P2 — NBsFz): a dir name might be a
|
|
19117
|
+
// tokenized identity OR a literal raw namespace name that merely LOOKS like a
|
|
19118
|
+
// token (e.g. a configured or dynamic name `ns-616c706861`). The round-trip check below
|
|
19119
|
+
// (`namespaceIdentityToken(decoded) === dirName`) is TAUTOLOGICAL for a
|
|
19120
|
+
// canonical token string, so it cannot tell a tokenized dir for `alpha` apart
|
|
19121
|
+
// from the legacy raw root of a namespace literally named `ns-616c706861`
|
|
19122
|
+
// (codex NRCve). A dir name that is itself a KNOWN namespace (configured or
|
|
19123
|
+
// catalog-owned at this exact storage root) is therefore preserved as the
|
|
19124
|
+
// literal namespace BEFORE attempting to decode it.
|
|
19125
|
+
if (this.configuredNamespaces().includes(dirName)) {
|
|
19126
|
+
return dirName;
|
|
19127
|
+
}
|
|
19128
|
+
this.loadNamespaceStorageDirHintsFromCatalog();
|
|
19129
|
+
const hintedNamespaces = this.namespaceStorageDirHints.get(resolvedStorageDir);
|
|
19130
|
+
if (hintedNamespaces?.has(dirName)) {
|
|
19131
|
+
return dirName;
|
|
19132
|
+
}
|
|
19133
|
+
if (hintedNamespaces?.size === 1) {
|
|
19134
|
+
const [hintedNamespace] = hintedNamespaces;
|
|
19135
|
+
if (hintedNamespace) return hintedNamespace;
|
|
19136
|
+
}
|
|
19137
|
+
const decoded = namespaceIdentityFromToken(dirName);
|
|
19138
|
+
if (decoded && namespaceIdentityToken(decoded) === dirName) {
|
|
19139
|
+
return decoded;
|
|
19140
|
+
}
|
|
19141
|
+
return dirName;
|
|
19142
|
+
}
|
|
19143
|
+
|
|
19144
|
+
/**
|
|
19145
|
+
* Record a namespace write in the catalog (issue #1499). Best-effort and
|
|
19146
|
+
* failure-tolerant: a catalog write error MUST NOT crash the primary memory
|
|
19147
|
+
* write (CLAUDE.md gotcha #13, rule #40). Fire-and-forget by design.
|
|
19148
|
+
*/
|
|
19149
|
+
private markCatalogWrite(namespace: string, storageDir?: string): void {
|
|
19150
|
+
if (!this.namespaceCatalog.enabled) return;
|
|
19151
|
+
this.rememberNamespaceStorageDirHint(namespace, storageDir);
|
|
19152
|
+
void this.namespaceCatalog
|
|
19153
|
+
.markWrite(namespace, { discoveredBy: "write", storageDir })
|
|
19154
|
+
.catch(() => undefined);
|
|
19155
|
+
}
|
|
19156
|
+
|
|
19157
|
+
/**
|
|
19158
|
+
* Public best-effort catalog write touch (issue #1499). User-facing explicit
|
|
19159
|
+
* captures (`memory_store`) and review-queue approvals persist via
|
|
19160
|
+
* `persistExplicitCapture()` → `storage.writeMemory()`, which bypasses the
|
|
19161
|
+
* extraction write path that calls `markCatalogWrite`. Without this their
|
|
19162
|
+
* namespaces never record `lastWriteAt`, so the catalog under-reports write
|
|
19163
|
+
* recency (round 5, codex P2). Fire-and-forget and failure-tolerant — a
|
|
19164
|
+
* catalog error must never affect the explicit write (gotcha #13, rule #40).
|
|
19165
|
+
*
|
|
19166
|
+
* An undefined/empty `namespace` means the write targeted the DEFAULT namespace
|
|
19167
|
+
* (`getStorage(undefined)` routes there), so we record it under the configured
|
|
19168
|
+
* default rather than skipping it (round 6, codex P2 — default `memory_store`
|
|
19169
|
+
* and inline-note writes were missing from `writtenSince`/maintenance).
|
|
19170
|
+
*/
|
|
19171
|
+
recordCatalogWrite(namespace?: string, storageDir?: string): void {
|
|
19172
|
+
const ns = namespace && namespace.trim().length > 0 ? namespace : this.config.defaultNamespace;
|
|
19173
|
+
if (!ns) return;
|
|
19174
|
+
this.markCatalogWrite(ns, storageDir);
|
|
19175
|
+
}
|
|
19176
|
+
|
|
19177
|
+
/** Record a namespace read in the catalog. Best-effort, failure-tolerant. */
|
|
19178
|
+
private markCatalogRead(namespace: string, storageDir?: string): void {
|
|
19179
|
+
if (!this.namespaceCatalog.enabled) return;
|
|
19180
|
+
this.rememberNamespaceStorageDirHint(namespace, storageDir);
|
|
19181
|
+
void this.namespaceCatalog
|
|
19182
|
+
.markRead(namespace, { discoveredBy: "read", storageDir })
|
|
19183
|
+
.catch(() => undefined);
|
|
18542
19184
|
}
|
|
18543
19185
|
|
|
18544
19186
|
private async readAllMemoriesForNamespaces(
|