@remnic/core 10.0.67 → 10.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +1 -1
- package/dist/{chunk-LFYEQK45.js → chunk-DDBB55L2.js} +417 -284
- package/dist/chunk-DDBB55L2.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/orchestration/buffer-drain-runner.js +1 -1
- package/dist/orchestration/extraction-persist.d.ts.map +1 -1
- package/dist/orchestration/promotion-dedup-window.d.ts +118 -0
- package/dist/orchestration/promotion-dedup-window.d.ts.map +1 -0
- package/dist/orchestrator.js +1 -1
- package/package.json +2 -2
- package/src/extraction-parallel-dedup.test.ts +185 -0
- package/src/extraction-shared-promotion-parallel-dedup.test.ts +212 -0
- package/src/orchestration/extraction-persist.ts +58 -61
- package/src/orchestration/promotion-dedup-window.ts +319 -0
- package/src/orchestration/provider-breaker.test.ts +22 -0
- package/dist/chunk-LFYEQK45.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-persist.d.ts","sourceRoot":"","sources":["../../src/orchestration/extraction-persist.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"extraction-persist.d.ts","sourceRoot":"","sources":["../../src/orchestration/extraction-persist.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAiBxG,OAAO,EAA6C,KAAK,cAAc,EAAE,MAAM,aAAa,CAAC;AAM7F,OAAO,EAUgE,KAAK,8BAA8B,EACxG,KAAK,4BAA4B,EAClC,MAAM,oBAAoB,CAAC;AAqD5B,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAGhF,OAAO,KAAK,EAEV,gBAAgB,EAQjB,MAAM,aAAa,CAAC;AAWrB,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAmB1E,6FAA6F;AAC7F,MAAM,WAAW,uBAAuB;IACtC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,4EAA4E;IAC5E,aAAa,EAAE,MAAM,CAAC;IACtB,kFAAkF;IAClF,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,qBAAa,4BAA4B;IAErC,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,EAAE,qBAAqB;IAE9C,OAAO,KAAK,MAAM,GAEjB;IACD,OAAO,KAAK,6BAA6B,GAGxC;IAEK,iBAAiB,CACrB,MAAM,EAAE,gBAAgB,EACxB,OAAO,EAAE,cAAc,EACvB,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,EACrC,aAAa,CAAC,EAAE,uBAAuB,EACvC,aAAa,CAAC,EAAE,MAAM,EACtB,qBAAqB,CAAC,EAAE,wBAAwB,GAAG,IAAI,EACvD,UAAU,CAAC,EAAE,MAAM,EACnB,SAAS,GAAE,8BAAuF,EAClG,aAAa,GAAE,4BAAmF,GACjG,OAAO,CAAC,uBAAuB,CAAC;CA8wFpC"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promotion destination writes under the fact-dedup commit window (codex P2,
|
|
3
|
+
* PRRT_kwDORJXyws6zzdG4 / #3311). The persist-time promotion gates run
|
|
4
|
+
* OUTSIDE the window that serializes source writes, so with
|
|
5
|
+
* extractionQueueConcurrency > 1 two sessions promoting the same fact from
|
|
6
|
+
* different source namespaces can both gate-miss and both write the shared
|
|
7
|
+
* destination — duplicate active copies, because writeMemory mints a fresh id
|
|
8
|
+
* per write and does not coalesce active duplicates. This module wraps each
|
|
9
|
+
* promotion destination write in `persistUnderFactDedupWindow` keyed on the
|
|
10
|
+
* DESTINATION storage dir and rechecks with the same connector/content
|
|
11
|
+
* predicate the promotion gates confirm with, so a twin promotion committed
|
|
12
|
+
* between the gate and the write is suppressed instead of duplicated.
|
|
13
|
+
*
|
|
14
|
+
* Split out of extraction-persist.ts for the structural ratchet (#1995);
|
|
15
|
+
* behavior is a verbatim move of the wrapped writes the persist closure
|
|
16
|
+
* previously inlined.
|
|
17
|
+
*/
|
|
18
|
+
import { type StorageManager } from "../index.js";
|
|
19
|
+
import type { ImportanceScore, MemoryFrontmatter, ProvenanceSource } from "../types.js";
|
|
20
|
+
import type { SealedMemoryEnvelope } from "../write-envelope.js";
|
|
21
|
+
import type { DependencyPropagationDeliveryPort } from "./dependency-propagation-delivery.js";
|
|
22
|
+
import { type FactDedupWindowOutcome } from "./fact-dedup-window.js";
|
|
23
|
+
/**
|
|
24
|
+
* The promotion fields the windowed write and its suppression-path bounds
|
|
25
|
+
* backfill read. Structurally satisfied by both promotion closures' options
|
|
26
|
+
* objects in extraction-persist.ts (profile-target and shared).
|
|
27
|
+
*/
|
|
28
|
+
export interface PromotionDedupWriteFields {
|
|
29
|
+
category: string;
|
|
30
|
+
sourceMemoryId: string;
|
|
31
|
+
importance?: ImportanceScore;
|
|
32
|
+
intentGoal?: string;
|
|
33
|
+
intentActionType?: string;
|
|
34
|
+
intentEntityTypes?: string[];
|
|
35
|
+
memoryKind?: MemoryFrontmatter["memoryKind"];
|
|
36
|
+
validAt?: string;
|
|
37
|
+
invalidAt?: string;
|
|
38
|
+
observedAt?: string;
|
|
39
|
+
eventTimeSource?: "extracted" | "assumed";
|
|
40
|
+
sources?: ProvenanceSource[];
|
|
41
|
+
provenance?: "verified" | "unverified" | "none";
|
|
42
|
+
toolScoped?: true;
|
|
43
|
+
}
|
|
44
|
+
export interface PromotionDedupWriteArgs {
|
|
45
|
+
/** Promotion DESTINATION storage — the commit window is keyed on its dir. */
|
|
46
|
+
destination: StorageManager;
|
|
47
|
+
/** Composed sealed envelope for the promoted copy. */
|
|
48
|
+
envelope: SealedMemoryEnvelope;
|
|
49
|
+
/** The promotion closure's options (read structurally). */
|
|
50
|
+
promotion: PromotionDedupWriteFields;
|
|
51
|
+
/** Fact hash-key content (attrs-suffixed); pre-citation raw for others. */
|
|
52
|
+
dedupContent: string;
|
|
53
|
+
rawContent: string;
|
|
54
|
+
/** promotionWithholdsToolScope() result (shared) / forwarded marker (profile). */
|
|
55
|
+
toolScoped: boolean;
|
|
56
|
+
/** Raw source connector: trimmed for the recheck, raw for bounds backfill. */
|
|
57
|
+
connector: string | undefined;
|
|
58
|
+
citationEnabled: boolean;
|
|
59
|
+
citationTemplate: string;
|
|
60
|
+
/** lifecycleCaps.temporalSupersession on the shared path; false on the
|
|
61
|
+
* profile path (its gate predicate has no entity filter). When true AND the
|
|
62
|
+
* envelope carries an entityRef with non-empty surviving attrs, the recheck
|
|
63
|
+
* additionally requires the same entity (PR #402 Uybg guard parity). */
|
|
64
|
+
sameEntityGuardEnabled: boolean;
|
|
65
|
+
/** Envelope-surviving structured attributes (PR #402 Uybg guard input). */
|
|
66
|
+
survivingAttrs?: Record<string, string> | null;
|
|
67
|
+
/** When set, the suppression path runs the post-write supersession pass
|
|
68
|
+
* against the winning copy (PRRT_kwDORJXyws6z18qb): a suppressed
|
|
69
|
+
* later-dated promotion must still retire a conflicting shared fact dated
|
|
70
|
+
* between the events, mirroring the gate's hash-dedup-hit pass. */
|
|
71
|
+
supersession?: {
|
|
72
|
+
/** Caller's lifecycleCaps.temporalSupersession — without it the pass
|
|
73
|
+
* must not run even though the event-time `enabled` gate passes
|
|
74
|
+
* (PRRT_kwDORJXyws6z2o1t). */
|
|
75
|
+
supersessionEnabled: boolean;
|
|
76
|
+
dependencyPropagationDelivery?: DependencyPropagationDeliveryPort;
|
|
77
|
+
namespaceScope?: string;
|
|
78
|
+
};
|
|
79
|
+
backfillTemporalBoundsOnDedupHit: ExtractionPersistDeps["backfillTemporalBoundsOnDedupHit"];
|
|
80
|
+
}
|
|
81
|
+
type ExtractionPersistDeps = {
|
|
82
|
+
backfillTemporalBoundsOnDedupHit: (targetStorage: StorageManager, dedupContent: string, bounds: {
|
|
83
|
+
invalidAt?: string;
|
|
84
|
+
validFrom?: string;
|
|
85
|
+
observedAt?: string;
|
|
86
|
+
eventTimeSource?: "extracted" | "assumed";
|
|
87
|
+
}, entityRef?: string, sourceConnector?: string) => Promise<void>;
|
|
88
|
+
};
|
|
89
|
+
/**
|
|
90
|
+
* In-window race confirm for a promotion destination. Authoritative index +
|
|
91
|
+
* miss is a clean NOT-duplicate answered WITHOUT the corpus scan, mirroring
|
|
92
|
+
* `confirmActiveFactDuplicate` and the window contract (one corpus confirm +
|
|
93
|
+
* one write, not a scan-length lock) — PRRT_kwDORJXyws6z18qS. On a hit, a
|
|
94
|
+
* non-authoritative index, or a probe failure, confirm against the corpus
|
|
95
|
+
* with the gates' predicate.
|
|
96
|
+
*/
|
|
97
|
+
export declare function confirmPromotionRaceDuplicate(args: {
|
|
98
|
+
storage: StorageManager;
|
|
99
|
+
dedupContent: string;
|
|
100
|
+
category: string;
|
|
101
|
+
connector: string | undefined;
|
|
102
|
+
citationEnabled: boolean;
|
|
103
|
+
citationTemplate: string;
|
|
104
|
+
entityKeyNorm?: string;
|
|
105
|
+
}): Promise<boolean>;
|
|
106
|
+
/**
|
|
107
|
+
* Run one promotion destination write inside the destination-dir fact-dedup
|
|
108
|
+
* commit window: recheck (facts only; the same confirm the persist-time gate
|
|
109
|
+
* used, now under the lock) and suppress as a duplicate when a concurrent
|
|
110
|
+
* promotion committed the same content first. A suppressed promotion backfills
|
|
111
|
+
* temporal bounds onto the winning copy (#1671 parity with the gates' skip
|
|
112
|
+
* paths) and reports `{ suppressed: true }`. Lock contention throws
|
|
113
|
+
* fail-closed to the caller — never an unlocked write; the durable source
|
|
114
|
+
* fact is unaffected.
|
|
115
|
+
*/
|
|
116
|
+
export declare function writePromotionUnderFactDedupWindow(args: PromotionDedupWriteArgs): Promise<FactDedupWindowOutcome<Awaited<ReturnType<StorageManager["writeSealedMemory"]>>>>;
|
|
117
|
+
export {};
|
|
118
|
+
//# sourceMappingURL=promotion-dedup-window.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promotion-dedup-window.d.ts","sourceRoot":"","sources":["../../src/orchestration/promotion-dedup-window.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAoB,KAAK,cAAc,EAAE,MAAM,aAAa,CAAC;AAEpE,OAAO,KAAK,EAAE,eAAe,EAAc,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpG,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAKjE,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,sCAAsC,CAAC;AAC9F,OAAO,EAA+B,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAElG;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,UAAU,CAAC,EAAE,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IAC1C,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,UAAU,CAAC,EAAE,UAAU,GAAG,YAAY,GAAG,MAAM,CAAC;IAChD,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,6EAA6E;IAC7E,WAAW,EAAE,cAAc,CAAC;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,2DAA2D;IAC3D,SAAS,EAAE,yBAAyB,CAAC;IACrC,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,kFAAkF;IAClF,UAAU,EAAE,OAAO,CAAC;IACpB,8EAA8E;IAC9E,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;6EAGyE;IACzE,sBAAsB,EAAE,OAAO,CAAC;IAChC,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC/C;;;wEAGoE;IACpE,YAAY,CAAC,EAAE;QACb;;uCAE+B;QAC/B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,6BAA6B,CAAC,EAAE,iCAAiC,CAAC;QAClE,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,gCAAgC,EAAE,qBAAqB,CAAC,kCAAkC,CAAC,CAAC;CAC7F;AAED,KAAK,qBAAqB,GAAG;IAC3B,gCAAgC,EAAE,CAChC,aAAa,EAAE,cAAc,EAC7B,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE;QACN,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,eAAe,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;KAC3C,EACD,SAAS,CAAC,EAAE,MAAM,EAClB,eAAe,CAAC,EAAE,MAAM,KACrB,OAAO,CAAC,IAAI,CAAC,CAAC;CACpB,CAAC;AA8CF;;;;;;;GAOG;AACH,wBAAsB,6BAA6B,CAAC,IAAI,EAAE;IACxD,OAAO,EAAE,cAAc,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,OAAO,CAAC,CAWnB;AA4BD;;;;;;;;;GASG;AACH,wBAAsB,kCAAkC,CACtD,IAAI,EAAE,uBAAuB,GAC5B,OAAO,CAAC,sBAAsB,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAuG3F"}
|
package/dist/orchestrator.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/core",
|
|
3
|
-
"version": "10.0.
|
|
3
|
+
"version": "10.0.69",
|
|
4
4
|
"description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -3376,7 +3376,7 @@
|
|
|
3376
3376
|
"core"
|
|
3377
3377
|
],
|
|
3378
3378
|
"peerDependencies": {
|
|
3379
|
-
"@remnic/coding-graph": "^10.0.
|
|
3379
|
+
"@remnic/coding-graph": "^10.0.69"
|
|
3380
3380
|
},
|
|
3381
3381
|
"peerDependenciesMeta": {
|
|
3382
3382
|
"@remnic/coding-graph": {
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue #3312 — non-fact dedup registration must share the fact-dedup commit
|
|
3
|
+
* window.
|
|
4
|
+
*
|
|
5
|
+
* writeMemory indexes content hashes for facts only (rule 44 #1579); every
|
|
6
|
+
* other category used to register its hash in a post-write block that ran
|
|
7
|
+
* after persistUnderFactDedupWindow had already released the per-dir commit
|
|
8
|
+
* window. With extractionQueueConcurrency > 1 a second session's in-window
|
|
9
|
+
* recheck could then miss and both sessions wrote the same preference —
|
|
10
|
+
* duplicate active copies. The fix registers the hash inside the window
|
|
11
|
+
* callback, so any later window holder observes the committed content.
|
|
12
|
+
*
|
|
13
|
+
* These tests drive the real persistExtraction path (no LLM network; every
|
|
14
|
+
* gate that would call one is off) and assert the durable outcome: exactly
|
|
15
|
+
* one active copy survives a concurrent same-content write pair.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import assert from "node:assert/strict";
|
|
19
|
+
import os from "node:os";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
22
|
+
import test from "node:test";
|
|
23
|
+
|
|
24
|
+
import { parseConfig } from "./config.js";
|
|
25
|
+
import { Orchestrator } from "./orchestrator.js";
|
|
26
|
+
import type { ExtractionResult, PluginConfig } from "./types.js";
|
|
27
|
+
import type { LocalLlmClient } from "./local-llm.js";
|
|
28
|
+
import type { PersistExtractionFn } from "./testing/orchestrator-lite.js";
|
|
29
|
+
import type { StorageManager } from "./storage.js";
|
|
30
|
+
|
|
31
|
+
// Fixture contract for the private Orchestrator surface this suite drives.
|
|
32
|
+
interface OrchestratorFixture {
|
|
33
|
+
localLlm?: unknown;
|
|
34
|
+
chatCompletion?: unknown;
|
|
35
|
+
getStorage?: unknown;
|
|
36
|
+
persistExtraction?: unknown;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Named owner contract for the private internals this suite drives.
|
|
40
|
+
interface OrchestratorInternals {
|
|
41
|
+
localLlm: LocalLlmClient;
|
|
42
|
+
getStorage: (ns: string) => Promise<StorageManager>;
|
|
43
|
+
persistExtraction: PersistExtractionFn;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function asInternals<T extends object>(value: OrchestratorFixture): T {
|
|
47
|
+
// SAFETY: the fixture provides the members the persist path reads; the
|
|
48
|
+
// remaining contract members are unused by these tests.
|
|
49
|
+
return value as T;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function makeHarness(): Promise<{
|
|
53
|
+
orchestrator: Orchestrator;
|
|
54
|
+
persist: OrchestratorInternals;
|
|
55
|
+
memoryDir: string;
|
|
56
|
+
}> {
|
|
57
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-parallel-dedup-"));
|
|
58
|
+
const config: PluginConfig = parseConfig({
|
|
59
|
+
openaiApiKey: "sk-test",
|
|
60
|
+
memoryDir,
|
|
61
|
+
workspaceDir: path.join(memoryDir, "workspace"),
|
|
62
|
+
// Minimize every gate that needs a model or corpus-scale machinery so the
|
|
63
|
+
// preference reaches the normal (non-chunked) write path.
|
|
64
|
+
extractionJudgeEnabled: false,
|
|
65
|
+
temporalSupersessionEnabled: false,
|
|
66
|
+
contradictionDetectionEnabled: false,
|
|
67
|
+
chunkingEnabled: false,
|
|
68
|
+
embeddingFallbackEnabled: false,
|
|
69
|
+
qmdEnabled: false,
|
|
70
|
+
extractionMinChars: 0,
|
|
71
|
+
extractionMinImportanceLevel: "trivial",
|
|
72
|
+
inlineSourceAttributionEnabled: false,
|
|
73
|
+
// Real deployment profile: the artifact write sits between the window
|
|
74
|
+
// release and hash registration, widening the exact gap under test.
|
|
75
|
+
verbatimArtifactsEnabled: true,
|
|
76
|
+
verbatimArtifactCategories: ["preference"],
|
|
77
|
+
});
|
|
78
|
+
const orchestrator = new Orchestrator(config);
|
|
79
|
+
const internals = asInternals<OrchestratorInternals>(orchestrator);
|
|
80
|
+
internals.localLlm = asInternals<LocalLlmClient>({ chatCompletion: async () => null });
|
|
81
|
+
return { orchestrator, persist: internals, memoryDir };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function preferenceResult(content: string): ExtractionResult {
|
|
85
|
+
const result = {
|
|
86
|
+
facts: [
|
|
87
|
+
{
|
|
88
|
+
content,
|
|
89
|
+
category: "preference",
|
|
90
|
+
tags: [],
|
|
91
|
+
confidence: 0.9,
|
|
92
|
+
},
|
|
93
|
+
],
|
|
94
|
+
entities: [],
|
|
95
|
+
relationships: [],
|
|
96
|
+
questions: [],
|
|
97
|
+
profileUpdates: [],
|
|
98
|
+
};
|
|
99
|
+
// SAFETY: literal carries the full extraction-result shape persistExtraction
|
|
100
|
+
// consumes for a single preference candidate.
|
|
101
|
+
return result as ExtractionResult;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function activePreferenceCopies(storage: StorageManager): Promise<number> {
|
|
105
|
+
const memories = await storage.readAllMemories();
|
|
106
|
+
return memories.filter(
|
|
107
|
+
(m) => m.frontmatter.category === "preference" && (m.frontmatter.status ?? "active") === "active",
|
|
108
|
+
).length;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const PREFERENCE = "Always answer with bullet points.";
|
|
112
|
+
|
|
113
|
+
test("a repeated identical preference persists once (serial baseline)", async () => {
|
|
114
|
+
const { persist, memoryDir } = await makeHarness();
|
|
115
|
+
try {
|
|
116
|
+
const storage = await persist.getStorage("default");
|
|
117
|
+
await storage.ensureDirectories();
|
|
118
|
+
|
|
119
|
+
const first = await persist.persistExtraction(
|
|
120
|
+
preferenceResult(PREFERENCE),
|
|
121
|
+
storage,
|
|
122
|
+
null,
|
|
123
|
+
{ sessionKey: "agent:test:a", principal: "test" },
|
|
124
|
+
undefined,
|
|
125
|
+
null,
|
|
126
|
+
"User said they always want bullet points.",
|
|
127
|
+
);
|
|
128
|
+
const second = await persist.persistExtraction(
|
|
129
|
+
preferenceResult(PREFERENCE),
|
|
130
|
+
storage,
|
|
131
|
+
null,
|
|
132
|
+
{ sessionKey: "agent:test:b", principal: "test" },
|
|
133
|
+
undefined,
|
|
134
|
+
null,
|
|
135
|
+
"User said they always want bullet points.",
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
assert.equal(first.persistedIds.length, 1, "first write persists");
|
|
139
|
+
assert.equal(second.persistedIds.length, 0, "second identical write is deduped");
|
|
140
|
+
assert.equal(await activePreferenceCopies(storage), 1);
|
|
141
|
+
} finally {
|
|
142
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("concurrent identical preferences leave exactly one active copy", async () => {
|
|
147
|
+
const { persist, memoryDir } = await makeHarness();
|
|
148
|
+
try {
|
|
149
|
+
const storage = await persist.getStorage("default");
|
|
150
|
+
await storage.ensureDirectories();
|
|
151
|
+
|
|
152
|
+
const sourceText = "User said they always want bullet points.";
|
|
153
|
+
// The race was probabilistic before the fix (the window released before
|
|
154
|
+
// registration); after it, this invariant holds for every interleaving.
|
|
155
|
+
const [a, b] = await Promise.all([
|
|
156
|
+
persist.persistExtraction(
|
|
157
|
+
preferenceResult(PREFERENCE),
|
|
158
|
+
storage,
|
|
159
|
+
null,
|
|
160
|
+
{ sessionKey: "agent:test:a", principal: "test" },
|
|
161
|
+
undefined,
|
|
162
|
+
null,
|
|
163
|
+
sourceText,
|
|
164
|
+
),
|
|
165
|
+
persist.persistExtraction(
|
|
166
|
+
preferenceResult(PREFERENCE),
|
|
167
|
+
storage,
|
|
168
|
+
null,
|
|
169
|
+
{ sessionKey: "agent:test:b", principal: "test" },
|
|
170
|
+
undefined,
|
|
171
|
+
null,
|
|
172
|
+
sourceText,
|
|
173
|
+
),
|
|
174
|
+
]);
|
|
175
|
+
|
|
176
|
+
assert.equal(
|
|
177
|
+
a.persistedIds.length + b.persistedIds.length,
|
|
178
|
+
1,
|
|
179
|
+
"exactly one concurrent write may persist",
|
|
180
|
+
);
|
|
181
|
+
assert.equal(await activePreferenceCopies(storage), 1);
|
|
182
|
+
} finally {
|
|
183
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
184
|
+
}
|
|
185
|
+
});
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Codex P2 PRRT_kwDORJXyws6zzdG4 (#3311) — promotion-target writes must share
|
|
3
|
+
* the fact-dedup commit window.
|
|
4
|
+
*
|
|
5
|
+
* With extractionQueueConcurrency > 1 and shared auto-promotion enabled, two
|
|
6
|
+
* sessions extracting the same fact into DIFFERENT source namespaces can both
|
|
7
|
+
* pass the shared-namespace dedup gate (`hasFactContentHash`) before either
|
|
8
|
+
* promotion writes: the persist-time gate runs outside the commit window that
|
|
9
|
+
* protects the source writes, and the promotion destination write ran without
|
|
10
|
+
* any window at all. writeMemory mints a fresh id per write, so both
|
|
11
|
+
* promotions land distinct active shared copies — exactly the duplicate class
|
|
12
|
+
* `persistUnderFactDedupWindow` exists to prevent for source writes.
|
|
13
|
+
*
|
|
14
|
+
* The fix wraps each promotion destination write in
|
|
15
|
+
* `persistUnderFactDedupWindow` keyed on the DESTINATION storage dir, so the
|
|
16
|
+
* second promotion's in-window recheck observes the first promotion's
|
|
17
|
+
* committed hash and suppresses instead of duplicating. The invariant these
|
|
18
|
+
* tests assert — exactly one active shared copy survives a concurrent
|
|
19
|
+
* same-fact promotion pair — holds for every interleaving once the window is
|
|
20
|
+
* in place, because recheck+write are atomic per destination dir; before the
|
|
21
|
+
* fix the duplicate depends on the interleaving (two symmetric pipelines in
|
|
22
|
+
* lockstep both gate-miss before either write commits).
|
|
23
|
+
*
|
|
24
|
+
* These tests drive the real persistExtraction path (no LLM network; every
|
|
25
|
+
* gate that would call one is off) with two real source namespaces and the
|
|
26
|
+
* real shared destination, and assert the durable outcome.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import assert from "node:assert/strict";
|
|
30
|
+
import os from "node:os";
|
|
31
|
+
import path from "node:path";
|
|
32
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
33
|
+
import test from "node:test";
|
|
34
|
+
|
|
35
|
+
import { parseConfig } from "./config.js";
|
|
36
|
+
import { Orchestrator } from "./orchestrator.js";
|
|
37
|
+
import type { ExtractionResult, PluginConfig } from "./types.js";
|
|
38
|
+
import type { LocalLlmClient } from "./local-llm.js";
|
|
39
|
+
import type { PersistExtractionFn } from "./testing/orchestrator-lite.js";
|
|
40
|
+
import type { StorageManager } from "./storage.js";
|
|
41
|
+
|
|
42
|
+
// Fixture contract for the private Orchestrator surface this suite drives.
|
|
43
|
+
interface OrchestratorFixture {
|
|
44
|
+
localLlm?: unknown;
|
|
45
|
+
chatCompletion?: unknown;
|
|
46
|
+
getStorage?: unknown;
|
|
47
|
+
persistExtraction?: unknown;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Named owner contract for the private internals this suite drives.
|
|
51
|
+
interface OrchestratorInternals {
|
|
52
|
+
localLlm: LocalLlmClient;
|
|
53
|
+
getStorage: (ns: string) => Promise<StorageManager>;
|
|
54
|
+
persistExtraction: PersistExtractionFn;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function asInternals<T extends object>(value: OrchestratorFixture): T {
|
|
58
|
+
// SAFETY: the fixture provides the members the persist path reads; the
|
|
59
|
+
// remaining contract members are unused by these tests.
|
|
60
|
+
return value as T;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function makeHarness(): Promise<{
|
|
64
|
+
orchestrator: Orchestrator;
|
|
65
|
+
persist: OrchestratorInternals;
|
|
66
|
+
memoryDir: string;
|
|
67
|
+
shared: StorageManager;
|
|
68
|
+
teamA: StorageManager;
|
|
69
|
+
teamB: StorageManager;
|
|
70
|
+
}> {
|
|
71
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-promotion-dedup-"));
|
|
72
|
+
const config: PluginConfig = parseConfig({
|
|
73
|
+
openaiApiKey: "sk-test",
|
|
74
|
+
memoryDir,
|
|
75
|
+
workspaceDir: path.join(memoryDir, "workspace"),
|
|
76
|
+
// Minimize every gate that needs a model or corpus-scale machinery so the
|
|
77
|
+
// fact reaches the normal (non-chunked) write path.
|
|
78
|
+
extractionJudgeEnabled: false,
|
|
79
|
+
temporalSupersessionEnabled: false,
|
|
80
|
+
contradictionDetectionEnabled: false,
|
|
81
|
+
chunkingEnabled: false,
|
|
82
|
+
embeddingFallbackEnabled: false,
|
|
83
|
+
qmdEnabled: false,
|
|
84
|
+
extractionMinChars: 0,
|
|
85
|
+
extractionMinImportanceLevel: "trivial",
|
|
86
|
+
inlineSourceAttributionEnabled: false,
|
|
87
|
+
// Shared auto-promotion from every source namespace into the default
|
|
88
|
+
// "shared" namespace (both default off — the deployment profile this
|
|
89
|
+
// finding targets).
|
|
90
|
+
namespacesEnabled: true,
|
|
91
|
+
autoPromoteToSharedEnabled: true,
|
|
92
|
+
});
|
|
93
|
+
const orchestrator = new Orchestrator(config);
|
|
94
|
+
const internals = asInternals<OrchestratorInternals>(orchestrator);
|
|
95
|
+
internals.localLlm = asInternals<LocalLlmClient>({ chatCompletion: async () => null });
|
|
96
|
+
const shared = await internals.getStorage("shared");
|
|
97
|
+
const teamA = await internals.getStorage("team-a");
|
|
98
|
+
const teamB = await internals.getStorage("team-b");
|
|
99
|
+
await Promise.all([shared, teamA, teamB].map((s) => s.ensureDirectories()));
|
|
100
|
+
return { orchestrator, persist: internals, memoryDir, shared, teamA, teamB };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function factResult(content: string): ExtractionResult {
|
|
104
|
+
const result = {
|
|
105
|
+
facts: [
|
|
106
|
+
{
|
|
107
|
+
content,
|
|
108
|
+
category: "fact",
|
|
109
|
+
// confidenceTier(0.96) === "explicit", which the default
|
|
110
|
+
// autoPromoteMinConfidenceTier ("explicit") admits.
|
|
111
|
+
confidence: 0.96,
|
|
112
|
+
tags: [],
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
entities: [],
|
|
116
|
+
relationships: [],
|
|
117
|
+
questions: [],
|
|
118
|
+
profileUpdates: [],
|
|
119
|
+
};
|
|
120
|
+
// SAFETY: literal carries the full extraction-result shape persistExtraction
|
|
121
|
+
// consumes for a single fact candidate.
|
|
122
|
+
return result as ExtractionResult;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function activeFactCopies(storage: StorageManager): Promise<number> {
|
|
126
|
+
const memories = await storage.readAllMemories();
|
|
127
|
+
return memories.filter(
|
|
128
|
+
(m) => m.frontmatter.category === "fact" && (m.frontmatter.status ?? "active") === "active",
|
|
129
|
+
).length;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const FACT = "Deployment rollbacks complete within five minutes.";
|
|
133
|
+
|
|
134
|
+
test("the same fact promoted from two namespaces lands once in shared (serial baseline)", async () => {
|
|
135
|
+
const { persist, memoryDir, shared, teamA, teamB } = await makeHarness();
|
|
136
|
+
try {
|
|
137
|
+
const first = await persist.persistExtraction(
|
|
138
|
+
factResult(FACT),
|
|
139
|
+
teamA,
|
|
140
|
+
null,
|
|
141
|
+
{ sessionKey: "agent:test:a", principal: "test" },
|
|
142
|
+
undefined,
|
|
143
|
+
null,
|
|
144
|
+
"User mentioned that deployment rollbacks complete within five minutes.",
|
|
145
|
+
);
|
|
146
|
+
const second = await persist.persistExtraction(
|
|
147
|
+
factResult(FACT),
|
|
148
|
+
teamB,
|
|
149
|
+
null,
|
|
150
|
+
{ sessionKey: "agent:test:b", principal: "test" },
|
|
151
|
+
undefined,
|
|
152
|
+
null,
|
|
153
|
+
"User mentioned that deployment rollbacks complete within five minutes.",
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
// Each source namespace keeps its own copy (cross-namespace source writes
|
|
157
|
+
// never dedup), and the second promotion dedups through the shared gate.
|
|
158
|
+
assert.equal(first.persistedIds.length, 1, "first source write persists");
|
|
159
|
+
assert.equal(second.persistedIds.length, 1, "second source write persists in its own namespace");
|
|
160
|
+
assert.equal(await activeFactCopies(teamA), 1);
|
|
161
|
+
assert.equal(await activeFactCopies(teamB), 1);
|
|
162
|
+
assert.equal(await activeFactCopies(shared), 1, "shared namespace holds exactly one promoted copy");
|
|
163
|
+
} finally {
|
|
164
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
test("concurrent same-fact promotions from different namespaces leave exactly one active shared copy", async () => {
|
|
169
|
+
const { persist, memoryDir, shared, teamA, teamB } = await makeHarness();
|
|
170
|
+
try {
|
|
171
|
+
const sourceText = "User mentioned that deployment rollbacks complete within five minutes.";
|
|
172
|
+
// Two real sessions, different source namespaces, same fact, one shared
|
|
173
|
+
// destination. Before the fix this interleaving (both pipelines gate-miss
|
|
174
|
+
// in lockstep before either promotion writes) landed two distinct active
|
|
175
|
+
// shared copies; after the fix the destination window serializes the
|
|
176
|
+
// promotions and the invariant holds for every interleaving.
|
|
177
|
+
const [a, b] = await Promise.all([
|
|
178
|
+
persist.persistExtraction(
|
|
179
|
+
factResult(FACT),
|
|
180
|
+
teamA,
|
|
181
|
+
null,
|
|
182
|
+
{ sessionKey: "agent:test:a", principal: "test" },
|
|
183
|
+
undefined,
|
|
184
|
+
null,
|
|
185
|
+
sourceText,
|
|
186
|
+
),
|
|
187
|
+
persist.persistExtraction(
|
|
188
|
+
factResult(FACT),
|
|
189
|
+
teamB,
|
|
190
|
+
null,
|
|
191
|
+
{ sessionKey: "agent:test:b", principal: "test" },
|
|
192
|
+
undefined,
|
|
193
|
+
null,
|
|
194
|
+
sourceText,
|
|
195
|
+
),
|
|
196
|
+
]);
|
|
197
|
+
|
|
198
|
+
// Each source namespace still persists its own copy — the window only
|
|
199
|
+
// serializes the shared DESTINATION.
|
|
200
|
+
assert.equal(await activeFactCopies(teamA), 1, "team-a keeps its source copy");
|
|
201
|
+
assert.equal(await activeFactCopies(teamB), 1, "team-b keeps its source copy");
|
|
202
|
+
assert.equal(a.persistedIds.length, 1);
|
|
203
|
+
assert.equal(b.persistedIds.length, 1);
|
|
204
|
+
assert.equal(
|
|
205
|
+
await activeFactCopies(shared),
|
|
206
|
+
1,
|
|
207
|
+
"exactly one active shared copy survives the concurrent promotion pair",
|
|
208
|
+
);
|
|
209
|
+
} finally {
|
|
210
|
+
await rm(memoryDir, { recursive: true, force: true });
|
|
211
|
+
}
|
|
212
|
+
});
|