@staix/agent-mem 0.20.2 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +4 -3
- package/package.json +2 -2
- package/src/cli.ts +8 -1
- package/src/commands-table.ts +2 -1
- package/src/hook-spool.ts +45 -3
- package/src/index.ts +25 -2
- package/src/memory/bridge.ts +66 -18
- package/src/memory/candidates.ts +71 -9
- package/src/memory/generation-index.ts +95 -18
- package/src/memory/generation-sync.ts +3 -0
- package/src/memory/generation.ts +33 -15
- package/src/memory/platform.ts +38 -3
- package/src/memory/policy.ts +28 -15
- package/src/memory/prune-publisher-lineage.ts +50 -12
- package/src/memory/public-prune-advance.ts +11 -5
- package/src/memory/public-prune-inputs.ts +17 -16
- package/src/memory/shard-sync-applications.ts +153 -0
- package/src/memory/shard-sync-format.ts +15 -0
- package/src/memory/shard-sync-graph.ts +6 -4
- package/src/memory/shard-sync-history.ts +238 -0
- package/src/memory/shard-sync.ts +67 -151
- package/src/memory/shard.ts +32 -6
- package/src/memory/store.ts +14 -1
- package/src/memory/sync-binding.ts +63 -0
- package/src/memory/sync-placement-active.ts +75 -0
- package/src/memory/sync-placement-cache.ts +32 -0
- package/src/memory/sync-placement-capture.ts +292 -0
- package/src/memory/sync-placement-dependencies.ts +255 -0
- package/src/memory/sync-placement-format.ts +182 -0
- package/src/memory/sync-placement-git.ts +95 -0
- package/src/memory/sync-placement-guard.ts +98 -0
- package/src/memory/sync-placement-inspection-evidence.ts +257 -0
- package/src/memory/sync-placement-inspection-pending.ts +66 -0
- package/src/memory/sync-placement-inspection.ts +236 -0
- package/src/memory/sync-placement-journal-format.ts +114 -0
- package/src/memory/sync-placement-journal-storage.ts +513 -0
- package/src/memory/sync-placement-journal.ts +153 -0
- package/src/memory/sync-placement-materialize.ts +149 -0
- package/src/memory/sync-placement-plan.ts +171 -0
- package/src/memory/sync-placement-projection.ts +79 -0
- package/src/memory/sync-placement-publication-plan.ts +189 -0
- package/src/memory/sync-placement-publication-storage.ts +87 -0
- package/src/memory/sync-placement-transport.ts +489 -0
- package/src/memory/sync-policy-inspection.ts +126 -0
- package/src/memory/sync-screen.ts +128 -33
- package/src/memory/sync.ts +20 -64
- package/src/setup.ts +69 -14
- package/src/ui/inbox-cache.ts +219 -0
- package/src/ui/inbox-worker-version.ts +14 -0
- package/src/ui/inbox-worker.ts +32 -0
- package/src/ui/inbox.ts +281 -0
- package/src/ui/panel.js +7 -3
- package/src/ui/plugin.ts +26 -53
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { validatePlacementProfile, validatePlacementShards } from "./sync-placement-format.ts";
|
|
1
2
|
/** Already-v2 monotone commit. Native rows stay unchanged; readers apply the proved growing mask on rebind. */
|
|
2
3
|
import { createHash } from "node:crypto";
|
|
3
4
|
import { constants, closeSync, fsyncSync, fstatSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
@@ -73,10 +74,15 @@ function checked(value: unknown, home: string): AdvanceIntent {
|
|
|
73
74
|
guard(value.schema === "agent-mem.public-prune-advance/v1" && value.home === home && hex(value.generation,32) && hex(value.expectedRevision,64));
|
|
74
75
|
for(const key of ["previousProfile","previousShards","finalProfile","finalShards"])guard(typeof value[key] === "string");
|
|
75
76
|
const previous=JSON.parse(value.previousProfile as string), oldShards=JSON.parse(value.previousShards as string), final=JSON.parse(value.finalProfile as string), shards=JSON.parse(value.finalShards as string), view=validatePublicPruneView(value.view);
|
|
76
|
-
|
|
77
|
-
|
|
77
|
+
const placed = previous?.schema === "agent-mem.profile/v3";
|
|
78
|
+
const profileFields = ["schema","kind","generation","foreignPruneFormat","pruneInputSetHash", ...(placed ? ["publicationFormat","placementHead"] : [])];
|
|
79
|
+
const shardFields: readonly string[] = [...SHARDED_V2_FIELDS, ...(placed ? ["publicationFormat","placementHead"] : [])];
|
|
80
|
+
if (placed) { validatePlacementShards(oldShards, validatePlacementProfile(previous)); validatePlacementShards(shards, validatePlacementProfile(final)); }
|
|
81
|
+
guard(object(previous) && exact(previous,profileFields) && previous.kind === "generation" && hex(previous.pruneInputSetHash,64) && previous.schema === (placed ? "agent-mem.profile/v3" : "agent-mem.profile/v2") && previous.generation === value.generation && previous.foreignPruneFormat === "agent-mem.public-prune-witness/v1" && hashJSON(previous) === value.expectedRevision);
|
|
82
|
+
guard(object(final) && exact(final,profileFields) && final.schema === previous.schema && final.kind === "generation" && final.generation === value.generation && final.foreignPruneFormat === previous.foreignPruneFormat && final.pruneInputSetHash === publicPruneViewHash(view));
|
|
83
|
+
if (placed) guard(final.publicationFormat === previous.publicationFormat && final.placementHead === previous.placementHead);
|
|
78
84
|
guard(view.receivingGeneration === value.generation && canonicalJSON(final)+"\n" === value.finalProfile && hashJSON(final) === value.finalProfileHash && hashJSON(shards) === value.finalShardsHash && canonicalJSON(shards)+"\n" === value.finalShards);
|
|
79
|
-
guard(object(oldShards) && object(shards) && Object.keys(oldShards).every(key=>
|
|
85
|
+
guard(object(oldShards) && object(shards) && Object.keys(oldShards).every(key=>shardFields.includes(key)) && Object.keys(shards).every(key=>shardFields.includes(key)) && oldShards.phase === "active" && oldShards.generation === value.generation && oldShards.schema === (placed ? "agent-mem.shards/v3" : "agent-mem.shards/v2") && oldShards.selectorRevision === value.expectedRevision && shards.selectorRevision === value.finalProfileHash);
|
|
80
86
|
const {selectorRevision:_old,...a}=oldShards,{selectorRevision:_new,...b}=shards;guard(canonicalJSON(a)===canonicalJSON(b));
|
|
81
87
|
guard(Array.isArray(value.bindings) && value.bindings.length <= 1024 && object(value.root) && exact(value.root,["dev","ino","aliasesHash","machineHash"]) && hex(value.root.aliasesHash,64) && hex(value.root.machineHash,64));
|
|
82
88
|
let prior="";for(const item of value.bindings){guard(object(item)&&exact(item,["key","dir","dev","ino","metadata"])&&typeof item.key==="string"&&item.key>prior&&item.dir===realmDir(home,item.key)&&hex(item.metadata,64));prior=item.key;}
|
|
@@ -91,7 +97,7 @@ function proofs(intent: AdvanceIntent): (()=>void)[] {
|
|
|
91
97
|
for(const [key,set] of Object.entries(intent.view.realms)){
|
|
92
98
|
if(old.realms[key]===set)continue;
|
|
93
99
|
const binding=intent.bindings.find(item=>item.key===key);guard(binding);
|
|
94
|
-
const load=createPublicPruneReplayCache({dir:binding.dir,realm:key==="user"?{scope:"user"}:{scope:"project",key:key.slice(8)},receivingGeneration:intent.generation,inputSetHash:set});
|
|
100
|
+
const load=createPublicPruneReplayCache({dir:binding.dir,realm:key==="user"?{scope:"user"}:{scope:"project",key:key.slice(8)},receivingGeneration:intent.generation,inputSetHash:set,...(previous.schema==="agent-mem.profile/v3"?{publicationFormat:"agent-mem.sync-placement/v1" as const}:{})});
|
|
95
101
|
const check=()=>{const proof:PublicPruneProjection=load();guard(old.realms[key]===undefined||proof.commitments.includes(old.realms[key]!));};check();checks.push(check);
|
|
96
102
|
}
|
|
97
103
|
return checks;
|
|
@@ -128,7 +134,7 @@ function run(intent: AdvanceIntent, provided: readonly MutationHandle[]|undefine
|
|
|
128
134
|
}
|
|
129
135
|
export async function advancePublicPruneView(options: PublicPruneAdvanceOptions):Promise<PublicPruneAdvanceResult>{
|
|
130
136
|
const home=realPath(options.memoryDir),root=owned(home,true),selection=readSelector(home),profile=selection.selector;
|
|
131
|
-
guard(profile?.schema==="agent-mem.profile/v2"&&selection.revision===options.expectedRevision);
|
|
137
|
+
guard((profile?.schema==="agent-mem.profile/v2"||profile?.schema==="agent-mem.profile/v3")&&selection.revision===options.expectedRevision);
|
|
132
138
|
const view=validatePublicPruneView(options.view);guard(view.receivingGeneration===profile.generation);
|
|
133
139
|
const previousProfile=raw(join(home,"profile.json")),previousShards=raw(join(home,"shards.json")),old=readPublicPruneViewForActivation(home,profile.pruneInputSetHash,profile.generation);
|
|
134
140
|
const bindings:Binding[]=[];
|
|
@@ -7,7 +7,7 @@ import { acquireMutation, assertMutationHandle, type MutationHandle } from "./mu
|
|
|
7
7
|
import { realPath } from "./platform.ts";
|
|
8
8
|
import { entryId, type Entry } from "./store.ts";
|
|
9
9
|
import { prunePayloadHash } from "./prune.ts";
|
|
10
|
-
import { shardSyncRef, type ShardSyncRealm } from "./shard-sync-format.ts";
|
|
10
|
+
import { decodePlacementTransportDescriptor, shardSyncRef, type ShardSyncRealm } from "./shard-sync-format.ts";
|
|
11
11
|
import { verifyPublicPruneWitness, type VerifiedPublicPruneWitness } from "./public-prune-witness.ts";
|
|
12
12
|
|
|
13
13
|
import { PUBLIC_PRUNE_INPUT_MAX_BYTES, PUBLIC_PRUNE_REPLAY_MAX_BYTES, PUBLIC_PRUNE_MAX_SETS, PublicPruneInputError, validatePublicPruneInputSet, type PublicPruneInputSetV1 } from "./public-prune-input-chain.ts";
|
|
@@ -20,7 +20,7 @@ export interface PublicPruneInputV1 {
|
|
|
20
20
|
schema: "agent-mem.public-prune-input/v1"; realm: ShardSyncRealm; receivingGeneration: string;
|
|
21
21
|
selectorRevision: string; witnessHash: string; source: Source;
|
|
22
22
|
}
|
|
23
|
-
export interface PublicPruneBinding { realm: ShardSyncRealm; receivingGeneration: string; selectorRevision: string; inputSetHash: string | null }
|
|
23
|
+
export interface PublicPruneBinding { publicationFormat?: "agent-mem.sync-placement/v1"; realm: ShardSyncRealm; receivingGeneration: string; selectorRevision: string; inputSetHash: string | null }
|
|
24
24
|
/** Complete Git object bytes, including the uncompressed '<type> <size>\0' header. */
|
|
25
25
|
export interface PublicPruneObject { oid: string; bytes: Uint8Array }
|
|
26
26
|
export interface PublicPruneInputOptions {
|
|
@@ -32,7 +32,7 @@ declare const planBrand: unique symbol;
|
|
|
32
32
|
export interface PublicPruneInputPlan { readonly [planBrand]: true }
|
|
33
33
|
export interface PublicPruneInputRegistration { inputHash?: string; inputSetHash: string; previous: string | null }
|
|
34
34
|
export interface PublicPruneReplayInput {
|
|
35
|
-
dir: string; realm: ShardSyncRealm; receivingGeneration: string; inputSetHash: string;
|
|
35
|
+
dir: string; realm: ShardSyncRealm; receivingGeneration: string; inputSetHash: string; publicationFormat?: "agent-mem.sync-placement/v1";
|
|
36
36
|
nativeTargets?: readonly string[]; ordinary?: readonly Entry[];
|
|
37
37
|
}
|
|
38
38
|
export interface PublicPruneProjection {
|
|
@@ -68,9 +68,10 @@ export function validatePublicPruneInput(v: unknown): PublicPruneInputV1 {
|
|
|
68
68
|
});
|
|
69
69
|
}
|
|
70
70
|
function binding(v: PublicPruneBinding): PublicPruneBinding {
|
|
71
|
-
guard(object(v) && exact(v, ["realm", "receivingGeneration", "selectorRevision", "inputSetHash"]));
|
|
71
|
+
guard(object(v) && exact(v, ["realm", "receivingGeneration", "selectorRevision", "inputSetHash", ...(v.publicationFormat === undefined ? [] : ["publicationFormat"])]));
|
|
72
|
+
guard(v.publicationFormat === undefined || v.publicationFormat === "agent-mem.sync-placement/v1");
|
|
72
73
|
guard(hex(v.receivingGeneration, 32) && (hex(v.selectorRevision, 64) || v.selectorRevision === "absent") && (v.inputSetHash === null || hex(v.inputSetHash, 64)));
|
|
73
|
-
return { realm: selectedRealm(v.realm), receivingGeneration: v.receivingGeneration, selectorRevision: v.selectorRevision, inputSetHash: v.inputSetHash };
|
|
74
|
+
return { realm: selectedRealm(v.realm), receivingGeneration: v.receivingGeneration, selectorRevision: v.selectorRevision, inputSetHash: v.inputSetHash, ...(v.publicationFormat ? {publicationFormat:v.publicationFormat} : {}) };
|
|
74
75
|
}
|
|
75
76
|
function json(bytes: Buffer): unknown {
|
|
76
77
|
guard(bytes.length > 0 && bytes.length <= PUBLIC_PRUNE_INPUT_MAX_BYTES, "resource");
|
|
@@ -99,7 +100,7 @@ function treeEntries(body: Buffer): Map<string, { mode: string; oid: string }> {
|
|
|
99
100
|
return entries;
|
|
100
101
|
}
|
|
101
102
|
/** Exact path closure only. Sibling object bodies are neither requested nor retained. */
|
|
102
|
-
function graph(input: PublicPruneInputV1, load: (oid: string) => Buffer): { proof: VerifiedPublicPruneWitness; used: Set<string> } {
|
|
103
|
+
function graph(input: PublicPruneInputV1, load: (oid: string) => Buffer, publicationFormat?: "agent-mem.sync-placement/v1"): { proof: VerifiedPublicPruneWitness; used: Set<string> } {
|
|
103
104
|
const used = new Set<string>(), cache = new Map<string, GitObject>(), trees = new Map<string, ReturnType<typeof treeEntries>>(); let total = 0;
|
|
104
105
|
const get = (oid: string, type: GitObject["type"]): GitObject => {
|
|
105
106
|
let item = cache.get(oid);
|
|
@@ -121,8 +122,8 @@ function graph(input: PublicPruneInputV1, load: (oid: string) => Buffer): { proo
|
|
|
121
122
|
}
|
|
122
123
|
return { oid, body: get(oid, "blob").body };
|
|
123
124
|
};
|
|
124
|
-
const
|
|
125
|
-
guard(
|
|
125
|
+
const version = decodePlacementTransportDescriptor(path("shard.json").body, input.realm);
|
|
126
|
+
guard(version === "v2" || version === "v3" && publicationFormat === "agent-mem.sync-placement/v1");
|
|
126
127
|
const manifest = path(input.source.manifestPath); guard(manifest.oid === input.source.manifestBlob);
|
|
127
128
|
const raw = json(manifest.body); guard(object(raw) && Array.isArray(raw.witnesses) && raw.witnesses.length <= 64 && object(raw.pruneOccurrence));
|
|
128
129
|
const refs = [raw.pruneOccurrence, ...raw.witnesses], packs = new Map<string, { pack: string; blob: string; bytes: Buffer }>();
|
|
@@ -151,7 +152,7 @@ export function preparePublicPruneInput(options: PublicPruneInputOptions): Publi
|
|
|
151
152
|
}
|
|
152
153
|
input = validatePublicPruneInput({ schema: "agent-mem.public-prune-input/v1", realm: expected.realm, receivingGeneration: expected.receivingGeneration,
|
|
153
154
|
selectorRevision: expected.selectorRevision, witnessHash: value.witnessHash, source: value.source });
|
|
154
|
-
const checked = graph(input, oid => { const bytes = objects.get(oid); guard(bytes); return bytes; });
|
|
155
|
+
const checked = graph(input, oid => { const bytes = objects.get(oid); guard(bytes); return bytes; }, expected.publicationFormat);
|
|
155
156
|
guard(checked.used.size === objects.size); // never retain whole sibling subtrees or unrelated private bodies
|
|
156
157
|
}
|
|
157
158
|
guard(canonicalJSON(binding(options.current())) === canonicalJSON(expected), "binding");
|
|
@@ -202,7 +203,7 @@ function replay(input: Omit<PublicPruneReplayInput, "dir">, load: Loader): Publi
|
|
|
202
203
|
for (const member of members) {
|
|
203
204
|
const descriptor = validatePublicPruneInput(json(data("input", member)));
|
|
204
205
|
guard(hashJSON(descriptor) === member && descriptor.receivingGeneration === input.receivingGeneration && canonicalJSON(descriptor.realm) === canonicalJSON(selected));
|
|
205
|
-
const { proof } = graph(descriptor, oid => data("object", oid)), prune = proof.manifest.prune, hash = prunePayloadHash(prune), key = `${canonicalJSON(selected)}:${prune.id}`;
|
|
206
|
+
const { proof } = graph(descriptor, oid => data("object", oid), input.publicationFormat), prune = proof.manifest.prune, hash = prunePayloadHash(prune), key = `${canonicalJSON(selected)}:${prune.id}`;
|
|
206
207
|
guard(!identities.has(key) || identities.get(key) === hash); identities.set(key, hash);
|
|
207
208
|
for (const ref of proof.manifest.witnesses) { guard(!rowHashes.has(ref.id) || rowHashes.get(ref.id) === ref.rowHash); rowHashes.set(ref.id, ref.rowHash); }
|
|
208
209
|
for (const target of prune.targets) targets.add(target.id);
|
|
@@ -312,7 +313,7 @@ export function persistPublicPruneInput(plan: PublicPruneInputPlan, options: { o
|
|
|
312
313
|
for (const suffix of ["", "-wal", "-shm", "-journal"]) { const stat = lstatSync(join(dir, "mutation-lock.db" + suffix), { throwIfNoEntry: false }); if (stat) privateStat(stat); }
|
|
313
314
|
const release = options.mutation ?? acquireMutation(dir); guard(release); assertMutationHandle(release, dir); try {
|
|
314
315
|
current(); assertMutationHandle(release, dir); recoverLinkedTemps(dir, release); current(); const previous = expected.inputSetHash;
|
|
315
|
-
const members = previous ? replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, inputSetHash: previous }, (kind, id) => read(dir, kind, id)).inputs : [];
|
|
316
|
+
const members = previous ? replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, ...(expected.publicationFormat ? {publicationFormat:expected.publicationFormat} : {}), inputSetHash: previous }, (kind, id) => read(dir, kind, id)).inputs : [];
|
|
316
317
|
const inputHash = prepared.input && hashJSON(prepared.input), includes = inputHash !== undefined && members.includes(inputHash);
|
|
317
318
|
const set: PublicPruneInputSetV1 = { schema: "agent-mem.public-prune-input-set/v1", receivingGeneration: expected.receivingGeneration,
|
|
318
319
|
previous, inputs: [...new Set([...members, ...(inputHash ? [inputHash] : [])])].sort() };
|
|
@@ -323,7 +324,7 @@ export function persistPublicPruneInput(plan: PublicPruneInputPlan, options: { o
|
|
|
323
324
|
if (prepared.input && inputHash) writes.set(`input:${inputHash}`, { kind: "input", id: inputHash, bytes: Buffer.from(canonicalJSON(prepared.input) + "\n") });
|
|
324
325
|
if (inputSetHash !== previous) writes.set(`set:${inputSetHash}`, { kind: "set", id: inputSetHash, bytes: Buffer.from(canonicalJSON(set) + "\n") });
|
|
325
326
|
// Preflight the complete future committed view and byte budget before the first archive write.
|
|
326
|
-
replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, inputSetHash }, (kind, id) => writes.get(`${kind}:${id}`)?.bytes ?? read(dir, kind, id));
|
|
327
|
+
replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, ...(expected.publicationFormat ? {publicationFormat:expected.publicationFormat} : {}), inputSetHash }, (kind, id) => writes.get(`${kind}:${id}`)?.bytes ?? read(dir, kind, id));
|
|
327
328
|
capacity(dir, writes); current();
|
|
328
329
|
for (const value of writes.values()) { assertMutationHandle(release, dir); write(dir, value.kind, value.id, value.bytes, options.observe); current(); assertMutationHandle(release, dir); }
|
|
329
330
|
return { ...(inputHash ? { inputHash } : {}), inputSetHash, previous };
|
|
@@ -386,12 +387,12 @@ function batchOverlay(batch: PreparedBatch): { writes: Map<string, ArchiveWrite>
|
|
|
386
387
|
};
|
|
387
388
|
const load: Loader = (kind, id) => writes.get(`${kind}:${id}`)?.bytes ?? read(batch.dir, kind, id);
|
|
388
389
|
let previous = expected.inputSetHash, lastInputHash: string | undefined;
|
|
389
|
-
let members = previous ? replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, inputSetHash: previous }, load).inputs : [];
|
|
390
|
+
let members = previous ? replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, ...(expected.publicationFormat ? {publicationFormat:expected.publicationFormat} : {}), inputSetHash: previous }, load).inputs : [];
|
|
390
391
|
for (const prepared of batch.inputs) {
|
|
391
392
|
guard(prepared.input); const input = prepared.input, inputHash = hashJSON(input); lastInputHash = inputHash;
|
|
392
393
|
// Every descriptor stays bound to the actual initial selector revision, never a simulated callback.
|
|
393
394
|
guard(canonicalJSON(prepared.binding) === canonicalJSON(expected));
|
|
394
|
-
const checked = graph(input, oid => { const bytes = prepared.objects.get(oid); guard(bytes); return bytes; });
|
|
395
|
+
const checked = graph(input, oid => { const bytes = prepared.objects.get(oid); guard(bytes); return bytes; }, expected.publicationFormat);
|
|
395
396
|
guard(checked.used.size === prepared.objects.size);
|
|
396
397
|
for (const [oid, bytes] of prepared.objects) add("object", oid, bytes);
|
|
397
398
|
add("input", inputHash, Buffer.from(canonicalJSON(input) + "\n"));
|
|
@@ -405,7 +406,7 @@ function batchOverlay(batch: PreparedBatch): { writes: Map<string, ArchiveWrite>
|
|
|
405
406
|
const set: PublicPruneInputSetV1 = { schema: "agent-mem.public-prune-input-set/v1", receivingGeneration: expected.receivingGeneration, previous: null, inputs: [] };
|
|
406
407
|
previous = hashJSON(set); add("set", previous, Buffer.from(canonicalJSON(set) + "\n"));
|
|
407
408
|
}
|
|
408
|
-
replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, inputSetHash: previous }, load);
|
|
409
|
+
replay({ realm: expected.realm, receivingGeneration: expected.receivingGeneration, ...(expected.publicationFormat ? {publicationFormat:expected.publicationFormat} : {}), inputSetHash: previous }, load);
|
|
409
410
|
// capacity removes already-complete candidates only from its private working copy.
|
|
410
411
|
capacity(batch.dir, writes);
|
|
411
412
|
return { writes, result: { inputSetHash: previous, previous: expected.inputSetHash, ...(lastInputHash ? { inputHash: lastInputHash } : {}) } };
|
|
@@ -449,7 +450,7 @@ export function persistPublicPruneInputBatch(batch: PublicPruneInputBatch, optio
|
|
|
449
450
|
batchCurrent(prepared, mutation); write(prepared.dir, item.kind, item.id, item.bytes, options?.observe); batchCurrent(prepared, mutation);
|
|
450
451
|
}
|
|
451
452
|
// Validate installed bytes too, including exact retries and interrupted prior attempts.
|
|
452
|
-
replay({ realm: prepared.binding.realm, receivingGeneration: prepared.binding.receivingGeneration, inputSetHash: result.inputSetHash }, (kind, id) => read(prepared.dir, kind, id));
|
|
453
|
+
replay({ realm: prepared.binding.realm, receivingGeneration: prepared.binding.receivingGeneration, ...(prepared.binding.publicationFormat ? {publicationFormat:prepared.binding.publicationFormat} : {}), inputSetHash: result.inputSetHash }, (kind, id) => read(prepared.dir, kind, id));
|
|
453
454
|
batchCurrent(prepared, mutation); return result;
|
|
454
455
|
});
|
|
455
456
|
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/** Shared v2/v3 public-prune application preflight and durable view installation. No row import or transport publication. */
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { canonicalJSON, readSelector } from "./generation.ts";
|
|
4
|
+
import { advancePublicPruneView } from "./public-prune-advance.ts";
|
|
5
|
+
import { readPublicPruneView, type PublicPruneViewV1 } from "./public-prune-view.ts";
|
|
6
|
+
import { collectPublicPruneGitGraph } from "./shard-sync-graph.ts";
|
|
7
|
+
import { PublicPruneInputError, PUBLIC_PRUNE_REPLAY_MAX_BYTES, PUBLIC_PRUNE_MAX_SETS, preparePublicPruneInputBatch,
|
|
8
|
+
persistPublicPruneInputBatch, readPublicPruneInputDescriptor, readPublicPruneInputSetChain, reducePublicPruneInputs,
|
|
9
|
+
type PublicPruneInputOptions } from "./public-prune-inputs.ts";
|
|
10
|
+
import { decodeShardPack, shardSyncRef, SHARD_PACK_MAX_BYTES } from "./shard-sync-format.ts";
|
|
11
|
+
import { shardPrepareHolds, shardSelectorRetention, type ShardSyncApplicationTarget, type ShardSyncApplicationPhase,
|
|
12
|
+
type ShardSyncHoldReceipt } from "./shard-sync.ts";
|
|
13
|
+
import type { ShardHistory } from "./shard-sync-history.ts";
|
|
14
|
+
import type { MutationHandle } from "./mutation.ts";
|
|
15
|
+
import type { SyncResult } from "./sync.ts";
|
|
16
|
+
import type { PruneRow } from "./prune.ts";
|
|
17
|
+
|
|
18
|
+
type Target = ShardSyncApplicationTarget;
|
|
19
|
+
export interface ShardApplicationGitObject { type: "commit" | "tree" | "blob"; body: Uint8Array }
|
|
20
|
+
export interface ShardSyncApplicationsOptions {
|
|
21
|
+
home: string; root: string; url: string; targets: Target[]; homeMutation: MutationHandle;
|
|
22
|
+
getRevision(): string; setRevision(revision: string): void; check(): void;
|
|
23
|
+
history(tip: string, target: Target): Promise<ShardHistory>;
|
|
24
|
+
/** Placement supplies an already audited in-memory candidate before installing Git objects. */
|
|
25
|
+
candidateHistory(target: Target): Promise<ShardHistory>;
|
|
26
|
+
assertAncestor(ancestor: string, descendant: string, target: Target): Promise<void>;
|
|
27
|
+
readObject(oid: string): Promise<ShardApplicationGitObject>;
|
|
28
|
+
/** Rebind the caller's acquisition cache to these same immutable raw bytes, avoiding a second retained body. */
|
|
29
|
+
retainGraphObject?(oid: string, rawBytes: Uint8Array): void | Promise<void>;
|
|
30
|
+
/** Re-prove source/policy stability and update the caller's metadata binding after a legitimate view commit. */
|
|
31
|
+
afterAdvance(): void | Promise<void>;
|
|
32
|
+
phase?(phase: ShardSyncApplicationPhase): void | Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
export class ShardSyncApplicationError extends Error {
|
|
35
|
+
constructor(message: string) { super(message); this.name = "ShardSyncApplicationError"; }
|
|
36
|
+
}
|
|
37
|
+
function refuse(message: string): never { throw new ShardSyncApplicationError(message); }
|
|
38
|
+
const digest = (value: Uint8Array | string) => createHash("sha256").update(value).digest("hex");
|
|
39
|
+
const oid = (value: unknown): value is string => typeof value === "string" && value.length === 40 && /^[a-f0-9]+$/.test(value);
|
|
40
|
+
const realmKey = (target: Target) => target.realm.scope === "user" ? "user" : `project:${target.realm.key}`;
|
|
41
|
+
export class ShardSyncApplications {
|
|
42
|
+
readonly #options: ShardSyncApplicationsOptions;
|
|
43
|
+
#boundView?: PublicPruneViewV1;
|
|
44
|
+
readonly #objects = new Map<string, ShardApplicationGitObject>();
|
|
45
|
+
readonly #graphs = new Map<string, Uint8Array>();
|
|
46
|
+
readonly #sets = new Set<string>();
|
|
47
|
+
#objectBytes = 0;
|
|
48
|
+
readonly application: NonNullable<SyncResult["pruneApplication"]> = { transport: "unconfirmed", proofComplete: 0, projectionActive: null, nativeApplied: null, held: 0, phase: "preflight" };
|
|
49
|
+
constructor(options: ShardSyncApplicationsOptions) { this.#options = options; }
|
|
50
|
+
get boundView(): Readonly<PublicPruneViewV1> | undefined { return this.#boundView && structuredClone(this.#boundView); }
|
|
51
|
+
async #checked<T>(work: () => T | Promise<T>): Promise<T> { this.#options.check(); const value = await work(); this.#options.check(); return value; }
|
|
52
|
+
async #proveHoldSource(target: Target, receipt: ShardSyncHoldReceipt): Promise<void> {
|
|
53
|
+
if (!target.remote) refuse("foreign prune has no selected public source");
|
|
54
|
+
await this.#checked(() => this.#options.assertAncestor(receipt.source.commit, target.remote!, target));
|
|
55
|
+
const original = await this.#checked(() => this.#options.history(receipt.source.commit, target)), bytes = original.files.get(receipt.source.pack);
|
|
56
|
+
if (!bytes || createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex") !== receipt.source.blob
|
|
57
|
+
|| !decodeShardPack(receipt.source.pack, bytes, target.realm, target.generation).some(row => canonicalJSON(row) === canonicalJSON(receipt.row))
|
|
58
|
+
|| receipt.source.ref !== shardSyncRef(target.realm)) refuse("shard sync prune hold source mismatch");
|
|
59
|
+
}
|
|
60
|
+
async loadExisting(): Promise<void> {
|
|
61
|
+
const o = this.#options; o.check();
|
|
62
|
+
const selected = readSelector(o.home).selector;
|
|
63
|
+
if ((selected?.schema !== "agent-mem.profile/v2" && selected?.schema !== "agent-mem.profile/v3")
|
|
64
|
+
|| o.targets.some(target => target.generation !== selected.generation)) refuse("selected public-prune profile unavailable");
|
|
65
|
+
this.#boundView = readPublicPruneView(o.home, selected.pruneInputSetHash, selected.generation);
|
|
66
|
+
for (const target of o.targets) {
|
|
67
|
+
target.registered = []; const set = this.#boundView.realms[realmKey(target)]; if (!set) continue;
|
|
68
|
+
// Shared across selected realms and retries; neither reopening an overlay nor repeating preflight resets the allowance.
|
|
69
|
+
const chain = readPublicPruneInputSetChain({ dir: target.store.dir, receivingGeneration: target.generation, inputSetHash: set, ...(target.store.profile.publicationFormat ? {publicationFormat:target.store.profile.publicationFormat} : {}) });
|
|
70
|
+
for (const commitment of chain.commitments) this.#sets.add(`${target.store.dir}:${commitment}`);
|
|
71
|
+
if (this.#sets.size > PUBLIC_PRUNE_MAX_SETS) throw new PublicPruneInputError("resource");
|
|
72
|
+
const projection = reducePublicPruneInputs({ dir: target.store.dir, realm: target.realm, receivingGeneration: target.generation, inputSetHash: set, ...(target.store.profile.publicationFormat ? {publicationFormat:target.store.profile.publicationFormat} : {}) });
|
|
73
|
+
for (const hash of projection.inputs) {
|
|
74
|
+
const descriptor = readPublicPruneInputDescriptor({ dir: target.store.dir, hash });
|
|
75
|
+
if (!target.remote || descriptor.source.remoteHash !== digest(o.url)) refuse("registered prune source history disappeared or its remote binding changed");
|
|
76
|
+
await this.#checked(() => o.assertAncestor(descriptor.source.commit, target.remote!, target));
|
|
77
|
+
const original = await this.#checked(() => o.history(descriptor.source.commit, target)), manifest = original.manifests.get(descriptor.source.manifestPath);
|
|
78
|
+
if (!manifest || !manifest.eligible || manifest.blob !== descriptor.source.manifestBlob || descriptor.source.ref !== shardSyncRef(target.realm)
|
|
79
|
+
|| digest(canonicalJSON(manifest.value)) !== descriptor.witnessHash) refuse("registered prune anchor no longer has exact audited evidence");
|
|
80
|
+
target.registered.push(descriptor);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
o.check();
|
|
84
|
+
}
|
|
85
|
+
async #readObject(object: string): Promise<ShardApplicationGitObject> {
|
|
86
|
+
if (!oid(object)) refuse("Git object identity invalid");
|
|
87
|
+
const cached = this.#objects.get(object); if (cached) return cached;
|
|
88
|
+
const value = await this.#checked(() => this.#options.readObject(object));
|
|
89
|
+
if (!value || !["commit", "tree", "blob"].includes(value.type) || !(value.body instanceof Uint8Array)) refuse("Git object type refused");
|
|
90
|
+
const limit = value.type === "commit" ? 65536 : SHARD_PACK_MAX_BYTES;
|
|
91
|
+
if (value.body.length > limit) refuse("Git object exceeds complete evidence bound");
|
|
92
|
+
const header = Buffer.from(`${value.type} ${value.body.length}\0`), encodedSize = header.length + value.body.length;
|
|
93
|
+
if (this.#objectBytes + encodedSize > PUBLIC_PRUNE_REPLAY_MAX_BYTES) throw new PublicPruneInputError("resource");
|
|
94
|
+
if (createHash("sha1").update(header).update(value.body).digest("hex") !== object) refuse("Git object identity mismatch");
|
|
95
|
+
this.#objectBytes += encodedSize; this.#objects.set(object, value); return value;
|
|
96
|
+
}
|
|
97
|
+
async prepare(): Promise<void> {
|
|
98
|
+
const o = this.#options; o.check(); if (!this.#boundView) refuse("public-prune application preflight is missing");
|
|
99
|
+
this.application.proofComplete = 0; this.application.held = 0; let plannedSets = this.#sets.size;
|
|
100
|
+
for (const target of o.targets) {
|
|
101
|
+
const current = await this.#checked(() => o.candidateHistory(target)); target.manifests = new Map(current.manifests);
|
|
102
|
+
target.eligible = new Set([...current.manifests.values()].filter(item => item.eligible).map(item => item.row.id)); this.application.proofComplete += target.eligible.size;
|
|
103
|
+
const initial = this.#boundView.realms[realmKey(target)] ?? null, inputs: NonNullable<PublicPruneInputOptions["input"]>[] = [];
|
|
104
|
+
for (const record of [...current.manifests.values()].sort((a,b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) {
|
|
105
|
+
if (!record.eligible || record.row.generation === target.generation) continue;
|
|
106
|
+
const witnessHash = digest(canonicalJSON(record.value)); if (target.registered.some(item => item.witnessHash === witnessHash)) continue;
|
|
107
|
+
if (++plannedSets > PUBLIC_PRUNE_MAX_SETS) throw new PublicPruneInputError("resource");
|
|
108
|
+
// A pure candidate is not publication evidence. Foreign anchors must still be in actual selected remote history.
|
|
109
|
+
if (!target.remote) refuse("foreign prune has no selected public source");
|
|
110
|
+
await this.#checked(() => o.assertAncestor(record.commit, target.remote!, target));
|
|
111
|
+
const original = await this.#checked(() => o.history(record.commit, target)), anchor = original.manifests.get(record.path);
|
|
112
|
+
if (!anchor || !anchor.eligible || anchor.blob !== record.blob || canonicalJSON(anchor.value) !== canonicalJSON(record.value)) refuse("foreign prune anchor lacks exact audited public evidence");
|
|
113
|
+
const acquired = await this.#checked(() => collectPublicPruneGitGraph({ commit: record.commit, manifestPath: record.path, manifestBlob: record.blob, realm: target.realm, ...(target.store.profile.publicationFormat ? {publicationFormat:target.store.profile.publicationFormat} : {}), readObject: object => this.#readObject(object) }));
|
|
114
|
+
const objects: { oid: string; bytes: Uint8Array }[] = [];
|
|
115
|
+
for (const item of acquired) {
|
|
116
|
+
const prior = this.#graphs.get(item.oid); if (prior && !Buffer.from(prior).equals(item.bytes)) refuse("conflicting Git graph bytes");
|
|
117
|
+
const bytes = prior ?? item.bytes; this.#graphs.set(item.oid, bytes);
|
|
118
|
+
const cached = this.#objects.get(item.oid); if (!cached) refuse("Git graph acquisition receipt unavailable");
|
|
119
|
+
this.#objects.set(item.oid, { type: cached.type, body: bytes.subarray(bytes.indexOf(0) + 1) });
|
|
120
|
+
if (o.retainGraphObject) await this.#checked(() => o.retainGraphObject!(item.oid, bytes));
|
|
121
|
+
objects.push({ oid: item.oid, bytes });
|
|
122
|
+
}
|
|
123
|
+
inputs.push({ witnessHash, source: { remoteHash: digest(o.url), ref: shardSyncRef(target.realm), commit: record.commit, manifestPath: record.path, manifestBlob: record.blob }, objects });
|
|
124
|
+
}
|
|
125
|
+
target.batch = undefined;
|
|
126
|
+
if (inputs.length) {
|
|
127
|
+
const binding = { realm: target.realm, receivingGeneration: target.generation, selectorRevision: o.getRevision(), inputSetHash: initial, ...(target.store.profile.publicationFormat ? {publicationFormat:target.store.profile.publicationFormat} : {}) };
|
|
128
|
+
target.batch = preparePublicPruneInputBatch({ dir: target.store.dir, binding, current: () => { o.check(); return { ...binding, selectorRevision: o.getRevision() }; }, inputs }, target.mutation!);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
await this.#checked(() => shardPrepareHolds(o.targets, o.root, o.url, (target, receipt) => this.#proveHoldSource(target, receipt)));
|
|
132
|
+
this.application.held = o.targets.reduce((sum, target) => sum + target.holds.length, 0); this.application.phase = "prepared";
|
|
133
|
+
}
|
|
134
|
+
async install(): Promise<void> {
|
|
135
|
+
const o = this.#options; o.check(); if (!this.#boundView) refuse("public-prune application preflight is missing");
|
|
136
|
+
const realms = { ...this.#boundView.realms };
|
|
137
|
+
for (const target of o.targets) if (target.batch) {
|
|
138
|
+
const installed = persistPublicPruneInputBatch(target.batch, { mutation: target.mutation!, observe: () => o.check() }); realms[realmKey(target)] = installed.inputSetHash;
|
|
139
|
+
if (o.phase) await this.#checked(() => o.phase!("inputs-installed"));
|
|
140
|
+
}
|
|
141
|
+
const view: PublicPruneViewV1 = { ...this.#boundView, realms };
|
|
142
|
+
if (canonicalJSON(view) !== canonicalJSON(this.#boundView)) {
|
|
143
|
+
for (const target of o.targets) shardSelectorRetention(target, true, true);
|
|
144
|
+
const result = await advancePublicPruneView({ memoryDir: o.home, view, expectedRevision: o.getRevision(), mutations: [o.homeMutation, ...o.targets.map(target => target.mutation!)] });
|
|
145
|
+
o.setRevision(result.revision); this.#boundView = view;
|
|
146
|
+
for (const target of o.targets) target.store.refresh();
|
|
147
|
+
await o.afterAdvance();
|
|
148
|
+
if (o.phase) await this.#checked(() => o.phase!("view-committed"));
|
|
149
|
+
}
|
|
150
|
+
this.application.projectionActive = o.targets.reduce((sum, target) => sum + [...target.eligible].filter(id => (target.rows.get(id) as PruneRow).generation !== target.generation).length, 0);
|
|
151
|
+
this.application.phase = "active"; o.check();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { PLACEMENT_DESCRIPTOR_SCHEMA, validatePlacementDescriptor } from "./sync-placement-format.ts";
|
|
2
|
+
import { PLACEMENT_STAGE1_REFUSAL } from "./sync-placement-guard.ts";
|
|
1
3
|
import { createHash } from "node:crypto";
|
|
2
4
|
import { legacyEntries } from "./bridge.ts";
|
|
3
5
|
import { canonicalJSON, validGeneration } from "./generation.ts";
|
|
@@ -50,10 +52,23 @@ function decodeMetadata(bytes: Uint8Array, label: string): Record<string, unknow
|
|
|
50
52
|
}
|
|
51
53
|
export function decodeShardDescriptor(bytes: Uint8Array, realm: ShardSyncRealm): "v1" | "v2" {
|
|
52
54
|
const label = shardSyncLabel(realm), value = decodeMetadata(bytes, label);
|
|
55
|
+
if (value.schema === PLACEMENT_DESCRIPTOR_SCHEMA) {
|
|
56
|
+
try { validatePlacementDescriptor(value, realm); } catch { throw new ShardSyncFormatError(label, "invalid placement descriptor"); }
|
|
57
|
+
throw new ShardSyncFormatError(label, PLACEMENT_STAGE1_REFUSAL);
|
|
58
|
+
}
|
|
53
59
|
const version = value.schema === SHARD_SYNC_SCHEMA ? "v1" : value.schema === SHARD_SYNC_V2_SCHEMA ? "v2" : undefined;
|
|
54
60
|
if (!version || canonicalJSON(value) + "\n" !== shardSyncDescriptor(realm, version)) throw new ShardSyncFormatError(label, "descriptor realm or schema mismatch");
|
|
55
61
|
return version;
|
|
56
62
|
}
|
|
63
|
+
/** Known wire metadata for activated placement audit/proof only. Legacy transports retain their v3 refusal. */
|
|
64
|
+
export function decodePlacementTransportDescriptor(bytes: Uint8Array, realm: ShardSyncRealm): "v1" | "v2" | "v3" {
|
|
65
|
+
const value = decodeMetadata(bytes, shardSyncLabel(realm));
|
|
66
|
+
if (value.schema === PLACEMENT_DESCRIPTOR_SCHEMA) {
|
|
67
|
+
try { validatePlacementDescriptor(value, realm); } catch { throw new ShardSyncFormatError(shardSyncLabel(realm), "invalid placement descriptor"); }
|
|
68
|
+
return "v3";
|
|
69
|
+
}
|
|
70
|
+
return decodeShardDescriptor(bytes, realm);
|
|
71
|
+
}
|
|
57
72
|
export function isShardManifestPath(path: string): boolean { return MANIFEST.exec(path)?.[0] === path; }
|
|
58
73
|
/** Shape and lineage classification belong to Layer 1 after complete pack acquisition. */
|
|
59
74
|
export function decodeShardManifest(path: string, bytes: Uint8Array): unknown {
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
/** Bounded selected-path acquisition only. Transport admission and receiving binding stay with the caller. */
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { PUBLIC_PRUNE_GRAPH_MAX_BYTES, PUBLIC_PRUNE_GRAPH_MAX_OBJECTS, PUBLIC_PRUNE_INPUT_MAX_BYTES, PublicPruneInputError, type PublicPruneObject } from "./public-prune-inputs.ts";
|
|
4
|
-
import {
|
|
4
|
+
import { decodePlacementTransportDescriptor, decodeShardManifest, SHARD_PACK_MAX_BYTES, type ShardSyncRealm } from "./shard-sync-format.ts";
|
|
5
5
|
import { PUBLIC_PRUNE_MAX_PACKS, PUBLIC_PRUNE_MAX_WITNESSES, verifyPublicPruneWitness } from "./public-prune-witness.ts";
|
|
6
6
|
|
|
7
7
|
type ObjectType = "commit" | "tree" | "blob";
|
|
8
8
|
export interface PublicPruneGitGraphOptions {
|
|
9
|
+
publicationFormat?: "agent-mem.sync-placement/v1";
|
|
9
10
|
commit: string; manifestPath: string; manifestBlob: string; realm: ShardSyncRealm;
|
|
10
11
|
readObject: (oid: string) => Promise<{ type: ObjectType; body: Uint8Array }>;
|
|
11
12
|
}
|
|
@@ -13,7 +14,7 @@ const hex = (v: unknown, n: number): v is string => typeof v === "string" && v.l
|
|
|
13
14
|
const object = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v);
|
|
14
15
|
function guard(ok: unknown, resource = false): asserts ok { if (!ok) throw new PublicPruneInputError(resource ? "resource" : "unavailable"); }
|
|
15
16
|
interface TreeEntry { mode: "40000" | "100644"; oid: string }
|
|
16
|
-
function
|
|
17
|
+
export function parsePublicPruneGitTree(body: Buffer): Map<string, TreeEntry> {
|
|
17
18
|
const entries = new Map<string, TreeEntry>();
|
|
18
19
|
for (let offset = 0; offset < body.length;) {
|
|
19
20
|
const space = body.indexOf(32, offset); guard(space > offset);
|
|
@@ -59,12 +60,13 @@ export async function collectPublicPruneGitGraph(options: PublicPruneGitGraphOpt
|
|
|
59
60
|
let oid = root;
|
|
60
61
|
for (let i = 0; i < parts.length; i++) {
|
|
61
62
|
let entries = trees.get(oid);
|
|
62
|
-
if (!entries) { entries =
|
|
63
|
+
if (!entries) { entries = parsePublicPruneGitTree((await get(oid, "tree")).body); trees.set(oid, entries); }
|
|
63
64
|
const entry = entries.get(parts[i]!); guard(entry && entry.mode === (i === parts.length - 1 ? "100644" : "40000")); oid = entry.oid;
|
|
64
65
|
}
|
|
65
66
|
const value = await get(oid, "blob"); return { oid, body: value.body };
|
|
66
67
|
};
|
|
67
|
-
|
|
68
|
+
const version = decodePlacementTransportDescriptor((await path("shard.json")).body, realm);
|
|
69
|
+
guard(version === "v2" || version === "v3" && options.publicationFormat === "agent-mem.sync-placement/v1");
|
|
68
70
|
const manifest = await path(manifestPath); guard(manifest.oid === manifestBlob);
|
|
69
71
|
const raw = decodeShardManifest(manifestPath, manifest.body);
|
|
70
72
|
guard(object(raw) && object(raw.pruneOccurrence) && Array.isArray(raw.witnesses));
|