@staix/agent-mem 0.20.3 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +30 -3
- package/package.json +2 -2
- package/src/cli.ts +36 -3
- package/src/commands-table.ts +3 -0
- package/src/hook-spool.ts +45 -3
- package/src/index.ts +31 -2
- package/src/memory/bridge.ts +66 -18
- package/src/memory/candidates.ts +71 -9
- package/src/memory/commands.ts +3 -2
- package/src/memory/generation-index.ts +95 -18
- package/src/memory/generation-sync.ts +3 -0
- package/src/memory/generation.ts +33 -15
- package/src/memory/platform.ts +38 -3
- package/src/memory/policy.ts +28 -15
- package/src/memory/prune-publisher-lineage.ts +50 -12
- package/src/memory/public-prune-activation.ts +4 -4
- package/src/memory/public-prune-advance.ts +11 -5
- package/src/memory/public-prune-inputs.ts +17 -16
- package/src/memory/shard-sync-applications.ts +153 -0
- package/src/memory/shard-sync-format.ts +15 -0
- package/src/memory/shard-sync-graph.ts +6 -4
- package/src/memory/shard-sync-history.ts +238 -0
- package/src/memory/shard-sync.ts +66 -151
- package/src/memory/shard.ts +32 -6
- package/src/memory/store.ts +14 -1
- package/src/memory/sync-binding.ts +63 -0
- package/src/memory/sync-cadence.ts +2 -1
- package/src/memory/sync-placement-abandon.ts +142 -0
- package/src/memory/sync-placement-activation-locks.ts +28 -0
- package/src/memory/sync-placement-activation-path.ts +2 -0
- package/src/memory/sync-placement-activation-proof.ts +68 -0
- package/src/memory/sync-placement-activation-storage.ts +167 -0
- package/src/memory/sync-placement-activation.ts +185 -0
- package/src/memory/sync-placement-active.ts +75 -0
- package/src/memory/sync-placement-cache.ts +32 -0
- package/src/memory/sync-placement-capture.ts +373 -0
- package/src/memory/sync-placement-dependencies.ts +255 -0
- package/src/memory/sync-placement-effective-writers.ts +84 -0
- package/src/memory/sync-placement-format.ts +182 -0
- package/src/memory/sync-placement-git.ts +95 -0
- package/src/memory/sync-placement-guard.ts +104 -0
- package/src/memory/sync-placement-inspection-evidence.ts +257 -0
- package/src/memory/sync-placement-inspection-pending.ts +66 -0
- package/src/memory/sync-placement-inspection.ts +240 -0
- package/src/memory/sync-placement-journal-format.ts +114 -0
- package/src/memory/sync-placement-journal-storage.ts +557 -0
- package/src/memory/sync-placement-journal.ts +160 -0
- package/src/memory/sync-placement-materialize.ts +149 -0
- package/src/memory/sync-placement-plan.ts +171 -0
- package/src/memory/sync-placement-prerequisites.ts +50 -0
- package/src/memory/sync-placement-projection.ts +79 -0
- package/src/memory/sync-placement-publication-plan.ts +189 -0
- package/src/memory/sync-placement-publication-storage.ts +87 -0
- package/src/memory/sync-placement-transport.ts +500 -0
- package/src/memory/sync-placement-writer-history.ts +278 -0
- package/src/memory/sync-placement-writer-renewal.ts +179 -0
- package/src/memory/sync-placement-writer-revision-format.ts +122 -0
- package/src/memory/sync-placement-writer-roots.ts +167 -0
- package/src/memory/sync-placement-writers.ts +135 -0
- package/src/memory/sync-policy-inspection.ts +126 -0
- package/src/memory/sync-screen.ts +128 -33
- package/src/memory/sync.ts +19 -64
- package/src/setup.ts +101 -14
- package/src/sync-placement-writer-context.ts +136 -0
- package/src/ui/inbox-cache.ts +219 -0
- package/src/ui/inbox-worker-version.ts +14 -0
- package/src/ui/inbox-worker.ts +32 -0
- package/src/ui/inbox.ts +281 -0
- package/src/ui/panel.js +7 -3
- package/src/ui/plugin.ts +26 -53
package/src/memory/candidates.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { assertFormatFloor } from "./formats.ts";
|
|
2
2
|
import { Database } from "bun:sqlite";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import { chmodSync, mkdirSync } from "node:fs";
|
|
4
|
+
import { chmodSync, lstatSync, mkdirSync } from "node:fs";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { CANDIDATE_SHOWN_GRACE_MS, hasShownColumn } from "./candidate-pending.ts";
|
|
7
7
|
import { oneLine } from "./note-add.ts";
|
|
@@ -130,6 +130,72 @@ function retryLocked(work: () => void): void {
|
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
type CandidateRow = { id: string; payload: string; at: number; status: CandidateStatus; note_id: string | null };
|
|
134
|
+
/** Canonical persisted decoding shared by the writer and its read-only inbox port. */
|
|
135
|
+
function decodeCandidate(row: CandidateRow, unknownKind: () => void): Candidate | undefined {
|
|
136
|
+
const input = JSON.parse(row.payload) as CandidateInput;
|
|
137
|
+
if (input && typeof input === "object" && !KINDS.includes(input.kind)) { unknownKind(); return undefined; }
|
|
138
|
+
validate(input);
|
|
139
|
+
if (hash({ schema: "agent-mem.candidate/v1", input }) !== row.id) throw new Error("candidate content hash mismatch");
|
|
140
|
+
return { id: row.id, createdAt: new Date(row.at).toISOString(), status: row.status, input, acceptedNoteId: row.note_id };
|
|
141
|
+
}
|
|
142
|
+
const CANDIDATE_EXPIRED_SQL = "at <= ? AND (status IN ('accepted', 'rejected') OR (shown_at IS NOT NULL AND shown_at <= ?))";
|
|
143
|
+
const CANDIDATE_PRIORITY_SQL = "CASE WHEN status IN ('pending', 'accepting') AND shown_at IS NULL THEN 2 WHEN status IN ('pending', 'accepting') THEN 1 ELSE 0 END DESC, at DESC";
|
|
144
|
+
export interface PendingCandidateReader { projects(): string[]; list(projects: readonly string[]): Candidate[]; nextExpiry(): number | null }
|
|
145
|
+
/** Internal inbox read port. It neither creates/migrates a database nor prunes or stamps any row.
|
|
146
|
+
* The port is valid only during the synchronous callback. No immutable fallback may hide current WAL frames. */
|
|
147
|
+
export function withPendingCandidates<T>(memoryDir: string, run: (reader: PendingCandidateReader) => T): T {
|
|
148
|
+
assertFormatFloor(memoryDir, "candidates");
|
|
149
|
+
const file = join(memoryDir, "candidates.db"), stat = lstatSync(file, { throwIfNoEntry: false });
|
|
150
|
+
if (!stat) {
|
|
151
|
+
let closed = false;
|
|
152
|
+
const assertOpen = () => { if (closed) throw new Error("pending candidate reader is closed"); };
|
|
153
|
+
try { return run({ projects() { assertOpen(); return []; }, list(projects) {
|
|
154
|
+
assertOpen(); if (!Array.isArray(projects) || projects.some(project => !text(project, 400))) throw new Error("invalid candidate project scope"); return [];
|
|
155
|
+
}, nextExpiry() { assertOpen(); return null; } }); } finally { closed = true; }
|
|
156
|
+
}
|
|
157
|
+
if (!stat.isFile() || stat.isSymbolicLink()) throw new Error("pending candidates unavailable");
|
|
158
|
+
const db = new Database(file, { readonly: true }); let closed = false;
|
|
159
|
+
try {
|
|
160
|
+
const shown = hasShownColumn(db), now = Date.now();
|
|
161
|
+
const live = (scope: boolean) => `WITH raw AS (SELECT rowid AS candidate_rowid, id, project, kind, payload, at, status, note_id, ${shown ? "shown_at" : "NULL AS shown_at"} FROM candidates${scope ? " WHERE project IN (SELECT value FROM json_each(?))" : ""}),
|
|
162
|
+
live AS (SELECT * FROM raw WHERE NOT (${CANDIDATE_EXPIRED_SQL})),
|
|
163
|
+
retained AS (SELECT *, row_number() OVER (PARTITION BY project ORDER BY ${CANDIDATE_PRIORITY_SQL}, candidate_rowid DESC) AS retention_rank FROM live)`;
|
|
164
|
+
const assertOpen = () => { if (closed) throw new Error("pending candidate reader is closed"); };
|
|
165
|
+
return run({
|
|
166
|
+
nextExpiry() {
|
|
167
|
+
assertOpen(); if (!shown) return null;
|
|
168
|
+
const row = db.query(`${live(false)} SELECT min(expiry) AS expiry FROM (
|
|
169
|
+
SELECT max(at + ?, shown_at + ?) AS expiry FROM retained WHERE retention_rank <= ? AND status = 'pending'
|
|
170
|
+
AND shown_at IS NOT NULL AND kind IN (SELECT value FROM json_each(?))) WHERE expiry > ?`)
|
|
171
|
+
.get(now - CANDIDATE_RETENTION.ageMs, now - CANDIDATE_SHOWN_GRACE_MS, CANDIDATE_RETENTION.ageMs,
|
|
172
|
+
CANDIDATE_SHOWN_GRACE_MS, CANDIDATE_RETENTION.count, JSON.stringify(KINDS), now) as { expiry: number | null };
|
|
173
|
+
if (row.expiry !== null && (!Number.isFinite(row.expiry) || row.expiry <= now)) throw new Error("pending candidate expiry unavailable");
|
|
174
|
+
return row.expiry;
|
|
175
|
+
},
|
|
176
|
+
projects() {
|
|
177
|
+
assertOpen();
|
|
178
|
+
const rows = db.query(`${live(false)} SELECT DISTINCT project FROM retained WHERE retention_rank <= ? AND status = 'pending' AND kind IN (SELECT value FROM json_each(?)) ORDER BY project`)
|
|
179
|
+
.all(now - CANDIDATE_RETENTION.ageMs, now - CANDIDATE_SHOWN_GRACE_MS, CANDIDATE_RETENTION.count, JSON.stringify(KINDS)) as { project: string }[];
|
|
180
|
+
if (rows.some(row => !text(row.project, 400))) throw new Error("pending candidate project unavailable");
|
|
181
|
+
return rows.map(row => row.project);
|
|
182
|
+
},
|
|
183
|
+
list(projects) {
|
|
184
|
+
assertOpen();
|
|
185
|
+
if (!Array.isArray(projects) || projects.some(project => !text(project, 400))) throw new Error("invalid candidate project scope");
|
|
186
|
+
if (!projects.length) return [];
|
|
187
|
+
const rows = db.query(`${live(true)} SELECT * FROM retained WHERE retention_rank <= ? AND status = 'pending' AND kind IN (SELECT value FROM json_each(?)) ORDER BY at DESC, candidate_rowid DESC LIMIT ?`)
|
|
188
|
+
.all(JSON.stringify([...new Set(projects)]), now - CANDIDATE_RETENTION.ageMs, now - CANDIDATE_SHOWN_GRACE_MS, CANDIDATE_RETENTION.count, JSON.stringify(KINDS), CANDIDATE_RETENTION.count) as (CandidateRow & { project: string })[];
|
|
189
|
+
return rows.flatMap(row => {
|
|
190
|
+
const candidate = decodeCandidate(row, () => {});
|
|
191
|
+
if (candidate && candidate.input.project !== row.project) throw new Error("candidate project binding mismatch");
|
|
192
|
+
return candidate ? [candidate] : [];
|
|
193
|
+
});
|
|
194
|
+
},
|
|
195
|
+
});
|
|
196
|
+
} finally { closed = true; db.close(); }
|
|
197
|
+
}
|
|
198
|
+
|
|
133
199
|
/** The queue holds the cap's worth of candidates nobody has seen. Named so a caller (the MCP tool) can map it. */
|
|
134
200
|
export class CandidateQueueFull extends Error {
|
|
135
201
|
readonly code = "candidate_queue_full";
|
|
@@ -184,20 +250,16 @@ export class CandidateStore {
|
|
|
184
250
|
this.retainConsolidationReviews();
|
|
185
251
|
// Candidates (AC4, no candidate expires unshown): by age only a decided one, or a pending one that was shown and has had its grace period, goes. By count three tiers decide who is
|
|
186
252
|
// kept: unshown pending first, then shown pending, then decided, so a cap eviction takes decided rows before a shown one (which is owed its grace) and shown ones before an unseen one.
|
|
187
|
-
this.db.query(
|
|
188
|
-
this.db.query(`DELETE FROM candidates WHERE rowid IN (SELECT rowid FROM (SELECT rowid, row_number() OVER (PARTITION BY project ORDER BY
|
|
253
|
+
this.db.query(`DELETE FROM candidates WHERE ${CANDIDATE_EXPIRED_SQL}`).run(now - CANDIDATE_RETENTION.ageMs, now - CANDIDATE_SHOWN_GRACE_MS);
|
|
254
|
+
this.db.query(`DELETE FROM candidates WHERE rowid IN (SELECT rowid FROM (SELECT rowid, row_number() OVER (PARTITION BY project ORDER BY ${CANDIDATE_PRIORITY_SQL}, rowid DESC) AS rank FROM candidates) WHERE rank > ?)`).run(CANDIDATE_RETENTION.count);
|
|
189
255
|
this.db.query("DELETE FROM counters WHERE at <= ?").run(now - COUNTER_RETENTION.ageMs);
|
|
190
256
|
this.db.query("DELETE FROM counters WHERE rowid IN (SELECT rowid FROM (SELECT rowid, row_number() OVER (PARTITION BY project ORDER BY at DESC, rowid DESC) AS rank FROM counters) WHERE rank > ?)").run(COUNTER_RETENTION.count);
|
|
191
257
|
}).immediate();
|
|
192
258
|
}
|
|
193
259
|
/** #199: rows of a kind this version does not know (a newer writer's) are skipped and counted here, never thrown. */
|
|
194
260
|
skippedUnknownKinds = 0;
|
|
195
|
-
private decode(row:
|
|
196
|
-
|
|
197
|
-
if (input && typeof input === "object" && !KINDS.includes(input.kind)) { this.skippedUnknownKinds++; return undefined; }
|
|
198
|
-
validate(input);
|
|
199
|
-
if (hash({ schema: "agent-mem.candidate/v1", input }) !== row.id) throw new Error("candidate content hash mismatch");
|
|
200
|
-
return { id: row.id, createdAt: new Date(row.at).toISOString(), status: row.status, input, acceptedNoteId: row.note_id };
|
|
261
|
+
private decode(row: CandidateRow): Candidate | undefined {
|
|
262
|
+
return decodeCandidate(row, () => { this.skippedUnknownKinds++; });
|
|
201
263
|
}
|
|
202
264
|
put(rawInput: CandidateInput, verdict: CandidateVerdict): string {
|
|
203
265
|
// `<private>` spans are never stored: dropped before the input is validated, hashed (so the id) or written.
|
package/src/memory/commands.ts
CHANGED
|
@@ -206,11 +206,12 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
206
206
|
const r = await syncMemory(store, { ...opts.sync, ...(remote ? { remote } : {}) });
|
|
207
207
|
if (json) return JSON.stringify(r, null, 2);
|
|
208
208
|
if (!r.ok) fail(`${names.memory} sync: ${r.error}`);
|
|
209
|
+
const placement = r.placementPublication, placementText = placement ? `; placement ${placement.outcome}, ${placement.exported} exported, ${placement.withheld} withheld this attempt` : "";
|
|
209
210
|
if (r.pruneApplication) {
|
|
210
211
|
const a = r.pruneApplication;
|
|
211
|
-
return `synced${r.pushed ? " and pushed" : ""}: ${r.entries} long-term entries transported; ${a.proofComplete} prune proofs complete, ${a.projectionActive ?? "pending"} foreign projections active, ${a.nativeApplied ?? "pending"} native prunes applied, ${a.held} held${r.warning ? `\nnote: ${r.warning}` : ""}`;
|
|
212
|
+
return `synced${r.pushed ? " and pushed" : ""}: ${r.entries} long-term entries transported; ${a.proofComplete} prune proofs complete, ${a.projectionActive ?? "pending"} foreign projections active, ${a.nativeApplied ?? "pending"} native prunes applied, ${a.held} held${placementText}${r.warning ? `\nnote: ${r.warning}` : ""}`;
|
|
212
213
|
}
|
|
213
|
-
return `synced${r.pushed ? " and pushed" : ""}: ${r.entries} long-term entries visible here${r.warning ? `\nnote: ${r.warning}` : ""}`;
|
|
214
|
+
return `synced${r.pushed ? " and pushed" : ""}: ${r.entries} long-term entries visible here${placementText}${r.warning ? `\nnote: ${r.warning}` : ""}`;
|
|
214
215
|
}
|
|
215
216
|
case "dedupe": {
|
|
216
217
|
const apply = rest.includes("--apply");
|
|
@@ -6,8 +6,8 @@ import type { Database } from "bun:sqlite";
|
|
|
6
6
|
import { createHash } from "node:crypto";
|
|
7
7
|
import { closeSync, constants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, unlinkSync, writeSync } from "node:fs";
|
|
8
8
|
import { join } from "node:path";
|
|
9
|
-
import { canonicalJSON, hashJSON, fsyncDirectory, readProfileJSON, selectorRevision, writeProfileJSON, type BoundProfile, validateVaultLinkRow, type VaultLink, rebindPublicPruneProfile } from "./generation.ts";
|
|
10
|
-
import { legacyEntries, legacyHead, readLegacySnapshot, SegmentReader, type LegacyCheckpoint, type LegacySegment } from "./bridge.ts";
|
|
9
|
+
import { canonicalJSON, hashJSON, fsyncDirectory, readProfileJSON, selectorRevision, writeProfileJSON, type BoundProfile, generationDescriptor, readSelector, validateVaultLinkRow, type VaultLink, rebindPublicPruneProfile } from "./generation.ts";
|
|
10
|
+
import { checkedEntry, completeLines, legacyEntries, legacyHead, readLegacySnapshot, SegmentReader, SegmentReadBudgetError, type LegacyCheckpoint, type LegacySegment } from "./bridge.ts";
|
|
11
11
|
import { reducePruneRows, validatePruneRow, type PrunePlan, type PruneRow, type PruneSnapshot, type PruneSourceRow, type PruneStorePort } from "./prune.ts";
|
|
12
12
|
import { PROMOTED_KINDS, type Entry } from "./store.ts";
|
|
13
13
|
|
|
@@ -89,33 +89,33 @@ export class GenerationIndex {
|
|
|
89
89
|
if (!(db.query("PRAGMA table_info(generation_segments)").all() as { name: string }[]).some(row => row.name === "stamp")) db.run("ALTER TABLE generation_segments ADD COLUMN stamp TEXT");
|
|
90
90
|
db.run("CREATE TABLE IF NOT EXISTS generation_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
|
|
91
91
|
}
|
|
92
|
-
private ownSegments(reader: SegmentReader = this.reader): LegacySegment[] {
|
|
92
|
+
private ownSegments(reader: SegmentReader = this.reader, full = this.fullCapture): LegacySegment[] {
|
|
93
93
|
const prior = this.options.db.query("SELECT source,path,offset,hash,commit_id,stamp FROM generation_segments WHERE source='generation'").all() as (LegacyCheckpoint & { stamp: string | null })[];
|
|
94
94
|
return ["log", "local"].flatMap(folder => {
|
|
95
95
|
const dir = join(this.profile.dir, folder);
|
|
96
96
|
if (lstatSync(dir).isSymbolicLink() || !lstatSync(dir).isDirectory()) return fail();
|
|
97
|
-
return
|
|
97
|
+
return reader.listFiles(dir).filter(name => name.endsWith(".jsonl")).map(name => {
|
|
98
98
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*\.jsonl$/.test(name)) return fail();
|
|
99
99
|
const path = `${folder}/${name}`, checkpoint = prior.find(row => row.path === path);
|
|
100
|
-
const data = reader.read(join(dir, name), checkpoint ? { ...checkpoint, stamp: checkpoint.stamp ?? undefined } : undefined,
|
|
100
|
+
const data = reader.read(join(dir, name), checkpoint ? { ...checkpoint, stamp: checkpoint.stamp ?? undefined } : undefined, full);
|
|
101
101
|
return { ...data, source: "generation", path, local: folder === "local" };
|
|
102
102
|
});
|
|
103
103
|
});
|
|
104
104
|
}
|
|
105
105
|
/** Authoring handles cannot read or append a partially resumed split; migration index bootstraps use captured limits. */
|
|
106
|
-
private shardFrontier(): string | undefined {
|
|
106
|
+
private shardFrontier(readOnly = false): string | undefined {
|
|
107
107
|
if (!this.profile.sharded || this.options.limits !== undefined) return undefined;
|
|
108
|
-
for (let attempt = 0; attempt < 6; attempt++) {
|
|
109
|
-
rebindPublicPruneProfile(this.profile);
|
|
108
|
+
for (let attempt = 0; attempt < (readOnly ? 1 : 6); attempt++) {
|
|
109
|
+
if (!readOnly) rebindPublicPruneProfile(this.profile);
|
|
110
110
|
const value = readProfileJSON(join(this.profile.home, "shards.json")) as { schema?: unknown; phase?: unknown; generation?: unknown; selectorRevision?: unknown } | undefined;
|
|
111
111
|
if (selectorRevision(this.profile.home) !== this.profile.selectorRevision) { if (this.profile.foreignPruneFormat) continue; return fail(); }
|
|
112
|
-
if (!value || value.schema !== (this.profile.foreignPruneFormat ? SHARDED_V2_SCHEMA : SHARDED_SCHEMA) || value.phase !== "active" || value.generation !== this.profile.generation || value.selectorRevision !== this.profile.selectorRevision) return fail();
|
|
112
|
+
if (!value || value.schema !== (this.profile.publicationFormat ? "agent-mem.shards/v3" : this.profile.foreignPruneFormat ? SHARDED_V2_SCHEMA : SHARDED_SCHEMA) || value.phase !== "active" || value.generation !== this.profile.generation || value.selectorRevision !== this.profile.selectorRevision) return fail();
|
|
113
113
|
return hashJSON(value);
|
|
114
114
|
}
|
|
115
115
|
return fail();
|
|
116
116
|
}
|
|
117
|
-
private capture(reader: SegmentReader = this.reader): { segments: LegacySegment[]; commit?: string } {
|
|
118
|
-
const shardBefore = this.shardFrontier();
|
|
117
|
+
private capture(reader: SegmentReader = this.reader, readOnly = false): { segments: LegacySegment[]; commit?: string } {
|
|
118
|
+
const shardBefore = this.shardFrontier(readOnly), full = this.fullCapture || readOnly;
|
|
119
119
|
const db = this.options.db;
|
|
120
120
|
const previous = db.query("SELECT source,path,offset,hash,commit_id,stamp FROM generation_segments WHERE source != 'generation'").all() as (LegacyCheckpoint & { commit_id: string | null; stamp: string | null })[];
|
|
121
121
|
const initial = readProfileJSON(join(this.profile.dir, "migration-sources.json")) as { sources?: LegacyCheckpoint[]; legacyCommit?: string } | undefined;
|
|
@@ -124,19 +124,23 @@ export class GenerationIndex {
|
|
|
124
124
|
const checkpoints = [...bySource.values()];
|
|
125
125
|
const persisted = new Set(previous.map(row => `${row.source}:${row.path}`));
|
|
126
126
|
const completeFrontier = checkpoints.every(row => persisted.has(`${row.source}:${row.path}`));
|
|
127
|
-
const
|
|
127
|
+
const indexedCommit = (db.query("SELECT value FROM generation_state WHERE key='legacyCommit'").get() as { value: string } | null)?.value;
|
|
128
|
+
const prior = indexedCommit ?? initial?.legacyCommit;
|
|
128
129
|
const legacyHome = this.profile.legacyHome ?? this.profile.home;
|
|
129
130
|
const generation = this.profile.sharded ? undefined : this.profile.generation;
|
|
130
|
-
const commit = this.options.commit ?? (generation ? legacyHead(legacyHome, generation) : undefined);
|
|
131
|
+
const commit = this.options.commit ?? (generation ? legacyHead(legacyHome, generation, readOnly ? reader.sourceByteLimit : undefined, readOnly ? reader.sourceFileLimit : undefined) : undefined);
|
|
132
|
+
// ponytail: read-only inspection accepts the indexed archived frontier only, without an ancestry walk.
|
|
133
|
+
// Reopen after ordinary refresh, or add a bounded read-only ancestry witness to support a newer frontier.
|
|
134
|
+
if (readOnly && (commit !== indexedCommit || checkpoints.some(row => row.source === "legacy-main" && row.commit !== commit))) return fail();
|
|
131
135
|
const legacy = this.profile.sharded
|
|
132
|
-
? readShardLegacySnapshot(this.profile, { checkpoints, ...(this.options.limits ? { limits: this.options.limits } : {}), reader, incremental: !
|
|
133
|
-
: readLegacySnapshot(legacyHome, { ...(generation ? { generation } : {}), ...(commit ? { commit } : {}), checkpoints, ...(generation && prior ? { previousCommit: prior } : {}), ...(this.options.limits ? { limits: this.options.limits } : {}), reader, incremental: !
|
|
134
|
-
const own = this.ownSegments(reader);
|
|
136
|
+
? readShardLegacySnapshot(this.profile, { checkpoints, ...(this.options.limits ? { limits: this.options.limits } : {}), reader, incremental: !full && !this.options.limits && completeFrontier })
|
|
137
|
+
: readLegacySnapshot(legacyHome, { ...(generation ? { generation } : {}), ...(commit ? { commit } : {}), checkpoints, ...(generation && prior ? { previousCommit: prior } : {}), ...(this.options.limits ? { limits: this.options.limits } : {}), reader, incremental: !full && !this.options.limits && completeFrontier });
|
|
138
|
+
const own = this.ownSegments(reader, full);
|
|
135
139
|
for (const old of db.query("SELECT path,offset,hash FROM generation_segments WHERE source='generation'").all() as { path: string; offset: number; hash: string }[]) {
|
|
136
140
|
const segment = own.find(s => s.path === old.path);
|
|
137
141
|
if (!segment || segment.offset < old.offset) fail();
|
|
138
142
|
}
|
|
139
|
-
if (this.shardFrontier() !== shardBefore) { if (this.profile.foreignPruneFormat) throw new PublicPruneViewChanged(); return fail(); }
|
|
143
|
+
if (this.shardFrontier(readOnly) !== shardBefore) { if (this.profile.foreignPruneFormat) throw new PublicPruneViewChanged(); return fail(); }
|
|
140
144
|
return { segments: [...legacy, ...own], ...(commit ? { commit } : {}) };
|
|
141
145
|
}
|
|
142
146
|
private index(captured: { segments: LegacySegment[]; commit?: string }): void {
|
|
@@ -193,7 +197,7 @@ export class GenerationIndex {
|
|
|
193
197
|
return;
|
|
194
198
|
}
|
|
195
199
|
if (!this.foreignCache || this.foreignCache.set !== set) this.foreignCache = { set, load: createPublicPruneReplayCache({ dir: profile.dir,
|
|
196
|
-
realm: profile.shardUser ? { scope: "user" } : { scope: "project", key: profile.shardKey! }, receivingGeneration: profile.generation, inputSetHash: set }) };
|
|
200
|
+
realm: profile.shardUser ? { scope: "user" } : { scope: "project", key: profile.shardKey! }, receivingGeneration: profile.generation, inputSetHash: set, ...(profile.publicationFormat ? {publicationFormat:profile.publicationFormat} : {}) }) };
|
|
197
201
|
const proof = this.foreignCache.load();
|
|
198
202
|
if (validateRows || this.foreignStamp !== set) for (const item of db.query("SELECT id,line FROM generation_rows").all() as {id:string;line:string}[]) {
|
|
199
203
|
const expected = proof.rowHashes.get(item.id); if (expected && hashJSON(JSON.parse(item.line)) !== expected) return fail();
|
|
@@ -253,6 +257,79 @@ export class GenerationIndex {
|
|
|
253
257
|
this.replayProvenance = false;
|
|
254
258
|
this.retainedTail = input.segments.some(segment => segment.tailBytes > 0);
|
|
255
259
|
}
|
|
260
|
+
/** Inspect canonical outbound sources without refreshing/rebinding the derived index.
|
|
261
|
+
* The returned check must run before the caller declares the whole opened-target inspection complete.
|
|
262
|
+
* Each capture accepts <=8MiB source bytes, <=20000 physical nonblank rows and <=16MiB segment reads.
|
|
263
|
+
* A fresh final capture verifies the same source again (<=32MiB segment verification I/O over both captures).
|
|
264
|
+
*/
|
|
265
|
+
inspectOutboundRows(visit: (row: Record<string, unknown>) => void, limits: { maxBytes: number; maxRows: number }): () => void {
|
|
266
|
+
const unavailable = (budget = false): never => { throw Object.assign(new Error("outbound generation input unavailable"), { code: budget ? "input-bound" : "input-unavailable" }); };
|
|
267
|
+
if (!Number.isSafeInteger(limits.maxBytes) || limits.maxBytes < 0 || limits.maxBytes > 8 * 1024 * 1024
|
|
268
|
+
|| !Number.isSafeInteger(limits.maxRows) || limits.maxRows < 0 || limits.maxRows > 20_000) return unavailable(true);
|
|
269
|
+
if (this.profile.kind !== "generation" || !this.profile.generation || this.options.limits !== undefined) return unavailable();
|
|
270
|
+
const profile = canonicalJSON(this.profile);
|
|
271
|
+
const stamp = (file: string): string | undefined => {
|
|
272
|
+
const stat = lstatSync(file, { bigint: true, throwIfNoEntry: false });
|
|
273
|
+
if (!stat) return undefined;
|
|
274
|
+
if (!stat.isFile() || stat.isSymbolicLink()) return fail();
|
|
275
|
+
return canonicalJSON([String(stat.dev), String(stat.ino), String(stat.size), String(stat.mtimeNs), String(stat.ctimeNs)]);
|
|
276
|
+
};
|
|
277
|
+
const epoch = (): string => {
|
|
278
|
+
if (canonicalJSON(this.profile) !== profile) return fail();
|
|
279
|
+
const selected = readSelector(this.profile.home);
|
|
280
|
+
if (selected.revision !== this.profile.selectorRevision || !this.profile.sharded && (selected.selector?.kind !== "generation" || selected.selector.generation !== this.profile.generation)) return fail();
|
|
281
|
+
const shards = this.shardFrontier(true);
|
|
282
|
+
if (!this.profile.sharded && readProfileJSON(join(this.profile.home, "shards.json")) !== undefined) return fail();
|
|
283
|
+
if (canonicalJSON(readProfileJSON(join(this.profile.dir, "generation.json"))) !== canonicalJSON(generationDescriptor(this.profile.generation!))) return fail();
|
|
284
|
+
const paths = [join(this.profile.home, "profile.json"), join(this.profile.home, "shards.json"), join(this.profile.dir, "generation.json"),
|
|
285
|
+
join(this.profile.dir, "migration-sources.json"), join(this.profile.dir, "project.json"), join(this.profile.legacyHome ?? this.profile.home, "machine.json"),
|
|
286
|
+
...(this.profile.legacyMainHome ? [join(this.profile.legacyMainHome, "machine.json")] : [])];
|
|
287
|
+
return canonicalJSON({ shards, metadata: [...new Set(paths)].map(file => {
|
|
288
|
+
const before = stamp(file), value = readProfileJSON(file);
|
|
289
|
+
if (stamp(file) !== before) return fail();
|
|
290
|
+
return [file, before, hashJSON(value ?? null)];
|
|
291
|
+
}) });
|
|
292
|
+
};
|
|
293
|
+
const protectedRead = <T>(read: () => T): T => {
|
|
294
|
+
try { return read(); } catch (error) { return unavailable(error instanceof SegmentReadBudgetError); }
|
|
295
|
+
};
|
|
296
|
+
const before = protectedRead(epoch);
|
|
297
|
+
const capture = () => {
|
|
298
|
+
if (epoch() !== before) return fail();
|
|
299
|
+
const reader = new SegmentReader(2 * limits.maxBytes, { maxSourceBytes: limits.maxBytes, maxRows: limits.maxRows });
|
|
300
|
+
const input = this.capture(reader, true);
|
|
301
|
+
if (input.segments.some(segment => segment.tailBytes !== 0) || epoch() !== before) return fail();
|
|
302
|
+
const identity = canonicalJSON({ commit: input.commit, segments: input.segments.map(({ bytes: _bytes, local, ...segment }) => ({ ...segment, local })), metrics: reader.inspectionMetrics() });
|
|
303
|
+
return { input, identity };
|
|
304
|
+
};
|
|
305
|
+
const captured = protectedRead(capture);
|
|
306
|
+
const rows = protectedRead(() => {
|
|
307
|
+
const rows = new Map<string, PruneSourceRow>();
|
|
308
|
+
for (const segment of captured.input.segments) for (const bytes of completeLines(segment.bytes)) {
|
|
309
|
+
if (!bytes.length) continue;
|
|
310
|
+
const line = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
|
311
|
+
if (!line.trim()) { if (segment.source === "generation") return fail(); continue; }
|
|
312
|
+
const parsed: unknown = JSON.parse(line);
|
|
313
|
+
const op = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as { op?: unknown }).op : undefined;
|
|
314
|
+
if (segment.source === "generation" && op === "prune") {
|
|
315
|
+
const row = validatePruneRow(parsed, this.profile.generation!, segment.local ? "local" : "synced");
|
|
316
|
+
if (segment.path !== `${segment.local ? "local" : "log"}/${row.machine}.jsonl`) return fail();
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
if (segment.source === "generation" && op === "vault_link") {
|
|
320
|
+
const row = validateVaultLinkRow(parsed, this.profile.generation!, segment.local ? "local" : "synced");
|
|
321
|
+
if (segment.path !== `local/${row.machine}.jsonl`) return fail();
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
const row = checkedEntry(parsed);
|
|
325
|
+
rows.set(row.id, coalesceGenerationSource(rows.get(row.id), row, segment));
|
|
326
|
+
}
|
|
327
|
+
return [...rows.values()].filter(row => row.placement === "synced" && (row.own || row.legacy)).sort((left, right) => left.row.id.localeCompare(right.row.id));
|
|
328
|
+
});
|
|
329
|
+
for (const source of rows) visit(structuredClone(source.row) as unknown as Record<string, unknown>);
|
|
330
|
+
return () => { protectedRead(() => { if (capture().identity !== captured.identity) return fail(); }); };
|
|
331
|
+
}
|
|
332
|
+
|
|
256
333
|
vaultWitnesses(ids: readonly string[]): Map<string, string> {
|
|
257
334
|
if (ids.length > 5000) throw new Error("vault scan budget exceeded");
|
|
258
335
|
return new Map((this.options.db.query("SELECT id,line FROM generation_rows WHERE id IN (SELECT value FROM json_each(?))").all(JSON.stringify(ids)) as { id: string; line: string }[])
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { placementRuntimeRefusal } from "./sync-placement-guard.ts";
|
|
1
2
|
import { randomBytes } from "node:crypto";
|
|
2
3
|
import { closeSync, existsSync, lstatSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { join } from "node:path";
|
|
@@ -74,6 +75,8 @@ function validatePublic(bytes: string, generation: string, path: string, checkPr
|
|
|
74
75
|
|
|
75
76
|
/** Separate generation repository, fixed refs and a one-way legacy cache; failures are bounded result receipts. */
|
|
76
77
|
export async function syncGenerationMemory(store: MemoryStore, opts: SyncOptions = {}): Promise<SyncResult> {
|
|
78
|
+
const placementRefusal = placementRuntimeRefusal(store.profile.home, store.profile);
|
|
79
|
+
if (placementRefusal) return { ok: false, pushed: false, entries: 0, error: placementRefusal };
|
|
77
80
|
let pushed = false, releaseMutation: (() => void) | undefined, releaseSync: (() => void) | undefined;
|
|
78
81
|
const result = (error: string): SyncResult => ({ ok: false, pushed, entries: 0, error });
|
|
79
82
|
const profile = store.profile, generation = profile.generation, dir = store.dir;
|
package/src/memory/generation.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { validatePlacementProfile, type PlacementProfile } from "./sync-placement-format.ts";
|
|
2
|
+
import { placementSelectorRefusal, SyncPlacementRuntimeError } from "./sync-placement-guard.ts";
|
|
1
3
|
import { assertNoPublicPruneIntent, readPublicPruneView, waitForPublicPruneWriter, PUBLIC_PRUNE_INTENT } from "./public-prune-view.ts";
|
|
2
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
5
|
import { types } from "node:util";
|
|
@@ -23,12 +25,15 @@ export interface BoundProfile {
|
|
|
23
25
|
/** Public legacy witnesses stay distinct from same-machine private bridge input. */
|
|
24
26
|
legacyMainHome?: string;
|
|
25
27
|
sharded?: boolean;
|
|
28
|
+
publicationFormat?: "agent-mem.sync-placement/v1";
|
|
29
|
+
placementHead?: string;
|
|
26
30
|
shardKey?: string;
|
|
27
31
|
shardUser?: boolean;
|
|
28
32
|
}
|
|
29
33
|
export type ProfileInput = "shared-v1" | { generation: string };
|
|
30
34
|
export type ProfileSelector = { schema: "agent-mem.profile/v1"; kind: "shared-v1" | "generation"; generation?: string }
|
|
31
|
-
| { schema: "agent-mem.profile/v2"; kind: "generation"; generation: string; foreignPruneFormat: "agent-mem.public-prune-witness/v1"; pruneInputSetHash: string }
|
|
35
|
+
| { schema: "agent-mem.profile/v2"; kind: "generation"; generation: string; foreignPruneFormat: "agent-mem.public-prune-witness/v1"; pruneInputSetHash: string }
|
|
36
|
+
| PlacementProfile;
|
|
32
37
|
export interface GenerationDescriptor { schema: "agent-mem.generation/v1"; generation: string; legacySchema: "agent-hub.memory/v1" }
|
|
33
38
|
export interface GenerationSeal { schema: "agent-mem.generation-seal/v1"; generation: string; descriptorHash: string; sourceDigest: string }
|
|
34
39
|
export const PROFILE_LOCK_TIMEOUT_MS = 3000;
|
|
@@ -51,23 +56,33 @@ export function readProfileJSON(path: string, limit = LOCAL_METADATA_MAX_BYTES):
|
|
|
51
56
|
} catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; return unavailable(); }
|
|
52
57
|
}
|
|
53
58
|
function keys(value: object, allowed: string[]): boolean { return Object.keys(value).every(key => allowed.includes(key)); }
|
|
59
|
+
/** Pure legacy marker shape, shared with the stage1 placement preflight; no selector effects. */
|
|
60
|
+
export function legacyProfileMarkerShape(value: unknown): value is ProfileSelector | undefined {
|
|
61
|
+
if (value === undefined) return true;
|
|
62
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
63
|
+
const selector = value as ProfileSelector;
|
|
64
|
+
if (selector.schema === "agent-mem.profile/v2") return Object.keys(selector).length === 5 && keys(selector, ["schema", "kind", "generation", "foreignPruneFormat", "pruneInputSetHash"])
|
|
65
|
+
&& selector.kind === "generation" && validGeneration(selector.generation) && selector.generation.length === 32 && selector.foreignPruneFormat === "agent-mem.public-prune-witness/v1"
|
|
66
|
+
&& typeof selector.pruneInputSetHash === "string" && selector.pruneInputSetHash.length === 64 && /^[a-f0-9]{64}$/.test(selector.pruneInputSetHash);
|
|
67
|
+
return selector.schema === "agent-mem.profile/v1" && keys(selector, ["schema", "kind", "generation"])
|
|
68
|
+
&& ["shared-v1", "generation"].includes(selector.kind) && (selector.kind === "generation" ? validGeneration(selector.generation) : selector.generation === undefined);
|
|
69
|
+
}
|
|
54
70
|
function selectorSnapshot(memoryDir: string, deadline: number): { selector?: ProfileSelector; revision: string } {
|
|
71
|
+
const initialRefusal = placementSelectorRefusal(memoryDir);
|
|
72
|
+
if (initialRefusal) throw new SyncPlacementRuntimeError(initialRefusal);
|
|
55
73
|
assertNoPublicPruneIntent(memoryDir, deadline);
|
|
74
|
+
// Legacy v2 readers still choose their snapshot after the existing writer wait, never its earlier preflight pair.
|
|
75
|
+
const refusal = placementSelectorRefusal(memoryDir);
|
|
76
|
+
if (refusal) throw new SyncPlacementRuntimeError(refusal);
|
|
56
77
|
const value = readProfileJSON(join(memoryDir, "profile.json"));
|
|
57
78
|
if (value === undefined) return { revision: "absent" };
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
if (selector.schema === "agent-mem.profile/v2") {
|
|
61
|
-
if (Object.keys(selector).length !== 5 || !keys(selector, ["schema", "kind", "generation", "foreignPruneFormat", "pruneInputSetHash"])
|
|
62
|
-
|| selector.kind !== "generation" || !validGeneration(selector.generation) || selector.generation.length !== 32
|
|
63
|
-
|| selector.foreignPruneFormat !== "agent-mem.public-prune-witness/v1" || typeof selector.pruneInputSetHash !== "string"
|
|
64
|
-
|| selector.pruneInputSetHash.length !== 64 || !/^[a-f0-9]+$/.test(selector.pruneInputSetHash)) return unavailable();
|
|
79
|
+
const selector: ProfileSelector = (value as {schema?:unknown})?.schema === "agent-mem.profile/v3"
|
|
80
|
+
? validatePlacementProfile(value) : legacyProfileMarkerShape(value) ? value! : unavailable();
|
|
81
|
+
if (selector.schema === "agent-mem.profile/v2" || selector.schema === "agent-mem.profile/v3") {
|
|
65
82
|
const shard = readProfileJSON(join(memoryDir, "shards.json")) as Record<string, unknown> | undefined;
|
|
66
|
-
if (!shard || shard.schema !== "agent-mem.shards/v2" || shard.phase !== "active" || shard.generation !== selector.generation || shard.selectorRevision !== hashJSON(selector)) return unavailable();
|
|
83
|
+
if (!shard || shard.schema !== (selector.schema === "agent-mem.profile/v3" ? "agent-mem.shards/v3" : "agent-mem.shards/v2") || shard.phase !== "active" || shard.generation !== selector.generation || shard.selectorRevision !== hashJSON(selector)) return unavailable();
|
|
67
84
|
readPublicPruneView(memoryDir, selector.pruneInputSetHash, selector.generation, deadline);
|
|
68
|
-
} else if (
|
|
69
|
-
|| !["shared-v1", "generation"].includes(selector.kind) || (selector.kind === "generation" ? !validGeneration(selector.generation) : selector.generation !== undefined)) return unavailable();
|
|
70
|
-
else if ((readProfileJSON(join(memoryDir, "shards.json")) as {schema?:unknown}|undefined)?.schema === "agent-mem.shards/v2") return unavailable();
|
|
85
|
+
} else if ((readProfileJSON(join(memoryDir, "shards.json")) as {schema?:unknown}|undefined)?.schema === "agent-mem.shards/v2") return unavailable();
|
|
71
86
|
return { selector, revision: hashJSON(selector) };
|
|
72
87
|
}
|
|
73
88
|
|
|
@@ -81,6 +96,7 @@ export function readSelector(memoryDir: string, absoluteDeadline?: number): { se
|
|
|
81
96
|
const current = readProfileJSON(join(memoryDir, "profile.json"));
|
|
82
97
|
if (result.revision === (current === undefined ? "absent" : hashJSON(current))) return result;
|
|
83
98
|
} catch (error) {
|
|
99
|
+
if (error instanceof SyncPlacementRuntimeError) throw error;
|
|
84
100
|
if (error instanceof Error && error.message === "public prune view unavailable") throw error; // A spent wait budget is never restarted.
|
|
85
101
|
if (!lstatSync(join(memoryDir, PUBLIC_PRUNE_INTENT), { throwIfNoEntry: false }) && !lstatSync(join(memoryDir, "profile.lock"), { throwIfNoEntry: false })) throw error;
|
|
86
102
|
waitForPublicPruneWriter(memoryDir, deadline);
|
|
@@ -97,8 +113,10 @@ export function rebindPublicPruneProfile(profile: BoundProfile): void {
|
|
|
97
113
|
for (let attempt = 0; attempt < 6; attempt++) {
|
|
98
114
|
const current = readSelector(profile.home), selector = current.selector;
|
|
99
115
|
if (current.revision === profile.selectorRevision) return;
|
|
116
|
+
if (selector?.schema !== "agent-mem.profile/v2" && selector?.schema !== "agent-mem.profile/v3") return unavailable();
|
|
100
117
|
if (!profile.sharded || profile.kind !== "generation" || !profile.generation || !profile.pruneInputSetHash
|
|
101
|
-
|| selector
|
|
118
|
+
|| (profile.publicationFormat ? selector.schema !== "agent-mem.profile/v3" || selector.publicationFormat !== profile.publicationFormat || selector.placementHead !== profile.placementHead : selector.schema !== "agent-mem.profile/v2")
|
|
119
|
+
|| selector.generation !== profile.generation || selector.foreignPruneFormat !== profile.foreignPruneFormat) return unavailable();
|
|
102
120
|
const previous = readPublicPruneView(profile.home, profile.pruneInputSetHash, profile.generation);
|
|
103
121
|
const next = readPublicPruneView(profile.home, selector.pruneInputSetHash, profile.generation);
|
|
104
122
|
if (!Object.keys(previous.realms).every(key => Object.hasOwn(next.realms, key))) return unavailable();
|
|
@@ -143,9 +161,9 @@ export function validateGeneration(memoryDir: string, generation: string): strin
|
|
|
143
161
|
export function resolveProfile(memoryDir: string, explicit?: ProfileInput): BoundProfile {
|
|
144
162
|
const home = existsSync(memoryDir) ? realPath(memoryDir) : canonicalPath(memoryDir), selected = readSelector(home);
|
|
145
163
|
if (explicit !== undefined && explicit !== "shared-v1" && (!explicit || typeof explicit !== "object" || !keys(explicit, ["generation"]) || !validGeneration(explicit.generation))) return unavailable();
|
|
146
|
-
if (selected.selector?.schema === "agent-mem.profile/v2" && explicit !== undefined) return unavailable();
|
|
164
|
+
if ((selected.selector?.schema === "agent-mem.profile/v2" || selected.selector?.schema === "agent-mem.profile/v3") && explicit !== undefined) return unavailable();
|
|
147
165
|
const generation = explicit === "shared-v1" ? undefined : explicit?.generation ?? (selected.selector?.kind === "generation" ? selected.selector.generation : undefined);
|
|
148
|
-
return generation ? { kind: "generation", home, dir: validateGeneration(home, generation), generation, selectorRevision: selected.revision, ...(selected.selector?.schema === "agent-mem.profile/v2" ? { foreignPruneFormat: selected.selector.foreignPruneFormat, pruneInputSetHash: selected.selector.pruneInputSetHash } : {}) }
|
|
166
|
+
return generation ? { kind: "generation", home, dir: validateGeneration(home, generation), generation, selectorRevision: selected.revision, ...((selected.selector?.schema === "agent-mem.profile/v2" || selected.selector?.schema === "agent-mem.profile/v3") ? { foreignPruneFormat: selected.selector.foreignPruneFormat, pruneInputSetHash: selected.selector.pruneInputSetHash, ...(selected.selector.schema === "agent-mem.profile/v3" ? { publicationFormat: selected.selector.publicationFormat, placementHead: selected.selector.placementHead } : {}) } : {}) }
|
|
149
167
|
: { kind: "shared-v1", home, dir: home, selectorRevision: selected.revision };
|
|
150
168
|
}
|
|
151
169
|
|
package/src/memory/platform.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Single-source platform helpers. Hub, local and Pi modules re-export these functions unchanged.
|
|
2
2
|
import { realpathSync } from "node:fs";
|
|
3
3
|
import { basename, dirname, join, resolve, sep } from "node:path";
|
|
4
|
-
import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from "node:child_process";
|
|
4
|
+
import type { SpawnSyncOptions, SpawnSyncOptionsWithBufferEncoding, SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from "node:child_process";
|
|
5
5
|
import { createRequire } from "node:module";
|
|
6
6
|
const require = createRequire(import.meta.url);
|
|
7
7
|
let processes: typeof import("node:child_process") | undefined;
|
|
@@ -91,8 +91,14 @@ export const memoryGitArgv = (args: string[], opts: { credentials?: boolean } =
|
|
|
91
91
|
["git", "--no-pager", ...SAFE_CONFIG.filter((c) => !opts.credentials || !CREDENTIAL_KEYS.has(c)).flatMap((c) => ["-c", c]), ...args];
|
|
92
92
|
|
|
93
93
|
/** The caller's env plus the no-prompt switch; existing keys (childEnv, GIT_OPTIONAL_LOCKS=0) pass through. */
|
|
94
|
-
export function memoryGitEnv(env: NodeJS.ProcessEnv = process.env, options: { repositoryIsolation?: boolean } = {}): NodeJS.ProcessEnv {
|
|
94
|
+
export function memoryGitEnv(env: NodeJS.ProcessEnv = process.env, options: { repositoryIsolation?: boolean; readOnlyDiagnostic?: boolean } = {}): NodeJS.ProcessEnv {
|
|
95
95
|
const result: NodeJS.ProcessEnv = { ...env, GIT_TERMINAL_PROMPT: "0" };
|
|
96
|
+
if (options.readOnlyDiagnostic) {
|
|
97
|
+
// Fixed diagnostic commands must not inherit tracing/test/config switches that can write or lazily fetch.
|
|
98
|
+
// This is an environment projection, not permission to execute arbitrary Git commands.
|
|
99
|
+
for (const key of Object.keys(result)) if (key.startsWith("GIT_")) delete result[key];
|
|
100
|
+
Object.assign(result, { GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1", GIT_OPTIONAL_LOCKS: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null" });
|
|
101
|
+
}
|
|
96
102
|
if (options.repositoryIsolation) {
|
|
97
103
|
for (const key of Object.keys(result)) if (["GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_GRAFT_FILE", "GIT_REPLACE_REF_BASE", "GIT_CONFIG_COUNT", "GIT_CONFIG_PARAMETERS"].includes(key) || /^GIT_CONFIG_(KEY|VALUE)_/.test(key)) delete result[key];
|
|
98
104
|
result.GIT_NO_REPLACE_OBJECTS = "1";
|
|
@@ -101,7 +107,9 @@ export function memoryGitEnv(env: NodeJS.ProcessEnv = process.env, options: { re
|
|
|
101
107
|
}
|
|
102
108
|
|
|
103
109
|
/** spawnSync form of the same call: the env option is merged with memoryGitEnv, everything else passes through. */
|
|
104
|
-
export function memoryGitSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns<string
|
|
110
|
+
export function memoryGitSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns<string>;
|
|
111
|
+
export function memoryGitSync(args: string[], options: SpawnSyncOptionsWithBufferEncoding): SpawnSyncReturns<Buffer>;
|
|
112
|
+
export function memoryGitSync(args: string[], options: SpawnSyncOptions): SpawnSyncReturns<string | Buffer> {
|
|
105
113
|
const argv = memoryGitArgv(args);
|
|
106
114
|
const timeout = gitTimeoutContext?.getStore();
|
|
107
115
|
return childProcess().spawnSync(argv[0]!, argv.slice(1), { ...(timeout !== undefined ? { timeout, killSignal: "SIGKILL" as const } : {}), ...options, env: memoryGitEnv(options.env ?? process.env) });
|
|
@@ -134,8 +142,35 @@ export function processSignature(pid: number): string | undefined {
|
|
|
134
142
|
} catch { return undefined; }
|
|
135
143
|
}
|
|
136
144
|
|
|
145
|
+
/** Nonblocking counterpart of the exact same pinned ps identity contract, for request-serving hosts. */
|
|
146
|
+
function processFieldAsync(pid: number, fields: string): Promise<string | undefined> {
|
|
147
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) return Promise.resolve(undefined);
|
|
148
|
+
return new Promise(resolve => {
|
|
149
|
+
try {
|
|
150
|
+
childProcess().execFile("ps", ["-p", String(pid), "-o", fields], { encoding: "utf8", timeout: 500, killSignal: "SIGKILL", maxBuffer: 16 * 1024,
|
|
151
|
+
env: { ...process.env, LC_ALL: "C", TZ: "UTC" } }, (error, text) => resolve(error ? undefined : text.trim() || undefined));
|
|
152
|
+
} catch { resolve(undefined); }
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
export async function processSignatureAsync(pid: number): Promise<string | undefined> {
|
|
156
|
+
const text = await processFieldAsync(pid, "lstart=,comm=");
|
|
157
|
+
return text ? createHash("sha256").update(text).digest("hex") : undefined;
|
|
158
|
+
}
|
|
159
|
+
export async function processGroupIdAsync(pid: number): Promise<number | undefined> {
|
|
160
|
+
const text = await processFieldAsync(pid, "pgid=");
|
|
161
|
+
const group = text && /^[0-9]+$/.test(text) ? Number(text) : undefined;
|
|
162
|
+
return group !== undefined && Number.isSafeInteger(group) && group > 0 ? group : undefined;
|
|
163
|
+
}
|
|
164
|
+
|
|
137
165
|
export type Liveness = "live" | "gone" | "unknown";
|
|
138
166
|
|
|
167
|
+
/** Observation only, never authority to signal an old group. Zombies/unreadable groups are not gone. */
|
|
168
|
+
export function processGroupLiveness(group: unknown): Liveness {
|
|
169
|
+
if (typeof group !== "number" || !Number.isSafeInteger(group) || group <= 0) return "unknown";
|
|
170
|
+
try { process.kill(-group, 0); return "live"; }
|
|
171
|
+
catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH" ? "gone" : "unknown"; }
|
|
172
|
+
}
|
|
173
|
+
|
|
139
174
|
/**
|
|
140
175
|
* #226: the one ownership check behind every recorded pid (hub manifest, registry claim, recovery runner, manager
|
|
141
176
|
* owner, launcher records). With a recorded `signature` the identity decides: equal is live, different is gone (the
|
package/src/memory/policy.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { nodePath, pathCandidates } from "./project.ts";
|
|
|
6
6
|
|
|
7
7
|
/** True means the complete value is public. Hosts may narrow, never replace, standalone policy. */
|
|
8
8
|
export interface MemoryPolicy { screen(value: unknown): Promise<boolean> }
|
|
9
|
+
export type StandalonePolicyDecline = "secret" | "personal-data" | "deny-path" | "private-or-invalid";
|
|
9
10
|
/**
|
|
10
11
|
* The standalone policy. `screenStandalone` is its own part only (secrets, default PII, configured PII patterns and deny paths) as a synchronous
|
|
11
12
|
* answer, for callers that cannot await (the store's promotion screen); it never asks the host, so a caller with a host uses `screen`.
|
|
@@ -18,6 +19,11 @@ export interface StandalonePolicy extends MemoryPolicy {
|
|
|
18
19
|
*/
|
|
19
20
|
declinesPersonal(text: string): boolean;
|
|
20
21
|
}
|
|
22
|
+
/** Additive inspection surface; existing StandalonePolicy implementations need no new method. */
|
|
23
|
+
export interface InspectableStandalonePolicy extends StandalonePolicy {
|
|
24
|
+
/** Safe standalone classification only; no input, configured literal or host verdict is returned. */
|
|
25
|
+
inspectStandalone(value: unknown): { ok: true } | { ok: false; category: StandalonePolicyDecline };
|
|
26
|
+
}
|
|
21
27
|
export interface StandalonePolicyOptions {
|
|
22
28
|
denyPaths?: readonly string[];
|
|
23
29
|
/** Case-insensitive literal substrings, not executable regular expressions. */
|
|
@@ -153,7 +159,7 @@ function mayNeedPathCheck(text: string, probe: DefaultPathProbe | undefined, den
|
|
|
153
159
|
}
|
|
154
160
|
|
|
155
161
|
/** No filesystem, model, network or storage work occurs while constructing or screening this policy. */
|
|
156
|
-
export function createStandalonePolicy(options: StandalonePolicyOptions = {}):
|
|
162
|
+
export function createStandalonePolicy(options: StandalonePolicyOptions = {}): InspectableStandalonePolicy {
|
|
157
163
|
if (!options || typeof options !== "object" || types.isProxy(options) || Object.getPrototypeOf(options) !== Object.prototype || Object.getOwnPropertySymbols(options).length) throw new Error("invalid standalone policy options");
|
|
158
164
|
const descriptors = Object.getOwnPropertyDescriptors(options);
|
|
159
165
|
for (const [key, descriptor] of Object.entries(descriptors)) if (!["denyPaths", "piiPatterns", "host"].includes(key) || !("value" in descriptor)) throw new Error("invalid standalone policy options");
|
|
@@ -163,34 +169,34 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): S
|
|
|
163
169
|
if (host !== undefined && (!host || typeof host.screen !== "function")) throw new Error("invalid host policy");
|
|
164
170
|
// ponytail: retain at most 512 exact strings / 64 KiB per immutable policy configuration.
|
|
165
171
|
// Evict the batch when full; a measured need for better hit rates can upgrade this to an LRU.
|
|
166
|
-
const textChecks = new Map<string,
|
|
172
|
+
const textChecks = new Map<string, StandalonePolicyDecline | null>(); let textBytes = 0;
|
|
167
173
|
let defaultDenyRevision = "";
|
|
168
174
|
// ponytail: the negative probe supports at most 64 dense default entries and 256-character simple name sources.
|
|
169
175
|
// Everything else falls back; extend the grammar only with differential proof, never an approximate verdict.
|
|
170
176
|
let pathProbe: DefaultPathProbe | undefined;
|
|
171
177
|
let defaultsWerePlain = false;
|
|
172
|
-
const
|
|
178
|
+
const textDecline = (text: string): StandalonePolicyDecline | null => {
|
|
173
179
|
const cached = textChecks.get(text);
|
|
174
180
|
if (cached !== undefined) return cached;
|
|
175
181
|
const check = () => {
|
|
176
|
-
if (looksSecret(text)) return
|
|
182
|
+
if (looksSecret(text)) return "secret" as const;
|
|
177
183
|
const unquoted = text.replace(/["']/g, "");
|
|
178
|
-
if (unquoted !== text && looksSecret(unquoted)) return
|
|
184
|
+
if (unquoted !== text && looksSecret(unquoted)) return "secret" as const;
|
|
179
185
|
const normalized = fold(text);
|
|
180
|
-
if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return
|
|
181
|
-
if (!mayNeedPathCheck(normalized, pathProbe, deny)) return
|
|
186
|
+
if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return "personal-data" as const;
|
|
187
|
+
if (!mayNeedPathCheck(normalized, pathProbe, deny)) return null;
|
|
182
188
|
const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
|
|
183
189
|
const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
|
|
184
|
-
return
|
|
190
|
+
return [...new Set(paths)].some(path => path && isDenied(posix.normalize(path), deny)) ? "deny-path" as const : null;
|
|
185
191
|
};
|
|
186
|
-
const
|
|
192
|
+
const category = check(), bytes = Buffer.byteLength(text);
|
|
187
193
|
if (bytes <= 1024) {
|
|
188
194
|
if (textChecks.size >= 512 || textBytes + bytes > 64 * 1024) { textChecks.clear(); textBytes = 0; }
|
|
189
|
-
textChecks.set(text,
|
|
195
|
+
textChecks.set(text, category); textBytes += bytes;
|
|
190
196
|
}
|
|
191
|
-
return
|
|
197
|
+
return category;
|
|
192
198
|
};
|
|
193
|
-
const standalone = (value: unknown): { ok:
|
|
199
|
+
const standalone = (value: unknown): { ok: true; value: unknown } | { ok: false; category: StandalonePolicyDecline } => {
|
|
194
200
|
try {
|
|
195
201
|
// The exported default lists can be tightened by a host between calls.
|
|
196
202
|
// A getter can change behavior without changing its first serialized value: validate shape every time.
|
|
@@ -202,12 +208,15 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): S
|
|
|
202
208
|
else if (changed || !defaultsWerePlain) pathProbe = defaultPathProbe();
|
|
203
209
|
defaultsWerePlain = defaultsPlain;
|
|
204
210
|
const complete = snapshot(value);
|
|
205
|
-
if (!complete) return { ok: false };
|
|
211
|
+
if (!complete) return { ok: false, category: "private-or-invalid" };
|
|
206
212
|
// Only pure text checks are reused. Shape/bounds/private flags and the host's detached
|
|
207
213
|
// snapshot are fresh on every call; no row or host verdict is cached.
|
|
208
|
-
for (const text of complete.texts)
|
|
214
|
+
for (const text of complete.texts) {
|
|
215
|
+
const category = textDecline(text);
|
|
216
|
+
if (category !== null) return { ok: false, category };
|
|
217
|
+
}
|
|
209
218
|
return { ok: true, value: complete.value };
|
|
210
|
-
} catch { return { ok: false }; }
|
|
219
|
+
} catch { return { ok: false, category: "private-or-invalid" }; }
|
|
211
220
|
};
|
|
212
221
|
const personal = (text: string): boolean => {
|
|
213
222
|
const normalized = fold(text);
|
|
@@ -218,6 +227,10 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): S
|
|
|
218
227
|
};
|
|
219
228
|
return {
|
|
220
229
|
screenStandalone: value => standalone(value).ok,
|
|
230
|
+
inspectStandalone: value => {
|
|
231
|
+
const result = standalone(value);
|
|
232
|
+
return result.ok ? { ok: true } : { ok: false, category: result.category };
|
|
233
|
+
},
|
|
221
234
|
declinesPersonal: text => personal(text),
|
|
222
235
|
async screen(value: unknown): Promise<boolean> {
|
|
223
236
|
try {
|