@chendpoc/pi-memory 0.2.4 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -43
- package/dist/adapters/llm/standalone.js +1 -1
- package/dist/cli/init.js +20 -3
- package/dist/cli/parseArgs.d.ts +5 -2
- package/dist/cli/parseArgs.js +13 -0
- package/dist/cli/schedulerSync.d.ts +2 -0
- package/dist/cli/schedulerSync.js +26 -0
- package/dist/cli/status.d.ts +4 -40
- package/dist/cli/status.js +6 -230
- package/dist/cli/theme.d.ts +2 -0
- package/dist/cli/theme.js +8 -0
- package/dist/cli.js +5 -0
- package/dist/commands/status.js +2 -2
- package/dist/compact/parseMemoryExport.js +2 -1
- package/dist/compact/register.js +1 -1
- package/dist/compact/runSummary.js +1 -16
- package/dist/compact/subagentDelta.js +3 -5
- package/dist/consolidate/index.d.ts +1 -0
- package/dist/consolidate/index.js +1 -0
- package/dist/consolidate/mergeEntries.js +3 -3
- package/dist/consolidate/mergeMemoryEntries.d.ts +8 -0
- package/dist/consolidate/mergeMemoryEntries.js +23 -0
- package/dist/consolidate/mergePrompt.js +2 -2
- package/dist/consolidate/mergeWithLlm.js +2 -2
- package/dist/consolidate/runJob.d.ts +2 -2
- package/dist/consolidate/runJob.js +6 -12
- package/dist/consolidate/scheduler.d.ts +2 -2
- package/dist/consolidate/scheduler.js +1 -1
- package/dist/constants/env.d.ts +1 -0
- package/dist/constants/env.js +1 -0
- package/dist/constants/paths.d.ts +3 -1
- package/dist/constants/paths.js +6 -1
- package/dist/extension/createMemoryRuntime.d.ts +3 -0
- package/dist/extension/createMemoryRuntime.js +203 -0
- package/dist/extension/index.d.ts +2 -0
- package/dist/extension/index.js +1 -0
- package/dist/extension/lifecycle.d.ts +28 -0
- package/dist/extension/lifecycle.js +52 -0
- package/dist/extension/messageUtils.d.ts +4 -0
- package/dist/extension/messageUtils.js +18 -0
- package/dist/extension/types.d.ts +35 -0
- package/dist/extension/types.js +1 -0
- package/dist/index.d.ts +7 -9
- package/dist/index.js +7 -9
- package/dist/pi-extension.js +26 -236
- package/dist/preflight/episodic.js +13 -30
- package/dist/preflight/queryIntent.js +1 -1
- package/dist/preflight/render.js +1 -1
- package/dist/preflight/strip.d.ts +0 -1
- package/dist/preflight/strip.js +0 -24
- package/dist/redaction/index.d.ts +4 -0
- package/dist/redaction/index.js +3 -0
- package/dist/redaction/patterns/constants.d.ts +6 -0
- package/dist/redaction/patterns/constants.js +6 -0
- package/dist/redaction/patterns/crypto.d.ts +3 -0
- package/dist/redaction/patterns/crypto.js +17 -0
- package/dist/redaction/patterns/generic.d.ts +3 -0
- package/dist/redaction/patterns/generic.js +38 -0
- package/dist/redaction/patterns/index.d.ts +16 -0
- package/dist/redaction/patterns/index.js +23 -0
- package/dist/redaction/patterns/llm.d.ts +3 -0
- package/dist/redaction/patterns/llm.js +144 -0
- package/dist/redaction/patterns/platform.d.ts +3 -0
- package/dist/redaction/patterns/platform.js +87 -0
- package/dist/redaction/patterns/types.d.ts +18 -0
- package/dist/redaction/patterns/types.js +1 -0
- package/dist/redaction/redactText.d.ts +9 -0
- package/dist/redaction/redactText.js +31 -0
- package/dist/redaction/types.d.ts +19 -0
- package/dist/redaction/types.js +1 -0
- package/dist/redaction/utils.d.ts +28 -0
- package/dist/redaction/utils.js +106 -0
- package/dist/scheduler/index.d.ts +3 -0
- package/dist/scheduler/index.js +3 -0
- package/dist/scheduler/launchd.d.ts +14 -0
- package/dist/scheduler/launchd.js +69 -0
- package/dist/scheduler/launchdPlist.d.ts +14 -0
- package/dist/scheduler/launchdPlist.js +62 -0
- package/dist/scheduler/sync.d.ts +36 -0
- package/dist/scheduler/sync.js +79 -0
- package/dist/shutdown/enqueue.d.ts +1 -1
- package/dist/shutdown/enqueue.js +2 -5
- package/dist/shutdown/readQueue.js +1 -1
- package/dist/shutdown/runDrainJob.js +8 -37
- package/dist/shutdown/sessionReader.js +1 -14
- package/dist/sidecar/client.d.ts +6 -2
- package/dist/sidecar/client.js +49 -14
- package/dist/{preflight → sidecar}/queryCache.d.ts +1 -1
- package/dist/sidecar/reindexBridge.js +2 -2
- package/dist/sidecar/server/server.js +1 -1
- package/dist/sidecar/server/vec/chunkQuery.d.ts +4 -0
- package/dist/sidecar/server/vec/chunkQuery.js +46 -0
- package/dist/sidecar/server/vec/chunkReindex.d.ts +5 -0
- package/dist/sidecar/server/vec/chunkReindex.js +40 -0
- package/dist/sidecar/server/vec/embeddingCodec.d.ts +2 -0
- package/dist/sidecar/server/vec/embeddingCodec.js +6 -0
- package/dist/sidecar/server/vec/schema.d.ts +20 -0
- package/dist/sidecar/server/vec/schema.js +61 -0
- package/dist/sidecar/server/vec/store.d.ts +2 -13
- package/dist/sidecar/server/vec/store.js +12 -139
- package/dist/sidecar/sidecarManager.js +4 -58
- package/dist/sidecar/spawnLock.d.ts +2 -0
- package/dist/sidecar/spawnLock.js +57 -0
- package/dist/sidecar/syncIndex.d.ts +3 -0
- package/dist/sidecar/syncIndex.js +12 -0
- package/dist/sidecar/warmup.js +1 -1
- package/dist/status/copy.d.ts +2 -0
- package/dist/status/copy.js +2 -0
- package/dist/status/format.d.ts +7 -0
- package/dist/status/format.js +133 -0
- package/dist/status/gather.d.ts +2 -0
- package/dist/status/gather.js +88 -0
- package/dist/status/index.d.ts +4 -0
- package/dist/status/index.js +3 -0
- package/dist/status/types.d.ts +33 -0
- package/dist/status/types.js +1 -0
- package/dist/store/consolidatePort.d.ts +11 -0
- package/dist/store/consolidatePort.js +1 -0
- package/dist/store/index.d.ts +2 -0
- package/dist/store/index.js +1 -0
- package/dist/store/ingestEntries.d.ts +16 -0
- package/dist/store/ingestEntries.js +22 -0
- package/dist/{init/workspace.d.ts → store/initWorkspace.d.ts} +1 -1
- package/dist/{init/workspace.js → store/initWorkspace.js} +7 -5
- package/dist/store/listeners.d.ts +11 -0
- package/dist/store/listeners.js +27 -0
- package/dist/store/markdown/insert.d.ts +3 -0
- package/dist/store/markdown/insert.js +23 -0
- package/dist/store/memoryStore.d.ts +9 -22
- package/dist/store/memoryStore.js +71 -205
- package/dist/store/resolveEntries.d.ts +11 -0
- package/dist/store/resolveEntries.js +23 -0
- package/dist/store/types.d.ts +0 -1
- package/dist/store/writePath.d.ts +20 -0
- package/dist/store/writePath.js +123 -0
- package/dist/ui/memoryStatusWidget.d.ts +4 -8
- package/dist/ui/memoryStatusWidget.js +5 -17
- package/dist/utils/async.d.ts +11 -0
- package/dist/utils/async.js +24 -0
- package/dist/utils/index.d.ts +5 -1
- package/dist/utils/index.js +5 -1
- package/dist/{ipc/jsonlFramer.d.ts → utils/jsonl.d.ts} +1 -1
- package/dist/{ipc/jsonlFramer.js → utils/jsonl.js} +1 -1
- package/dist/utils/memory/index.d.ts +10 -0
- package/dist/utils/memory/index.js +43 -0
- package/dist/utils/paths.d.ts +4 -0
- package/dist/utils/paths.js +13 -3
- package/dist/utils/scheduler.d.ts +1 -1
- package/dist/utils/scheduler.js +6 -6
- package/dist/{preflight/session.d.ts → utils/session/index.d.ts} +1 -0
- package/dist/{preflight/session.js → utils/session/index.js} +5 -2
- package/doc/LAUNCH-KIT.md +229 -0
- package/doc/README-zh.md +445 -0
- package/doc/ROADMAP-zh.md +114 -0
- package/doc/ROADMAP.md +114 -0
- package/package.json +16 -4
- package/scripts/postinstall.mjs +11 -1
- package/templates/com.pi.memory.consolidate.plist.example +41 -0
- package/templates/consolidate.cmd.example +15 -0
- package/templates/crontab.example +14 -0
- package/templates/schtasks.example.txt +34 -0
- package/dist/consolidate/entryKey.d.ts +0 -5
- package/dist/consolidate/entryKey.js +0 -4
- /package/dist/{preflight → sidecar}/queryCache.js +0 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ParsedEntry, StoreMemoryEntry } from "./types.js";
|
|
2
|
+
export type MemoryStoreForIngest = {
|
|
3
|
+
listEntries(): Promise<ParsedEntry[]>;
|
|
4
|
+
appendMany(entries: StoreMemoryEntry[], opts?: {
|
|
5
|
+
mode?: "ifAbsent";
|
|
6
|
+
}): Promise<number>;
|
|
7
|
+
};
|
|
8
|
+
export type IngestMemoryExportResult = {
|
|
9
|
+
appended: number;
|
|
10
|
+
};
|
|
11
|
+
/** Parse Memory Export from summary text and append new facts to Ground Truth. */
|
|
12
|
+
export declare function ingestMemoryExport(opts: {
|
|
13
|
+
store: MemoryStoreForIngest;
|
|
14
|
+
summary: string;
|
|
15
|
+
isSubagent: boolean;
|
|
16
|
+
}): Promise<IngestMemoryExportResult>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { parseMemoryExport } from "../compact/parseMemoryExport.js";
|
|
2
|
+
import { filterCompactionDelta, shouldSkipSubagentCompactionIngest, } from "../compact/subagentDelta.js";
|
|
3
|
+
/** Parse Memory Export from summary text and append new facts to Ground Truth. */
|
|
4
|
+
export async function ingestMemoryExport(opts) {
|
|
5
|
+
const parsed = parseMemoryExport(opts.summary);
|
|
6
|
+
if (parsed.length === 0) {
|
|
7
|
+
return { appended: 0 };
|
|
8
|
+
}
|
|
9
|
+
let entries = parsed;
|
|
10
|
+
if (opts.isSubagent) {
|
|
11
|
+
const existing = await opts.store.listEntries();
|
|
12
|
+
entries = filterCompactionDelta(parsed, existing);
|
|
13
|
+
if (shouldSkipSubagentCompactionIngest(parsed, entries)) {
|
|
14
|
+
return { appended: 0 };
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
if (entries.length === 0) {
|
|
18
|
+
return { appended: 0 };
|
|
19
|
+
}
|
|
20
|
+
const appended = await opts.store.appendMany(entries, { mode: "ifAbsent" });
|
|
21
|
+
return { appended };
|
|
22
|
+
}
|
|
@@ -6,7 +6,7 @@ export type InitMemoryWorkspaceResult = {
|
|
|
6
6
|
reason?: "already_initialized";
|
|
7
7
|
};
|
|
8
8
|
/**
|
|
9
|
-
* Ensure the memory data directory exists and MEMORY.md follows the canonical template.
|
|
9
|
+
* Ensure the memory data directory exists, `logs/` is present, and MEMORY.md follows the canonical template.
|
|
10
10
|
* Never overwrites a non-empty MEMORY.md.
|
|
11
11
|
*/
|
|
12
12
|
export declare function initializeMemoryWorkspace(agentDir: string): Promise<InitMemoryWorkspaceResult>;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { DEFAULT_MEMORY_FILE } from "../constants/memory.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { PI_LOGS_SUBDIR } from "../constants/paths.js";
|
|
3
|
+
import { ensureDir, joinPath, readText } from "../utils/fs.js";
|
|
4
|
+
import { MarkdownMemoryBackend } from "./backend.js";
|
|
5
|
+
import { defaultMemoryTemplate } from "./markdown/template.js";
|
|
6
|
+
import { resolveAgentDir } from "./paths.js";
|
|
6
7
|
/**
|
|
7
|
-
* Ensure the memory data directory exists and MEMORY.md follows the canonical template.
|
|
8
|
+
* Ensure the memory data directory exists, `logs/` is present, and MEMORY.md follows the canonical template.
|
|
8
9
|
* Never overwrites a non-empty MEMORY.md.
|
|
9
10
|
*/
|
|
10
11
|
export async function initializeMemoryWorkspace(agentDir) {
|
|
@@ -12,6 +13,7 @@ export async function initializeMemoryWorkspace(agentDir) {
|
|
|
12
13
|
const memoryFile = joinPath(resolved, DEFAULT_MEMORY_FILE);
|
|
13
14
|
const backend = new MarkdownMemoryBackend(memoryFile);
|
|
14
15
|
await backend.ensureAgentDir();
|
|
16
|
+
await ensureDir(joinPath(resolved, PI_LOGS_SUBDIR));
|
|
15
17
|
const existing = await backend.readText(memoryFile);
|
|
16
18
|
if (existing.trim()) {
|
|
17
19
|
return {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type NotifyAfterWriteOptions = {
|
|
2
|
+
skipConsolidateCheck?: boolean;
|
|
3
|
+
};
|
|
4
|
+
export type StoreListeners = {
|
|
5
|
+
onSyncToSidecar(listener: () => void): () => void;
|
|
6
|
+
onConsolidateCheck(listener: () => void): () => void;
|
|
7
|
+
notifySyncToSidecar(): void;
|
|
8
|
+
notifyAfterWrite(opts?: NotifyAfterWriteOptions): void;
|
|
9
|
+
};
|
|
10
|
+
/** Sidecar sync + debounced consolidate-check subscriptions after Ground Truth writes. */
|
|
11
|
+
export declare function createStoreListeners(isConsolidating: () => boolean): StoreListeners;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** Sidecar sync + debounced consolidate-check subscriptions after Ground Truth writes. */
|
|
2
|
+
export function createStoreListeners(isConsolidating) {
|
|
3
|
+
const syncToSidecarListeners = new Set();
|
|
4
|
+
const consolidateCheckListeners = new Set();
|
|
5
|
+
return {
|
|
6
|
+
onSyncToSidecar(listener) {
|
|
7
|
+
syncToSidecarListeners.add(listener);
|
|
8
|
+
return () => syncToSidecarListeners.delete(listener);
|
|
9
|
+
},
|
|
10
|
+
onConsolidateCheck(listener) {
|
|
11
|
+
consolidateCheckListeners.add(listener);
|
|
12
|
+
return () => consolidateCheckListeners.delete(listener);
|
|
13
|
+
},
|
|
14
|
+
notifySyncToSidecar() {
|
|
15
|
+
for (const listener of syncToSidecarListeners)
|
|
16
|
+
listener();
|
|
17
|
+
},
|
|
18
|
+
notifyAfterWrite(opts) {
|
|
19
|
+
for (const listener of syncToSidecarListeners)
|
|
20
|
+
listener();
|
|
21
|
+
if (!opts?.skipConsolidateCheck && !isConsolidating()) {
|
|
22
|
+
for (const listener of consolidateCheckListeners)
|
|
23
|
+
listener();
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { formatEntryLine, formatSectionHeader } from "./format.js";
|
|
2
|
+
/** Insert one entry line under its section in MEMORY markdown. */
|
|
3
|
+
export function insertEntryIntoMarkdown(content, entry) {
|
|
4
|
+
const lines = content.split("\n");
|
|
5
|
+
const header = formatSectionHeader(entry.section);
|
|
6
|
+
const headerIdx = lines.findIndex((line) => line.trim() === header);
|
|
7
|
+
const line = formatEntryLine(entry);
|
|
8
|
+
if (headerIdx === -1) {
|
|
9
|
+
const trimmed = content.trimEnd();
|
|
10
|
+
return `${trimmed}\n\n${header}\n\n${line}\n`;
|
|
11
|
+
}
|
|
12
|
+
let insertAt = headerIdx + 1;
|
|
13
|
+
while (insertAt < lines.length && lines[insertAt]?.trim() === "")
|
|
14
|
+
insertAt++;
|
|
15
|
+
while (insertAt < lines.length) {
|
|
16
|
+
const current = lines[insertAt];
|
|
17
|
+
if (current.startsWith("## "))
|
|
18
|
+
break;
|
|
19
|
+
insertAt++;
|
|
20
|
+
}
|
|
21
|
+
const next = [...lines.slice(0, insertAt), line, ...lines.slice(insertAt)];
|
|
22
|
+
return `${next.join("\n").trimEnd()}\n`;
|
|
23
|
+
}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { type TimeInput } from "../utils/time.js";
|
|
2
|
-
import type {
|
|
3
|
-
|
|
2
|
+
import type { ConsolidateStoreAccess } from "./consolidatePort.js";
|
|
3
|
+
import type { IndexDocument, IntegrityReport, MemoryStats, MemoryStoreOptions, ParsedEntry, ResolvedMemory, StoreMemoryEntry } from "./types.js";
|
|
4
|
+
export declare class MemoryStore implements ConsolidateStoreAccess {
|
|
4
5
|
private readonly paths;
|
|
5
6
|
private readonly backend;
|
|
6
7
|
private readonly maxLines;
|
|
7
8
|
private readonly fallbackMaxChars;
|
|
8
|
-
private readonly
|
|
9
|
-
private readonly consolidateCheckListeners;
|
|
9
|
+
private readonly listeners;
|
|
10
10
|
private consolidating;
|
|
11
11
|
constructor(opts: MemoryStoreOptions);
|
|
12
|
+
private collectResolvedOpts;
|
|
13
|
+
private writePathDeps;
|
|
12
14
|
get agentDir(): string;
|
|
13
15
|
ensureInitialized(): Promise<void>;
|
|
14
16
|
isEmpty(): Promise<boolean>;
|
|
@@ -22,9 +24,8 @@ export declare class MemoryStore {
|
|
|
22
24
|
appendUser(entry: Omit<StoreMemoryEntry, "userAuthored">): Promise<void>;
|
|
23
25
|
appendMany(entries: StoreMemoryEntry[], opts?: {
|
|
24
26
|
mode?: "ifAbsent";
|
|
25
|
-
}): Promise<
|
|
27
|
+
}): Promise<number>;
|
|
26
28
|
appendIfAbsent(entry: StoreMemoryEntry): Promise<boolean>;
|
|
27
|
-
/** Fire-and-forget: parse Memory Export from compact summary → appendIfAbsent. */
|
|
28
29
|
appendFromCompaction(opts: {
|
|
29
30
|
compactionId: string;
|
|
30
31
|
summary: string;
|
|
@@ -38,28 +39,14 @@ export declare class MemoryStore {
|
|
|
38
39
|
}): Promise<void>;
|
|
39
40
|
rewrite(content: string): Promise<void>;
|
|
40
41
|
shouldConsolidate(at?: TimeInput, cronFired?: boolean): Promise<boolean>;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
onComplete?: () => void | Promise<void>;
|
|
44
|
-
}): void;
|
|
45
|
-
forceConsolidate(llm: LlmClient): Promise<void>;
|
|
42
|
+
isConsolidating(): boolean;
|
|
43
|
+
rewriteMemoryUnderLock(updateEntries: (entries: ParsedEntry[]) => Promise<ParsedEntry[]>): Promise<void>;
|
|
46
44
|
hasProcessedCompaction(compactionId: string): Promise<boolean>;
|
|
47
45
|
markCompactionProcessed(compactionId: string): Promise<void>;
|
|
48
46
|
verifyIntegrity(): Promise<IntegrityReport>;
|
|
49
|
-
/** Register a listener to sync MEMORY.md changes to the sidecar vector index. */
|
|
50
47
|
onSyncToSidecar(listener: () => void): () => void;
|
|
51
|
-
/** Register a listener invoked after writes to check shouldConsolidate. */
|
|
52
48
|
onConsolidateCheck(listener: () => void): () => void;
|
|
53
|
-
private notifyAfterWrite;
|
|
54
|
-
private notifySyncToSidecar;
|
|
55
|
-
private notifyConsolidateCheck;
|
|
56
|
-
private appendUnlocked;
|
|
57
|
-
private appendIfAbsentUnlocked;
|
|
58
|
-
private appendToOverflowUnlocked;
|
|
59
|
-
private insertEntryIntoMarkdown;
|
|
60
|
-
private rewriteEntriesUnlocked;
|
|
61
49
|
private readResolvedUnlocked;
|
|
62
|
-
private normalizeEntry;
|
|
63
50
|
private newEntryId;
|
|
64
51
|
private newAutoFilePath;
|
|
65
52
|
private readGcTimestamp;
|
|
@@ -1,26 +1,24 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
|
-
import {
|
|
3
|
-
import { filterCompactionDelta, shouldSkipSubagentCompactionIngest, } from "../compact/subagentDelta.js";
|
|
4
|
-
import { dedupeEntries } from "../consolidate/mergeEntries.js";
|
|
5
|
-
import { mergeEntriesWithLlm } from "../consolidate/mergeWithLlm.js";
|
|
6
|
-
import { MarkdownMemoryBackend } from "./backend.js";
|
|
7
|
-
import { AUTO_FILE_PREFIX, CONSOLIDATE_GC_INTERVAL_DAYS, CONSOLIDATE_OVERFLOW_FILE_THRESHOLD, DEFAULT_FALLBACK_MAX_CHARS, DEFAULT_MAX_LINES, DEFAULT_MEMORY_FILE, } from "../constants/memory.js";
|
|
8
|
-
import { countLines, formatEntryLine, formatSectionHeader } from "./markdown/format.js";
|
|
9
|
-
import { listOverflowPointers, parseMemoryMarkdown } from "./markdown/parse.js";
|
|
10
|
-
import { initializeMemoryWorkspace } from "../init/workspace.js";
|
|
2
|
+
import { CONSOLIDATE_GC_INTERVAL_DAYS, CONSOLIDATE_OVERFLOW_FILE_THRESHOLD, DEFAULT_FALLBACK_MAX_CHARS, DEFAULT_MAX_LINES, DEFAULT_MEMORY_FILE, AUTO_FILE_PREFIX, } from "../constants/memory.js";
|
|
11
3
|
import { readChunkingConfig } from "../config/chunking.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
4
|
+
import { initializeMemoryWorkspace } from "./initWorkspace.js";
|
|
5
|
+
import { joinPath, readText, readTextRequired, writeText } from "../utils/fs.js";
|
|
14
6
|
import { daysSince, formatLocalDate, formatTimestamp } from "../utils/time.js";
|
|
7
|
+
import { ingestMemoryExport } from "./ingestEntries.js";
|
|
8
|
+
import { MarkdownMemoryBackend } from "./backend.js";
|
|
9
|
+
import { buildIndexDocuments } from "./indexChunks.js";
|
|
10
|
+
import { createStoreListeners } from "./listeners.js";
|
|
11
|
+
import { countLines } from "./markdown/format.js";
|
|
12
|
+
import { listOverflowPointers } from "./markdown/parse.js";
|
|
15
13
|
import { getAgentPaths, resolveAgentDir } from "./paths.js";
|
|
16
|
-
import {
|
|
14
|
+
import { collectResolvedEntries } from "./resolveEntries.js";
|
|
15
|
+
import { appendIfAbsentUnlocked, rewriteEntriesUnlocked, tryAppendUnlocked, } from "./writePath.js";
|
|
17
16
|
export class MemoryStore {
|
|
18
17
|
paths;
|
|
19
18
|
backend;
|
|
20
19
|
maxLines;
|
|
21
20
|
fallbackMaxChars;
|
|
22
|
-
|
|
23
|
-
consolidateCheckListeners = new Set();
|
|
21
|
+
listeners = createStoreListeners(() => this.consolidating);
|
|
24
22
|
consolidating = false;
|
|
25
23
|
constructor(opts) {
|
|
26
24
|
const agentDir = resolveAgentDir(opts.agentDir);
|
|
@@ -29,6 +27,24 @@ export class MemoryStore {
|
|
|
29
27
|
this.maxLines = opts.maxLines ?? DEFAULT_MAX_LINES;
|
|
30
28
|
this.fallbackMaxChars = opts.defaultFallbackMaxChars ?? DEFAULT_FALLBACK_MAX_CHARS;
|
|
31
29
|
}
|
|
30
|
+
collectResolvedOpts() {
|
|
31
|
+
return {
|
|
32
|
+
backend: this.backend,
|
|
33
|
+
agentDir: this.paths.agentDir,
|
|
34
|
+
memoryFile: this.paths.memoryFile,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
writePathDeps() {
|
|
38
|
+
return {
|
|
39
|
+
backend: this.backend,
|
|
40
|
+
memoryFile: this.paths.memoryFile,
|
|
41
|
+
agentDir: this.paths.agentDir,
|
|
42
|
+
maxLines: this.maxLines,
|
|
43
|
+
createId: () => this.newEntryId(),
|
|
44
|
+
newAutoFilePath: () => this.newAutoFilePath(),
|
|
45
|
+
readResolvedUnlocked: () => this.readResolvedUnlocked(),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
32
48
|
get agentDir() {
|
|
33
49
|
return this.paths.agentDir;
|
|
34
50
|
}
|
|
@@ -60,22 +76,7 @@ export class MemoryStore {
|
|
|
60
76
|
}
|
|
61
77
|
async readResolved() {
|
|
62
78
|
await this.ensureInitialized();
|
|
63
|
-
|
|
64
|
-
const entries = [...parseMemoryMarkdown(main, this.paths.memoryFile)];
|
|
65
|
-
for (const fileName of listOverflowPointers(main)) {
|
|
66
|
-
const path = this.backend.autoFilePath(this.paths.agentDir, fileName);
|
|
67
|
-
const overflow = await this.backend.readText(path);
|
|
68
|
-
entries.push(...parseMemoryMarkdown(overflow, path));
|
|
69
|
-
}
|
|
70
|
-
const autoFiles = await this.backend.listAutoFiles(this.paths.agentDir);
|
|
71
|
-
for (const fileName of autoFiles) {
|
|
72
|
-
if (listOverflowPointers(main).includes(fileName))
|
|
73
|
-
continue;
|
|
74
|
-
const path = this.backend.autoFilePath(this.paths.agentDir, fileName);
|
|
75
|
-
const orphan = await this.backend.readText(path);
|
|
76
|
-
entries.push(...parseMemoryMarkdown(orphan, path));
|
|
77
|
-
}
|
|
78
|
-
return { content: main, entries };
|
|
79
|
+
return collectResolvedEntries(this.collectResolvedOpts());
|
|
79
80
|
}
|
|
80
81
|
async readForFallback(maxChars = this.fallbackMaxChars) {
|
|
81
82
|
const resolved = await this.readResolved();
|
|
@@ -95,43 +96,48 @@ export class MemoryStore {
|
|
|
95
96
|
return buildIndexDocuments(resolved.entries, readChunkingConfig());
|
|
96
97
|
}
|
|
97
98
|
async append(entry) {
|
|
99
|
+
let written = false;
|
|
98
100
|
await this.backend.withMemoryLock(async () => {
|
|
99
|
-
await this.
|
|
101
|
+
written = await tryAppendUnlocked(this.writePathDeps(), entry);
|
|
100
102
|
});
|
|
101
|
-
|
|
103
|
+
if (written)
|
|
104
|
+
this.listeners.notifyAfterWrite();
|
|
102
105
|
}
|
|
103
106
|
async appendUser(entry) {
|
|
107
|
+
let written = false;
|
|
104
108
|
await this.backend.withMemoryLock(async () => {
|
|
105
|
-
await this.
|
|
109
|
+
written = await tryAppendUnlocked(this.writePathDeps(), { ...entry, userAuthored: true });
|
|
106
110
|
});
|
|
107
|
-
|
|
111
|
+
if (written)
|
|
112
|
+
this.listeners.notifyAfterWrite();
|
|
108
113
|
}
|
|
109
114
|
async appendMany(entries, opts) {
|
|
115
|
+
let written = 0;
|
|
116
|
+
const deps = this.writePathDeps();
|
|
110
117
|
await this.backend.withMemoryLock(async () => {
|
|
111
118
|
for (const entry of entries) {
|
|
112
119
|
if (opts?.mode === "ifAbsent") {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
continue;
|
|
120
|
+
if (await appendIfAbsentUnlocked(deps, entry))
|
|
121
|
+
written += 1;
|
|
116
122
|
}
|
|
117
|
-
else {
|
|
118
|
-
|
|
123
|
+
else if (await tryAppendUnlocked(deps, entry)) {
|
|
124
|
+
written += 1;
|
|
119
125
|
}
|
|
120
126
|
}
|
|
121
127
|
});
|
|
122
|
-
if (
|
|
123
|
-
this.notifyAfterWrite();
|
|
128
|
+
if (written > 0)
|
|
129
|
+
this.listeners.notifyAfterWrite();
|
|
130
|
+
return written;
|
|
124
131
|
}
|
|
125
132
|
async appendIfAbsent(entry) {
|
|
126
133
|
let added = false;
|
|
127
134
|
await this.backend.withMemoryLock(async () => {
|
|
128
|
-
added = await this.
|
|
135
|
+
added = await appendIfAbsentUnlocked(this.writePathDeps(), entry);
|
|
129
136
|
});
|
|
130
137
|
if (added)
|
|
131
|
-
this.notifyAfterWrite();
|
|
138
|
+
this.listeners.notifyAfterWrite();
|
|
132
139
|
return added;
|
|
133
140
|
}
|
|
134
|
-
/** Fire-and-forget: parse Memory Export from compact summary → appendIfAbsent. */
|
|
135
141
|
appendFromCompaction(opts) {
|
|
136
142
|
void this.ingestCompactionSummary(opts).catch(() => { });
|
|
137
143
|
}
|
|
@@ -139,26 +145,16 @@ export class MemoryStore {
|
|
|
139
145
|
if (await this.hasProcessedCompaction(opts.compactionId))
|
|
140
146
|
return;
|
|
141
147
|
await this.ensureInitialized();
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
await this.markCompactionProcessed(opts.compactionId);
|
|
148
|
-
await opts.onComplete?.();
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
if (delta.length > 0) {
|
|
152
|
-
await this.appendMany(delta, { mode: "ifAbsent" });
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
else if (parsed.length > 0) {
|
|
156
|
-
await this.appendMany(parsed, { mode: "ifAbsent" });
|
|
157
|
-
}
|
|
148
|
+
await ingestMemoryExport({
|
|
149
|
+
store: this,
|
|
150
|
+
summary: opts.summary,
|
|
151
|
+
isSubagent: !!opts.subagent,
|
|
152
|
+
});
|
|
158
153
|
await this.markCompactionProcessed(opts.compactionId);
|
|
159
154
|
await opts.onComplete?.();
|
|
160
155
|
}
|
|
161
156
|
async updateEntry(id, patch) {
|
|
157
|
+
// Path B: correction detector should reuse prepareEntryForWrite when content is patched.
|
|
162
158
|
await this.backend.withMemoryLock(async () => {
|
|
163
159
|
const resolved = await this.readResolvedUnlocked();
|
|
164
160
|
const target = resolved.entries.find((entry) => entry.id === id);
|
|
@@ -173,9 +169,9 @@ export class MemoryStore {
|
|
|
173
169
|
timestamp: patch.timestamp ?? target.timestamp,
|
|
174
170
|
userAuthored: patch.userAuthored ?? target.userAuthored,
|
|
175
171
|
};
|
|
176
|
-
await this.
|
|
172
|
+
await rewriteEntriesUnlocked(this.writePathDeps(), resolved.entries.map((entry) => (entry.id === id ? { ...entry, ...next } : entry)));
|
|
177
173
|
});
|
|
178
|
-
this.notifyAfterWrite();
|
|
174
|
+
this.listeners.notifyAfterWrite();
|
|
179
175
|
}
|
|
180
176
|
async removeEntry(id, opts) {
|
|
181
177
|
await this.backend.withMemoryLock(async () => {
|
|
@@ -186,15 +182,15 @@ export class MemoryStore {
|
|
|
186
182
|
if (target.userAuthored && !opts?.force) {
|
|
187
183
|
throw new Error(`Cannot remove user-authored entry without force: ${id}`);
|
|
188
184
|
}
|
|
189
|
-
await this.
|
|
185
|
+
await rewriteEntriesUnlocked(this.writePathDeps(), resolved.entries.filter((entry) => entry.id !== id));
|
|
190
186
|
});
|
|
191
|
-
this.notifyAfterWrite();
|
|
187
|
+
this.listeners.notifyAfterWrite();
|
|
192
188
|
}
|
|
193
189
|
async rewrite(content) {
|
|
194
190
|
await this.backend.withMemoryLock(async () => {
|
|
195
191
|
await this.backend.writeText(this.paths.memoryFile, content);
|
|
196
192
|
});
|
|
197
|
-
this.notifyAfterWrite({ skipConsolidateCheck: true });
|
|
193
|
+
this.listeners.notifyAfterWrite({ skipConsolidateCheck: true });
|
|
198
194
|
}
|
|
199
195
|
async shouldConsolidate(at, cronFired = false) {
|
|
200
196
|
const stats = await this.getStats();
|
|
@@ -206,37 +202,26 @@ export class MemoryStore {
|
|
|
206
202
|
return false;
|
|
207
203
|
return daysSince(stats.lastConsolidatedAt, at) >= CONSOLIDATE_GC_INTERVAL_DAYS;
|
|
208
204
|
}
|
|
209
|
-
|
|
205
|
+
isConsolidating() {
|
|
206
|
+
return this.consolidating;
|
|
207
|
+
}
|
|
208
|
+
async rewriteMemoryUnderLock(updateEntries) {
|
|
210
209
|
if (this.consolidating)
|
|
211
210
|
return;
|
|
212
211
|
this.consolidating = true;
|
|
213
212
|
try {
|
|
214
213
|
await this.backend.withMemoryLock(async () => {
|
|
215
214
|
const resolved = await this.readResolvedUnlocked();
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
entries = await mergeEntriesWithLlm(entries, llm);
|
|
219
|
-
}
|
|
220
|
-
catch {
|
|
221
|
-
// rule-based dedupe only
|
|
222
|
-
}
|
|
223
|
-
await this.rewriteEntriesUnlocked(entries);
|
|
215
|
+
const entries = await updateEntries(resolved.entries);
|
|
216
|
+
await rewriteEntriesUnlocked(this.writePathDeps(), entries);
|
|
224
217
|
await writeText(this.paths.memoryGcFile, `${formatTimestamp()}\n`);
|
|
225
218
|
});
|
|
226
|
-
this.notifySyncToSidecar();
|
|
219
|
+
this.listeners.notifySyncToSidecar();
|
|
227
220
|
}
|
|
228
221
|
finally {
|
|
229
222
|
this.consolidating = false;
|
|
230
223
|
}
|
|
231
224
|
}
|
|
232
|
-
consolidateInBackground(llm, opts = {}) {
|
|
233
|
-
void this.consolidate(llm)
|
|
234
|
-
.then(() => opts.onComplete?.())
|
|
235
|
-
.catch(() => { });
|
|
236
|
-
}
|
|
237
|
-
async forceConsolidate(llm) {
|
|
238
|
-
await this.consolidate(llm);
|
|
239
|
-
}
|
|
240
225
|
async hasProcessedCompaction(compactionId) {
|
|
241
226
|
const state = await this.readCompactionState();
|
|
242
227
|
return state.processed.includes(compactionId);
|
|
@@ -263,133 +248,14 @@ export class MemoryStore {
|
|
|
263
248
|
}
|
|
264
249
|
return { ok: issues.length === 0, issues };
|
|
265
250
|
}
|
|
266
|
-
/** Register a listener to sync MEMORY.md changes to the sidecar vector index. */
|
|
267
251
|
onSyncToSidecar(listener) {
|
|
268
|
-
this.
|
|
269
|
-
return () => this.syncToSidecarListeners.delete(listener);
|
|
252
|
+
return this.listeners.onSyncToSidecar(listener);
|
|
270
253
|
}
|
|
271
|
-
/** Register a listener invoked after writes to check shouldConsolidate. */
|
|
272
254
|
onConsolidateCheck(listener) {
|
|
273
|
-
this.
|
|
274
|
-
return () => this.consolidateCheckListeners.delete(listener);
|
|
275
|
-
}
|
|
276
|
-
notifyAfterWrite(opts) {
|
|
277
|
-
this.notifySyncToSidecar();
|
|
278
|
-
if (!opts?.skipConsolidateCheck && !this.consolidating) {
|
|
279
|
-
this.notifyConsolidateCheck();
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
notifySyncToSidecar() {
|
|
283
|
-
for (const listener of this.syncToSidecarListeners)
|
|
284
|
-
listener();
|
|
285
|
-
}
|
|
286
|
-
notifyConsolidateCheck() {
|
|
287
|
-
for (const listener of this.consolidateCheckListeners)
|
|
288
|
-
listener();
|
|
289
|
-
}
|
|
290
|
-
async appendUnlocked(entry) {
|
|
291
|
-
const normalized = this.normalizeEntry(entry);
|
|
292
|
-
const main = await this.backend.readText(this.paths.memoryFile);
|
|
293
|
-
if (countLines(main) >= this.maxLines) {
|
|
294
|
-
await this.appendToOverflowUnlocked(normalized, main);
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
const next = this.insertEntryIntoMarkdown(main, normalized);
|
|
298
|
-
if (countLines(next) > this.maxLines) {
|
|
299
|
-
await this.appendToOverflowUnlocked(normalized, main);
|
|
300
|
-
return;
|
|
301
|
-
}
|
|
302
|
-
await this.backend.writeText(this.paths.memoryFile, next);
|
|
303
|
-
}
|
|
304
|
-
async appendIfAbsentUnlocked(entry) {
|
|
305
|
-
const resolved = await this.readResolvedUnlocked();
|
|
306
|
-
const exists = resolved.entries.some((item) => item.section === entry.section && item.content.trim() === entry.content.trim());
|
|
307
|
-
if (exists)
|
|
308
|
-
return false;
|
|
309
|
-
await this.appendUnlocked(entry);
|
|
310
|
-
return true;
|
|
311
|
-
}
|
|
312
|
-
async appendToOverflowUnlocked(entry, main) {
|
|
313
|
-
const autoFiles = await this.backend.listAutoFiles(this.paths.agentDir);
|
|
314
|
-
let targetName = autoFiles.at(-1);
|
|
315
|
-
let targetPath = targetName
|
|
316
|
-
? this.backend.autoFilePath(this.paths.agentDir, targetName)
|
|
317
|
-
: this.newAutoFilePath();
|
|
318
|
-
if (!targetName) {
|
|
319
|
-
targetName = pathBasename(targetPath);
|
|
320
|
-
await this.backend.writeText(targetPath, `${formatSectionHeader(entry.section)}\n\n`);
|
|
321
|
-
}
|
|
322
|
-
let overflowContent = await this.backend.readText(targetPath);
|
|
323
|
-
const line = formatEntryLine(entry);
|
|
324
|
-
overflowContent = overflowContent.trimEnd() + `\n${line}\n`;
|
|
325
|
-
await this.backend.writeText(targetPath, overflowContent);
|
|
326
|
-
const pointer = `- (overflow) → ${targetName}`;
|
|
327
|
-
if (!main.includes(pointer)) {
|
|
328
|
-
const withPointer = `${main.trimEnd()}\n${pointer}\n`;
|
|
329
|
-
await this.backend.writeText(this.paths.memoryFile, withPointer);
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
insertEntryIntoMarkdown(content, entry) {
|
|
333
|
-
const lines = content.split("\n");
|
|
334
|
-
const header = formatSectionHeader(entry.section);
|
|
335
|
-
const headerIdx = lines.findIndex((line) => line.trim() === header);
|
|
336
|
-
const line = formatEntryLine(entry);
|
|
337
|
-
if (headerIdx === -1) {
|
|
338
|
-
const trimmed = content.trimEnd();
|
|
339
|
-
return `${trimmed}\n\n${header}\n\n${line}\n`;
|
|
340
|
-
}
|
|
341
|
-
let insertAt = headerIdx + 1;
|
|
342
|
-
while (insertAt < lines.length && lines[insertAt]?.trim() === "")
|
|
343
|
-
insertAt++;
|
|
344
|
-
while (insertAt < lines.length) {
|
|
345
|
-
const current = lines[insertAt];
|
|
346
|
-
if (current.startsWith("## "))
|
|
347
|
-
break;
|
|
348
|
-
insertAt++;
|
|
349
|
-
}
|
|
350
|
-
const next = [...lines.slice(0, insertAt), line, ...lines.slice(insertAt)];
|
|
351
|
-
return `${next.join("\n").trimEnd()}\n`;
|
|
352
|
-
}
|
|
353
|
-
async rewriteEntriesUnlocked(entries) {
|
|
354
|
-
const grouped = new Map();
|
|
355
|
-
for (const section of MEMORY_SECTIONS)
|
|
356
|
-
grouped.set(section, []);
|
|
357
|
-
for (const entry of entries) {
|
|
358
|
-
grouped.get(entry.section)?.push(entry);
|
|
359
|
-
}
|
|
360
|
-
const lines = [];
|
|
361
|
-
for (const section of MEMORY_SECTIONS) {
|
|
362
|
-
lines.push(formatSectionHeader(section), "");
|
|
363
|
-
for (const entry of grouped.get(section) ?? []) {
|
|
364
|
-
lines.push(formatEntryLine({
|
|
365
|
-
id: entry.id,
|
|
366
|
-
section: entry.section,
|
|
367
|
-
content: entry.content,
|
|
368
|
-
userAuthored: entry.userAuthored,
|
|
369
|
-
timestamp: entry.timestamp,
|
|
370
|
-
}));
|
|
371
|
-
}
|
|
372
|
-
lines.push("");
|
|
373
|
-
}
|
|
374
|
-
await this.backend.writeText(this.paths.memoryFile, `${lines.join("\n").trimEnd()}\n`);
|
|
375
|
-
const autoFiles = await this.backend.listAutoFiles(this.paths.agentDir);
|
|
376
|
-
await Promise.all(autoFiles.map((fileName) => this.backend.deleteAutoFile(this.backend.autoFilePath(this.paths.agentDir, fileName))));
|
|
255
|
+
return this.listeners.onConsolidateCheck(listener);
|
|
377
256
|
}
|
|
378
257
|
async readResolvedUnlocked() {
|
|
379
|
-
|
|
380
|
-
const entries = [...parseMemoryMarkdown(main, this.paths.memoryFile)];
|
|
381
|
-
for (const fileName of listOverflowPointers(main)) {
|
|
382
|
-
const path = this.backend.autoFilePath(this.paths.agentDir, fileName);
|
|
383
|
-
entries.push(...parseMemoryMarkdown(await this.backend.readText(path), path));
|
|
384
|
-
}
|
|
385
|
-
return { content: main, entries };
|
|
386
|
-
}
|
|
387
|
-
normalizeEntry(entry) {
|
|
388
|
-
return {
|
|
389
|
-
...entry,
|
|
390
|
-
id: entry.id || this.newEntryId(),
|
|
391
|
-
timestamp: entry.timestamp || formatTimestamp(),
|
|
392
|
-
};
|
|
258
|
+
return collectResolvedEntries(this.collectResolvedOpts());
|
|
393
259
|
}
|
|
394
260
|
newEntryId() {
|
|
395
261
|
return randomBytes(6).toString("hex");
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { MarkdownMemoryBackend } from "./backend.js";
|
|
2
|
+
import type { ResolvedMemory } from "./types.js";
|
|
3
|
+
export type CollectResolvedEntriesOpts = {
|
|
4
|
+
backend: MarkdownMemoryBackend;
|
|
5
|
+
agentDir: string;
|
|
6
|
+
memoryFile: string;
|
|
7
|
+
/** Include auto-*.md files not referenced by MEMORY.md overflow pointers. Default true. */
|
|
8
|
+
includeOrphans?: boolean;
|
|
9
|
+
};
|
|
10
|
+
/** Read main MEMORY.md plus overflow / orphan auto files into a single entry list. */
|
|
11
|
+
export declare function collectResolvedEntries(opts: CollectResolvedEntriesOpts): Promise<ResolvedMemory>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { listOverflowPointers, parseMemoryMarkdown } from "./markdown/parse.js";
|
|
2
|
+
/** Read main MEMORY.md plus overflow / orphan auto files into a single entry list. */
|
|
3
|
+
export async function collectResolvedEntries(opts) {
|
|
4
|
+
const main = await opts.backend.readText(opts.memoryFile);
|
|
5
|
+
const entries = [...parseMemoryMarkdown(main, opts.memoryFile)];
|
|
6
|
+
const pointers = listOverflowPointers(main);
|
|
7
|
+
for (const fileName of pointers) {
|
|
8
|
+
const path = opts.backend.autoFilePath(opts.agentDir, fileName);
|
|
9
|
+
const overflow = await opts.backend.readText(path);
|
|
10
|
+
entries.push(...parseMemoryMarkdown(overflow, path));
|
|
11
|
+
}
|
|
12
|
+
if (opts.includeOrphans !== false) {
|
|
13
|
+
const autoFiles = await opts.backend.listAutoFiles(opts.agentDir);
|
|
14
|
+
for (const fileName of autoFiles) {
|
|
15
|
+
if (pointers.includes(fileName))
|
|
16
|
+
continue;
|
|
17
|
+
const path = opts.backend.autoFilePath(opts.agentDir, fileName);
|
|
18
|
+
const orphan = await opts.backend.readText(path);
|
|
19
|
+
entries.push(...parseMemoryMarkdown(orphan, path));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return { content: main, entries };
|
|
23
|
+
}
|