@chendpoc/pi-memory 0.2.4 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -43
- package/dist/adapters/llm/standalone.js +1 -1
- package/dist/cli/init.js +20 -3
- package/dist/cli/parseArgs.d.ts +5 -2
- package/dist/cli/parseArgs.js +13 -0
- package/dist/cli/schedulerSync.d.ts +2 -0
- package/dist/cli/schedulerSync.js +26 -0
- package/dist/cli/status.d.ts +4 -40
- package/dist/cli/status.js +6 -230
- package/dist/cli/theme.d.ts +2 -0
- package/dist/cli/theme.js +8 -0
- package/dist/cli.js +5 -0
- package/dist/commands/status.js +2 -2
- package/dist/compact/parseMemoryExport.js +2 -1
- package/dist/compact/register.js +1 -1
- package/dist/compact/runSummary.js +1 -16
- package/dist/compact/subagentDelta.js +3 -5
- package/dist/consolidate/index.d.ts +1 -0
- package/dist/consolidate/index.js +1 -0
- package/dist/consolidate/mergeEntries.js +3 -3
- package/dist/consolidate/mergeMemoryEntries.d.ts +8 -0
- package/dist/consolidate/mergeMemoryEntries.js +23 -0
- package/dist/consolidate/mergePrompt.js +2 -2
- package/dist/consolidate/mergeWithLlm.js +2 -2
- package/dist/consolidate/runJob.d.ts +2 -2
- package/dist/consolidate/runJob.js +6 -12
- package/dist/consolidate/scheduler.d.ts +2 -2
- package/dist/consolidate/scheduler.js +1 -1
- package/dist/constants/env.d.ts +1 -0
- package/dist/constants/env.js +1 -0
- package/dist/constants/paths.d.ts +3 -1
- package/dist/constants/paths.js +6 -1
- package/dist/extension/createMemoryRuntime.d.ts +3 -0
- package/dist/extension/createMemoryRuntime.js +203 -0
- package/dist/extension/index.d.ts +2 -0
- package/dist/extension/index.js +1 -0
- package/dist/extension/lifecycle.d.ts +28 -0
- package/dist/extension/lifecycle.js +52 -0
- package/dist/extension/messageUtils.d.ts +4 -0
- package/dist/extension/messageUtils.js +18 -0
- package/dist/extension/types.d.ts +35 -0
- package/dist/extension/types.js +1 -0
- package/dist/index.d.ts +7 -9
- package/dist/index.js +7 -9
- package/dist/pi-extension.js +26 -236
- package/dist/preflight/episodic.js +13 -30
- package/dist/preflight/queryIntent.js +1 -1
- package/dist/preflight/render.js +1 -1
- package/dist/preflight/strip.d.ts +0 -1
- package/dist/preflight/strip.js +0 -24
- package/dist/redaction/index.d.ts +4 -0
- package/dist/redaction/index.js +3 -0
- package/dist/redaction/patterns/constants.d.ts +6 -0
- package/dist/redaction/patterns/constants.js +6 -0
- package/dist/redaction/patterns/crypto.d.ts +3 -0
- package/dist/redaction/patterns/crypto.js +17 -0
- package/dist/redaction/patterns/generic.d.ts +3 -0
- package/dist/redaction/patterns/generic.js +38 -0
- package/dist/redaction/patterns/index.d.ts +16 -0
- package/dist/redaction/patterns/index.js +23 -0
- package/dist/redaction/patterns/llm.d.ts +3 -0
- package/dist/redaction/patterns/llm.js +144 -0
- package/dist/redaction/patterns/platform.d.ts +3 -0
- package/dist/redaction/patterns/platform.js +87 -0
- package/dist/redaction/patterns/types.d.ts +18 -0
- package/dist/redaction/patterns/types.js +1 -0
- package/dist/redaction/redactText.d.ts +9 -0
- package/dist/redaction/redactText.js +31 -0
- package/dist/redaction/types.d.ts +19 -0
- package/dist/redaction/types.js +1 -0
- package/dist/redaction/utils.d.ts +28 -0
- package/dist/redaction/utils.js +106 -0
- package/dist/scheduler/index.d.ts +3 -0
- package/dist/scheduler/index.js +3 -0
- package/dist/scheduler/launchd.d.ts +14 -0
- package/dist/scheduler/launchd.js +69 -0
- package/dist/scheduler/launchdPlist.d.ts +14 -0
- package/dist/scheduler/launchdPlist.js +62 -0
- package/dist/scheduler/sync.d.ts +36 -0
- package/dist/scheduler/sync.js +79 -0
- package/dist/shutdown/enqueue.d.ts +1 -1
- package/dist/shutdown/enqueue.js +2 -5
- package/dist/shutdown/readQueue.js +1 -1
- package/dist/shutdown/runDrainJob.js +8 -37
- package/dist/shutdown/sessionReader.js +1 -14
- package/dist/sidecar/client.d.ts +6 -2
- package/dist/sidecar/client.js +49 -14
- package/dist/{preflight → sidecar}/queryCache.d.ts +1 -1
- package/dist/sidecar/reindexBridge.js +2 -2
- package/dist/sidecar/server/server.js +1 -1
- package/dist/sidecar/server/vec/chunkQuery.d.ts +4 -0
- package/dist/sidecar/server/vec/chunkQuery.js +46 -0
- package/dist/sidecar/server/vec/chunkReindex.d.ts +5 -0
- package/dist/sidecar/server/vec/chunkReindex.js +40 -0
- package/dist/sidecar/server/vec/embeddingCodec.d.ts +2 -0
- package/dist/sidecar/server/vec/embeddingCodec.js +6 -0
- package/dist/sidecar/server/vec/schema.d.ts +20 -0
- package/dist/sidecar/server/vec/schema.js +61 -0
- package/dist/sidecar/server/vec/store.d.ts +2 -13
- package/dist/sidecar/server/vec/store.js +12 -139
- package/dist/sidecar/sidecarManager.js +4 -58
- package/dist/sidecar/spawnLock.d.ts +2 -0
- package/dist/sidecar/spawnLock.js +57 -0
- package/dist/sidecar/syncIndex.d.ts +3 -0
- package/dist/sidecar/syncIndex.js +12 -0
- package/dist/sidecar/warmup.js +1 -1
- package/dist/status/copy.d.ts +2 -0
- package/dist/status/copy.js +2 -0
- package/dist/status/format.d.ts +7 -0
- package/dist/status/format.js +133 -0
- package/dist/status/gather.d.ts +2 -0
- package/dist/status/gather.js +88 -0
- package/dist/status/index.d.ts +4 -0
- package/dist/status/index.js +3 -0
- package/dist/status/types.d.ts +33 -0
- package/dist/status/types.js +1 -0
- package/dist/store/consolidatePort.d.ts +11 -0
- package/dist/store/consolidatePort.js +1 -0
- package/dist/store/index.d.ts +2 -0
- package/dist/store/index.js +1 -0
- package/dist/store/ingestEntries.d.ts +16 -0
- package/dist/store/ingestEntries.js +22 -0
- package/dist/{init/workspace.d.ts → store/initWorkspace.d.ts} +1 -1
- package/dist/{init/workspace.js → store/initWorkspace.js} +7 -5
- package/dist/store/listeners.d.ts +11 -0
- package/dist/store/listeners.js +27 -0
- package/dist/store/markdown/insert.d.ts +3 -0
- package/dist/store/markdown/insert.js +23 -0
- package/dist/store/memoryStore.d.ts +9 -22
- package/dist/store/memoryStore.js +71 -205
- package/dist/store/resolveEntries.d.ts +11 -0
- package/dist/store/resolveEntries.js +23 -0
- package/dist/store/types.d.ts +0 -1
- package/dist/store/writePath.d.ts +20 -0
- package/dist/store/writePath.js +123 -0
- package/dist/ui/memoryStatusWidget.d.ts +4 -8
- package/dist/ui/memoryStatusWidget.js +5 -17
- package/dist/utils/async.d.ts +11 -0
- package/dist/utils/async.js +24 -0
- package/dist/utils/index.d.ts +5 -1
- package/dist/utils/index.js +5 -1
- package/dist/{ipc/jsonlFramer.d.ts → utils/jsonl.d.ts} +1 -1
- package/dist/{ipc/jsonlFramer.js → utils/jsonl.js} +1 -1
- package/dist/utils/memory/index.d.ts +10 -0
- package/dist/utils/memory/index.js +43 -0
- package/dist/utils/paths.d.ts +4 -0
- package/dist/utils/paths.js +13 -3
- package/dist/utils/scheduler.d.ts +1 -1
- package/dist/utils/scheduler.js +6 -6
- package/dist/{preflight/session.d.ts → utils/session/index.d.ts} +1 -0
- package/dist/{preflight/session.js → utils/session/index.js} +5 -2
- package/doc/LAUNCH-KIT.md +229 -0
- package/doc/README-zh.md +445 -0
- package/doc/ROADMAP-zh.md +114 -0
- package/doc/ROADMAP.md +114 -0
- package/package.json +16 -4
- package/scripts/postinstall.mjs +11 -1
- package/templates/com.pi.memory.consolidate.plist.example +41 -0
- package/templates/consolidate.cmd.example +15 -0
- package/templates/crontab.example +14 -0
- package/templates/schtasks.example.txt +34 -0
- package/dist/consolidate/entryKey.d.ts +0 -5
- package/dist/consolidate/entryKey.js +0 -4
- /package/dist/{preflight → sidecar}/queryCache.js +0 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { ENV_KEYS } from "../constants/env.js";
|
|
5
|
+
import { LAUNCHD_LABEL } from "../constants/paths.js";
|
|
6
|
+
import { resolveMemoryAgentDir } from "../config/agentDir.js";
|
|
7
|
+
import { debugMemory } from "../utils/debugLog.js";
|
|
8
|
+
import { isMacOS } from "../utils/platform.js";
|
|
9
|
+
import { defaultConsolidateSchedulerPaths } from "../utils/scheduler.js";
|
|
10
|
+
import { syncLaunchdMaintenanceJob } from "./launchd.js";
|
|
11
|
+
export function isSchedulerSyncDisabled(env = process.env) {
|
|
12
|
+
const value = env[ENV_KEYS.SKIP_SCHEDULER_SYNC];
|
|
13
|
+
return value === "1" || value === "true";
|
|
14
|
+
}
|
|
15
|
+
/** Whether this process can attempt a user LaunchAgent sync (macOS + uid + home). */
|
|
16
|
+
export function canSyncLaunchdInProcess(env = process.env) {
|
|
17
|
+
if (!isMacOS())
|
|
18
|
+
return { ok: false, reason: "unsupported-platform" };
|
|
19
|
+
if (process.getuid?.() === undefined)
|
|
20
|
+
return { ok: false, reason: "no-user-id" };
|
|
21
|
+
if (!homedir()?.trim())
|
|
22
|
+
return { ok: false, reason: "no-home-directory" };
|
|
23
|
+
if (isSchedulerSyncDisabled(env))
|
|
24
|
+
return { ok: false, reason: "PI_MEMORY_SKIP_SCHEDULER_SYNC" };
|
|
25
|
+
return { ok: true };
|
|
26
|
+
}
|
|
27
|
+
export function resolvePackageCliPath(moduleUrl = import.meta.url) {
|
|
28
|
+
return join(dirname(fileURLToPath(moduleUrl)), "..", "cli.js");
|
|
29
|
+
}
|
|
30
|
+
function formatSyncError(error) {
|
|
31
|
+
return error instanceof Error ? error.message : String(error);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Install or refresh the OS maintenance scheduler (macOS launchd today).
|
|
35
|
+
* Never throws — automatic callers (postinstall, session_start, init) treat failures as best-effort.
|
|
36
|
+
*/
|
|
37
|
+
export async function syncMaintenanceScheduler(opts = {}) {
|
|
38
|
+
const env = opts.env ?? process.env;
|
|
39
|
+
const capability = canSyncLaunchdInProcess(env);
|
|
40
|
+
if (!capability.ok) {
|
|
41
|
+
return { status: "skipped", reason: capability.reason };
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
const agentDir = opts.agentDir ?? resolveMemoryAgentDir({ env });
|
|
45
|
+
const paths = defaultConsolidateSchedulerPaths(agentDir);
|
|
46
|
+
const nodePath = opts.nodePath ?? process.execPath;
|
|
47
|
+
const cliPath = opts.cliPath ?? resolvePackageCliPath(import.meta.url);
|
|
48
|
+
const result = await syncLaunchdMaintenanceJob({
|
|
49
|
+
label: LAUNCHD_LABEL,
|
|
50
|
+
nodePath,
|
|
51
|
+
cliPath,
|
|
52
|
+
envFile: paths.envFile,
|
|
53
|
+
agentDir: paths.agentDir,
|
|
54
|
+
logsDir: paths.logsDir,
|
|
55
|
+
stdoutLog: paths.stdoutLog,
|
|
56
|
+
stderrLog: paths.stderrLog,
|
|
57
|
+
});
|
|
58
|
+
debugMemory("scheduler", "launchd sync complete", {
|
|
59
|
+
label: result.label,
|
|
60
|
+
changed: result.changed,
|
|
61
|
+
bootstrapped: result.bootstrapped,
|
|
62
|
+
removedLegacy: result.removedLegacy.join(",") || "none",
|
|
63
|
+
}, env);
|
|
64
|
+
return {
|
|
65
|
+
status: "synced",
|
|
66
|
+
platform: "launchd",
|
|
67
|
+
label: result.label,
|
|
68
|
+
plistPath: result.plistPath,
|
|
69
|
+
changed: result.changed,
|
|
70
|
+
bootstrapped: result.bootstrapped,
|
|
71
|
+
removedLegacy: result.removedLegacy,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
const message = formatSyncError(error);
|
|
76
|
+
debugMemory("scheduler", "launchd sync failed (best-effort)", { message }, env);
|
|
77
|
+
return { status: "failed", reason: "launchd-sync-error", message };
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { SessionShutdownEvent } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
export { readParentSession } from "../utils/session/index.js";
|
|
2
3
|
export type ShutdownQueueEntry = {
|
|
3
4
|
sessionFile: string;
|
|
4
5
|
parentSession?: string;
|
|
@@ -8,4 +9,3 @@ export type ShutdownQueueEntry = {
|
|
|
8
9
|
};
|
|
9
10
|
export declare function shutdownQueuePath(agentDir: string): string;
|
|
10
11
|
export declare function enqueueShutdownMetadata(agentDir: string, entry: ShutdownQueueEntry): Promise<void>;
|
|
11
|
-
export declare function readParentSession(header: Record<string, unknown> | null): string | undefined;
|
package/dist/shutdown/enqueue.js
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
import { SHUTDOWN_QUEUE_FILE } from "../constants/memory.js";
|
|
2
|
-
import { serializeJsonlFrame } from "../
|
|
2
|
+
import { serializeJsonlFrame } from "../utils/jsonl.js";
|
|
3
3
|
import { appendText, joinPath } from "../utils/fs.js";
|
|
4
|
+
export { readParentSession } from "../utils/session/index.js";
|
|
4
5
|
export function shutdownQueuePath(agentDir) {
|
|
5
6
|
return joinPath(agentDir, SHUTDOWN_QUEUE_FILE);
|
|
6
7
|
}
|
|
7
8
|
export async function enqueueShutdownMetadata(agentDir, entry) {
|
|
8
9
|
await appendText(shutdownQueuePath(agentDir), serializeJsonlFrame(entry));
|
|
9
10
|
}
|
|
10
|
-
export function readParentSession(header) {
|
|
11
|
-
const parent = header?.parentSession ?? header?.parent_session;
|
|
12
|
-
return typeof parent === "string" && parent.trim().length > 0 ? parent.trim() : undefined;
|
|
13
|
-
}
|
|
@@ -1,9 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { reindex } from "../sidecar/client.js";
|
|
4
|
-
import { ensureSidecarRunning } from "../sidecar/sidecarManager.js";
|
|
5
|
-
import { resolveSidecarPaths } from "../sidecar/paths.js";
|
|
6
|
-
import { sidecarQueryCache } from "../preflight/queryCache.js";
|
|
1
|
+
import { syncSidecarIndex } from "../sidecar/syncIndex.js";
|
|
2
|
+
import { ingestMemoryExport } from "../store/ingestEntries.js";
|
|
7
3
|
import { canRead, pathExists } from "../utils/fs.js";
|
|
8
4
|
import { buildShutdownMemoryExportPrompt, wrapShutdownExportMarkdown, } from "./extractPrompt.js";
|
|
9
5
|
import { markShutdownProcessed, readShutdownProcessedState, isShutdownProcessed } from "./processed.js";
|
|
@@ -104,39 +100,14 @@ async function drainQueueEntry(opts) {
|
|
|
104
100
|
return { appended };
|
|
105
101
|
}
|
|
106
102
|
async function ingestFromSummary(opts) {
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
await markShutdownProcessed(opts.agentDir, opts.sessionFile);
|
|
113
|
-
return 0;
|
|
114
|
-
}
|
|
115
|
-
let entries = parsed;
|
|
116
|
-
if (opts.subagent) {
|
|
117
|
-
const existing = await opts.store.listEntries();
|
|
118
|
-
entries = filterCompactionDelta(parsed, existing);
|
|
119
|
-
if (shouldSkipSubagentCompactionIngest(parsed, entries)) {
|
|
120
|
-
if (opts.compactionId) {
|
|
121
|
-
await opts.store.markCompactionProcessed(opts.compactionId);
|
|
122
|
-
}
|
|
123
|
-
await markShutdownProcessed(opts.agentDir, opts.sessionFile);
|
|
124
|
-
return 0;
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
if (entries.length > 0) {
|
|
128
|
-
await opts.store.appendMany(entries, { mode: "ifAbsent" });
|
|
129
|
-
}
|
|
103
|
+
const { appended } = await ingestMemoryExport({
|
|
104
|
+
store: opts.store,
|
|
105
|
+
summary: opts.summary,
|
|
106
|
+
isSubagent: !!opts.subagent,
|
|
107
|
+
});
|
|
130
108
|
if (opts.compactionId) {
|
|
131
109
|
await opts.store.markCompactionProcessed(opts.compactionId);
|
|
132
110
|
}
|
|
133
111
|
await markShutdownProcessed(opts.agentDir, opts.sessionFile);
|
|
134
|
-
return
|
|
135
|
-
}
|
|
136
|
-
async function syncSidecarIndex(agentDir, store) {
|
|
137
|
-
const sidecar = resolveSidecarPaths(agentDir);
|
|
138
|
-
await ensureSidecarRunning(sidecar);
|
|
139
|
-
const result = await reindex(sidecar.socketPath, await store.exportForIndex());
|
|
140
|
-
sidecarQueryCache.onReindexComplete(agentDir, result.index_generation);
|
|
141
|
-
return result.index_generation;
|
|
112
|
+
return appended;
|
|
142
113
|
}
|
|
@@ -1,19 +1,6 @@
|
|
|
1
1
|
import { buildSessionContext, convertToLlm, getLatestCompactionEntry, parseSessionEntries, serializeConversation, } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
2
|
+
import { stripPrivateMemoryFromMessages } from "../utils/memory/index.js";
|
|
3
3
|
import { readTextRequired } from "../utils/fs.js";
|
|
4
|
-
function stripPrivateMemoryFromMessages(messages) {
|
|
5
|
-
return messages.map((message) => {
|
|
6
|
-
if (message.role !== "user")
|
|
7
|
-
return message;
|
|
8
|
-
if (typeof message.content === "string") {
|
|
9
|
-
return { ...message, content: stripPrivateMemory(message.content) };
|
|
10
|
-
}
|
|
11
|
-
return {
|
|
12
|
-
...message,
|
|
13
|
-
content: message.content.map((block) => block.type === "text" ? { ...block, text: stripPrivateMemory(block.text) } : block),
|
|
14
|
-
};
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
4
|
function toSessionEntries(fileEntries) {
|
|
18
5
|
return fileEntries.filter((entry) => entry.type !== "session");
|
|
19
6
|
}
|
package/dist/sidecar/client.d.ts
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { type IndexDocument, type IndexStats, type SidecarResponse } from "./protocol.js";
|
|
2
|
-
export
|
|
2
|
+
export type SidecarRequestOptions = {
|
|
3
|
+
timeoutMs?: number;
|
|
4
|
+
signal?: AbortSignal;
|
|
5
|
+
};
|
|
6
|
+
export declare function sidecarRequest<T extends SidecarResponse>(socketPath: string, frame: Record<string, unknown>, options?: SidecarRequestOptions): Promise<T>;
|
|
3
7
|
export declare function ping(socketPath: string): Promise<boolean>;
|
|
4
8
|
export declare function fetchIndexStats(socketPath: string): Promise<{
|
|
5
9
|
stats: IndexStats;
|
|
6
10
|
} | {
|
|
7
11
|
error: string;
|
|
8
12
|
}>;
|
|
9
|
-
export declare function query(socketPath: string, queryText: string,
|
|
13
|
+
export declare function query(socketPath: string, queryText: string, options?: SidecarRequestOptions): Promise<{
|
|
10
14
|
type: "result";
|
|
11
15
|
request_id: string;
|
|
12
16
|
results: import("./protocol.js").MemoryEntry[];
|
package/dist/sidecar/client.js
CHANGED
|
@@ -2,45 +2,80 @@
|
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import net from "node:net";
|
|
4
4
|
import { SIDECAR_PING_TIMEOUT_MS, SIDECAR_QUERY_TIMEOUT_MS, SIDECAR_REINDEX_TIMEOUT_MS, } from "../constants/timing.js";
|
|
5
|
-
import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../
|
|
5
|
+
import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../utils/jsonl.js";
|
|
6
6
|
import { isErrorResponse } from "./protocol.js";
|
|
7
|
-
export function sidecarRequest(socketPath, frame,
|
|
7
|
+
export function sidecarRequest(socketPath, frame, options = {}) {
|
|
8
|
+
const timeoutMs = options.timeoutMs ?? SIDECAR_QUERY_TIMEOUT_MS;
|
|
9
|
+
const { signal } = options;
|
|
8
10
|
return new Promise((resolve, reject) => {
|
|
11
|
+
if (signal?.aborted) {
|
|
12
|
+
reject(abortError(signal));
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
9
15
|
const socket = net.connect(socketPath);
|
|
10
16
|
const framer = new JsonlFramer();
|
|
11
|
-
|
|
17
|
+
let settled = false;
|
|
18
|
+
const cleanup = () => {
|
|
19
|
+
clearTimeout(timer);
|
|
20
|
+
signal?.removeEventListener("abort", onAbort);
|
|
21
|
+
};
|
|
22
|
+
const fail = (error) => {
|
|
23
|
+
if (settled)
|
|
24
|
+
return;
|
|
25
|
+
settled = true;
|
|
26
|
+
cleanup();
|
|
12
27
|
socket.destroy();
|
|
13
|
-
reject(
|
|
28
|
+
reject(error);
|
|
29
|
+
};
|
|
30
|
+
const succeed = (value) => {
|
|
31
|
+
if (settled)
|
|
32
|
+
return;
|
|
33
|
+
settled = true;
|
|
34
|
+
cleanup();
|
|
35
|
+
resolve(value);
|
|
36
|
+
};
|
|
37
|
+
const onAbort = () => {
|
|
38
|
+
fail(abortError(signal));
|
|
39
|
+
};
|
|
40
|
+
const timer = setTimeout(() => {
|
|
41
|
+
fail(new Error("Sidecar request timed out"));
|
|
14
42
|
}, timeoutMs);
|
|
43
|
+
if (signal) {
|
|
44
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
45
|
+
}
|
|
15
46
|
socket.on("connect", () => {
|
|
16
47
|
socket.write(serializeJsonlFrame(frame));
|
|
17
48
|
});
|
|
18
49
|
socket.on("data", (chunk) => {
|
|
19
50
|
for (const line of framer.push(chunk.toString())) {
|
|
20
|
-
clearTimeout(timer);
|
|
21
51
|
socket.end();
|
|
22
52
|
let response;
|
|
23
53
|
try {
|
|
24
54
|
response = parseJsonlLine(line);
|
|
25
55
|
}
|
|
26
56
|
catch {
|
|
27
|
-
|
|
57
|
+
fail(new Error("Invalid JSON response from sidecar"));
|
|
28
58
|
return;
|
|
29
59
|
}
|
|
30
60
|
if (isErrorResponse(response)) {
|
|
31
|
-
|
|
61
|
+
fail(new Error(response.error));
|
|
32
62
|
return;
|
|
33
63
|
}
|
|
34
|
-
|
|
64
|
+
succeed(response);
|
|
35
65
|
return;
|
|
36
66
|
}
|
|
37
67
|
});
|
|
38
|
-
socket.on("error",
|
|
68
|
+
socket.on("error", (error) => {
|
|
69
|
+
fail(error instanceof Error ? error : new Error(String(error)));
|
|
70
|
+
});
|
|
39
71
|
});
|
|
40
72
|
}
|
|
73
|
+
function abortError(signal) {
|
|
74
|
+
return signal.reason instanceof Error ? signal.reason : new Error("Aborted");
|
|
75
|
+
}
|
|
41
76
|
export async function ping(socketPath) {
|
|
42
77
|
try {
|
|
43
|
-
const res = await sidecarRequest(socketPath, { type: "ping" }, SIDECAR_PING_TIMEOUT_MS);
|
|
78
|
+
const res = await sidecarRequest(socketPath, { type: "ping" }, { timeoutMs: SIDECAR_PING_TIMEOUT_MS });
|
|
44
79
|
return res.type === "pong";
|
|
45
80
|
}
|
|
46
81
|
catch {
|
|
@@ -49,7 +84,7 @@ export async function ping(socketPath) {
|
|
|
49
84
|
}
|
|
50
85
|
export async function fetchIndexStats(socketPath) {
|
|
51
86
|
try {
|
|
52
|
-
const res = await sidecarRequest(socketPath, { type: "stats" }, SIDECAR_PING_TIMEOUT_MS);
|
|
87
|
+
const res = await sidecarRequest(socketPath, { type: "stats" }, { timeoutMs: SIDECAR_PING_TIMEOUT_MS });
|
|
53
88
|
if (res.type !== "stats_ok")
|
|
54
89
|
return { error: "unexpected sidecar response" };
|
|
55
90
|
const { type: _type, ...stats } = res;
|
|
@@ -60,11 +95,11 @@ export async function fetchIndexStats(socketPath) {
|
|
|
60
95
|
return { error: message };
|
|
61
96
|
}
|
|
62
97
|
}
|
|
63
|
-
export async function query(socketPath, queryText,
|
|
98
|
+
export async function query(socketPath, queryText, options = {}) {
|
|
64
99
|
const request_id = randomUUID();
|
|
65
|
-
return sidecarRequest(socketPath, { type: "query", request_id, query: queryText },
|
|
100
|
+
return sidecarRequest(socketPath, { type: "query", request_id, query: queryText }, options);
|
|
66
101
|
}
|
|
67
102
|
export async function reindex(socketPath, documents = []) {
|
|
68
103
|
const request_id = randomUUID();
|
|
69
|
-
return sidecarRequest(socketPath, { type: "reindex", request_id, documents }, SIDECAR_REINDEX_TIMEOUT_MS);
|
|
104
|
+
return sidecarRequest(socketPath, { type: "reindex", request_id, documents }, { timeoutMs: SIDECAR_REINDEX_TIMEOUT_MS });
|
|
70
105
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import debounce from "
|
|
1
|
+
import { debounce } from "es-toolkit";
|
|
2
2
|
import { DEFAULT_REINDEX_DEBOUNCE_MS } from "../constants/timing.js";
|
|
3
|
-
import { sidecarQueryCache } from "
|
|
3
|
+
import { sidecarQueryCache } from "./queryCache.js";
|
|
4
4
|
import { ensureSidecarRunning } from "./sidecarManager.js";
|
|
5
5
|
import { reindex } from "./client.js";
|
|
6
6
|
export function createReindexScheduler(opts) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createServer } from "node:net";
|
|
2
2
|
import { writeFileSync } from "node:fs";
|
|
3
3
|
import { SIDECAR_PID_SUFFIX } from "../../constants/paths.js";
|
|
4
|
-
import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../../
|
|
4
|
+
import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../../utils/jsonl.js";
|
|
5
5
|
import { ensureDirSync, pathDirname } from "../../utils/fs.js";
|
|
6
6
|
import { cleanupSocketFiles, removeSocketFile, secureSocketPath } from "../../utils/socket.js";
|
|
7
7
|
import { handleQuery } from "./query.js";
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { readRetrievalConfig } from "../../../config/retrieval.js";
|
|
2
|
+
import { blobToEmbedding } from "./embeddingCodec.js";
|
|
3
|
+
import { getEmbedder } from "./embedder.js";
|
|
4
|
+
import { cosineSimilarity, distanceToRelevance, mmrSelect } from "./mmr.js";
|
|
5
|
+
import { clearChunksIfEmbeddingMismatch } from "./schema.js";
|
|
6
|
+
export async function queryChunks(db, queryText, topK) {
|
|
7
|
+
const retrieval = readRetrievalConfig();
|
|
8
|
+
const limit = topK ?? retrieval.topK;
|
|
9
|
+
const embedder = getEmbedder();
|
|
10
|
+
clearChunksIfEmbeddingMismatch(db, embedder);
|
|
11
|
+
const queryEmbedding = await embedder.embed(queryText);
|
|
12
|
+
const rows = db
|
|
13
|
+
.prepare("SELECT chunk_id, content, source, timestamp, embedding FROM memory_chunks")
|
|
14
|
+
.all();
|
|
15
|
+
if (rows.length === 0)
|
|
16
|
+
return [];
|
|
17
|
+
const candidates = [];
|
|
18
|
+
for (const row of rows) {
|
|
19
|
+
const embedding = blobToEmbedding(row.embedding);
|
|
20
|
+
if (embedding.length !== queryEmbedding.length)
|
|
21
|
+
continue;
|
|
22
|
+
const similarity = cosineSimilarity(queryEmbedding, embedding);
|
|
23
|
+
if (similarity < retrieval.minRelevance)
|
|
24
|
+
continue;
|
|
25
|
+
candidates.push({
|
|
26
|
+
chunkId: row.chunk_id,
|
|
27
|
+
content: row.content,
|
|
28
|
+
source: row.source,
|
|
29
|
+
timestamp: row.timestamp,
|
|
30
|
+
distance: 1 - similarity,
|
|
31
|
+
embedding,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
if (candidates.length === 0)
|
|
35
|
+
return [];
|
|
36
|
+
candidates.sort((a, b) => a.distance - b.distance);
|
|
37
|
+
const poolSize = Math.min(limit * retrieval.candidatePoolMultiplier, candidates.length);
|
|
38
|
+
const pool = candidates.slice(0, poolSize);
|
|
39
|
+
const selected = mmrSelect(queryEmbedding, pool, limit, retrieval.mmrLambda);
|
|
40
|
+
return selected.map((item) => ({
|
|
41
|
+
content: item.content,
|
|
42
|
+
source: item.source,
|
|
43
|
+
timestamp: item.timestamp,
|
|
44
|
+
relevance: distanceToRelevance(item.distance),
|
|
45
|
+
}));
|
|
46
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { IndexDocument } from "../../protocol.js";
|
|
2
|
+
import { type ReindexOutcome } from "./schema.js";
|
|
3
|
+
type VecDatabase = import("better-sqlite3").Database;
|
|
4
|
+
export declare function reindexChunks(db: VecDatabase, documents: IndexDocument[]): Promise<ReindexOutcome>;
|
|
5
|
+
export {};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { getEmbedder } from "./embedder.js";
|
|
2
|
+
import { embeddingToBlob } from "./embeddingCodec.js";
|
|
3
|
+
import { bumpIndexGeneration, clearChunksIfEmbeddingMismatch, getIndexGeneration, writeEmbeddingMeta, } from "./schema.js";
|
|
4
|
+
export async function reindexChunks(db, documents) {
|
|
5
|
+
const embedder = getEmbedder();
|
|
6
|
+
clearChunksIfEmbeddingMismatch(db, embedder);
|
|
7
|
+
if (documents.length === 0) {
|
|
8
|
+
const indexGeneration = getIndexGeneration(db);
|
|
9
|
+
writeEmbeddingMeta(db, embedder);
|
|
10
|
+
return { indexed: 0, indexGeneration };
|
|
11
|
+
}
|
|
12
|
+
const embeddings = await embedder.embedBatch(documents.map((doc) => doc.content));
|
|
13
|
+
const sync = db.transaction((docs, vectors) => {
|
|
14
|
+
const incomingIds = new Set(docs.map((doc) => doc.id));
|
|
15
|
+
const existing = db.prepare("SELECT chunk_id FROM memory_chunks").all();
|
|
16
|
+
const deleteChunk = db.prepare("DELETE FROM memory_chunks WHERE chunk_id = ?");
|
|
17
|
+
for (const row of existing) {
|
|
18
|
+
if (!incomingIds.has(row.chunk_id)) {
|
|
19
|
+
deleteChunk.run(row.chunk_id);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const upsert = db.prepare(`
|
|
23
|
+
INSERT INTO memory_chunks(chunk_id, content, source, timestamp, embedding)
|
|
24
|
+
VALUES (?, ?, ?, ?, ?)
|
|
25
|
+
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
26
|
+
content = excluded.content,
|
|
27
|
+
source = excluded.source,
|
|
28
|
+
timestamp = excluded.timestamp,
|
|
29
|
+
embedding = excluded.embedding
|
|
30
|
+
`);
|
|
31
|
+
for (let i = 0; i < docs.length; i++) {
|
|
32
|
+
const doc = docs[i];
|
|
33
|
+
upsert.run(doc.id, doc.content, doc.source, doc.timestamp, embeddingToBlob(vectors[i]));
|
|
34
|
+
}
|
|
35
|
+
const indexGeneration = bumpIndexGeneration(db);
|
|
36
|
+
writeEmbeddingMeta(db, embedder);
|
|
37
|
+
return { indexed: docs.length, indexGeneration };
|
|
38
|
+
});
|
|
39
|
+
return sync(documents, embeddings);
|
|
40
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Embedder } from "../../../adapters/embed/types.js";
|
|
2
|
+
type VecDatabase = import("better-sqlite3").Database;
|
|
3
|
+
export type StoredEmbeddingMeta = {
|
|
4
|
+
model: string;
|
|
5
|
+
provider: string;
|
|
6
|
+
dim: number;
|
|
7
|
+
};
|
|
8
|
+
export type ReindexOutcome = {
|
|
9
|
+
indexed: number;
|
|
10
|
+
indexGeneration: number;
|
|
11
|
+
};
|
|
12
|
+
export declare function initVecSchema(db: VecDatabase): void;
|
|
13
|
+
export declare function getStoredEmbeddingMeta(db: VecDatabase): StoredEmbeddingMeta | null;
|
|
14
|
+
export declare function embeddingMetaMatches(db: VecDatabase, embedder: Embedder): boolean;
|
|
15
|
+
export declare function getIndexGeneration(db: VecDatabase): number;
|
|
16
|
+
export declare function getChunkCount(db: VecDatabase): number;
|
|
17
|
+
export declare function clearChunksIfEmbeddingMismatch(db: VecDatabase, embedder: Embedder): void;
|
|
18
|
+
export declare function bumpIndexGeneration(db: VecDatabase): number;
|
|
19
|
+
export declare function writeEmbeddingMeta(db: VecDatabase, embedder: Embedder): void;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export function initVecSchema(db) {
|
|
2
|
+
db.exec(`
|
|
3
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
4
|
+
key TEXT PRIMARY KEY,
|
|
5
|
+
value TEXT NOT NULL
|
|
6
|
+
);
|
|
7
|
+
CREATE TABLE IF NOT EXISTS memory_chunks (
|
|
8
|
+
chunk_id TEXT PRIMARY KEY,
|
|
9
|
+
content TEXT NOT NULL,
|
|
10
|
+
source TEXT NOT NULL,
|
|
11
|
+
timestamp TEXT NOT NULL,
|
|
12
|
+
embedding BLOB NOT NULL
|
|
13
|
+
);
|
|
14
|
+
`);
|
|
15
|
+
}
|
|
16
|
+
export function getStoredEmbeddingMeta(db) {
|
|
17
|
+
const read = (key) => db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(key);
|
|
18
|
+
const model = read("embedding_model");
|
|
19
|
+
const provider = read("embedding_provider");
|
|
20
|
+
const dimRaw = read("embedding_dim");
|
|
21
|
+
if (!model || !provider || !dimRaw)
|
|
22
|
+
return null;
|
|
23
|
+
const dim = Number.parseInt(dimRaw, 10);
|
|
24
|
+
if (!Number.isFinite(dim))
|
|
25
|
+
return null;
|
|
26
|
+
return { model, provider, dim };
|
|
27
|
+
}
|
|
28
|
+
export function embeddingMetaMatches(db, embedder) {
|
|
29
|
+
const stored = getStoredEmbeddingMeta(db);
|
|
30
|
+
if (!stored)
|
|
31
|
+
return true;
|
|
32
|
+
return (stored.model === embedder.model &&
|
|
33
|
+
stored.provider === embedder.provider &&
|
|
34
|
+
stored.dim === embedder.dim);
|
|
35
|
+
}
|
|
36
|
+
export function getIndexGeneration(db) {
|
|
37
|
+
const raw = db.prepare("SELECT value FROM meta WHERE key = 'index_generation'").pluck().get();
|
|
38
|
+
return Number(raw ?? "0");
|
|
39
|
+
}
|
|
40
|
+
export function getChunkCount(db) {
|
|
41
|
+
const row = db.prepare("SELECT COUNT(*) AS count FROM memory_chunks").get();
|
|
42
|
+
return row.count;
|
|
43
|
+
}
|
|
44
|
+
export function clearChunksIfEmbeddingMismatch(db, embedder) {
|
|
45
|
+
if (embeddingMetaMatches(db, embedder))
|
|
46
|
+
return;
|
|
47
|
+
db.prepare("DELETE FROM memory_chunks").run();
|
|
48
|
+
}
|
|
49
|
+
export function bumpIndexGeneration(db) {
|
|
50
|
+
const generation = getIndexGeneration(db) + 1;
|
|
51
|
+
db
|
|
52
|
+
.prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
|
|
53
|
+
.run("index_generation", String(generation));
|
|
54
|
+
return generation;
|
|
55
|
+
}
|
|
56
|
+
export function writeEmbeddingMeta(db, embedder) {
|
|
57
|
+
const upsert = db.prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value");
|
|
58
|
+
upsert.run("embedding_model", embedder.model);
|
|
59
|
+
upsert.run("embedding_provider", embedder.provider);
|
|
60
|
+
upsert.run("embedding_dim", String(embedder.dim));
|
|
61
|
+
}
|
|
@@ -1,25 +1,14 @@
|
|
|
1
1
|
import type { Embedder } from "../../../adapters/embed/types.js";
|
|
2
2
|
import type { IndexDocument, MemoryEntry } from "../../protocol.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
provider: string;
|
|
6
|
-
dim: number;
|
|
7
|
-
};
|
|
8
|
-
export type ReindexOutcome = {
|
|
9
|
-
indexed: number;
|
|
10
|
-
indexGeneration: number;
|
|
11
|
-
};
|
|
3
|
+
import { type ReindexOutcome, type StoredEmbeddingMeta } from "./schema.js";
|
|
4
|
+
export type { ReindexOutcome, StoredEmbeddingMeta };
|
|
12
5
|
export declare class VecStore {
|
|
13
6
|
private db;
|
|
14
7
|
constructor(dbPath: string);
|
|
15
|
-
private initSchema;
|
|
16
8
|
getStoredEmbeddingMeta(): StoredEmbeddingMeta | null;
|
|
17
9
|
embeddingMetaMatches(embedder: Embedder): boolean;
|
|
18
10
|
getIndexGeneration(): number;
|
|
19
11
|
getChunkCount(): number;
|
|
20
|
-
private clearChunksIfEmbeddingMismatch;
|
|
21
|
-
private bumpIndexGeneration;
|
|
22
|
-
private writeEmbeddingMeta;
|
|
23
12
|
reindex(documents: IndexDocument[]): Promise<ReindexOutcome>;
|
|
24
13
|
query(queryText: string, topK?: number): Promise<MemoryEntry[]>;
|
|
25
14
|
close(): void;
|