@chendpoc/pi-memory 0.2.4 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/README.md +92 -43
  2. package/dist/adapters/llm/standalone.js +1 -1
  3. package/dist/cli/init.js +20 -3
  4. package/dist/cli/parseArgs.d.ts +5 -2
  5. package/dist/cli/parseArgs.js +13 -0
  6. package/dist/cli/schedulerSync.d.ts +2 -0
  7. package/dist/cli/schedulerSync.js +26 -0
  8. package/dist/cli/status.d.ts +4 -40
  9. package/dist/cli/status.js +6 -230
  10. package/dist/cli/theme.d.ts +2 -0
  11. package/dist/cli/theme.js +8 -0
  12. package/dist/cli.js +5 -0
  13. package/dist/commands/status.js +2 -2
  14. package/dist/compact/parseMemoryExport.js +2 -1
  15. package/dist/compact/register.js +1 -1
  16. package/dist/compact/runSummary.js +1 -16
  17. package/dist/compact/subagentDelta.js +3 -5
  18. package/dist/consolidate/index.d.ts +1 -0
  19. package/dist/consolidate/index.js +1 -0
  20. package/dist/consolidate/mergeEntries.js +3 -3
  21. package/dist/consolidate/mergeMemoryEntries.d.ts +8 -0
  22. package/dist/consolidate/mergeMemoryEntries.js +23 -0
  23. package/dist/consolidate/mergePrompt.js +2 -2
  24. package/dist/consolidate/mergeWithLlm.js +2 -2
  25. package/dist/consolidate/runJob.d.ts +2 -2
  26. package/dist/consolidate/runJob.js +6 -12
  27. package/dist/consolidate/scheduler.d.ts +2 -2
  28. package/dist/consolidate/scheduler.js +1 -1
  29. package/dist/constants/env.d.ts +1 -0
  30. package/dist/constants/env.js +1 -0
  31. package/dist/constants/paths.d.ts +3 -1
  32. package/dist/constants/paths.js +6 -1
  33. package/dist/extension/createMemoryRuntime.d.ts +3 -0
  34. package/dist/extension/createMemoryRuntime.js +203 -0
  35. package/dist/extension/index.d.ts +2 -0
  36. package/dist/extension/index.js +1 -0
  37. package/dist/extension/lifecycle.d.ts +28 -0
  38. package/dist/extension/lifecycle.js +52 -0
  39. package/dist/extension/messageUtils.d.ts +4 -0
  40. package/dist/extension/messageUtils.js +18 -0
  41. package/dist/extension/types.d.ts +35 -0
  42. package/dist/extension/types.js +1 -0
  43. package/dist/index.d.ts +7 -9
  44. package/dist/index.js +7 -9
  45. package/dist/pi-extension.js +26 -236
  46. package/dist/preflight/episodic.js +13 -30
  47. package/dist/preflight/queryIntent.js +1 -1
  48. package/dist/preflight/render.js +1 -1
  49. package/dist/preflight/strip.d.ts +0 -1
  50. package/dist/preflight/strip.js +0 -24
  51. package/dist/redaction/index.d.ts +4 -0
  52. package/dist/redaction/index.js +3 -0
  53. package/dist/redaction/patterns/constants.d.ts +6 -0
  54. package/dist/redaction/patterns/constants.js +6 -0
  55. package/dist/redaction/patterns/crypto.d.ts +3 -0
  56. package/dist/redaction/patterns/crypto.js +17 -0
  57. package/dist/redaction/patterns/generic.d.ts +3 -0
  58. package/dist/redaction/patterns/generic.js +38 -0
  59. package/dist/redaction/patterns/index.d.ts +16 -0
  60. package/dist/redaction/patterns/index.js +23 -0
  61. package/dist/redaction/patterns/llm.d.ts +3 -0
  62. package/dist/redaction/patterns/llm.js +144 -0
  63. package/dist/redaction/patterns/platform.d.ts +3 -0
  64. package/dist/redaction/patterns/platform.js +87 -0
  65. package/dist/redaction/patterns/types.d.ts +18 -0
  66. package/dist/redaction/patterns/types.js +1 -0
  67. package/dist/redaction/redactText.d.ts +9 -0
  68. package/dist/redaction/redactText.js +31 -0
  69. package/dist/redaction/types.d.ts +19 -0
  70. package/dist/redaction/types.js +1 -0
  71. package/dist/redaction/utils.d.ts +28 -0
  72. package/dist/redaction/utils.js +106 -0
  73. package/dist/scheduler/index.d.ts +3 -0
  74. package/dist/scheduler/index.js +3 -0
  75. package/dist/scheduler/launchd.d.ts +14 -0
  76. package/dist/scheduler/launchd.js +69 -0
  77. package/dist/scheduler/launchdPlist.d.ts +14 -0
  78. package/dist/scheduler/launchdPlist.js +62 -0
  79. package/dist/scheduler/sync.d.ts +36 -0
  80. package/dist/scheduler/sync.js +79 -0
  81. package/dist/shutdown/enqueue.d.ts +1 -1
  82. package/dist/shutdown/enqueue.js +2 -5
  83. package/dist/shutdown/readQueue.js +1 -1
  84. package/dist/shutdown/runDrainJob.js +8 -37
  85. package/dist/shutdown/sessionReader.js +1 -14
  86. package/dist/sidecar/client.d.ts +6 -2
  87. package/dist/sidecar/client.js +49 -14
  88. package/dist/{preflight → sidecar}/queryCache.d.ts +1 -1
  89. package/dist/sidecar/reindexBridge.js +2 -2
  90. package/dist/sidecar/server/server.js +1 -1
  91. package/dist/sidecar/server/vec/chunkQuery.d.ts +4 -0
  92. package/dist/sidecar/server/vec/chunkQuery.js +46 -0
  93. package/dist/sidecar/server/vec/chunkReindex.d.ts +5 -0
  94. package/dist/sidecar/server/vec/chunkReindex.js +40 -0
  95. package/dist/sidecar/server/vec/embeddingCodec.d.ts +2 -0
  96. package/dist/sidecar/server/vec/embeddingCodec.js +6 -0
  97. package/dist/sidecar/server/vec/schema.d.ts +20 -0
  98. package/dist/sidecar/server/vec/schema.js +61 -0
  99. package/dist/sidecar/server/vec/store.d.ts +2 -13
  100. package/dist/sidecar/server/vec/store.js +12 -139
  101. package/dist/sidecar/sidecarManager.js +4 -58
  102. package/dist/sidecar/spawnLock.d.ts +2 -0
  103. package/dist/sidecar/spawnLock.js +57 -0
  104. package/dist/sidecar/syncIndex.d.ts +3 -0
  105. package/dist/sidecar/syncIndex.js +12 -0
  106. package/dist/sidecar/warmup.js +1 -1
  107. package/dist/status/copy.d.ts +2 -0
  108. package/dist/status/copy.js +2 -0
  109. package/dist/status/format.d.ts +7 -0
  110. package/dist/status/format.js +133 -0
  111. package/dist/status/gather.d.ts +2 -0
  112. package/dist/status/gather.js +88 -0
  113. package/dist/status/index.d.ts +4 -0
  114. package/dist/status/index.js +3 -0
  115. package/dist/status/types.d.ts +33 -0
  116. package/dist/status/types.js +1 -0
  117. package/dist/store/consolidatePort.d.ts +11 -0
  118. package/dist/store/consolidatePort.js +1 -0
  119. package/dist/store/index.d.ts +2 -0
  120. package/dist/store/index.js +1 -0
  121. package/dist/store/ingestEntries.d.ts +16 -0
  122. package/dist/store/ingestEntries.js +22 -0
  123. package/dist/{init/workspace.d.ts → store/initWorkspace.d.ts} +1 -1
  124. package/dist/{init/workspace.js → store/initWorkspace.js} +7 -5
  125. package/dist/store/listeners.d.ts +11 -0
  126. package/dist/store/listeners.js +27 -0
  127. package/dist/store/markdown/insert.d.ts +3 -0
  128. package/dist/store/markdown/insert.js +23 -0
  129. package/dist/store/memoryStore.d.ts +9 -22
  130. package/dist/store/memoryStore.js +71 -205
  131. package/dist/store/resolveEntries.d.ts +11 -0
  132. package/dist/store/resolveEntries.js +23 -0
  133. package/dist/store/types.d.ts +0 -1
  134. package/dist/store/writePath.d.ts +20 -0
  135. package/dist/store/writePath.js +123 -0
  136. package/dist/ui/memoryStatusWidget.d.ts +4 -8
  137. package/dist/ui/memoryStatusWidget.js +5 -17
  138. package/dist/utils/async.d.ts +11 -0
  139. package/dist/utils/async.js +24 -0
  140. package/dist/utils/index.d.ts +5 -1
  141. package/dist/utils/index.js +5 -1
  142. package/dist/{ipc/jsonlFramer.d.ts → utils/jsonl.d.ts} +1 -1
  143. package/dist/{ipc/jsonlFramer.js → utils/jsonl.js} +1 -1
  144. package/dist/utils/memory/index.d.ts +10 -0
  145. package/dist/utils/memory/index.js +43 -0
  146. package/dist/utils/paths.d.ts +4 -0
  147. package/dist/utils/paths.js +13 -3
  148. package/dist/utils/scheduler.d.ts +1 -1
  149. package/dist/utils/scheduler.js +6 -6
  150. package/dist/{preflight/session.d.ts → utils/session/index.d.ts} +1 -0
  151. package/dist/{preflight/session.js → utils/session/index.js} +5 -2
  152. package/doc/LAUNCH-KIT.md +229 -0
  153. package/doc/README-zh.md +445 -0
  154. package/doc/ROADMAP-zh.md +114 -0
  155. package/doc/ROADMAP.md +114 -0
  156. package/package.json +16 -4
  157. package/scripts/postinstall.mjs +11 -1
  158. package/templates/com.pi.memory.consolidate.plist.example +41 -0
  159. package/templates/consolidate.cmd.example +15 -0
  160. package/templates/crontab.example +14 -0
  161. package/templates/schtasks.example.txt +34 -0
  162. package/dist/consolidate/entryKey.d.ts +0 -5
  163. package/dist/consolidate/entryKey.js +0 -4
  164. /package/dist/{preflight → sidecar}/queryCache.js +0 -0
@@ -0,0 +1,79 @@
1
+ import { homedir } from "node:os";
2
+ import { dirname, join } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import { ENV_KEYS } from "../constants/env.js";
5
+ import { LAUNCHD_LABEL } from "../constants/paths.js";
6
+ import { resolveMemoryAgentDir } from "../config/agentDir.js";
7
+ import { debugMemory } from "../utils/debugLog.js";
8
+ import { isMacOS } from "../utils/platform.js";
9
+ import { defaultConsolidateSchedulerPaths } from "../utils/scheduler.js";
10
+ import { syncLaunchdMaintenanceJob } from "./launchd.js";
11
+ export function isSchedulerSyncDisabled(env = process.env) {
12
+ const value = env[ENV_KEYS.SKIP_SCHEDULER_SYNC];
13
+ return value === "1" || value === "true";
14
+ }
15
+ /** Whether this process can attempt a user LaunchAgent sync (macOS + uid + home). */
16
+ export function canSyncLaunchdInProcess(env = process.env) {
17
+ if (!isMacOS())
18
+ return { ok: false, reason: "unsupported-platform" };
19
+ if (process.getuid?.() === undefined)
20
+ return { ok: false, reason: "no-user-id" };
21
+ if (!homedir()?.trim())
22
+ return { ok: false, reason: "no-home-directory" };
23
+ if (isSchedulerSyncDisabled(env))
24
+ return { ok: false, reason: "PI_MEMORY_SKIP_SCHEDULER_SYNC" };
25
+ return { ok: true };
26
+ }
27
+ export function resolvePackageCliPath(moduleUrl = import.meta.url) {
28
+ return join(dirname(fileURLToPath(moduleUrl)), "..", "cli.js");
29
+ }
30
+ function formatSyncError(error) {
31
+ return error instanceof Error ? error.message : String(error);
32
+ }
33
+ /**
34
+ * Install or refresh the OS maintenance scheduler (macOS launchd today).
35
+ * Never throws — automatic callers (postinstall, session_start, init) treat failures as best-effort.
36
+ */
37
+ export async function syncMaintenanceScheduler(opts = {}) {
38
+ const env = opts.env ?? process.env;
39
+ const capability = canSyncLaunchdInProcess(env);
40
+ if (!capability.ok) {
41
+ return { status: "skipped", reason: capability.reason };
42
+ }
43
+ try {
44
+ const agentDir = opts.agentDir ?? resolveMemoryAgentDir({ env });
45
+ const paths = defaultConsolidateSchedulerPaths(agentDir);
46
+ const nodePath = opts.nodePath ?? process.execPath;
47
+ const cliPath = opts.cliPath ?? resolvePackageCliPath(import.meta.url);
48
+ const result = await syncLaunchdMaintenanceJob({
49
+ label: LAUNCHD_LABEL,
50
+ nodePath,
51
+ cliPath,
52
+ envFile: paths.envFile,
53
+ agentDir: paths.agentDir,
54
+ logsDir: paths.logsDir,
55
+ stdoutLog: paths.stdoutLog,
56
+ stderrLog: paths.stderrLog,
57
+ });
58
+ debugMemory("scheduler", "launchd sync complete", {
59
+ label: result.label,
60
+ changed: result.changed,
61
+ bootstrapped: result.bootstrapped,
62
+ removedLegacy: result.removedLegacy.join(",") || "none",
63
+ }, env);
64
+ return {
65
+ status: "synced",
66
+ platform: "launchd",
67
+ label: result.label,
68
+ plistPath: result.plistPath,
69
+ changed: result.changed,
70
+ bootstrapped: result.bootstrapped,
71
+ removedLegacy: result.removedLegacy,
72
+ };
73
+ }
74
+ catch (error) {
75
+ const message = formatSyncError(error);
76
+ debugMemory("scheduler", "launchd sync failed (best-effort)", { message }, env);
77
+ return { status: "failed", reason: "launchd-sync-error", message };
78
+ }
79
+ }
@@ -1,4 +1,5 @@
1
1
  import type { SessionShutdownEvent } from "@earendil-works/pi-coding-agent";
2
+ export { readParentSession } from "../utils/session/index.js";
2
3
  export type ShutdownQueueEntry = {
3
4
  sessionFile: string;
4
5
  parentSession?: string;
@@ -8,4 +9,3 @@ export type ShutdownQueueEntry = {
8
9
  };
9
10
  export declare function shutdownQueuePath(agentDir: string): string;
10
11
  export declare function enqueueShutdownMetadata(agentDir: string, entry: ShutdownQueueEntry): Promise<void>;
11
- export declare function readParentSession(header: Record<string, unknown> | null): string | undefined;
@@ -1,13 +1,10 @@
1
1
  import { SHUTDOWN_QUEUE_FILE } from "../constants/memory.js";
2
- import { serializeJsonlFrame } from "../ipc/jsonlFramer.js";
2
+ import { serializeJsonlFrame } from "../utils/jsonl.js";
3
3
  import { appendText, joinPath } from "../utils/fs.js";
4
+ export { readParentSession } from "../utils/session/index.js";
4
5
  export function shutdownQueuePath(agentDir) {
5
6
  return joinPath(agentDir, SHUTDOWN_QUEUE_FILE);
6
7
  }
7
8
  export async function enqueueShutdownMetadata(agentDir, entry) {
8
9
  await appendText(shutdownQueuePath(agentDir), serializeJsonlFrame(entry));
9
10
  }
10
- export function readParentSession(header) {
11
- const parent = header?.parentSession ?? header?.parent_session;
12
- return typeof parent === "string" && parent.trim().length > 0 ? parent.trim() : undefined;
13
- }
@@ -1,4 +1,4 @@
1
- import { parseJsonlLine } from "../ipc/jsonlFramer.js";
1
+ import { parseJsonlLine } from "../utils/jsonl.js";
2
2
  import { readText } from "../utils/fs.js";
3
3
  import { shutdownQueuePath } from "./enqueue.js";
4
4
  /** Latest queue row per session file (deduped). */
@@ -1,9 +1,5 @@
1
- import { filterCompactionDelta, shouldSkipSubagentCompactionIngest, } from "../compact/subagentDelta.js";
2
- import { parseMemoryExport } from "../compact/parseMemoryExport.js";
3
- import { reindex } from "../sidecar/client.js";
4
- import { ensureSidecarRunning } from "../sidecar/sidecarManager.js";
5
- import { resolveSidecarPaths } from "../sidecar/paths.js";
6
- import { sidecarQueryCache } from "../preflight/queryCache.js";
1
+ import { syncSidecarIndex } from "../sidecar/syncIndex.js";
2
+ import { ingestMemoryExport } from "../store/ingestEntries.js";
7
3
  import { canRead, pathExists } from "../utils/fs.js";
8
4
  import { buildShutdownMemoryExportPrompt, wrapShutdownExportMarkdown, } from "./extractPrompt.js";
9
5
  import { markShutdownProcessed, readShutdownProcessedState, isShutdownProcessed } from "./processed.js";
@@ -104,39 +100,14 @@ async function drainQueueEntry(opts) {
104
100
  return { appended };
105
101
  }
106
102
  async function ingestFromSummary(opts) {
107
- const parsed = parseMemoryExport(opts.summary);
108
- if (parsed.length === 0) {
109
- if (opts.compactionId) {
110
- await opts.store.markCompactionProcessed(opts.compactionId);
111
- }
112
- await markShutdownProcessed(opts.agentDir, opts.sessionFile);
113
- return 0;
114
- }
115
- let entries = parsed;
116
- if (opts.subagent) {
117
- const existing = await opts.store.listEntries();
118
- entries = filterCompactionDelta(parsed, existing);
119
- if (shouldSkipSubagentCompactionIngest(parsed, entries)) {
120
- if (opts.compactionId) {
121
- await opts.store.markCompactionProcessed(opts.compactionId);
122
- }
123
- await markShutdownProcessed(opts.agentDir, opts.sessionFile);
124
- return 0;
125
- }
126
- }
127
- if (entries.length > 0) {
128
- await opts.store.appendMany(entries, { mode: "ifAbsent" });
129
- }
103
+ const { appended } = await ingestMemoryExport({
104
+ store: opts.store,
105
+ summary: opts.summary,
106
+ isSubagent: !!opts.subagent,
107
+ });
130
108
  if (opts.compactionId) {
131
109
  await opts.store.markCompactionProcessed(opts.compactionId);
132
110
  }
133
111
  await markShutdownProcessed(opts.agentDir, opts.sessionFile);
134
- return entries.length;
135
- }
136
- async function syncSidecarIndex(agentDir, store) {
137
- const sidecar = resolveSidecarPaths(agentDir);
138
- await ensureSidecarRunning(sidecar);
139
- const result = await reindex(sidecar.socketPath, await store.exportForIndex());
140
- sidecarQueryCache.onReindexComplete(agentDir, result.index_generation);
141
- return result.index_generation;
112
+ return appended;
142
113
  }
@@ -1,19 +1,6 @@
1
1
  import { buildSessionContext, convertToLlm, getLatestCompactionEntry, parseSessionEntries, serializeConversation, } from "@earendil-works/pi-coding-agent";
2
- import { stripPrivateMemory } from "../preflight/strip.js";
2
+ import { stripPrivateMemoryFromMessages } from "../utils/memory/index.js";
3
3
  import { readTextRequired } from "../utils/fs.js";
4
- function stripPrivateMemoryFromMessages(messages) {
5
- return messages.map((message) => {
6
- if (message.role !== "user")
7
- return message;
8
- if (typeof message.content === "string") {
9
- return { ...message, content: stripPrivateMemory(message.content) };
10
- }
11
- return {
12
- ...message,
13
- content: message.content.map((block) => block.type === "text" ? { ...block, text: stripPrivateMemory(block.text) } : block),
14
- };
15
- });
16
- }
17
4
  function toSessionEntries(fileEntries) {
18
5
  return fileEntries.filter((entry) => entry.type !== "session");
19
6
  }
@@ -1,12 +1,16 @@
1
1
  import { type IndexDocument, type IndexStats, type SidecarResponse } from "./protocol.js";
2
- export declare function sidecarRequest<T extends SidecarResponse>(socketPath: string, frame: Record<string, unknown>, timeoutMs?: number): Promise<T>;
2
+ export type SidecarRequestOptions = {
3
+ timeoutMs?: number;
4
+ signal?: AbortSignal;
5
+ };
6
+ export declare function sidecarRequest<T extends SidecarResponse>(socketPath: string, frame: Record<string, unknown>, options?: SidecarRequestOptions): Promise<T>;
3
7
  export declare function ping(socketPath: string): Promise<boolean>;
4
8
  export declare function fetchIndexStats(socketPath: string): Promise<{
5
9
  stats: IndexStats;
6
10
  } | {
7
11
  error: string;
8
12
  }>;
9
- export declare function query(socketPath: string, queryText: string, timeoutMs?: number): Promise<{
13
+ export declare function query(socketPath: string, queryText: string, options?: SidecarRequestOptions): Promise<{
10
14
  type: "result";
11
15
  request_id: string;
12
16
  results: import("./protocol.js").MemoryEntry[];
@@ -2,45 +2,80 @@
2
2
  import { randomUUID } from "node:crypto";
3
3
  import net from "node:net";
4
4
  import { SIDECAR_PING_TIMEOUT_MS, SIDECAR_QUERY_TIMEOUT_MS, SIDECAR_REINDEX_TIMEOUT_MS, } from "../constants/timing.js";
5
- import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../ipc/jsonlFramer.js";
5
+ import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../utils/jsonl.js";
6
6
  import { isErrorResponse } from "./protocol.js";
7
- export function sidecarRequest(socketPath, frame, timeoutMs = SIDECAR_QUERY_TIMEOUT_MS) {
7
+ export function sidecarRequest(socketPath, frame, options = {}) {
8
+ const timeoutMs = options.timeoutMs ?? SIDECAR_QUERY_TIMEOUT_MS;
9
+ const { signal } = options;
8
10
  return new Promise((resolve, reject) => {
11
+ if (signal?.aborted) {
12
+ reject(abortError(signal));
13
+ return;
14
+ }
9
15
  const socket = net.connect(socketPath);
10
16
  const framer = new JsonlFramer();
11
- const timer = setTimeout(() => {
17
+ let settled = false;
18
+ const cleanup = () => {
19
+ clearTimeout(timer);
20
+ signal?.removeEventListener("abort", onAbort);
21
+ };
22
+ const fail = (error) => {
23
+ if (settled)
24
+ return;
25
+ settled = true;
26
+ cleanup();
12
27
  socket.destroy();
13
- reject(new Error("Sidecar request timed out"));
28
+ reject(error);
29
+ };
30
+ const succeed = (value) => {
31
+ if (settled)
32
+ return;
33
+ settled = true;
34
+ cleanup();
35
+ resolve(value);
36
+ };
37
+ const onAbort = () => {
38
+ fail(abortError(signal));
39
+ };
40
+ const timer = setTimeout(() => {
41
+ fail(new Error("Sidecar request timed out"));
14
42
  }, timeoutMs);
43
+ if (signal) {
44
+ signal.addEventListener("abort", onAbort, { once: true });
45
+ }
15
46
  socket.on("connect", () => {
16
47
  socket.write(serializeJsonlFrame(frame));
17
48
  });
18
49
  socket.on("data", (chunk) => {
19
50
  for (const line of framer.push(chunk.toString())) {
20
- clearTimeout(timer);
21
51
  socket.end();
22
52
  let response;
23
53
  try {
24
54
  response = parseJsonlLine(line);
25
55
  }
26
56
  catch {
27
- reject(new Error("Invalid JSON response from sidecar"));
57
+ fail(new Error("Invalid JSON response from sidecar"));
28
58
  return;
29
59
  }
30
60
  if (isErrorResponse(response)) {
31
- reject(new Error(response.error));
61
+ fail(new Error(response.error));
32
62
  return;
33
63
  }
34
- resolve(response);
64
+ succeed(response);
35
65
  return;
36
66
  }
37
67
  });
38
- socket.on("error", reject);
68
+ socket.on("error", (error) => {
69
+ fail(error instanceof Error ? error : new Error(String(error)));
70
+ });
39
71
  });
40
72
  }
73
+ function abortError(signal) {
74
+ return signal.reason instanceof Error ? signal.reason : new Error("Aborted");
75
+ }
41
76
  export async function ping(socketPath) {
42
77
  try {
43
- const res = await sidecarRequest(socketPath, { type: "ping" }, SIDECAR_PING_TIMEOUT_MS);
78
+ const res = await sidecarRequest(socketPath, { type: "ping" }, { timeoutMs: SIDECAR_PING_TIMEOUT_MS });
44
79
  return res.type === "pong";
45
80
  }
46
81
  catch {
@@ -49,7 +84,7 @@ export async function ping(socketPath) {
49
84
  }
50
85
  export async function fetchIndexStats(socketPath) {
51
86
  try {
52
- const res = await sidecarRequest(socketPath, { type: "stats" }, SIDECAR_PING_TIMEOUT_MS);
87
+ const res = await sidecarRequest(socketPath, { type: "stats" }, { timeoutMs: SIDECAR_PING_TIMEOUT_MS });
53
88
  if (res.type !== "stats_ok")
54
89
  return { error: "unexpected sidecar response" };
55
90
  const { type: _type, ...stats } = res;
@@ -60,11 +95,11 @@ export async function fetchIndexStats(socketPath) {
60
95
  return { error: message };
61
96
  }
62
97
  }
63
- export async function query(socketPath, queryText, timeoutMs = SIDECAR_QUERY_TIMEOUT_MS) {
98
+ export async function query(socketPath, queryText, options = {}) {
64
99
  const request_id = randomUUID();
65
- return sidecarRequest(socketPath, { type: "query", request_id, query: queryText }, timeoutMs);
100
+ return sidecarRequest(socketPath, { type: "query", request_id, query: queryText }, options);
66
101
  }
67
102
  export async function reindex(socketPath, documents = []) {
68
103
  const request_id = randomUUID();
69
- return sidecarRequest(socketPath, { type: "reindex", request_id, documents }, SIDECAR_REINDEX_TIMEOUT_MS);
104
+ return sidecarRequest(socketPath, { type: "reindex", request_id, documents }, { timeoutMs: SIDECAR_REINDEX_TIMEOUT_MS });
70
105
  }
@@ -1,4 +1,4 @@
1
- import type { MemoryEntry } from "../sidecar/protocol.js";
1
+ import type { MemoryEntry } from "./protocol.js";
2
2
  declare class SidecarQueryCache {
3
3
  private readonly generationByAgent;
4
4
  private readonly lru;
@@ -1,6 +1,6 @@
1
- import debounce from "lodash/debounce.js";
1
+ import { debounce } from "es-toolkit";
2
2
  import { DEFAULT_REINDEX_DEBOUNCE_MS } from "../constants/timing.js";
3
- import { sidecarQueryCache } from "../preflight/queryCache.js";
3
+ import { sidecarQueryCache } from "./queryCache.js";
4
4
  import { ensureSidecarRunning } from "./sidecarManager.js";
5
5
  import { reindex } from "./client.js";
6
6
  export function createReindexScheduler(opts) {
@@ -1,7 +1,7 @@
1
1
  import { createServer } from "node:net";
2
2
  import { writeFileSync } from "node:fs";
3
3
  import { SIDECAR_PID_SUFFIX } from "../../constants/paths.js";
4
- import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../../ipc/jsonlFramer.js";
4
+ import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../../utils/jsonl.js";
5
5
  import { ensureDirSync, pathDirname } from "../../utils/fs.js";
6
6
  import { cleanupSocketFiles, removeSocketFile, secureSocketPath } from "../../utils/socket.js";
7
7
  import { handleQuery } from "./query.js";
@@ -0,0 +1,4 @@
1
+ import type { MemoryEntry } from "../../protocol.js";
2
+ type VecDatabase = import("better-sqlite3").Database;
3
+ export declare function queryChunks(db: VecDatabase, queryText: string, topK?: number): Promise<MemoryEntry[]>;
4
+ export {};
@@ -0,0 +1,46 @@
1
+ import { readRetrievalConfig } from "../../../config/retrieval.js";
2
+ import { blobToEmbedding } from "./embeddingCodec.js";
3
+ import { getEmbedder } from "./embedder.js";
4
+ import { cosineSimilarity, distanceToRelevance, mmrSelect } from "./mmr.js";
5
+ import { clearChunksIfEmbeddingMismatch } from "./schema.js";
6
+ export async function queryChunks(db, queryText, topK) {
7
+ const retrieval = readRetrievalConfig();
8
+ const limit = topK ?? retrieval.topK;
9
+ const embedder = getEmbedder();
10
+ clearChunksIfEmbeddingMismatch(db, embedder);
11
+ const queryEmbedding = await embedder.embed(queryText);
12
+ const rows = db
13
+ .prepare("SELECT chunk_id, content, source, timestamp, embedding FROM memory_chunks")
14
+ .all();
15
+ if (rows.length === 0)
16
+ return [];
17
+ const candidates = [];
18
+ for (const row of rows) {
19
+ const embedding = blobToEmbedding(row.embedding);
20
+ if (embedding.length !== queryEmbedding.length)
21
+ continue;
22
+ const similarity = cosineSimilarity(queryEmbedding, embedding);
23
+ if (similarity < retrieval.minRelevance)
24
+ continue;
25
+ candidates.push({
26
+ chunkId: row.chunk_id,
27
+ content: row.content,
28
+ source: row.source,
29
+ timestamp: row.timestamp,
30
+ distance: 1 - similarity,
31
+ embedding,
32
+ });
33
+ }
34
+ if (candidates.length === 0)
35
+ return [];
36
+ candidates.sort((a, b) => a.distance - b.distance);
37
+ const poolSize = Math.min(limit * retrieval.candidatePoolMultiplier, candidates.length);
38
+ const pool = candidates.slice(0, poolSize);
39
+ const selected = mmrSelect(queryEmbedding, pool, limit, retrieval.mmrLambda);
40
+ return selected.map((item) => ({
41
+ content: item.content,
42
+ source: item.source,
43
+ timestamp: item.timestamp,
44
+ relevance: distanceToRelevance(item.distance),
45
+ }));
46
+ }
@@ -0,0 +1,5 @@
1
+ import type { IndexDocument } from "../../protocol.js";
2
+ import { type ReindexOutcome } from "./schema.js";
3
+ type VecDatabase = import("better-sqlite3").Database;
4
+ export declare function reindexChunks(db: VecDatabase, documents: IndexDocument[]): Promise<ReindexOutcome>;
5
+ export {};
@@ -0,0 +1,40 @@
1
+ import { getEmbedder } from "./embedder.js";
2
+ import { embeddingToBlob } from "./embeddingCodec.js";
3
+ import { bumpIndexGeneration, clearChunksIfEmbeddingMismatch, getIndexGeneration, writeEmbeddingMeta, } from "./schema.js";
4
+ export async function reindexChunks(db, documents) {
5
+ const embedder = getEmbedder();
6
+ clearChunksIfEmbeddingMismatch(db, embedder);
7
+ if (documents.length === 0) {
8
+ const indexGeneration = getIndexGeneration(db);
9
+ writeEmbeddingMeta(db, embedder);
10
+ return { indexed: 0, indexGeneration };
11
+ }
12
+ const embeddings = await embedder.embedBatch(documents.map((doc) => doc.content));
13
+ const sync = db.transaction((docs, vectors) => {
14
+ const incomingIds = new Set(docs.map((doc) => doc.id));
15
+ const existing = db.prepare("SELECT chunk_id FROM memory_chunks").all();
16
+ const deleteChunk = db.prepare("DELETE FROM memory_chunks WHERE chunk_id = ?");
17
+ for (const row of existing) {
18
+ if (!incomingIds.has(row.chunk_id)) {
19
+ deleteChunk.run(row.chunk_id);
20
+ }
21
+ }
22
+ const upsert = db.prepare(`
23
+ INSERT INTO memory_chunks(chunk_id, content, source, timestamp, embedding)
24
+ VALUES (?, ?, ?, ?, ?)
25
+ ON CONFLICT(chunk_id) DO UPDATE SET
26
+ content = excluded.content,
27
+ source = excluded.source,
28
+ timestamp = excluded.timestamp,
29
+ embedding = excluded.embedding
30
+ `);
31
+ for (let i = 0; i < docs.length; i++) {
32
+ const doc = docs[i];
33
+ upsert.run(doc.id, doc.content, doc.source, doc.timestamp, embeddingToBlob(vectors[i]));
34
+ }
35
+ const indexGeneration = bumpIndexGeneration(db);
36
+ writeEmbeddingMeta(db, embedder);
37
+ return { indexed: docs.length, indexGeneration };
38
+ });
39
+ return sync(documents, embeddings);
40
+ }
@@ -0,0 +1,2 @@
1
+ export declare function embeddingToBlob(embedding: Float32Array): Buffer;
2
+ export declare function blobToEmbedding(blob: Buffer): Float32Array;
@@ -0,0 +1,6 @@
1
+ export function embeddingToBlob(embedding) {
2
+ return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
3
+ }
4
+ export function blobToEmbedding(blob) {
5
+ return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
6
+ }
@@ -0,0 +1,20 @@
1
+ import type { Embedder } from "../../../adapters/embed/types.js";
2
+ type VecDatabase = import("better-sqlite3").Database;
3
+ export type StoredEmbeddingMeta = {
4
+ model: string;
5
+ provider: string;
6
+ dim: number;
7
+ };
8
+ export type ReindexOutcome = {
9
+ indexed: number;
10
+ indexGeneration: number;
11
+ };
12
+ export declare function initVecSchema(db: VecDatabase): void;
13
+ export declare function getStoredEmbeddingMeta(db: VecDatabase): StoredEmbeddingMeta | null;
14
+ export declare function embeddingMetaMatches(db: VecDatabase, embedder: Embedder): boolean;
15
+ export declare function getIndexGeneration(db: VecDatabase): number;
16
+ export declare function getChunkCount(db: VecDatabase): number;
17
+ export declare function clearChunksIfEmbeddingMismatch(db: VecDatabase, embedder: Embedder): void;
18
+ export declare function bumpIndexGeneration(db: VecDatabase): number;
19
+ export declare function writeEmbeddingMeta(db: VecDatabase, embedder: Embedder): void;
20
+ export {};
@@ -0,0 +1,61 @@
1
+ export function initVecSchema(db) {
2
+ db.exec(`
3
+ CREATE TABLE IF NOT EXISTS meta (
4
+ key TEXT PRIMARY KEY,
5
+ value TEXT NOT NULL
6
+ );
7
+ CREATE TABLE IF NOT EXISTS memory_chunks (
8
+ chunk_id TEXT PRIMARY KEY,
9
+ content TEXT NOT NULL,
10
+ source TEXT NOT NULL,
11
+ timestamp TEXT NOT NULL,
12
+ embedding BLOB NOT NULL
13
+ );
14
+ `);
15
+ }
16
+ export function getStoredEmbeddingMeta(db) {
17
+ const read = (key) => db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(key);
18
+ const model = read("embedding_model");
19
+ const provider = read("embedding_provider");
20
+ const dimRaw = read("embedding_dim");
21
+ if (!model || !provider || !dimRaw)
22
+ return null;
23
+ const dim = Number.parseInt(dimRaw, 10);
24
+ if (!Number.isFinite(dim))
25
+ return null;
26
+ return { model, provider, dim };
27
+ }
28
+ export function embeddingMetaMatches(db, embedder) {
29
+ const stored = getStoredEmbeddingMeta(db);
30
+ if (!stored)
31
+ return true;
32
+ return (stored.model === embedder.model &&
33
+ stored.provider === embedder.provider &&
34
+ stored.dim === embedder.dim);
35
+ }
36
+ export function getIndexGeneration(db) {
37
+ const raw = db.prepare("SELECT value FROM meta WHERE key = 'index_generation'").pluck().get();
38
+ return Number(raw ?? "0");
39
+ }
40
+ export function getChunkCount(db) {
41
+ const row = db.prepare("SELECT COUNT(*) AS count FROM memory_chunks").get();
42
+ return row.count;
43
+ }
44
+ export function clearChunksIfEmbeddingMismatch(db, embedder) {
45
+ if (embeddingMetaMatches(db, embedder))
46
+ return;
47
+ db.prepare("DELETE FROM memory_chunks").run();
48
+ }
49
+ export function bumpIndexGeneration(db) {
50
+ const generation = getIndexGeneration(db) + 1;
51
+ db
52
+ .prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
53
+ .run("index_generation", String(generation));
54
+ return generation;
55
+ }
56
+ export function writeEmbeddingMeta(db, embedder) {
57
+ const upsert = db.prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value");
58
+ upsert.run("embedding_model", embedder.model);
59
+ upsert.run("embedding_provider", embedder.provider);
60
+ upsert.run("embedding_dim", String(embedder.dim));
61
+ }
@@ -1,25 +1,14 @@
1
1
  import type { Embedder } from "../../../adapters/embed/types.js";
2
2
  import type { IndexDocument, MemoryEntry } from "../../protocol.js";
3
- export type StoredEmbeddingMeta = {
4
- model: string;
5
- provider: string;
6
- dim: number;
7
- };
8
- export type ReindexOutcome = {
9
- indexed: number;
10
- indexGeneration: number;
11
- };
3
+ import { type ReindexOutcome, type StoredEmbeddingMeta } from "./schema.js";
4
+ export type { ReindexOutcome, StoredEmbeddingMeta };
12
5
  export declare class VecStore {
13
6
  private db;
14
7
  constructor(dbPath: string);
15
- private initSchema;
16
8
  getStoredEmbeddingMeta(): StoredEmbeddingMeta | null;
17
9
  embeddingMetaMatches(embedder: Embedder): boolean;
18
10
  getIndexGeneration(): number;
19
11
  getChunkCount(): number;
20
- private clearChunksIfEmbeddingMismatch;
21
- private bumpIndexGeneration;
22
- private writeEmbeddingMeta;
23
12
  reindex(documents: IndexDocument[]): Promise<ReindexOutcome>;
24
13
  query(queryText: string, topK?: number): Promise<MemoryEntry[]>;
25
14
  close(): void;