claude-memory-layer 1.0.23 → 1.0.25

package/src/hooks/semantic-daemon-client.ts

@@ -0,0 +1,208 @@
+import { spawn } from 'child_process';
+import * as fs from 'fs';
+import * as net from 'net';
+import * as os from 'os';
+import * as path from 'path';
+
+interface SemanticRequest {
+  sessionId: string;
+  prompt: string;
+  topK: number;
+  minScore: number;
+}
+
+interface SemanticMemory {
+  type: string;
+  content: string;
+  id?: string;
+  score?: number;
+}
+
+interface SemanticDaemonRequest {
+  type: 'retrieve';
+  sessionId: string;
+  prompt: string;
+  topK: number;
+  minScore: number;
+}
+
+interface SemanticDaemonResponse {
+  ok: boolean;
+  memories?: SemanticMemory[];
+  error?: string;
+}
+
+const DEFAULT_SOCKET_PATH = path.join(
+  os.homedir(),
+  '.claude-code',
+  'memory',
+  'semantic-daemon.sock'
+);
+
+const DAEMON_SOCKET_PATH = process.env.CLAUDE_MEMORY_SEMANTIC_SOCKET || DEFAULT_SOCKET_PATH;
+const DAEMON_START_TIMEOUT_MS = parseInt(process.env.CLAUDE_MEMORY_SEMANTIC_DAEMON_START_MS || '1500');
+
+let daemonStartPromise: Promise<void> | null = null;
+
+export async function retrieveSemanticMemories(
+  request: SemanticRequest,
+  timeoutMs: number
+): Promise<SemanticMemory[]> {
+  const payload: SemanticDaemonRequest = {
+    type: 'retrieve',
+    sessionId: request.sessionId,
+    prompt: request.prompt,
+    topK: request.topK,
+    minScore: request.minScore
+  };
+
+  try {
+    return await requestFromDaemon(payload, timeoutMs);
+  } catch (error) {
+    if (!isConnectionError(error)) {
+      throw error;
+    }
+
+    await ensureDaemonRunning();
+    return requestFromDaemon(payload, timeoutMs).catch((retryError) => {
+      if (process.env.CLAUDE_MEMORY_DEBUG) {
+        console.error('[semantic-client] retry failed after daemon start:', retryError);
+      }
+      throw retryError;
+    });
+  }
+}
+
+function requestFromDaemon(
+  payload: SemanticDaemonRequest,
+  timeoutMs: number
+): Promise<SemanticMemory[]> {
+  return new Promise((resolve, reject) => {
+    const client = net.createConnection(DAEMON_SOCKET_PATH);
+    client.setEncoding('utf8');
+
+    let settled = false;
+    let responseRaw = '';
+    const timer = setTimeout(() => {
+      const timeoutError = new Error(`semantic daemon timeout (${timeoutMs}ms)`);
+      (timeoutError as NodeJS.ErrnoException).code = 'ETIMEDOUT';
+      settle(timeoutError);
+      client.destroy();
+    }, timeoutMs);
+
+    const settle = (error?: Error, memories?: SemanticMemory[]) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (error) {
+        reject(error);
+      } else {
+        resolve(memories || []);
+      }
+    };
+
+    client.on('connect', () => {
+      client.end(JSON.stringify(payload));
+    });
+
+    client.on('data', (chunk) => {
+      responseRaw += chunk;
+      if (responseRaw.length > 4 * 1024 * 1024) {
+        settle(new Error('semantic daemon response too large'));
+        client.destroy();
+      }
+    });
+
+    client.on('end', () => {
+      try {
+        const parsed = JSON.parse(responseRaw || '{}') as SemanticDaemonResponse;
+        if (!parsed.ok) {
+          settle(new Error(parsed.error || 'semantic daemon error'));
+          return;
+        }
+        settle(undefined, parsed.memories || []);
+      } catch (error) {
+        settle(error as Error);
+      }
+    });
+
+    client.on('error', (error) => {
+      settle(error as Error);
+    });
+  });
+}
+
+export async function ensureDaemonRunning(): Promise<void> {
+  if (daemonStartPromise) {
+    return daemonStartPromise;
+  }
+
+  daemonStartPromise = (async () => {
+    if (await canConnect()) {
+      return;
+    }
+
+    const daemonScriptPath = getDaemonScriptPath();
+    if (!fs.existsSync(daemonScriptPath)) {
+      throw new Error(`semantic daemon script not found: ${daemonScriptPath}`);
+    }
+
+    const daemonDir = path.dirname(DAEMON_SOCKET_PATH);
+    if (!fs.existsSync(daemonDir)) {
+      fs.mkdirSync(daemonDir, { recursive: true });
+    }
+
+    const child = spawn(process.execPath, [daemonScriptPath], {
+      detached: true,
+      stdio: 'ignore',
+      env: process.env
+    });
+    child.unref();
+
+    const startDeadline = Date.now() + DAEMON_START_TIMEOUT_MS;
+    while (Date.now() < startDeadline) {
+      if (await canConnect()) {
+        return;
+      }
+      await sleep(60);
+    }
+
+    throw new Error(`semantic daemon start timeout (${DAEMON_START_TIMEOUT_MS}ms)`);
+  })();
+
+  try {
+    await daemonStartPromise;
+  } finally {
+    daemonStartPromise = null;
+  }
+}
+
+function getDaemonScriptPath(): string {
+  return path.join(path.dirname(new URL(import.meta.url).pathname), 'semantic-daemon.js');
+}
+
+function canConnect(): Promise<boolean> {
+  return new Promise((resolve) => {
+    let settled = false;
+    const client = net.createConnection(DAEMON_SOCKET_PATH);
+    const finalize = (ok: boolean) => {
+      if (settled) return;
+      settled = true;
+      client.destroy();
+      resolve(ok);
+    };
+
+    client.on('connect', () => finalize(true));
+    client.on('error', () => finalize(false));
+    setTimeout(() => finalize(false), 120).unref();
+  });
+}
+
+function isConnectionError(error: unknown): boolean {
+  const code = (error as NodeJS.ErrnoException | undefined)?.code;
+  return code === 'ENOENT' || code === 'ECONNREFUSED' || code === 'EPIPE' || code === 'ECONNRESET';
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/hooks/semantic-daemon.ts

@@ -0,0 +1,276 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs';
+import * as net from 'net';
+import * as os from 'os';
+import * as path from 'path';
+import { MemoryService, getProjectStoragePath, getSessionProject } from '../services/memory-service.js';
+
+interface SemanticDaemonRequest {
+  type?: 'retrieve';
+  sessionId?: string;
+  prompt?: string;
+  topK?: number;
+  minScore?: number;
+}
+
+interface SemanticMemory {
+  type: string;
+  content: string;
+  id?: string;
+  score?: number;
+}
+
+interface SemanticDaemonResponse {
+  ok: boolean;
+  memories?: SemanticMemory[];
+  error?: string;
+}
+
+const SOCKET_PATH = process.env.CLAUDE_MEMORY_SEMANTIC_SOCKET || path.join(
+  os.homedir(),
+  '.claude-code',
+  'memory',
+  'semantic-daemon.sock'
+);
+
+const IDLE_TIMEOUT_MS = parseInt(process.env.CLAUDE_MEMORY_SEMANTIC_DAEMON_IDLE_MS || '600000');
+const serviceCache = new Map<string, MemoryService>();
+
+let server: net.Server | null = null;
+let idleTimer: NodeJS.Timeout | null = null;
+let shuttingDown = false;
+
+function scheduleIdleShutdown(): void {
+  if (idleTimer) {
+    clearTimeout(idleTimer);
+  }
+
+  idleTimer = setTimeout(() => {
+    shutdown(0).catch(() => {
+      process.exit(0);
+    });
+  }, IDLE_TIMEOUT_MS);
+  idleTimer.unref();
+}
+
+function parseRequest(raw: string): SemanticDaemonRequest {
+  try {
+    return JSON.parse(raw) as SemanticDaemonRequest;
+  } catch {
+    return {};
+  }
+}
+
+function isValidRequest(input: SemanticDaemonRequest): input is Required<SemanticDaemonRequest> {
+  return input.type === 'retrieve'
+    && typeof input.sessionId === 'string'
+    && input.sessionId.length > 0
+    && typeof input.prompt === 'string'
+    && input.prompt.length > 0
+    && Number.isFinite(input.topK)
+    && Number.isFinite(input.minScore);
+}
+
+function makeErrorResponse(error: unknown): SemanticDaemonResponse {
+  return { ok: false, error: error instanceof Error ? error.message : 'unknown daemon error' };
+}
+
+function isVectorSessionFilterError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message.toLowerCase() : '';
+  return message.includes('no field named sessionid');
+}
+
+function getServiceForSession(sessionId: string): MemoryService {
+  const projectInfo = getSessionProject(sessionId);
+  const key = projectInfo?.projectHash || '__global__';
+
+  if (serviceCache.has(key)) {
+    return serviceCache.get(key)!;
+  }
+
+  const service = new MemoryService({
+    storagePath: projectInfo
+      ? getProjectStoragePath(projectInfo.projectPath)
+      : path.join(os.homedir(), '.claude-code', 'memory'),
+    projectHash: projectInfo?.projectHash,
+    projectPath: projectInfo?.projectPath,
+    readOnly: false,
+    embeddingOnly: true,
+    analyticsEnabled: false,
+    sharedStoreConfig: { enabled: false }
+  });
+
+  serviceCache.set(key, service);
+  return service;
+}
+
+async function handleRequest(raw: string): Promise<SemanticDaemonResponse> {
+  const input = parseRequest(raw);
+  if (!isValidRequest(input)) {
+    return { ok: false, error: 'invalid request' };
+  }
+
+  try {
+    const service = getServiceForSession(input.sessionId);
+    let result;
+    try {
+      result = await service.retrieveMemories(input.prompt, {
+        topK: input.topK,
+        minScore: input.minScore,
+        sessionId: input.sessionId,
+        intentRewrite: true,
+        adaptiveRerank: true,
+        projectScopeMode: 'strict'
+      });
+    } catch (error) {
+      if (!isVectorSessionFilterError(error)) {
+        throw error;
+      }
+
+      // LanceDB field-case mismatch can fail sessionId filtering.
+      // Retry without session filter and keep project strict scoping.
+      result = await service.retrieveMemories(input.prompt, {
+        topK: input.topK,
+        minScore: input.minScore,
+        intentRewrite: true,
+        adaptiveRerank: true,
+        projectScopeMode: 'strict'
+      });
+    }
+
+    const memories = result.memories.map((m) => ({
+      type: m.event.eventType,
+      content: m.event.content,
+      id: m.event.id,
+      score: m.score
+    }));
+
+    return { ok: true, memories };
+  } catch (error) {
+    return makeErrorResponse(error);
+  }
+}
+
+function createServer(): net.Server {
+  return net.createServer({ allowHalfOpen: true }, (socket) => {
+    scheduleIdleShutdown();
+    socket.setEncoding('utf8');
+
+    let requestRaw = '';
+
+    socket.on('data', (chunk) => {
+      requestRaw += chunk;
+      if (requestRaw.length > 1024 * 1024) {
+        socket.end(JSON.stringify({ ok: false, error: 'request too large' }));
+      }
+    });
+
+    socket.on('end', async () => {
+      const response = await handleRequest(requestRaw);
+      socket.end(JSON.stringify(response));
+      scheduleIdleShutdown();
+    });
+
+    socket.on('error', () => {
+      // Ignore per-socket errors to keep daemon process alive.
+    });
+  });
+}
+
+async function socketInUse(p: string): Promise<boolean> {
+  if (!fs.existsSync(p)) return false;
+  return new Promise((resolve) => {
+    let settled = false;
+    const client = net.createConnection(p);
+    const done = (alive: boolean) => {
+      if (settled) return;
+      settled = true;
+      client.destroy();
+      resolve(alive);
+    };
+    client.on('connect', () => done(true));
+    client.on('error', () => done(false));
+    setTimeout(() => done(false), 120).unref();
+  });
+}
+
+async function listenServer(): Promise<void> {
+  const socketDir = path.dirname(SOCKET_PATH);
+  if (!fs.existsSync(socketDir)) {
+    fs.mkdirSync(socketDir, { recursive: true });
+  }
+
+  if (await socketInUse(SOCKET_PATH)) {
+    process.exit(0);
+  }
+
+  if (fs.existsSync(SOCKET_PATH)) {
+    try {
+      fs.unlinkSync(SOCKET_PATH);
+    } catch {
+      // Ignore stale socket unlink failures.
+    }
+  }
+
+  server = createServer();
+
+  await new Promise<void>((resolve, reject) => {
+    if (!server) {
+      reject(new Error('daemon server not initialized'));
+      return;
+    }
+
+    server.once('error', reject);
+    server.listen(SOCKET_PATH, () => {
+      server?.off('error', reject);
+      resolve();
+    });
+  });
+}
+
+async function shutdown(code: number): Promise<void> {
+  if (shuttingDown) return;
+  shuttingDown = true;
+
+  if (idleTimer) {
+    clearTimeout(idleTimer);
+  }
+
+  const closePromises: Promise<void>[] = [];
+  for (const service of serviceCache.values()) {
+    closePromises.push(service.shutdown().catch(() => undefined));
+  }
+  await Promise.all(closePromises);
+  serviceCache.clear();
+
+  if (server) {
+    await new Promise<void>((resolve) => {
+      server?.close(() => resolve());
+    });
+  }
+
+  if (fs.existsSync(SOCKET_PATH)) {
+    try {
+      fs.unlinkSync(SOCKET_PATH);
+    } catch {
+      // Ignore socket cleanup failure.
+    }
+  }
+
+  process.exit(code);
+}
+
+async function main(): Promise<void> {
+  await listenServer();
+  scheduleIdleShutdown();
+}
+
+process.on('SIGINT', () => { shutdown(0).catch(() => process.exit(0)); });
+process.on('SIGTERM', () => { shutdown(0).catch(() => process.exit(0)); });
+process.on('uncaughtException', () => { shutdown(1).catch(() => process.exit(1)); });
+process.on('unhandledRejection', () => { shutdown(1).catch(() => process.exit(1)); });
+
+main().catch(() => {
+  process.exit(1);
+});

package/src/hooks/session-start.ts

@@ -8,6 +8,7 @@ import {
   getLightweightMemoryService,
   registerSession
 } from '../services/memory-service.js';
+import { ensureDaemonRunning } from './semantic-daemon-client.js';
 import type { SessionStartInput, SessionStartOutput } from '../core/types.js';
 
 async function main(): Promise<void> {
@@ -18,6 +19,12 @@ async function main(): Promise<void> {
   // Register session with project path for other hooks to find
   registerSession(input.session_id, input.cwd);
 
+  // Start semantic daemon in the background (non-blocking) so VectorWorker
+  // can process any pending embedding_outbox items immediately.
+  ensureDaemonRunning().catch(() => {
+    // Ignore - daemon will start on first prompt if needed
+  });
+
   // Use lightweight service to avoid starting background workers in hook process
   const memoryService = getLightweightMemoryService(input.session_id);
 
@@ -25,6 +32,10 @@ async function main(): Promise<void> {
     // Start session in memory service
     await memoryService.startSession(input.session_id, input.cwd);
 
+    // Backfill session summaries for recent sessions that ended without Stop hook
+    // (crash, force-close, etc.). Run in background - non-blocking.
+    memoryService.backfillMissingSummaries(input.session_id, 5).catch(() => {});
+
     // Get recent context for this project (now automatically scoped)
     const recentEvents = await memoryService.getRecentEvents(10);
 

package/src/hooks/stop.ts

@@ -17,7 +17,7 @@ import * as fs from 'fs';
 import * as readline from 'readline';
 import { getLightweightMemoryService } from '../services/memory-service.js';
 import { applyPrivacyFilter } from '../core/privacy/index.js';
-import { readTurnState, clearTurnState } from '../core/turn-state.js';
+import { readTurnState, clearTurnState, writeLastAssistantSnippet } from '../core/turn-state.js';
 import type { StopInput, Config } from '../core/types.js';
 
 // Default privacy config
@@ -94,8 +94,16 @@ async function main(): Promise<void> {
     // Read assistant messages from transcript
     const assistantMessages = await extractAssistantMessages(input.transcript_path);
 
+    const MIN_AGENT_RESPONSE_LEN = parseInt(
+      process.env.CLAUDE_MEMORY_AGENT_RESPONSE_MIN_LEN || '150'
+    );
+    const lastIdx = assistantMessages.length - 1;
+
     // Store each assistant response
-    for (const text of assistantMessages) {
+    for (let i = 0; i < assistantMessages.length; i++) {
+      const text = assistantMessages[i];
+      const isLast = i === lastIdx;
+
       // Apply privacy filter
       const filterResult = applyPrivacyFilter(text, DEFAULT_PRIVACY_CONFIG);
       let content = filterResult.content;
@@ -105,8 +113,9 @@ async function main(): Promise<void> {
         content = content.slice(0, 5000) + '...[truncated]';
       }
 
-      // Skip very short responses (likely just tool calls)
-      if (content.trim().length < 10) continue;
+      // Skip very short responses (likely just tool calls or transition messages)
+      // Always store the last message (may be the final answer)
+      if (!isLast && content.trim().length < MIN_AGENT_RESPONSE_LEN) continue;
 
       await memoryService.storeAgentResponse(
         input.session_id,
@@ -118,9 +127,29 @@ async function main(): Promise<void> {
       );
     }
 
+    // Save last assistant response snippet for next-turn retrieval context enrichment
+    if (assistantMessages.length > 0) {
+      const lastMessage = assistantMessages[assistantMessages.length - 1];
+      writeLastAssistantSnippet(input.session_id, lastMessage);
+    }
+
     // Clean up turn state file after processing
     clearTurnState(input.session_id);
 
+    // Evaluate helpfulness of retrieved memories for this session
+    try {
+      await memoryService.evaluateSessionHelpfulness(input.session_id);
+    } catch {
+      // non-critical
+    }
+
+    // Generate session summary from recent events (rule-based, no LLM needed)
+    try {
+      await memoryService.generateSessionSummary(input.session_id);
+    } catch {
+      // non-critical
+    }
+
     // Embeddings enqueued in SQLite - will be processed by vector worker when server runs
     await memoryService.processPendingEmbeddings();