claude-memory-layer 1.0.23 → 1.0.24

package/specs/selective-tool-observation/spec.md

@@ -0,0 +1,127 @@
+# Spec: Selective Storage Filtering
+
+## 개요
+
+모든 이벤트 타입에 걸쳐 메모리 가치가 낮은 데이터를 선별적으로 필터링하여
+저장량 55% 감소, 임베딩 backlog 해소, retrieval 품질 향상을 목표로 한다.
+
+## 목표
+
+- 전체 이벤트 저장량 **-55%** (10,536 → ~4,693)
+- 임베딩 pending 증가 속도 감소
+- retrieval signal-to-noise 향상
+- Ctrl+C, 메뉴번호 같은 쓰레기 데이터 제거
+
+## 비목표
+
+- 저장 스키마 변경 없음
+- 기존 저장된 이벤트 소급 삭제 없음
+- session_summary 로직 변경 없음
+
+---
+
+## 필터 규칙 1: tool_observation (post-tool-use.ts)
+
+### Blocklist 확장
+
+**추가 제외 도구** (현재: TodoWrite, TodoRead만 제외):
+
+```
+Read, Grep, Glob, ToolSearch,
+WebFetch, WebSearch, NotebookRead,
+Skill, EnterPlanMode,
+mcp__*  (MCP 도구 전체, 조건부 예외 적용)
+```
+
+**항상 저장 (allowlist)**:
+- `Write`, `Edit`, `MultiEdit` — 파일 변경 기록
+- `Agent`, `Task` — 서브태스크 결과
+- `Bash` — 조건부 (output 필터 적용)
+- `ExitPlanMode` — 계획 완료 기록 (조건부)
+
+### Output-level 필터 (Bash 등 조건부 도구)
+
+| 조건 | 동작 |
+|------|------|
+| `stderr` 존재 | 저장 (에러 컨텍스트) |
+| `stdout` 길이 ≥ 100 chars | 저장 |
+| Write/Edit/Agent/Task | 길이 무관 저장 |
+| 그 외 | 스킵 |
+
+### 환경변수
+
+```bash
+CLAUDE_MEMORY_TOOL_BLOCKLIST="Read,Grep,Glob,..."   # 커스텀 blocklist
+CLAUDE_MEMORY_TOOL_MIN_OUTPUT_LEN=100               # Bash 최소 출력 길이
+```
+
+---
+
+## 필터 규칙 2: agent_response (stop.ts)
+
+### Min-length 필터
+
+**150자 미만 agent_response는 저장 안 함**
+
+근거: 50자 미만 608개 (27%), 50~200자 587개 (26%) 가 도구 체인 전환 메시지.
+독립적 retrieval 가치 없음.
+
+```bash
+CLAUDE_MEMORY_AGENT_RESPONSE_MIN_LEN=150  # 기본값
+```
+
+**예외 (짧아도 저장):**
+- 세션의 마지막 agent_response (최종 답변일 가능성)
+
+---
+
+## 필터 규칙 3: user_prompt (importer + hook)
+
+### 임포터에 shouldStorePrompt() 적용
+
+현재 import 시 transcript의 모든 user 메시지를 무조건 저장.
+Ctrl+C(`\x03`), 숫자 `'1'`, `'go'` 등이 저장되는 원인.
+
+**변경:** `session-history-importer.ts`에서 각 user_prompt 저장 전
+`shouldStorePrompt()` 동일 조건 적용:
+- 길이 < 15자 → 스킵
+- `/`로 시작 → 스킵
+- 제어문자 포함 → 스킵
+- 한글/영문 2글자 이상 포함 여부 확인
+
+---
+
+## 적용 파일
+
+| 파일 | 변경 |
+|------|------|
+| `src/hooks/post-tool-use.ts` | blocklist 확장 + output-level 필터 |
+| `src/hooks/stop.ts` | agent_response min-length 필터 |
+| `src/services/session-history-importer.ts` | shouldStorePrompt() 임포트 적용 |
+
+---
+
+## 판단 흐름
+
+```
+[PostToolUse]
+  tool_name이 blocklist? → 스킵
+  tool_name이 allowlist(Write/Edit/Agent/Task)? → 저장
+  Bash/기타: output length ≥ 100 OR stderr 있음? → 저장 else 스킵
+
+[Stop - agent_response]
+  마지막 메시지? → 저장
+  length ≥ 150? → 저장 else 스킵
+
+[Importer - user_prompt]
+  shouldStorePrompt() 통과? → 저장 else 스킵
+```
+
+---
+
+## 성공 지표
+
+- 신규 세션 tool_observation 비율 < 40% (현재 68.5%)
+- agent_response 저장 비율 < 50% (현재 전량 저장)
+- user_prompt 쓰레기 입력 0건
+- 임베딩 pending 증가 속도 현재 대비 -50%

package/src/cli/index.ts

@@ -442,6 +442,7 @@ program
     const service = getMemoryServiceForProject(projectPath);
 
     try {
+      await service.initialize();
       console.log('⏳ Processing pending embeddings...');
       const count = await service.processPendingEmbeddings();
       console.log(`✅ Processed ${count} embeddings`);

package/src/core/embedder.ts

@@ -46,6 +46,13 @@ export class Embedder {
     }
   }
 
+  // ~4 chars per token; 512 tokens * 4 = 2048, use 2000 to be safe
+  private static readonly MAX_CHARS = 2000;
+
+  private truncate(text: string): string {
+    return text.length > Embedder.MAX_CHARS ? text.slice(0, Embedder.MAX_CHARS) : text;
+  }
+
   /**
    * Generate embedding for a single text
    */
@@ -56,10 +63,11 @@ export class Embedder {
       throw new Error('Embedding pipeline not initialized');
     }
 
-    const output = await this.pipeline(text, {
+    const output = await this.pipeline(this.truncate(text), {
       pooling: 'mean',
       normalize: true,
-      truncation: true
+      truncation: true,
+      max_length: 512
     });
 
     const vector = Array.from(output.data as Float32Array);
@@ -89,10 +97,11 @@ export class Embedder {
       const batch = texts.slice(i, i + batchSize);
 
       for (const text of batch) {
-        const output = await this.pipeline(text, {
+        const output = await this.pipeline(this.truncate(text), {
           pooling: 'mean',
           normalize: true,
-          truncation: true
+          truncation: true,
+          max_length: 512
         });
 
         const vector = Array.from(output.data as Float32Array);

package/src/core/sqlite-event-store.ts

@@ -1145,6 +1145,22 @@ export class SQLiteEventStore {
     );
   }
 
+  /**
+   * Get session IDs that have unevaluated retrievals (measured_at IS NULL).
+   * Excludes the current session. Used to backfill sessions that ended without Stop hook.
+   */
+  async getUnevaluatedSessions(currentSessionId: string, limit = 5): Promise<string[]> {
+    await this.initialize();
+    const rows = sqliteAll<{ session_id: string }>(
+      this.db,
+      `SELECT DISTINCT session_id FROM memory_helpfulness
+       WHERE measured_at IS NULL AND session_id != ?
+       ORDER BY created_at DESC LIMIT ?`,
+      [currentSessionId, limit]
+    );
+    return rows.map((r) => r.session_id);
+  }
+
   /**
    * Evaluate helpfulness for all retrievals in a session
    * Called at session end - uses behavioral signals to compute score

package/src/core/turn-state.ts

@@ -122,6 +122,54 @@ export function clearTurnState(sessionId: string): void {
   }
 }
 
+// ---------------------------------------------------------------------------
+// Last Assistant Snippet State
+// Persists the last ~500 chars of the assistant's response so the next
+// UserPromptSubmit can enrich the retrieval query with conversation context.
+// ---------------------------------------------------------------------------
+
+const LAST_RESPONSE_SNIPPET_CHARS = 500;
+
+interface LastResponseState {
+  sessionId: string;
+  snippet: string;
+  createdAt: string;
+}
+
+function getLastResponsePath(sessionId: string): string {
+  return path.join(TURN_STATE_DIR, `.last-response-${sessionId}.json`);
+}
+
+export function writeLastAssistantSnippet(sessionId: string, text: string): void {
+  try {
+    if (!fs.existsSync(TURN_STATE_DIR)) {
+      fs.mkdirSync(TURN_STATE_DIR, { recursive: true });
+    }
+    const snippet = text.slice(0, LAST_RESPONSE_SNIPPET_CHARS);
+    const state: LastResponseState = { sessionId, snippet, createdAt: new Date().toISOString() };
+    const filePath = getLastResponsePath(sessionId);
+    const tempPath = filePath + '.tmp';
+    fs.writeFileSync(tempPath, JSON.stringify(state));
+    fs.renameSync(tempPath, filePath);
+  } catch {
+    // non-critical
+  }
+}
+
+export function readLastAssistantSnippet(sessionId: string): string | null {
+  try {
+    const filePath = getLastResponsePath(sessionId);
+    if (!fs.existsSync(filePath)) return null;
+    const state: LastResponseState = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+    if (state.sessionId !== sessionId) return null;
+    // Ignore if older than 2 hours (stale session)
+    if (Date.now() - new Date(state.createdAt).getTime() > 2 * 60 * 60 * 1000) return null;
+    return state.snippet || null;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Clean up stale turn state files (older than 1 hour).
  * Can be called periodically to prevent file accumulation.

package/src/core/types.ts

@@ -185,6 +185,7 @@ export const ConfigSchema = z.object({
   toolObservation: z.object({
     enabled: z.boolean().default(true),
     excludedTools: z.array(z.string()).default(['TodoWrite', 'TodoRead']),
+    minOutputLength: z.number().default(100),
     maxOutputLength: z.number().default(10000),
     maxOutputLines: z.number().default(100),
     storeOnlyOnSuccess: z.boolean().default(false)

package/src/hooks/post-tool-use.ts

@@ -20,12 +20,42 @@ import type { PostToolUseInput, ToolObservationPayload, Config } from '../core/t
 // Default config
 const DEFAULT_CONFIG: Config['toolObservation'] = {
   enabled: true,
-  excludedTools: ['TodoWrite', 'TodoRead'],
+  excludedTools: [
+    // Trivial meta tools
+    'TodoWrite', 'TodoRead',
+    // Reproducible query tools (no storage value)
+    'Read', 'Grep', 'Glob',
+    'ToolSearch', 'WebFetch', 'WebSearch', 'NotebookRead',
+    // Low-value system tools
+    'Skill', 'EnterPlanMode',
+  ],
+  minOutputLength: parseInt(process.env.CLAUDE_MEMORY_TOOL_MIN_OUTPUT_LEN || '100'),
   maxOutputLength: 10000,
   maxOutputLines: 100,
   storeOnlyOnSuccess: false
 };
 
+// Tools that are always stored regardless of output length
+const ALWAYS_STORE_TOOLS = new Set([
+  'Write', 'Edit', 'MultiEdit', 'Agent', 'Task', 'ExitPlanMode'
+]);
+
+/**
+ * Determine if a tool output is significant enough to store.
+ * Always-store tools bypass the length check.
+ * Other tools require non-empty stderr or output length >= minLen.
+ */
+function hasSignificantOutput(
+  toolName: string,
+  output: string,
+  response: PostToolUseInput['tool_response'],
+  minLen: number
+): boolean {
+  if (ALWAYS_STORE_TOOLS.has(toolName)) return true;
+  if (response?.stderr && response.stderr.trim().length > 0) return true;
+  return output.trim().length >= minLen;
+}
+
 const DEFAULT_PRIVACY_CONFIG: Config['privacy'] = {
   excludePatterns: ['password', 'secret', 'api_key', 'token', 'bearer'],
   anonymize: false,
@@ -77,9 +107,15 @@ async function main(): Promise<void> {
   const inputData = await readStdin();
   const input: PostToolUseInput = JSON.parse(inputData);
 
-  const config = DEFAULT_CONFIG;
+  const config = { ...DEFAULT_CONFIG };
   const privacyConfig = DEFAULT_PRIVACY_CONFIG;
 
+  // Allow env-based blocklist override
+  const envBlocklist = process.env.CLAUDE_MEMORY_TOOL_BLOCKLIST;
+  if (envBlocklist !== undefined) {
+    config.excludedTools = envBlocklist.split(',').map((s) => s.trim()).filter(Boolean);
+  }
+
   // 1. Check if tool observation is enabled
   if (!config.enabled) {
     console.log(JSON.stringify({}));
@@ -102,6 +138,15 @@ async function main(): Promise<void> {
     return;
   }
 
+  // 4.5. Output-level filter: skip low-signal outputs
+  if (!hasSignificantOutput(
+    input.tool_name, toolOutput, input.tool_response,
+    config.minOutputLength ?? 100
+  )) {
+    console.log(JSON.stringify({}));
+    return;
+  }
+
   try {
     const memoryService = getLightweightMemoryService(input.session_id);
 

package/src/hooks/semantic-daemon-client.ts

@@ -0,0 +1,208 @@
+import { spawn } from 'child_process';
+import * as fs from 'fs';
+import * as net from 'net';
+import * as os from 'os';
+import * as path from 'path';
+
+interface SemanticRequest {
+  sessionId: string;
+  prompt: string;
+  topK: number;
+  minScore: number;
+}
+
+interface SemanticMemory {
+  type: string;
+  content: string;
+  id?: string;
+  score?: number;
+}
+
+interface SemanticDaemonRequest {
+  type: 'retrieve';
+  sessionId: string;
+  prompt: string;
+  topK: number;
+  minScore: number;
+}
+
+interface SemanticDaemonResponse {
+  ok: boolean;
+  memories?: SemanticMemory[];
+  error?: string;
+}
+
+const DEFAULT_SOCKET_PATH = path.join(
+  os.homedir(),
+  '.claude-code',
+  'memory',
+  'semantic-daemon.sock'
+);
+
+const DAEMON_SOCKET_PATH = process.env.CLAUDE_MEMORY_SEMANTIC_SOCKET || DEFAULT_SOCKET_PATH;
+const DAEMON_START_TIMEOUT_MS = parseInt(process.env.CLAUDE_MEMORY_SEMANTIC_DAEMON_START_MS || '1500');
+
+let daemonStartPromise: Promise<void> | null = null;
+
+export async function retrieveSemanticMemories(
+  request: SemanticRequest,
+  timeoutMs: number
+): Promise<SemanticMemory[]> {
+  const payload: SemanticDaemonRequest = {
+    type: 'retrieve',
+    sessionId: request.sessionId,
+    prompt: request.prompt,
+    topK: request.topK,
+    minScore: request.minScore
+  };
+
+  try {
+    return await requestFromDaemon(payload, timeoutMs);
+  } catch (error) {
+    if (!isConnectionError(error)) {
+      throw error;
+    }
+
+    await ensureDaemonRunning();
+    return requestFromDaemon(payload, timeoutMs).catch((retryError) => {
+      if (process.env.CLAUDE_MEMORY_DEBUG) {
+        console.error('[semantic-client] retry failed after daemon start:', retryError);
+      }
+      throw retryError;
+    });
+  }
+}
+
+function requestFromDaemon(
+  payload: SemanticDaemonRequest,
+  timeoutMs: number
+): Promise<SemanticMemory[]> {
+  return new Promise((resolve, reject) => {
+    const client = net.createConnection(DAEMON_SOCKET_PATH);
+    client.setEncoding('utf8');
+
+    let settled = false;
+    let responseRaw = '';
+    const timer = setTimeout(() => {
+      const timeoutError = new Error(`semantic daemon timeout (${timeoutMs}ms)`);
+      (timeoutError as NodeJS.ErrnoException).code = 'ETIMEDOUT';
+      settle(timeoutError);
+      client.destroy();
+    }, timeoutMs);
+
+    const settle = (error?: Error, memories?: SemanticMemory[]) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (error) {
+        reject(error);
+      } else {
+        resolve(memories || []);
+      }
+    };
+
+    client.on('connect', () => {
+      client.end(JSON.stringify(payload));
+    });
+
+    client.on('data', (chunk) => {
+      responseRaw += chunk;
+      if (responseRaw.length > 4 * 1024 * 1024) {
+        settle(new Error('semantic daemon response too large'));
+        client.destroy();
+      }
+    });
+
+    client.on('end', () => {
+      try {
+        const parsed = JSON.parse(responseRaw || '{}') as SemanticDaemonResponse;
+        if (!parsed.ok) {
+          settle(new Error(parsed.error || 'semantic daemon error'));
+          return;
+        }
+        settle(undefined, parsed.memories || []);
+      } catch (error) {
+        settle(error as Error);
+      }
+    });
+
+    client.on('error', (error) => {
+      settle(error as Error);
+    });
+  });
+}
+
+export async function ensureDaemonRunning(): Promise<void> {
+  if (daemonStartPromise) {
+    return daemonStartPromise;
+  }
+
+  daemonStartPromise = (async () => {
+    if (await canConnect()) {
+      return;
+    }
+
+    const daemonScriptPath = getDaemonScriptPath();
+    if (!fs.existsSync(daemonScriptPath)) {
+      throw new Error(`semantic daemon script not found: ${daemonScriptPath}`);
+    }
+
+    const daemonDir = path.dirname(DAEMON_SOCKET_PATH);
+    if (!fs.existsSync(daemonDir)) {
+      fs.mkdirSync(daemonDir, { recursive: true });
+    }
+
+    const child = spawn(process.execPath, [daemonScriptPath], {
+      detached: true,
+      stdio: 'ignore',
+      env: process.env
+    });
+    child.unref();
+
+    const startDeadline = Date.now() + DAEMON_START_TIMEOUT_MS;
+    while (Date.now() < startDeadline) {
+      if (await canConnect()) {
+        return;
+      }
+      await sleep(60);
+    }
+
+    throw new Error(`semantic daemon start timeout (${DAEMON_START_TIMEOUT_MS}ms)`);
+  })();
+
+  try {
+    await daemonStartPromise;
+  } finally {
+    daemonStartPromise = null;
+  }
+}
+
+function getDaemonScriptPath(): string {
+  return path.join(path.dirname(new URL(import.meta.url).pathname), 'semantic-daemon.js');
+}
+
+function canConnect(): Promise<boolean> {
+  return new Promise((resolve) => {
+    let settled = false;
+    const client = net.createConnection(DAEMON_SOCKET_PATH);
+    const finalize = (ok: boolean) => {
+      if (settled) return;
+      settled = true;
+      client.destroy();
+      resolve(ok);
+    };
+
+    client.on('connect', () => finalize(true));
+    client.on('error', () => finalize(false));
+    setTimeout(() => finalize(false), 120).unref();
+  });
+}
+
+function isConnectionError(error: unknown): boolean {
+  const code = (error as NodeJS.ErrnoException | undefined)?.code;
+  return code === 'ENOENT' || code === 'ECONNREFUSED' || code === 'EPIPE' || code === 'ECONNRESET';
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}