claude-memory-layer 1.0.27 → 1.0.28

package/src/extensions/vector/embedder.ts

@@ -0,0 +1,197 @@
+/**
+ * Local Embedding Generator using @huggingface/transformers
+ * AXIOMMIND Principle 7: Standard JSON format for vectors
+ */
+
+export interface EmbeddingResult {
+  vector: number[];
+  model: string;
+  dimensions: number;
+}
+
+type FeatureExtractionPipelineFactory = (
+  task: 'feature-extraction',
+  model: string
+) => Promise<NonNullable<Embedder['pipeline']>>;
+
+export class Embedder {
+  private pipeline: ((input: string, options?: Record<string, unknown>) => Promise<{ data: Float32Array }>) | null = null;
+  private readonly modelName: string;
+  private activeModelName: string;
+  private initialized = false;
+
+  constructor(modelName: string = 'jinaai/jina-embeddings-v5-text-nano-text-matching') {
+    this.modelName = modelName;
+    this.activeModelName = modelName;
+  }
+
+  /**
+   * Initialize the embedding pipeline
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    const pipeline = await withSuppressedKnownTransformersWarnings(() => loadTransformersPipeline());
+
+    try {
+      this.pipeline = await withSuppressedKnownTransformersWarnings(() => pipeline('feature-extraction', this.modelName));
+      this.activeModelName = this.modelName;
+      this.initialized = true;
+      return;
+    } catch (primaryError) {
+      const fallbackModel = process.env.CLAUDE_MEMORY_EMBEDDING_FALLBACK_MODEL || 'onnx-community/embeddinggemma-300m-ONNX';
+      if (fallbackModel === this.modelName) {
+        throw primaryError;
+      }
+
+      console.warn(`[Embedder] Primary model failed (${this.modelName}). Falling back to ${fallbackModel}`);
+      this.pipeline = await withSuppressedKnownTransformersWarnings(() => pipeline('feature-extraction', fallbackModel));
+      this.activeModelName = fallbackModel;
+      this.initialized = true;
+    }
+  }
+
+  // ~4 chars per token; 512 tokens * 4 = 2048, use 2000 to be safe
+  private static readonly MAX_CHARS = 2000;
+
+  private truncate(text: string): string {
+    return text.length > Embedder.MAX_CHARS ? text.slice(0, Embedder.MAX_CHARS) : text;
+  }
+
+  /**
+   * Generate embedding for a single text
+   */
+  async embed(text: string): Promise<EmbeddingResult> {
+    await this.initialize();
+
+    if (!this.pipeline) {
+      throw new Error('Embedding pipeline not initialized');
+    }
+
+    const output = await this.pipeline(this.truncate(text), {
+      pooling: 'mean',
+      normalize: true,
+      truncation: true,
+      max_length: 512
+    });
+
+    const vector = Array.from(output.data);
+
+    return {
+      vector,
+      model: this.activeModelName,
+      dimensions: vector.length
+    };
+  }
+
+  /**
+   * Generate embeddings for multiple texts in batch
+   */
+  async embedBatch(texts: string[]): Promise<EmbeddingResult[]> {
+    await this.initialize();
+
+    if (!this.pipeline) {
+      throw new Error('Embedding pipeline not initialized');
+    }
+
+    const results: EmbeddingResult[] = [];
+
+    // Process in batches of 32 for memory efficiency
+    const batchSize = 32;
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+
+      for (const text of batch) {
+        const output = await this.pipeline(this.truncate(text), {
+          pooling: 'mean',
+          normalize: true,
+          truncation: true,
+          max_length: 512
+        });
+
+        const vector = Array.from(output.data);
+
+        results.push({
+          vector,
+          model: this.activeModelName,
+          dimensions: vector.length
+        });
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Get embedding dimensions for the current model
+   */
+  async getDimensions(): Promise<number> {
+    const result = await this.embed('test');
+    return result.dimensions;
+  }
+
+  /**
+   * Check if embedder is ready
+   */
+  isReady(): boolean {
+    return this.initialized && this.pipeline !== null;
+  }
+
+  /**
+   * Get model name
+   */
+  getModelName(): string {
+    return this.activeModelName;
+  }
+}
+
+// Singleton instance for reuse
+let defaultEmbedder: Embedder | null = null;
+
+export function getDefaultEmbedder(): Embedder {
+  const envModel = process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
+  if (!defaultEmbedder) {
+    defaultEmbedder = new Embedder(envModel || undefined);
+  }
+  return defaultEmbedder;
+}
+
+let transformersWarningSuppressionDepth = 0;
+let originalConsoleWarn: typeof console.warn | null = null;
+
+export async function withSuppressedKnownTransformersWarnings<T>(fn: () => Promise<T>): Promise<T> {
+  if (transformersWarningSuppressionDepth === 0) {
+    originalConsoleWarn = console.warn;
+    console.warn = (...args: unknown[]) => {
+      const message = args.map(String).join(' ');
+      if (isKnownBenignTransformersWarning(message)) return;
+      (originalConsoleWarn ?? console.warn)(...args);
+    };
+  }
+  transformersWarningSuppressionDepth += 1;
+
+  try {
+    return await fn();
+  } finally {
+    transformersWarningSuppressionDepth -= 1;
+    if (transformersWarningSuppressionDepth === 0 && originalConsoleWarn) {
+      console.warn = originalConsoleWarn;
+      originalConsoleWarn = null;
+    }
+  }
+}
+
+export function isKnownBenignTransformersWarning(message: string): boolean {
+  return message.includes('Unknown model class "eurobert"') ||
+    message.includes('dtype not specified for "model"');
+}
+
+async function loadTransformersPipeline(): Promise<FeatureExtractionPipelineFactory> {
+  // Keep @huggingface/transformers lazy so importing MemoryService or pure
+  // adapter helpers does not eagerly dlopen onnxruntime native bindings.
+  const dynamicImport = new Function('specifier', 'return import(specifier)') as (
+    specifier: string
+  ) => Promise<{ pipeline: unknown }>;
+  const transformers = await dynamicImport('@huggingface/transformers');
+  return transformers.pipeline as FeatureExtractionPipelineFactory;
+}

package/src/extensions/vector/index.ts

@@ -0,0 +1 @@
+export * from './embedder.js';

package/src/hooks/post-tool-use.ts

@@ -1,242 +1,9 @@
 #!/usr/bin/env node
 /**
- * PostToolUse Hook
- * Called after each tool execution - stores tool observations
+ * Compatibility entrypoint for the Claude post-tool-use hook.
  *
- * Actual Claude Code input format:
- * {
- *   session_id, tool_name, tool_input, tool_use_id,
- *   tool_response: { stdout?, stderr?, content?, interrupted?, isImage? },
- *   cwd, transcript_path, permission_mode, hook_event_name
- * }
+ * Implementation lives in the Claude adapter layer so core stays platform-agnostic.
  */
-
-import { getLightweightMemoryService } from '../services/memory-service.js';
-import { applyPrivacyFilter, maskSensitiveInput, truncateOutput } from '../core/privacy/index.js';
-import { extractMetadata } from '../core/metadata-extractor.js';
-import { readTurnState } from '../core/turn-state.js';
-import type { PostToolUseInput, ToolObservationPayload, Config } from '../core/types.js';
-
-// Default config
-const DEFAULT_CONFIG: Config['toolObservation'] = {
-  enabled: true,
-  excludedTools: [
-    // Trivial meta tools
-    'TodoWrite', 'TodoRead',
-    // Reproducible query tools (no storage value)
-    'Read', 'Grep', 'Glob',
-    'ToolSearch', 'WebFetch', 'WebSearch', 'NotebookRead',
-    // Low-value system tools
-    'Skill', 'EnterPlanMode',
-  ],
-  minOutputLength: parseInt(process.env.CLAUDE_MEMORY_TOOL_MIN_OUTPUT_LEN || '100'),
-  maxOutputLength: 10000,
-  maxOutputLines: 100,
-  storeOnlyOnSuccess: false
-};
-
-// Tools that are always stored regardless of output length
-const ALWAYS_STORE_TOOLS = new Set([
-  'Write', 'Edit', 'MultiEdit', 'Agent', 'Task', 'ExitPlanMode'
-]);
-
-// Keywords that indicate a Bash output is worth storing
-const IMPORTANT_BASH_KEYWORDS = [
-  'error', 'failed', 'exception', 'traceback', 'panic',
-  'warning', 'deprecated',
-  'test passed', 'test failed', 'tests passed', 'tests failed',
-  'coverage', 'assert',
-  'published', 'deployed', 'built successfully', 'build complete',
-  'successfully installed', 'successfully created',
-];
-
-/**
- * For Bash commands, only store output that is significant:
- * - Has stderr content
- * - Contains important keywords (errors, test results, deploy events)
- * - Output is very long (> 2000 chars), indicating meaningful work
- */
-function isBashSignificant(output: string, response: PostToolUseInput['tool_response']): boolean {
-  if (response?.stderr && response.stderr.trim().length > 20) return true;
-  const lower = output.toLowerCase();
-  if (IMPORTANT_BASH_KEYWORDS.some((kw) => lower.includes(kw))) return true;
-  return output.trim().length > 2000;
-}
-
-/**
- * Determine if a tool output is significant enough to store.
- * Always-store tools bypass the length check.
- * Bash uses keyword-based significance detection.
- * Other tools require non-empty stderr or output length >= minLen.
- */
-function hasSignificantOutput(
-  toolName: string,
-  output: string,
-  response: PostToolUseInput['tool_response'],
-  minLen: number
-): boolean {
-  if (ALWAYS_STORE_TOOLS.has(toolName)) return true;
-  if (toolName === 'Bash') return isBashSignificant(output, response);
-  if (response?.stderr && response.stderr.trim().length > 0) return true;
-  return output.trim().length >= minLen;
-}
-
-const DEFAULT_PRIVACY_CONFIG: Config['privacy'] = {
-  excludePatterns: ['password', 'secret', 'api_key', 'token', 'bearer'],
-  anonymize: false,
-  privateTags: {
-    enabled: true,
-    marker: '[PRIVATE]',
-    preserveLineCount: false,
-    supportedFormats: ['xml']
-  }
-};
-
-/**
- * Extract text output from tool_response object
- */
-function extractToolOutput(response: PostToolUseInput['tool_response']): string {
-  if (!response) return '';
-
-  // Bash tools: stdout + stderr
-  if (response.stdout !== undefined) {
-    const parts: string[] = [];
-    if (response.stdout) parts.push(response.stdout);
-    if (response.stderr) parts.push(`[stderr] ${response.stderr}`);
-    return parts.join('\n') || '';
-  }
-
-  // Other tools may have content field
-  if (response.content !== undefined) {
-    return typeof response.content === 'string'
-      ? response.content
-      : JSON.stringify(response.content);
-  }
-
-  // Fallback: stringify the whole response
-  return JSON.stringify(response);
-}
-
-/**
- * Determine if the tool execution was successful
- */
-function isToolSuccess(response: PostToolUseInput['tool_response']): boolean {
-  if (!response) return false;
-  if (response.interrupted) return false;
-  // If stderr has content but stdout also has content, still consider success
-  return true;
-}
-
-async function main(): Promise<void> {
-  // Read input from stdin
-  const inputData = await readStdin();
-  const input: PostToolUseInput = JSON.parse(inputData);
-
-  const config = { ...DEFAULT_CONFIG };
-  const privacyConfig = DEFAULT_PRIVACY_CONFIG;
-
-  // Allow env-based blocklist override
-  const envBlocklist = process.env.CLAUDE_MEMORY_TOOL_BLOCKLIST;
-  if (envBlocklist !== undefined) {
-    config.excludedTools = envBlocklist.split(',').map((s) => s.trim()).filter(Boolean);
-  }
-
-  // 1. Check if tool observation is enabled
-  if (!config.enabled) {
-    console.log(JSON.stringify({}));
-    return;
-  }
-
-  // 2. Check if tool is excluded
-  if (config.excludedTools?.includes(input.tool_name)) {
-    console.log(JSON.stringify({}));
-    return;
-  }
-
-  // 3. Extract output from tool_response object
-  const toolOutput = extractToolOutput(input.tool_response);
-  const success = isToolSuccess(input.tool_response);
-
-  // 4. Check success filter
-  if (!success && config.storeOnlyOnSuccess) {
-    console.log(JSON.stringify({}));
-    return;
-  }
-
-  // 4.5. Output-level filter: skip low-signal outputs
-  if (!hasSignificantOutput(
-    input.tool_name, toolOutput, input.tool_response,
-    config.minOutputLength ?? 100
-  )) {
-    console.log(JSON.stringify({}));
-    return;
-  }
-
-  try {
-    const memoryService = getLightweightMemoryService(input.session_id);
-
-    // 5. Mask sensitive data in input
-    const maskedInput = maskSensitiveInput(input.tool_input);
-
-    // 6. Apply privacy filter to output
-    const filterResult = applyPrivacyFilter(toolOutput, privacyConfig);
-    const maskedOutput = filterResult.content;
-
-    // 7. Truncate output
-    const truncatedOutput = truncateOutput(maskedOutput, {
-      maxLength: config.maxOutputLength,
-      maxLines: config.maxOutputLines
-    });
-
-    // 8. Extract metadata
-    const metadata = extractMetadata(
-      input.tool_name,
-      maskedInput,
-      toolOutput,
-      success
-    );
-
-    // 8.5. Read current turn_id from state file
-    const turnId = readTurnState(input.session_id);
-
-    // 9. Create payload (include turnId in metadata for grouping)
-    const payload: ToolObservationPayload = {
-      toolName: input.tool_name,
-      toolInput: maskedInput,
-      toolOutput: truncatedOutput,
-      durationMs: 0, // Claude Code doesn't provide timing info
-      success,
-      errorMessage: input.tool_response?.stderr || undefined,
-      metadata: {
-        ...metadata,
-        ...(turnId ? { turnId } : {})
-      }
-    };
-
-    // 10. Store observation
-    await memoryService.storeToolObservation(input.session_id, payload);
-
-    // Output empty (hook doesn't return context)
-    console.log(JSON.stringify({}));
-  } catch (error) {
-    if (process.env.CLAUDE_MEMORY_DEBUG) {
-      console.error('PostToolUse hook error:', error);
-    }
-    console.log(JSON.stringify({}));
-  }
-}
-
-function readStdin(): Promise<string> {
-  return new Promise((resolve) => {
-    let data = '';
-    process.stdin.setEncoding('utf8');
-    process.stdin.on('data', (chunk) => {
-      data += chunk;
-    });
-    process.stdin.on('end', () => {
-      resolve(data);
-    });
-  });
-}
+import { main } from '../adapters/claude/hooks/post-tool-use.js';
 
 main().catch(console.error);

package/src/hooks/semantic-daemon-client.ts

@@ -1,208 +1 @@
-import { spawn } from 'child_process';
-import * as fs from 'fs';
-import * as net from 'net';
-import * as os from 'os';
-import * as path from 'path';
-
-interface SemanticRequest {
-  sessionId: string;
-  prompt: string;
-  topK: number;
-  minScore: number;
-}
-
-interface SemanticMemory {
-  type: string;
-  content: string;
-  id?: string;
-  score?: number;
-}
-
-interface SemanticDaemonRequest {
-  type: 'retrieve';
-  sessionId: string;
-  prompt: string;
-  topK: number;
-  minScore: number;
-}
-
-interface SemanticDaemonResponse {
-  ok: boolean;
-  memories?: SemanticMemory[];
-  error?: string;
-}
-
-const DEFAULT_SOCKET_PATH = path.join(
-  os.homedir(),
-  '.claude-code',
-  'memory',
-  'semantic-daemon.sock'
-);
-
-const DAEMON_SOCKET_PATH = process.env.CLAUDE_MEMORY_SEMANTIC_SOCKET || DEFAULT_SOCKET_PATH;
-const DAEMON_START_TIMEOUT_MS = parseInt(process.env.CLAUDE_MEMORY_SEMANTIC_DAEMON_START_MS || '1500');
-
-let daemonStartPromise: Promise<void> | null = null;
-
-export async function retrieveSemanticMemories(
-  request: SemanticRequest,
-  timeoutMs: number
-): Promise<SemanticMemory[]> {
-  const payload: SemanticDaemonRequest = {
-    type: 'retrieve',
-    sessionId: request.sessionId,
-    prompt: request.prompt,
-    topK: request.topK,
-    minScore: request.minScore
-  };
-
-  try {
-    return await requestFromDaemon(payload, timeoutMs);
-  } catch (error) {
-    if (!isConnectionError(error)) {
-      throw error;
-    }
-
-    await ensureDaemonRunning();
-    return requestFromDaemon(payload, timeoutMs).catch((retryError) => {
-      if (process.env.CLAUDE_MEMORY_DEBUG) {
-        console.error('[semantic-client] retry failed after daemon start:', retryError);
-      }
-      throw retryError;
-    });
-  }
-}
-
-function requestFromDaemon(
-  payload: SemanticDaemonRequest,
-  timeoutMs: number
-): Promise<SemanticMemory[]> {
-  return new Promise((resolve, reject) => {
-    const client = net.createConnection(DAEMON_SOCKET_PATH);
-    client.setEncoding('utf8');
-
-    let settled = false;
-    let responseRaw = '';
-    const timer = setTimeout(() => {
-      const timeoutError = new Error(`semantic daemon timeout (${timeoutMs}ms)`);
-      (timeoutError as NodeJS.ErrnoException).code = 'ETIMEDOUT';
-      settle(timeoutError);
-      client.destroy();
-    }, timeoutMs);
-
-    const settle = (error?: Error, memories?: SemanticMemory[]) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      if (error) {
-        reject(error);
-      } else {
-        resolve(memories || []);
-      }
-    };
-
-    client.on('connect', () => {
-      client.end(JSON.stringify(payload));
-    });
-
-    client.on('data', (chunk) => {
-      responseRaw += chunk;
-      if (responseRaw.length > 4 * 1024 * 1024) {
-        settle(new Error('semantic daemon response too large'));
-        client.destroy();
-      }
-    });
-
-    client.on('end', () => {
-      try {
-        const parsed = JSON.parse(responseRaw || '{}') as SemanticDaemonResponse;
-        if (!parsed.ok) {
-          settle(new Error(parsed.error || 'semantic daemon error'));
-          return;
-        }
-        settle(undefined, parsed.memories || []);
-      } catch (error) {
-        settle(error as Error);
-      }
-    });
-
-    client.on('error', (error) => {
-      settle(error as Error);
-    });
-  });
-}
-
-export async function ensureDaemonRunning(): Promise<void> {
-  if (daemonStartPromise) {
-    return daemonStartPromise;
-  }
-
-  daemonStartPromise = (async () => {
-    if (await canConnect()) {
-      return;
-    }
-
-    const daemonScriptPath = getDaemonScriptPath();
-    if (!fs.existsSync(daemonScriptPath)) {
-      throw new Error(`semantic daemon script not found: ${daemonScriptPath}`);
-    }
-
-    const daemonDir = path.dirname(DAEMON_SOCKET_PATH);
-    if (!fs.existsSync(daemonDir)) {
-      fs.mkdirSync(daemonDir, { recursive: true });
-    }
-
-    const child = spawn(process.execPath, [daemonScriptPath], {
-      detached: true,
-      stdio: 'ignore',
-      env: process.env
-    });
-    child.unref();
-
-    const startDeadline = Date.now() + DAEMON_START_TIMEOUT_MS;
-    while (Date.now() < startDeadline) {
-      if (await canConnect()) {
-        return;
-      }
-      await sleep(60);
-    }
-
-    throw new Error(`semantic daemon start timeout (${DAEMON_START_TIMEOUT_MS}ms)`);
-  })();
-
-  try {
-    await daemonStartPromise;
-  } finally {
-    daemonStartPromise = null;
-  }
-}
-
-function getDaemonScriptPath(): string {
-  return path.join(path.dirname(new URL(import.meta.url).pathname), 'semantic-daemon.js');
-}
-
-function canConnect(): Promise<boolean> {
-  return new Promise((resolve) => {
-    let settled = false;
-    const client = net.createConnection(DAEMON_SOCKET_PATH);
-    const finalize = (ok: boolean) => {
-      if (settled) return;
-      settled = true;
-      client.destroy();
-      resolve(ok);
-    };
-
-    client.on('connect', () => finalize(true));
-    client.on('error', () => finalize(false));
-    setTimeout(() => finalize(false), 120).unref();
-  });
-}
-
-function isConnectionError(error: unknown): boolean {
-  const code = (error as NodeJS.ErrnoException | undefined)?.code;
-  return code === 'ENOENT' || code === 'ECONNREFUSED' || code === 'EPIPE' || code === 'ECONNRESET';
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
+export * from '../adapters/claude/hooks/semantic-daemon-client.js';