claude-memory-layer 1.0.0

package/src/core/metadata-extractor.ts

@@ -0,0 +1,203 @@
+/**
+ * Metadata Extractor
+ * Extracts tool-specific metadata from tool inputs and outputs
+ */
+
+import type { ToolMetadata } from './types.js';
+
+/**
+ * Get file type from path
+ */
+function getFileType(filePath: string): string | undefined {
+  const ext = filePath.split('.').pop()?.toLowerCase();
+  if (!ext) return undefined;
+
+  const typeMap: Record<string, string> = {
+    ts: 'typescript',
+    tsx: 'typescript',
+    js: 'javascript',
+    jsx: 'javascript',
+    py: 'python',
+    rb: 'ruby',
+    go: 'go',
+    rs: 'rust',
+    java: 'java',
+    kt: 'kotlin',
+    swift: 'swift',
+    c: 'c',
+    cpp: 'cpp',
+    h: 'header',
+    hpp: 'header',
+    cs: 'csharp',
+    php: 'php',
+    html: 'html',
+    css: 'css',
+    scss: 'scss',
+    json: 'json',
+    yaml: 'yaml',
+    yml: 'yaml',
+    xml: 'xml',
+    md: 'markdown',
+    sql: 'sql',
+    sh: 'shell',
+    bash: 'shell',
+    zsh: 'shell'
+  };
+
+  return typeMap[ext];
+}
+
+/**
+ * Count lines in content
+ */
+function countLines(content: string): number {
+  return content.split('\n').length;
+}
+
+/**
+ * Extract bash command (without arguments that might contain secrets)
+ */
+function extractCommand(fullCommand: string): string {
+  // Get first word (command name)
+  const parts = fullCommand.trim().split(/\s+/);
+  const command = parts[0];
+
+  // For common commands, include safe arguments
+  const safeCommands = ['git', 'npm', 'yarn', 'pnpm', 'node', 'python', 'go', 'cargo', 'make'];
+  if (safeCommands.includes(command) && parts.length > 1) {
+    // Include subcommand for these
+    return `${command} ${parts[1]}`;
+  }
+
+  return command;
+}
+
+/**
+ * Extract metadata from tool usage
+ */
+export function extractMetadata(
+  toolName: string,
+  input: Record<string, unknown>,
+  output: string,
+  success: boolean
+): ToolMetadata {
+  switch (toolName) {
+    case 'Read': {
+      const filePath = input.file_path as string | undefined;
+      return {
+        filePath,
+        fileType: filePath ? getFileType(filePath) : undefined,
+        lineCount: success ? countLines(output) : undefined
+      };
+    }
+
+    case 'Write': {
+      const filePath = input.file_path as string | undefined;
+      const content = input.content as string | undefined;
+      return {
+        filePath,
+        fileType: filePath ? getFileType(filePath) : undefined,
+        lineCount: content ? countLines(content) : undefined
+      };
+    }
+
+    case 'Edit': {
+      const filePath = input.file_path as string | undefined;
+      return {
+        filePath,
+        fileType: filePath ? getFileType(filePath) : undefined
+      };
+    }
+
+    case 'Bash': {
+      const fullCommand = input.command as string | undefined;
+      return {
+        command: fullCommand ? extractCommand(fullCommand) : undefined
+      };
+    }
+
+    case 'Grep': {
+      const pattern = input.pattern as string | undefined;
+      // Count matches from output
+      const matchCount = success
+        ? (output.match(/\n/g) || []).length + (output.trim() ? 1 : 0)
+        : undefined;
+      return {
+        pattern,
+        matchCount
+      };
+    }
+
+    case 'Glob': {
+      const pattern = input.pattern as string | undefined;
+      const matchCount = success
+        ? (output.match(/\n/g) || []).length + (output.trim() ? 1 : 0)
+        : undefined;
+      return {
+        pattern,
+        matchCount
+      };
+    }
+
+    case 'WebFetch': {
+      const url = input.url as string | undefined;
+      // Try to extract status code from output
+      const statusMatch = output.match(/status:\s*(\d{3})/i);
+      return {
+        url,
+        statusCode: statusMatch ? parseInt(statusMatch[1], 10) : undefined
+      };
+    }
+
+    case 'WebSearch': {
+      return {};
+    }
+
+    case 'NotebookEdit': {
+      const notebookPath = input.notebook_path as string | undefined;
+      return {
+        filePath: notebookPath,
+        fileType: 'jupyter'
+      };
+    }
+
+    default:
+      return {};
+  }
+}
+
+/**
+ * Create embedding content for tool observation
+ */
+export function createToolObservationEmbedding(
+  toolName: string,
+  metadata: ToolMetadata,
+  success: boolean
+): string {
+  const parts: string[] = [];
+
+  parts.push(`Tool: ${toolName}`);
+
+  if (metadata.filePath) {
+    parts.push(`File: ${metadata.filePath}`);
+  }
+  if (metadata.command) {
+    parts.push(`Command: ${metadata.command}`);
+  }
+  if (metadata.pattern) {
+    parts.push(`Pattern: ${metadata.pattern}`);
+  }
+  if (metadata.url) {
+    // Only include domain for privacy
+    try {
+      const url = new URL(metadata.url);
+      parts.push(`URL: ${url.hostname}`);
+    } catch {
+      // Invalid URL, skip
+    }
+  }
+
+  parts.push(`Result: ${success ? 'Success' : 'Failed'}`);
+
+  return parts.join('\n');
+}

package/src/core/privacy/filter.ts

@@ -0,0 +1,179 @@
+/**
+ * Privacy Filter
+ * Combines pattern-based filtering with private tag parsing
+ */
+
+import { parsePrivateTagsSafe, hasUnmatchedOpenTag } from './tag-parser.js';
+import type { Config } from '../types.js';
+
+export interface FilterResult {
+  content: string;
+  metadata: {
+    hasPrivateTags: boolean;
+    privateTagCount: number;
+    patternMatchCount: number;
+    originalLength: number;
+    filteredLength: number;
+    hasUnmatchedTags: boolean;
+  };
+}
+
+// Sensitive data patterns
+const SENSITIVE_PATTERNS = [
+  /password\s*[:=]\s*['"]?[^\s'"]+/gi,
+  /api[_-]?key\s*[:=]\s*['"]?[^\s'"]+/gi,
+  /secret\s*[:=]\s*['"]?[^\s'"]+/gi,
+  /token\s*[:=]\s*['"]?[^\s'"]+/gi,
+  /bearer\s+[a-zA-Z0-9\-_.]+/gi,
+  /AWS[_-]?ACCESS[_-]?KEY[_-]?ID\s*[:=]\s*['"]?[A-Z0-9]+/gi,
+  /AWS[_-]?SECRET[_-]?ACCESS[_-]?KEY\s*[:=]\s*['"]?[^\s'"]+/gi,
+  /-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----[\s\S]*?-----END\s+(RSA\s+)?PRIVATE\s+KEY-----/g,
+  /ghp_[a-zA-Z0-9]{36}/g,  // GitHub Personal Access Token
+  /sk-[a-zA-Z0-9]{48}/g,   // OpenAI API Key
+];
+
+/**
+ * Apply privacy filter to content
+ */
+export function applyPrivacyFilter(
+  content: string,
+  config: Config['privacy']
+): FilterResult {
+  let filtered = content;
+  let privateTagCount = 0;
+  let patternMatchCount = 0;
+  const hasUnmatchedTags = hasUnmatchedOpenTag(content);
+
+  // 1. Private tag filtering
+  if (config.privateTags?.enabled !== false) {
+    const tagResult = parsePrivateTagsSafe(filtered, {
+      formats: config.privateTags?.supportedFormats || ['xml'],
+      marker: config.privateTags?.marker || '[PRIVATE]'
+    });
+    filtered = tagResult.filtered;
+    privateTagCount = tagResult.stats.count;
+  }
+
+  // 2. Built-in sensitive pattern filtering
+  for (const pattern of SENSITIVE_PATTERNS) {
+    // Reset lastIndex for global regex
+    pattern.lastIndex = 0;
+    const matches = filtered.match(pattern);
+    if (matches) {
+      patternMatchCount += matches.length;
+      filtered = filtered.replace(pattern, '[REDACTED]');
+    }
+  }
+
+  // 3. Custom pattern filtering from config
+  for (const patternStr of config.excludePatterns || []) {
+    try {
+      const regex = new RegExp(
+        `(${patternStr})\\s*[:=]\\s*['"]?[^\\s'"]+`,
+        'gi'
+      );
+      const matches = filtered.match(regex);
+      if (matches) {
+        patternMatchCount += matches.length;
+        filtered = filtered.replace(regex, '[REDACTED]');
+      }
+    } catch {
+      // Invalid regex pattern, skip
+    }
+  }
+
+  // 4. Clean up consecutive markers
+  filtered = filtered.replace(/(\[PRIVATE\]\s*)+/g, '[PRIVATE]\n');
+  filtered = filtered.replace(/(\[REDACTED\]\s*)+/g, '[REDACTED] ');
+
+  return {
+    content: filtered,
+    metadata: {
+      hasPrivateTags: privateTagCount > 0,
+      privateTagCount,
+      patternMatchCount,
+      originalLength: content.length,
+      filteredLength: filtered.length,
+      hasUnmatchedTags
+    }
+  };
+}
+
+/**
+ * Mask sensitive data in tool input (recursively)
+ */
+export function maskSensitiveInput(
+  input: Record<string, unknown>
+): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+
+  for (const [key, value] of Object.entries(input)) {
+    // Check if key suggests sensitive data
+    const sensitiveKeys = ['password', 'secret', 'key', 'token', 'auth', 'credential'];
+    const isSensitiveKey = sensitiveKeys.some(k =>
+      key.toLowerCase().includes(k)
+    );
+
+    if (isSensitiveKey && typeof value === 'string') {
+      result[key] = '[REDACTED]';
+    } else if (typeof value === 'string') {
+      // Apply pattern filtering to string values
+      let filtered = value;
+      for (const pattern of SENSITIVE_PATTERNS) {
+        pattern.lastIndex = 0;
+        filtered = filtered.replace(pattern, '[REDACTED]');
+      }
+      result[key] = filtered;
+    } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      result[key] = maskSensitiveInput(value as Record<string, unknown>);
+    } else if (Array.isArray(value)) {
+      result[key] = value.map(item =>
+        typeof item === 'object' && item !== null
+          ? maskSensitiveInput(item as Record<string, unknown>)
+          : item
+      );
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Truncate output with head + tail strategy
+ */
+export function truncateOutput(
+  output: string,
+  options: { maxLength?: number; maxLines?: number }
+): string {
+  const { maxLength = 10000, maxLines = 100 } = options;
+
+  // Split into lines
+  const lines = output.split('\n');
+
+  // Apply line limit first
+  if (lines.length > maxLines) {
+    const headLines = Math.ceil(maxLines / 2);
+    const tailLines = Math.floor(maxLines / 2);
+    const head = lines.slice(0, headLines);
+    const tail = lines.slice(-tailLines);
+    const truncatedLines = [
+      ...head,
+      `\n... [${lines.length - maxLines} lines truncated] ...\n`,
+      ...tail
+    ];
+    output = truncatedLines.join('\n');
+  }
+
+  // Apply character limit
+  if (output.length > maxLength) {
+    const headChars = Math.ceil(maxLength / 2);
+    const tailChars = Math.floor(maxLength / 2);
+    output = output.slice(0, headChars) +
+      `\n... [${output.length - maxLength} characters truncated] ...\n` +
+      output.slice(-tailChars);
+  }
+
+  return output;
+}

package/src/core/privacy/index.ts

@@ -0,0 +1,20 @@
+/**
+ * Privacy Module
+ * Exports privacy-related utilities
+ */
+
+export {
+  parsePrivateTags,
+  parsePrivateTagsSafe,
+  hasUnmatchedOpenTag,
+  type PrivateSection,
+  type ParseResult,
+  type ParseOptions
+} from './tag-parser.js';
+
+export {
+  applyPrivacyFilter,
+  maskSensitiveInput,
+  truncateOutput,
+  type FilterResult
+} from './filter.js';

package/src/core/privacy/tag-parser.ts

@@ -0,0 +1,145 @@
+/**
+ * Private Tag Parser
+ * Parses and removes <private> tags from content
+ */
+
+export interface PrivateSection {
+  start: number;
+  end: number;
+  content: string;
+  format: 'xml' | 'bracket' | 'comment';
+}
+
+export interface ParseResult {
+  filtered: string;
+  sections: PrivateSection[];
+  stats: {
+    count: number;
+    totalLength: number;
+  };
+}
+
+export interface ParseOptions {
+  formats: Array<'xml' | 'bracket' | 'comment'>;
+  marker: string;
+}
+
+// Tag patterns for different formats
+const TAG_PATTERNS: Record<string, RegExp> = {
+  xml: /<private>([\s\S]*?)<\/private>/gi,
+  bracket: /\[private\]([\s\S]*?)\[\/private\]/gi,
+  comment: /<!--\s*private\s*-->([\s\S]*?)<!--\s*\/private\s*-->/gi
+};
+
+/**
+ * Parse and remove private tags from text
+ */
+export function parsePrivateTags(
+  text: string,
+  options: ParseOptions
+): ParseResult {
+  const sections: PrivateSection[] = [];
+  let filtered = text;
+
+  // Find all private sections for each format
+  for (const format of options.formats) {
+    const pattern = TAG_PATTERNS[format];
+    if (!pattern) continue;
+
+    // Reset lastIndex for global regex
+    pattern.lastIndex = 0;
+
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      sections.push({
+        start: match.index,
+        end: match.index + match[0].length,
+        content: match[1],
+        format: format as PrivateSection['format']
+      });
+    }
+  }
+
+  // Remove all tags and replace with marker
+  for (const format of options.formats) {
+    const pattern = TAG_PATTERNS[format];
+    if (!pattern) continue;
+
+    // Need to create new regex for replacement (global flag issues)
+    const replacePattern = new RegExp(pattern.source, 'gi');
+
+    filtered = filtered.replace(replacePattern, (_match, content: string) => {
+      // Empty tags are completely removed
+      if (!content.trim()) return '';
+      return options.marker;
+    });
+  }
+
+  return {
+    filtered,
+    sections,
+    stats: {
+      count: sections.length,
+      totalLength: sections.reduce((sum, s) => sum + s.content.length, 0)
+    }
+  };
+}
+
+/**
+ * Parse private tags safely, protecting code blocks
+ */
+export function parsePrivateTagsSafe(
+  text: string,
+  options: ParseOptions
+): ParseResult {
+  // 1. Extract and protect code blocks
+  const codeBlocks: string[] = [];
+  const textWithPlaceholders = text.replace(
+    /```[\s\S]*?```/g,
+    (match) => {
+      codeBlocks.push(match);
+      return `__CODE_BLOCK_${codeBlocks.length - 1}__`;
+    }
+  );
+
+  // Also protect inline code
+  const inlineCode: string[] = [];
+  const textWithAllPlaceholders = textWithPlaceholders.replace(
+    /`[^`]+`/g,
+    (match) => {
+      inlineCode.push(match);
+      return `__INLINE_CODE_${inlineCode.length - 1}__`;
+    }
+  );
+
+  // 2. Parse private tags
+  const result = parsePrivateTags(textWithAllPlaceholders, options);
+
+  // 3. Restore inline code
+  result.filtered = result.filtered.replace(
+    /__INLINE_CODE_(\d+)__/g,
+    (_, idx) => inlineCode[Number(idx)]
+  );
+
+  // 4. Restore code blocks
+  result.filtered = result.filtered.replace(
+    /__CODE_BLOCK_(\d+)__/g,
+    (_, idx) => codeBlocks[Number(idx)]
+  );
+
+  return result;
+}
+
+/**
+ * Check if text has unclosed private tags
+ */
+export function hasUnmatchedOpenTag(text: string): boolean {
+  // Check for opening tags without closing
+  const openXml = (text.match(/<private>/gi) || []).length;
+  const closeXml = (text.match(/<\/private>/gi) || []).length;
+
+  const openBracket = (text.match(/\[private\]/gi) || []).length;
+  const closeBracket = (text.match(/\[\/private\]/gi) || []).length;
+
+  return openXml !== closeXml || openBracket !== closeBracket;
+}