memorylink 1.0.0

package/src/memorylink.ts

@@ -0,0 +1,308 @@
+/**
+ * MemoryLink - Programmatic API
+ * Use MemoryLink from your own code
+ * 
+ * v1.0 UPDATE: Now uses JSONL format with file locking
+ */
+
+import {
+  isInitialized,
+  findRoot,
+  getFilePath,
+  readLines,
+  getTimestamp,
+  generateId,
+  appendJsonlEntry,
+  readJsonlEntries,
+  isJsonlFormat,
+  checkSecretsBeforeWrite,
+} from './utils.js';
+import { isFileProtected } from './commands/protect.js';
+import { getStatusReport } from './commands/status.js';
+import type { 
+  LearningStatus, 
+  StatusReport,
+  MemoryEntry,
+  ChangeEntry,
+  LearningEntry,
+  MemoryType,
+} from './types.js';
+
+/**
+ * MemoryLink class for programmatic usage
+ */
+class MemoryLink {
+  private root: string | null;
+  private defaultAgent: string;
+
+  constructor(projectPath?: string, agent: string = 'sdk') {
+    this.root = projectPath || findRoot();
+    this.defaultAgent = agent;
+  }
+
+  /**
+   * Check if MemoryLink is initialized
+   */
+  isInitialized(): boolean {
+    return this.root !== null && isInitialized(this.root);
+  }
+
+  /**
+   * Get the project root
+   */
+  getRoot(): string | null {
+    return this.root;
+  }
+
+  /**
+   * Get status report
+   */
+  getStatus(): StatusReport | null {
+    return getStatusReport(this.root || undefined);
+  }
+
+  /**
+   * Check if a file is protected
+   */
+  isProtected(filePath: string): boolean {
+    return isFileProtected(filePath, this.root || undefined);
+  }
+
+  /**
+   * Add a memory entry (JSONL format with locking)
+   */
+  async addMemory(
+    details: string, 
+    options: {
+      tags?: string[];
+      type?: MemoryType;
+      agent?: string;
+      force?: boolean;
+    } = {}
+  ): Promise<MemoryEntry> {
+    if (!this.root || !isInitialized(this.root)) {
+      throw new Error('MemoryLink is not initialized');
+    }
+
+    const { tags = [], type = 'note', agent = this.defaultAgent, force = false } = options;
+
+    // Secret detection
+    if (!force && !checkSecretsBeforeWrite(details, force)) {
+      throw new Error('Secret detected in content. Use force=true to override.');
+    }
+
+    const entry: MemoryEntry = {
+      id: generateId(),
+      ts: getTimestamp(),
+      agent,
+      type,
+      details,
+      tags,
+      status: 'ACTIVE',
+    };
+
+    const memoryPath = getFilePath('MEMORY', this.root);
+    await appendJsonlEntry(memoryPath, entry);
+    
+    return entry;
+  }
+
+  /**
+   * Log a file change (JSONL format with locking)
+   */
+  async logChange(
+    file: string, 
+    reason: string, 
+    options: {
+      agent?: string;
+      action?: 'created' | 'edited' | 'deleted' | 'renamed';
+      force?: boolean;
+    } = {}
+  ): Promise<ChangeEntry> {
+    if (!this.root || !isInitialized(this.root)) {
+      throw new Error('MemoryLink is not initialized');
+    }
+
+    const { agent = this.defaultAgent, action = 'edited', force = false } = options;
+
+    // Secret detection
+    if (!force && !checkSecretsBeforeWrite(reason, force)) {
+      throw new Error('Secret detected in content. Use force=true to override.');
+    }
+
+    const entry: ChangeEntry = {
+      id: generateId(),
+      ts: getTimestamp(),
+      agent,
+      file,
+      action,
+      reason,
+      status: 'ACTIVE',
+    };
+
+    const changesPath = getFilePath('CHANGES', this.root);
+    await appendJsonlEntry(changesPath, entry);
+    
+    return entry;
+  }
+
+  /**
+   * Add a learning (JSONL format with locking)
+   */
+  async addLearning(
+    problem: string, 
+    solution: string, 
+    outcome: LearningStatus,
+    options: {
+      tags?: string[];
+      reason?: string;
+      agent?: string;
+      force?: boolean;
+    } = {}
+  ): Promise<LearningEntry> {
+    if (!this.root || !isInitialized(this.root)) {
+      throw new Error('MemoryLink is not initialized');
+    }
+
+    const { tags = [], reason, agent = this.defaultAgent, force = false } = options;
+
+    // Secret detection
+    const content = `${problem} ${solution} ${reason || ''}`;
+    if (!force && !checkSecretsBeforeWrite(content, force)) {
+      throw new Error('Secret detected in content. Use force=true to override.');
+    }
+
+    const entry: LearningEntry = {
+      id: generateId(),
+      ts: getTimestamp(),
+      agent,
+      problem,
+      solution,
+      outcome,
+      reason,
+      tags,
+      status: 'ACTIVE',
+    };
+
+    const learningsPath = getFilePath('LEARNINGS', this.root);
+    await appendJsonlEntry(learningsPath, entry);
+    
+    return entry;
+  }
+
+  /**
+   * Get all memory entries (JSONL format)
+   */
+  getMemories(): MemoryEntry[] {
+    if (!this.root || !isInitialized(this.root)) {
+      return [];
+    }
+
+    const memoryPath = getFilePath('MEMORY', this.root);
+    
+    if (!isJsonlFormat(memoryPath)) {
+      // Return empty for legacy format - user should migrate
+      console.warn('Legacy format detected. Run: memorylink migrate');
+      return [];
+    }
+
+    return readJsonlEntries<MemoryEntry>(memoryPath)
+      .filter(e => e.status === 'ACTIVE');
+  }
+
+  /**
+   * Get all change entries (JSONL format)
+   */
+  getChanges(): ChangeEntry[] {
+    if (!this.root || !isInitialized(this.root)) {
+      return [];
+    }
+
+    const changesPath = getFilePath('CHANGES', this.root);
+    
+    if (!isJsonlFormat(changesPath)) {
+      console.warn('Legacy format detected. Run: memorylink migrate');
+      return [];
+    }
+
+    return readJsonlEntries<ChangeEntry>(changesPath);
+  }
+
+  /**
+   * Get all learning entries (JSONL format)
+   */
+  getLearnings(): LearningEntry[] {
+    if (!this.root || !isInitialized(this.root)) {
+      return [];
+    }
+
+    const learningsPath = getFilePath('LEARNINGS', this.root);
+    
+    if (!isJsonlFormat(learningsPath)) {
+      console.warn('Legacy format detected. Run: memorylink migrate');
+      return [];
+    }
+
+    return readJsonlEntries<LearningEntry>(learningsPath)
+      .filter(e => e.status === 'ACTIVE');
+  }
+
+  /**
+   * Get successful learnings only
+   */
+  getSuccessfulLearnings(): LearningEntry[] {
+    return this.getLearnings().filter(e => e.outcome === 'SUCCESS');
+  }
+
+  /**
+   * Get failed learnings only
+   */
+  getFailedLearnings(): LearningEntry[] {
+    return this.getLearnings().filter(e => e.outcome === 'FAILED');
+  }
+
+  /**
+   * Get protected files list
+   */
+  getProtectedFiles(): string[] {
+    if (!this.root || !isInitialized(this.root)) {
+      return [];
+    }
+
+    const protectedPath = getFilePath('PROTECTED', this.root);
+    const lines = readLines(protectedPath).filter(l => !l.startsWith('#'));
+
+    return lines.map(line => line.split('#')[0].trim()).filter(Boolean);
+  }
+
+  /**
+   * Search across all logs by keyword
+   */
+  search(keyword: string): {
+    memories: MemoryEntry[];
+    learnings: LearningEntry[];
+    changes: ChangeEntry[];
+  } {
+    const kw = keyword.toLowerCase();
+    
+    const memories = this.getMemories().filter(e => 
+      e.details.toLowerCase().includes(kw) ||
+      e.tags?.some(t => t.toLowerCase().includes(kw))
+    );
+
+    const learnings = this.getLearnings().filter(e =>
+      e.problem.toLowerCase().includes(kw) ||
+      e.solution.toLowerCase().includes(kw) ||
+      e.tags?.some(t => t.toLowerCase().includes(kw))
+    );
+
+    const changes = this.getChanges().filter(e =>
+      e.file.toLowerCase().includes(kw) ||
+      e.reason.toLowerCase().includes(kw)
+    );
+
+    return { memories, learnings, changes };
+  }
+}
+
+export default MemoryLink;

package/src/search/local-embeddings.ts

@@ -0,0 +1,94 @@
+/**
+ * MemoryLink - Local Embeddings
+ * Local-first embeddings using @xenova/transformers (optional)
+ * 
+ * v1.3: Week 3-4 - Local semantic search
+ */
+
+import type { Embedding } from './vector-search.js';
+
+/**
+ * Check if local embeddings are available
+ */
+export function isLocalEmbeddingsAvailable(): boolean {
+  try {
+    // Try to require transformers (optional dependency)
+    require.resolve('@xenova/transformers');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Generate embeddings using local model (Xenova Transformers)
+ * Falls back to error if not available
+ */
+export async function generateEmbeddingLocal(text: string): Promise<Embedding> {
+  if (!isLocalEmbeddingsAvailable()) {
+    throw new Error('@xenova/transformers not installed. Install with: npm install @xenova/transformers');
+  }
+
+  try {
+    // Dynamic import to avoid requiring if not used
+    // @ts-ignore - Optional dependency, may not be installed
+    const { pipeline } = await import('@xenova/transformers');
+    
+    // Use a lightweight model (all-MiniLM-L6-v2 is ~80MB)
+    const extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+      quantized: true, // Use quantized model for smaller size
+    });
+
+    const output = await extractor(text, { pooling: 'mean', normalize: true });
+    
+    // Convert tensor to array
+    const embedding = Array.from(output.data);
+    return embedding as Embedding;
+  } catch (error) {
+    throw new Error(`Failed to generate local embedding: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+/**
+ * Generate embeddings for multiple texts (batch) using local model
+ */
+export async function generateEmbeddingsLocal(texts: string[]): Promise<Embedding[]> {
+  if (!isLocalEmbeddingsAvailable()) {
+    throw new Error('@xenova/transformers not installed. Install with: npm install @xenova/transformers');
+  }
+
+  try {
+    // @ts-ignore - Optional dependency, may not be installed
+    const { pipeline } = await import('@xenova/transformers');
+    const extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+      quantized: true,
+    });
+
+    // Process in batches to avoid memory issues
+    const batchSize = 10;
+    const allEmbeddings: Embedding[] = [];
+
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      const output = await extractor(batch, { pooling: 'mean', normalize: true });
+      
+      // Convert tensor to array
+      // Output shape: [batch_size, embedding_dim] (e.g., [10, 384])
+      const outputArray = Array.from(output.data);
+      const embeddingDim = output.dims[1];
+      
+      // Extract each embedding from the batch
+      for (let j = 0; j < batch.length; j++) {
+        const startIdx = j * embeddingDim;
+        const endIdx = startIdx + embeddingDim;
+        const embedding = outputArray.slice(startIdx, endIdx);
+        allEmbeddings.push(embedding as Embedding);
+      }
+    }
+
+    return allEmbeddings;
+  } catch (error) {
+    throw new Error(`Failed to generate local embeddings: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+