memorylink 1.0.0

package/src/utils/streaming.ts

@@ -0,0 +1,109 @@
+/**
+ * MemoryLink - Streaming Utilities
+ * Stream JSONL files line-by-line for memory efficiency
+ * 
+ * v1.0.1: Added pagination and streaming support
+ */
+
+import * as fs from 'fs';
+import * as readline from 'readline';
+import * as path from 'path';
+
+/**
+ * Stream JSONL entries from a file with pagination
+ * Memory-efficient: reads one line at a time
+ */
+export async function* streamJsonlEntries<T>(
+  filePath: string,
+  options: {
+    page?: number;
+    perPage?: number;
+    filter?: (entry: T) => boolean;
+  } = {}
+): AsyncGenerator<T, void, unknown> {
+  const { page = 1, perPage, filter } = options;
+  
+  if (!fs.existsSync(filePath)) {
+    return; // File doesn't exist, nothing to stream
+  }
+
+  const fileStream = fs.createReadStream(filePath);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  let currentLine = 0;
+  let currentPage = 1;
+  let entriesInPage = 0;
+  const startLine = perPage ? (page - 1) * perPage : 0;
+  const endLine = perPage ? startLine + perPage : Infinity;
+
+  try {
+    for await (const line of rl) {
+      if (!line.trim()) {
+        continue; // Skip empty lines
+      }
+
+      try {
+        const entry = JSON.parse(line) as T;
+        
+        // Apply filter if provided
+        if (filter && !filter(entry)) {
+          continue;
+        }
+
+        // Pagination logic
+        if (perPage) {
+          if (currentLine >= startLine && currentLine < endLine) {
+            yield entry;
+            entriesInPage++;
+          }
+          currentLine++;
+          
+          if (currentLine >= endLine) {
+            break; // Reached end of requested page
+          }
+        } else {
+          // No pagination - yield all entries
+          yield entry;
+        }
+      } catch (e) {
+        // Skip invalid JSON lines
+        continue;
+      }
+    }
+  } finally {
+    rl.close();
+    fileStream.close();
+  }
+}
+
+/**
+ * Count total entries in a JSONL file (streaming, memory-efficient)
+ */
+export async function countJsonlEntries(
+  filePath: string,
+  filter?: (entry: any) => boolean
+): Promise<number> {
+  let count = 0;
+  
+  for await (const entry of streamJsonlEntries(filePath, { filter })) {
+    count++;
+  }
+  
+  return count;
+}
+
+/**
+ * Get total pages for a JSONL file
+ */
+export async function getTotalPages(
+  filePath: string,
+  perPage: number,
+  filter?: (entry: any) => boolean
+): Promise<number> {
+  const total = await countJsonlEntries(filePath, filter);
+  return Math.ceil(total / perPage);
+}
+

package/src/utils/tag-suggestions.ts

@@ -0,0 +1,117 @@
+/**
+ * MemoryLink - Tag Suggestions
+ * Auto-suggest tags based on content analysis
+ * 
+ * MINOR FEATURE: Quick win for better UX
+ */
+
+import { normalizeTag } from '../utils.js';
+import type { MemoryEntry } from '../types.js';
+
+/**
+ * Common technical keywords that make good tags
+ */
+const TECHNICAL_KEYWORDS = [
+  // Languages
+  'typescript', 'javascript', 'python', 'java', 'go', 'rust', 'cpp', 'csharp', 'php', 'ruby',
+  // Frameworks
+  'react', 'vue', 'angular', 'next', 'express', 'django', 'flask', 'spring', 'laravel',
+  // Concepts
+  'auth', 'authentication', 'authorization', 'api', 'database', 'db', 'sql', 'nosql',
+  'frontend', 'backend', 'fullstack', 'devops', 'deployment', 'testing', 'security',
+  'performance', 'optimization', 'caching', 'pagination', 'validation', 'error-handling',
+  // Tools
+  'git', 'docker', 'kubernetes', 'aws', 'azure', 'gcp', 'ci-cd', 'github', 'gitlab',
+  // Patterns
+  'singleton', 'factory', 'observer', 'mvc', 'mvp', 'mvvm', 'rest', 'graphql',
+];
+
+/**
+ * Extract potential tags from text
+ */
+export function suggestTags(text: string, existingTags: string[] = []): string[] {
+  const lowerText = text.toLowerCase();
+  const suggested: string[] = [];
+  const existingSet = new Set(existingTags.map(t => t.toLowerCase()));
+
+  // 1. Extract technical keywords
+  for (const keyword of TECHNICAL_KEYWORDS) {
+    if (lowerText.includes(keyword) && !existingSet.has(keyword)) {
+      suggested.push(normalizeTag(keyword));
+    }
+  }
+
+  // 2. Extract quoted terms (often important concepts)
+  const quotedMatches = text.match(/"([^"]+)"/g);
+  if (quotedMatches) {
+    for (const match of quotedMatches) {
+      const term = match.replace(/"/g, '').trim();
+      if (term.length > 2 && term.length < 30 && !existingSet.has(term.toLowerCase())) {
+        suggested.push(normalizeTag(term));
+      }
+    }
+  }
+
+  // 3. Extract capitalized words (often proper nouns/concepts)
+  const capitalizedWords = text.match(/\b[A-Z][a-z]+\b/g);
+  if (capitalizedWords) {
+    for (const word of capitalizedWords) {
+      const normalized = normalizeTag(word);
+      if (normalized.length > 2 && !existingSet.has(normalized) && !suggested.includes(normalized)) {
+        // Only add if it appears multiple times (likely important)
+        const count = (text.match(new RegExp(`\\b${word}\\b`, 'gi')) || []).length;
+        if (count >= 2) {
+          suggested.push(normalized);
+        }
+      }
+    }
+  }
+
+  // 4. Extract common patterns (e.g., "JWT", "API", "URL")
+  const acronyms = text.match(/\b[A-Z]{2,}\b/g);
+  if (acronyms) {
+    for (const acronym of acronyms) {
+      const normalized = normalizeTag(acronym);
+      if (!existingSet.has(normalized) && !suggested.includes(normalized)) {
+        suggested.push(normalized);
+      }
+    }
+  }
+
+  // Limit to top 5 suggestions
+  return suggested.slice(0, 5);
+}
+
+
+/**
+ * Get similar existing tags (for tag normalization suggestions)
+ */
+export function findSimilarTags(
+  tag: string,
+  allTags: string[]
+): Array<{ tag: string; similarity: number }> {
+  const normalized = normalizeTag(tag);
+  const similar: Array<{ tag: string; similarity: number }> = [];
+
+  for (const existingTag of allTags) {
+    const existingNormalized = normalizeTag(existingTag);
+    
+    // Exact match after normalization
+    if (existingNormalized === normalized) {
+      similar.push({ tag: existingTag, similarity: 1.0 });
+      continue;
+    }
+
+    // Check if one contains the other
+    if (existingNormalized.includes(normalized) || normalized.includes(existingNormalized)) {
+      const similarity = Math.min(existingNormalized.length, normalized.length) /
+                        Math.max(existingNormalized.length, normalized.length);
+      if (similarity > 0.7) {
+        similar.push({ tag: existingTag, similarity });
+      }
+    }
+  }
+
+  return similar.sort((a, b) => b.similarity - a.similarity).slice(0, 3);
+}
+