@sylphx/flow 1.0.1 → 1.0.3

package/src/services/storage/separated-storage.ts

@@ -0,0 +1,467 @@
+/**
+ * Separated storage implementation - 分離式存儲
+ * Memory 數據存儲在 memory.db (應該上 Git)
+ * Cache 數據存儲在 cache.db (不應該上 Git)
+ *
+ * This file now acts as a compatibility layer that delegates to the specialized storage modules
+ */
+
+import {
+  CacheStorage,
+  type CodebaseFileEntry,
+  type TfidfDocumentEntry,
+  type TfidfIdfEntry,
+  type TfidfTermEntry,
+} from './cache-storage.js';
+import { type MemoryEntry, MemoryStorage } from './memory-storage.js';
+
+export type {
+  CodebaseFileEntry,
+  TfidfDocumentEntry,
+  TfidfIdfEntry,
+  TfidfTermEntry,
+} from './cache-storage.js';
+// Re-export interfaces for backward compatibility
+export type { MemoryEntry } from './memory-storage.js';
+
+/**
+ * Separated storage implementation - Compatibility Layer
+ *
+ * This class maintains backward compatibility while delegating to the specialized
+ * storage modules. New code should use MemoryStorage and CacheStorage directly.
+ */
+export class SeparatedMemoryStorage {
+  private memoryStorage: MemoryStorage;
+  private cacheStorage: CacheStorage;
+
+  constructor() {
+    this.memoryStorage = new MemoryStorage();
+    this.cacheStorage = new CacheStorage();
+  }
+
+  async initialize(): Promise<void> {
+    await Promise.all([this.memoryStorage.initialize(), this.cacheStorage.initialize()]);
+  }
+
+  // === Memory Operations (delegated to MemoryStorage) ===
+
+  /**
+   * Set a memory entry
+   */
+  async set(key: string, value: any, namespace = 'default'): Promise<void> {
+    return this.memoryStorage.set(key, value, namespace);
+  }
+
+  /**
+   * Get a memory entry
+   */
+  async get(key: string, namespace = 'default'): Promise<MemoryEntry | null> {
+    return this.memoryStorage.get(key, namespace);
+  }
+
+  /**
+   * Get all memory entries
+   */
+  async getAll(namespace?: string): Promise<MemoryEntry[]> {
+    return this.memoryStorage.getAll(namespace);
+  }
+
+  /**
+   * Search memory entries
+   */
+  async search(pattern: string, namespace?: string): Promise<MemoryEntry[]> {
+    return this.memoryStorage.search(pattern, namespace);
+  }
+
+  /**
+   * Delete a memory entry
+   */
+  async delete(key: string, namespace = 'default'): Promise<boolean> {
+    return this.memoryStorage.delete(key, namespace);
+  }
+
+  /**
+   * Clear all memory entries in a namespace
+   */
+  async clear(namespace = 'default'): Promise<void> {
+    return this.memoryStorage.clear(namespace);
+  }
+
+  /**
+   * Get memory statistics
+   */
+  async getStats(namespace?: string): Promise<{
+    totalEntries: number;
+    namespaces: string[];
+  }> {
+    return this.memoryStorage.getStats(namespace);
+  }
+
+  // === Codebase Files Operations (delegated to CacheStorage) ===
+
+  /**
+   * Store a codebase file entry
+   */
+  async setCodebaseFile(
+    path: string,
+    mtime: number,
+    hash: string,
+    content?: string,
+    language?: string,
+    size?: number
+  ): Promise<void> {
+    const indexedAt = new Date().toISOString();
+    return this.cacheStorage.setCodebaseFile({
+      path,
+      mtime,
+      hash,
+      content,
+      language,
+      size,
+      indexedAt,
+    });
+  }
+
+  /**
+   * Get a codebase file entry
+   */
+  async getCodebaseFile(path: string): Promise<CodebaseFileEntry | null> {
+    return this.cacheStorage.getCodebaseFile(path);
+  }
+
+  /**
+   * Get all codebase files
+   */
+  async getAllCodebaseFiles(): Promise<CodebaseFileEntry[]> {
+    return this.cacheStorage.getAllCodebaseFiles();
+  }
+
+  /**
+   * Delete a codebase file entry
+   */
+  async deleteCodebaseFile(path: string): Promise<boolean> {
+    return this.cacheStorage.deleteCodebaseFile(path);
+  }
+
+  /**
+   * Clear all codebase files
+   */
+  async clearCodebaseFiles(): Promise<void> {
+    return this.cacheStorage.clearCodebaseFiles();
+  }
+
+  // === Metadata Operations (delegated to CacheStorage) ===
+
+  /**
+   * Set metadata value
+   */
+  async setMetadata(key: string, value: string): Promise<void> {
+    return this.cacheStorage.setMetadata(key, value);
+  }
+
+  /**
+   * Get metadata value
+   */
+  async getMetadata(key: string): Promise<string | null> {
+    return this.cacheStorage.getMetadata(key);
+  }
+
+  /**
+   * Get all metadata
+   */
+  async getAllMetadata(): Promise<Record<string, string>> {
+    return this.cacheStorage.getAllMetadata();
+  }
+
+  /**
+   * Delete metadata
+   */
+  async deleteMetadata(key: string): Promise<boolean> {
+    return this.cacheStorage.deleteMetadata(key);
+  }
+
+  // === TF-IDF Terms Operations (delegated to CacheStorage) ===
+
+  /**
+   * Store TF-IDF terms for a file
+   */
+  async setTfidfTerms(
+    terms: Array<{ filePath: string; term: string; frequency: number }>
+  ): Promise<void> {
+    return this.cacheStorage.setTfidfTerms(terms);
+  }
+
+  /**
+   * Get TF-IDF terms for a file
+   */
+  async getTfidfTerms(filePath: string): Promise<TfidfTermEntry[]> {
+    return this.cacheStorage.getTfidfTerms(filePath);
+  }
+
+  /**
+   * Get TF-IDF terms by term
+   */
+  async getTfidfTermsByTerm(term: string): Promise<TfidfTermEntry[]> {
+    return this.cacheStorage.getTfidfTermsByTerm(term);
+  }
+
+  /**
+   * Clear all TF-IDF terms
+   */
+  async clearTfidfTerms(): Promise<void> {
+    return this.cacheStorage.clearTfidfTerms();
+  }
+
+  // === TF-IDF Documents Operations (delegated to CacheStorage) ===
+
+  /**
+   * Store TF-IDF document vector
+   */
+  async setTfidfDocument(
+    filePath: string,
+    magnitude: number,
+    termCount: number,
+    rawTerms: string
+  ): Promise<void> {
+    return this.cacheStorage.setTfidfDocument({
+      filePath,
+      magnitude,
+      termCount,
+      rawTerms,
+    });
+  }
+
+  /**
+   * Get TF-IDF document (alias for getTfidfDocument for compatibility)
+   */
+  async getTFIDFDocument(filePath: string): Promise<TfidfDocumentEntry | null> {
+    return this.cacheStorage.getTfidfDocument(filePath);
+  }
+
+  /**
+   * Get all TF-IDF documents
+   */
+  async getAllTfidfDocuments(): Promise<TfidfDocumentEntry[]> {
+    return this.cacheStorage.getAllTfidfDocuments();
+  }
+
+  /**
+   * Clear all TF-IDF documents
+   */
+  async clearTfidfDocuments(): Promise<void> {
+    return this.cacheStorage.clearTfidfDocuments();
+  }
+
+  // === TF-IDF IDF Operations (delegated to CacheStorage) ===
+
+  /**
+   * Store IDF value
+   */
+  async setTfidfIdf(term: string, idfValue: number): Promise<void> {
+    return this.cacheStorage.setTfidfIdf({ term, idfValue });
+  }
+
+  /**
+   * Get IDF value
+   */
+  async getTfidfIdf(term: string): Promise<TfidfIdfEntry | null> {
+    return this.cacheStorage.getTfidfIdf(term);
+  }
+
+  /**
+   * Get all IDF values
+   */
+  async getAllTfidfIdf(): Promise<TfidfIdfEntry[]> {
+    return this.cacheStorage.getAllTfidfIdf();
+  }
+
+  /**
+   * Clear all IDF values
+   */
+  async clearTfidfIdf(): Promise<void> {
+    return this.cacheStorage.clearTfidfIdf();
+  }
+
+  // === Utility Operations ===
+
+  /**
+   * Get comprehensive cache statistics
+   */
+  async getCacheStats(): Promise<{
+    codebaseFiles: number;
+    metadataEntries: number;
+    tfidfTerms: number;
+    tfidfDocuments: number;
+    tfidfIdfValues: number;
+  }> {
+    return this.cacheStorage.getStats();
+  }
+
+  /**
+   * Clear all cache data
+   */
+  async clearAllCache(): Promise<void> {
+    return this.cacheStorage.clearAll();
+  }
+
+  /**
+   * Get comprehensive statistics for both memory and cache
+   */
+  async getAllStats(): Promise<{
+    memory: {
+      totalEntries: number;
+      namespaces: string[];
+    };
+    cache: {
+      codebaseFiles: number;
+      metadataEntries: number;
+      tfidfTerms: number;
+      tfidfDocuments: number;
+      tfidfIdfValues: number;
+    };
+  }> {
+    const [memoryStats, cacheStats] = await Promise.all([
+      this.memoryStorage.getStats(),
+      this.cacheStorage.getStats(),
+    ]);
+
+    return {
+      memory: memoryStats,
+      cache: cacheStats,
+    };
+  }
+
+  /**
+   * Clear all data (both memory and cache)
+   */
+  async clearAll(): Promise<void> {
+    await Promise.all([this.memoryStorage.clear('default'), this.cacheStorage.clearAll()]);
+  }
+
+  // === Additional methods for UnifiedSearchService compatibility ===
+
+  /**
+   * Get codebase index statistics
+   */
+  async getCodebaseIndexStats(): Promise<{
+    indexedAt?: string;
+    totalFiles: number;
+    totalTerms: number;
+  }> {
+    const metadata = await this.cacheStorage.getAllMetadata();
+    return {
+      indexedAt: metadata.indexedAt,
+      totalFiles: Number.parseInt(metadata.totalFiles || '0', 10),
+      totalTerms: Number.parseInt(metadata.totalTerms || '0', 10),
+    };
+  }
+
+  /**
+   * Get IDF values (alias for getAllTfidfIdf)
+   */
+  async getIDFValues(): Promise<Record<string, number>> {
+    const entries = await this.cacheStorage.getAllTfidfIdf();
+    const idfValues: Record<string, number> = {};
+    for (const entry of entries) {
+      idfValues[entry.term] = entry.idfValue;
+    }
+    return idfValues;
+  }
+
+  /**
+   * Upsert TF-IDF document (alias for setTfidfDocument)
+   */
+  async upsertTFIDFDocument(
+    filePath: string,
+    document: {
+      magnitude: number;
+      termCount: number;
+      rawTerms: Record<string, number>;
+    }
+  ): Promise<void> {
+    return this.cacheStorage.setTfidfDocument({
+      filePath,
+      magnitude: document.magnitude,
+      termCount: document.termCount,
+      rawTerms: JSON.stringify(document.rawTerms),
+    });
+  }
+
+  /**
+   * Upsert codebase file (alias for setCodebaseFile)
+   */
+  async upsertCodebaseFile(file: {
+    path: string;
+    mtime: number;
+    hash: string;
+    content?: string;
+    language?: string;
+    size?: number;
+    indexedAt?: string;
+  }): Promise<void> {
+    return this.cacheStorage.setCodebaseFile({
+      ...file,
+      indexedAt: file.indexedAt || new Date().toISOString(),
+    });
+  }
+
+  /**
+   * Set IDF values (alias for multiple setTfidfIdf calls)
+   */
+  async setIDFValues(idfValues: Record<string, number>): Promise<void> {
+    const promises = Object.entries(idfValues).map(([term, idfValue]) =>
+      this.cacheStorage.setTfidfIdf({ term, idfValue })
+    );
+    await Promise.all(promises);
+  }
+
+  /**
+   * Clear codebase index
+   */
+  async clearCodebaseIndex(): Promise<void> {
+    await Promise.all([
+      this.cacheStorage.clearCodebaseFiles(),
+      this.cacheStorage.clearTfidfTerms(),
+      this.cacheStorage.clearTfidfDocuments(),
+      this.cacheStorage.clearTfidfIdf(),
+    ]);
+  }
+
+  /**
+   * Get codebase metadata (alias for getMetadata)
+   */
+  async getCodebaseMetadata(key: string): Promise<string | null> {
+    return this.cacheStorage.getMetadata(key);
+  }
+
+  /**
+   * Set codebase metadata (alias for setMetadata)
+   */
+  async setCodebaseMetadata(key: string, value: string): Promise<void> {
+    return this.cacheStorage.setMetadata(key, value);
+  }
+
+  /**
+   * Get TF-IDF terms for a file
+   */
+  async getTFIDFTerms(filePath: string): Promise<Record<string, number>> {
+    const entries = await this.cacheStorage.getTfidfTerms(filePath);
+    const terms: Record<string, number> = {};
+    for (const entry of entries) {
+      terms[entry.term] = entry.frequency;
+    }
+    return terms;
+  }
+
+  /**
+   * Set TF-IDF terms (alias for setTfidfTerms)
+   */
+  async setTFIDFTerms(filePath: string, terms: Record<string, number>): Promise<void> {
+    const termsWithFilePath = Object.entries(terms).map(([term, frequency]) => ({
+      filePath,
+      term,
+      frequency,
+    }));
+    return this.cacheStorage.setTfidfTerms(termsWithFilePath);
+  }
+}

package/src/services/storage/vector-storage.ts

@@ -0,0 +1,13 @@
+/**
+ * Vector storage interface and implementation
+ * Uses LanceDB for high-performance local vector database
+ */
+
+// Re-export from the LanceDB implementation
+export {
+  generateMockEmbedding,
+  type VectorDocument,
+  type VectorSearchResult,
+  VectorStorage,
+  type VectorStorageMetadata,
+} from './lancedb-vector-storage.js';

package/src/shared/agents/index.ts

@@ -0,0 +1,63 @@
+/**
+ * Agent configuration and selection utilities
+ */
+
+import type { AgentConfig, AgentConfigs } from '../types/index.js';
+
+/**
+ * Get list of supported agents
+ * @param configs - Agent configurations
+ * @returns Array of supported agent keys
+ */
+export function getSupportedAgents(configs: AgentConfigs): string[] {
+  return Object.keys(configs);
+}
+
+/**
+ * Get configuration for a specific agent
+ * @param configs - Agent configurations
+ * @param agent - Agent key
+ * @returns Agent configuration
+ * @throws Error if agent not found
+ */
+export function getAgentConfig(configs: AgentConfigs, agent: string): AgentConfig {
+  const config = configs[agent];
+  if (!config) {
+    throw new Error(`Agent configuration not found: ${agent}`);
+  }
+  return config;
+}
+
+/**
+ * Prompt user to select an agent (currently defaults to first)
+ * @param configs - Agent configurations
+ * @param toolName - Name of the tool for display
+ * @returns Selected agent key
+ */
+export async function promptForAgent(configs: AgentConfigs, toolName: string): Promise<string> {
+  const supportedAgents = getSupportedAgents(configs);
+
+  console.log(`\n📝 ${toolName}`);
+  console.log('================');
+  console.log('Available agents:');
+  supportedAgents.forEach((agent, index) => {
+    const config = getAgentConfig(configs, agent);
+    console.log(`  ${index + 1}. ${config.name} - ${config.description}`);
+  });
+
+  // For now, default to first agent
+  // In a real implementation, you might want to use readline or a CLI prompt library
+  return supportedAgents[0];
+}
+
+/**
+ * Detect which agent tool to use
+ * @param configs - Agent configurations
+ * @param defaultAgent - Default agent to use
+ * @returns Agent key
+ */
+export function detectAgentTool(_configs: AgentConfigs, defaultAgent = 'opencode'): string {
+  // Simple detection logic - could be enhanced
+  // For now, return default
+  return defaultAgent;
+}

package/src/shared/files/index.ts

@@ -0,0 +1,99 @@
+/**
+ * File collection and information utilities
+ */
+
+import path from 'node:path';
+import {
+  deletePathSafe,
+  getFileInfo,
+  readDirectorySafe,
+  readFileSafe,
+} from '../../utils/file-operations.js';
+import type { ProcessResult } from '../types/index.js';
+
+/**
+ * Collect files from directory with specified extensions
+ * @param dir - Directory to search
+ * @param extensions - File extensions to include
+ * @returns Array of file paths relative to directory
+ */
+export async function collectFiles(dir: string, extensions: string[]): Promise<string[]> {
+  try {
+    const allFiles = await readDirectorySafe(dir, {
+      recursive: true,
+      includeFiles: true,
+      includeDirectories: false,
+    });
+
+    return allFiles
+      .filter((filePath) => extensions.some((ext) => filePath.endsWith(ext)))
+      .map((filePath) => path.relative(dir, filePath))
+      .sort();
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Get local file information including content and modification time
+ * @param filePath - Path to file
+ * @returns File info with content and mtime, or null if file doesn't exist
+ */
+export async function getLocalFileInfo(
+  filePath: string
+): Promise<{ content: string; mtime: Date } | null> {
+  const info = await getFileInfo(filePath);
+
+  if (!info.exists || !info.isFile) {
+    return null;
+  }
+
+  const content = await readFileSafe(filePath);
+
+  if (content === null) {
+    return null;
+  }
+
+  return {
+    content,
+    mtime: info.mtime!,
+  };
+}
+
+/**
+ * Clear obsolete files from target directory
+ * @param targetDir - Target directory
+ * @param expectedFiles - Set of expected file names
+ * @param extensions - Valid file extensions
+ * @param results - Array to store process results
+ */
+export async function clearObsoleteFiles(
+  targetDir: string,
+  expectedFiles: Set<string>,
+  extensions: string[],
+  results: ProcessResult[]
+): Promise<void> {
+  try {
+    const items = await readDirectorySafe(targetDir, {
+      recursive: false,
+      includeFiles: true,
+      includeDirectories: false,
+    });
+
+    for (const itemPath of items) {
+      const fileName = path.basename(itemPath);
+      const hasValidExtension = extensions.some((ext) => fileName.endsWith(ext));
+
+      if (hasValidExtension && !expectedFiles.has(fileName)) {
+        await deletePathSafe(itemPath);
+        results.push({
+          file: fileName,
+          status: 'skipped',
+          action: 'Removed obsolete file',
+        });
+      }
+    }
+  } catch {
+    // Directory doesn't exist, nothing to clear
+  }
+}

package/src/shared/index.ts

@@ -0,0 +1,32 @@
+/**
+ * Shared utilities and types barrel export
+ * Provides clean access to all shared functionality
+ */
+
+// Agent configuration
+export {
+  detectAgentTool,
+  getAgentConfig,
+  getSupportedAgents,
+  promptForAgent,
+} from './agents/index.js';
+// File operations
+export {
+  clearObsoleteFiles,
+  collectFiles,
+  getLocalFileInfo,
+} from './files/index.js';
+// Logging
+export { log } from './logging/index.js';
+// Processing utilities
+export {
+  displayResults,
+  processBatch,
+} from './processing/index.js';
+// Types
+export type {
+  AgentConfig,
+  AgentConfigs,
+  CommonOptions,
+  ProcessResult,
+} from './types/index.js';

package/src/shared/logging/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Logging utilities for console output with colors
+ */
+
+/**
+ * Log a message with color
+ * @param message - Message to log
+ * @param color - Color name (red, green, yellow, blue, magenta, cyan, white)
+ */
+export function log(message: string, color = 'white'): void {
+  const colors = {
+    red: '\x1b[31m',
+    green: '\x1b[32m',
+    yellow: '\x1b[33m',
+    blue: '\x1b[34m',
+    magenta: '\x1b[35m',
+    cyan: '\x1b[36m',
+    white: '\x1b[37m',
+    reset: '\x1b[0m',
+  };
+
+  const colorCode = colors[color as keyof typeof colors] || colors.white;
+  console.log(`${colorCode}${message}${colors.reset}`);
+}