@mnemonik/scanner 1.0.0 → 2.0.0

package/src/daemon.ts

@@ -4,34 +4,170 @@ import { join } from 'path';
 import { CodeScanner, type CodeChunk } from '@mnemonik/shared';
 import { MnemonikClient, type ScanPushFile } from './client.js';
 import { FileWatcher } from './watcher.js';
+import { ProjectDiscovery, type DiscoveredProject } from './discovery.js';
 
 export interface DaemonConfig {
-  projectId: string;
-  projectRoot: string;
   serverUrl: string;
   apiKey: string;
+  roots: string[];
+  refreshIntervalMs?: number;
+  maxConcurrentScans?: number;
+}
+
+interface WatchedProject {
+  projectId: string;
+  path: string;
+  name?: string;
+  watcher: FileWatcher;
+  pendingRetries: Set<string>;
+  retryTimer: ReturnType<typeof setInterval> | null;
 }
 
 export class ScannerDaemon {
   private client: MnemonikClient;
   private scanner: CodeScanner;
-  private watcher: FileWatcher | null = null;
-  private pendingRetries: Set<string> = new Set();
-  private retryTimer: ReturnType<typeof setInterval> | null = null;
+  private projects = new Map<string, WatchedProject>();
+  private refreshTimer: ReturnType<typeof setInterval> | null = null;
+  private discovery: ProjectDiscovery;
+  private refreshIntervalMs: number;
+  private maxConcurrentScans: number;
 
   constructor(private config: DaemonConfig) {
     this.client = new MnemonikClient(config.serverUrl, config.apiKey);
     this.scanner = new CodeScanner();
+    this.discovery = new ProjectDiscovery(config.roots);
+    this.refreshIntervalMs = config.refreshIntervalMs ?? 300_000; // 5 min
+    this.maxConcurrentScans = config.maxConcurrentScans ?? 5;
   }
 
   async start(): Promise<void> {
-    console.log(`[scanner] Starting daemon for project: ${this.config.projectId}`);
+    console.log(`[scanner] Starting daemon`);
     console.log(`[scanner] Server: ${this.config.serverUrl}`);
-    console.log(`[scanner] Root: ${this.config.projectRoot}`);
+    console.log(`[scanner] Roots: ${this.config.roots.join(', ')}`);
 
     await this.waitForServer();
-    await this.initialScan();
-    await this.startWatching();
+    await this.refreshProjects();
+
+    this.refreshTimer = setInterval(() => {
+      this.refreshProjects().catch((err) => {
+        console.warn('[scanner] Refresh failed:', (err as Error).message);
+      });
+    }, this.refreshIntervalMs);
+    this.refreshTimer.unref();
+
+    console.log('[scanner] Watching for changes.');
+  }
+
+  async stop(): Promise<void> {
+    if (this.refreshTimer) {
+      clearInterval(this.refreshTimer);
+      this.refreshTimer = null;
+    }
+
+    for (const project of this.projects.values()) {
+      project.watcher.stop();
+      if (project.retryTimer) clearInterval(project.retryTimer);
+    }
+    this.projects.clear();
+    console.log('[scanner] Daemon stopped');
+  }
+
+  getWatchedProjects(): Array<{ projectId: string; path: string; name?: string }> {
+    return Array.from(this.projects.values()).map((p) => ({
+      projectId: p.projectId,
+      path: p.path,
+      name: p.name,
+    }));
+  }
+
+  /**
+   * Discover projects from configured roots and reconcile with current watch list.
+   */
+  async refreshProjects(): Promise<void> {
+    const discovered = await this.discovery.discover();
+    const discoveredMap = new Map(discovered.map((d) => [d.projectId, d]));
+
+    // Remove projects no longer discovered
+    for (const [projectId, project] of this.projects) {
+      if (!discoveredMap.has(projectId)) {
+        console.log(`[scanner] Project removed: ${project.name ?? projectId} (${project.path})`);
+        project.watcher.stop();
+        if (project.retryTimer) clearInterval(project.retryTimer);
+        this.projects.delete(projectId);
+      }
+    }
+
+    // Add new projects or handle path changes
+    for (const discovered_project of discoveredMap.values()) {
+      const existing = this.projects.get(discovered_project.projectId);
+
+      if (!existing) {
+        // New project
+        await this.addProject(discovered_project);
+      } else if (existing.path !== discovered_project.path) {
+        // Path changed (folder renamed/moved)
+        console.log(
+          `[scanner] Project moved: ${existing.name ?? existing.projectId} ` +
+            `${existing.path} → ${discovered_project.path}`
+        );
+        existing.watcher.stop();
+        if (existing.retryTimer) clearInterval(existing.retryTimer);
+        this.projects.delete(discovered_project.projectId);
+        await this.addProject(discovered_project);
+      }
+    }
+
+    console.log(`[scanner] Watching ${this.projects.size} project(s)`);
+  }
+
+  private async addProject(discovered: DiscoveredProject): Promise<void> {
+    const label = discovered.projectName ?? discovered.projectId.slice(0, 8);
+    console.log(`[scanner] Adding project: ${label} (${discovered.path})`);
+
+    try {
+      await this.initialScan(discovered.projectId, discovered.path);
+    } catch (err) {
+      console.warn(
+        `[scanner] Initial scan failed for ${label}:`,
+        (err as Error).message
+      );
+    }
+
+    const watcher = new FileWatcher(
+      discovered.path,
+      (changedFiles) => this.handleChanges(discovered.projectId, discovered.path, changedFiles),
+      500,
+      (err) => {
+        console.warn(
+          `[scanner] Root watcher error for ${label}: ${err.message}. Removing project.`
+        );
+        const project = this.projects.get(discovered.projectId);
+        if (project) {
+          project.watcher.stop();
+          if (project.retryTimer) clearInterval(project.retryTimer);
+          this.projects.delete(discovered.projectId);
+        }
+      }
+    );
+
+    try {
+      await watcher.start();
+    } catch (err) {
+      console.warn(
+        `[scanner] Failed to start watcher for ${label}:`,
+        (err as Error).message
+      );
+      return;
+    }
+
+    this.projects.set(discovered.projectId, {
+      projectId: discovered.projectId,
+      path: discovered.path,
+      name: discovered.projectName,
+      watcher,
+      pendingRetries: new Set(),
+      retryTimer: null,
+    });
   }
 
   private async waitForServer(): Promise<void> {
@@ -58,83 +194,46 @@ export class ScannerDaemon {
     throw new Error(`Server unreachable after ${maxRetries} attempts`);
   }
 
-  getProjectRoot(): string {
-    return this.config.projectRoot;
-  }
-
-  getProjectId(): string {
-    return this.config.projectId;
-  }
-
-  async stop(): Promise<void> {
-    this.watcher?.stop();
-    if (this.retryTimer) clearInterval(this.retryTimer);
-    console.log('[scanner] Daemon stopped');
-  }
-
-  private startRetryLoop(): void {
-    if (this.retryTimer) return;
-    console.log(`[scanner] ${this.pendingRetries.size} file(s) queued for retry`);
-    this.retryTimer = setInterval(async () => {
-      if (this.pendingRetries.size === 0) {
-        if (this.retryTimer) clearInterval(this.retryTimer);
-        this.retryTimer = null;
-        return;
-      }
-      const files = [...this.pendingRetries];
-      this.pendingRetries.clear();
-      await this.handleChanges(files);
-    }, 10_000);
-    this.retryTimer.unref();
-  }
-
-  private async initialScan(): Promise<void> {
-    console.log('[scanner] Starting initial scan...');
+  private async initialScan(projectId: string, projectRoot: string): Promise<void> {
     const startTime = Date.now();
 
-    const chunks = await this.scanner.scanDirectory(this.config.projectRoot);
-    console.log(`[scanner] Scanned ${chunks.length} chunks locally`);
-
-    const serverHashes = await this.client.getStatus(this.config.projectId);
-    console.log(`[scanner] Server knows ${serverHashes.size} files`);
+    const chunks = await this.scanner.scanDirectory(projectRoot);
+    const serverHashes = await this.client.getStatus(projectId);
 
-    const files = await this.groupChunksByFile(chunks);
+    const files = await this.groupChunksByFile(chunks, projectRoot);
     const filesToPush = files.filter((f) => {
       const serverHash = serverHashes.get(f.path);
       return !serverHash || serverHash !== f.hash;
     });
 
-    if (filesToPush.length === 0) {
-      console.log('[scanner] All files up to date, nothing to push');
-    } else {
-      console.log(`[scanner] Pushing ${filesToPush.length} changed files...`);
-      await this.client.pushFiles(this.config.projectId, filesToPush);
-      console.log(`[scanner] Push complete`);
+    if (filesToPush.length > 0) {
+      console.log(`[scanner] Pushing ${filesToPush.length} changed files for ${projectId.slice(0, 8)}...`);
+      await this.client.pushFiles(projectId, filesToPush);
     }
 
     const duration = ((Date.now() - startTime) / 1000).toFixed(1);
-    console.log(`[scanner] Initial scan complete in ${duration}s`);
-  }
-
-  private async startWatching(): Promise<void> {
-    this.watcher = new FileWatcher(
-      this.config.projectRoot,
-      (changedFiles) => this.handleChanges(changedFiles),
-      500
+    console.log(
+      `[scanner] Scan complete for ${projectId.slice(0, 8)}: ` +
+        `${chunks.length} chunks, ${filesToPush.length} pushed (${duration}s)`
     );
-    await this.watcher.start();
   }
 
-  private async handleChanges(changedFiles: string[]): Promise<void> {
+  private async handleChanges(
+    projectId: string,
+    projectRoot: string,
+    changedFiles: string[]
+  ): Promise<void> {
+    const project = this.projects.get(projectId);
+    if (!project) return;
+
     try {
-      const absPaths = changedFiles.map((rel) => join(this.config.projectRoot, rel));
-      const chunks = await this.scanner.scanFiles(absPaths, this.config.projectRoot);
+      const absPaths = changedFiles.map((rel) => join(projectRoot, rel));
+      const chunks = await this.scanner.scanFiles(absPaths, projectRoot);
 
       if (chunks.length === 0) return;
 
-      const files = await this.groupChunksByFile(chunks);
+      const files = await this.groupChunksByFile(chunks, projectRoot);
 
-      // Push in batches, only retrying files from failed batches
       const batchSize = 25;
       const succeededPaths = new Set<string>();
       let hadFailure = false;
@@ -142,39 +241,55 @@ export class ScannerDaemon {
       for (let i = 0; i < files.length; i += batchSize) {
         const batch = files.slice(i, i + batchSize);
         try {
-          await this.client.pushFiles(this.config.projectId, batch);
+          await this.client.pushFiles(projectId, batch);
           for (const f of batch) succeededPaths.add(f.path);
         } catch {
           hadFailure = true;
-          for (const f of batch) this.pendingRetries.add(f.path);
+          for (const f of batch) project.pendingRetries.add(f.path);
         }
       }
 
       if (succeededPaths.size > 0) {
         console.log(
-          `[scanner] Pushed ${succeededPaths.size} changed file(s): ${[...succeededPaths].join(', ')}`
+          `[scanner] [${projectId.slice(0, 8)}] Pushed ${succeededPaths.size} file(s): ${[...succeededPaths].join(', ')}`
         );
       }
       if (hadFailure) {
-        console.warn(`[scanner] ${this.pendingRetries.size} file(s) failed, queued for retry`);
-        this.startRetryLoop();
+        console.warn(
+          `[scanner] [${projectId.slice(0, 8)}] ${project.pendingRetries.size} file(s) failed, queued for retry`
+        );
+        this.startRetryLoop(project);
       }
     } catch (err) {
-      console.error('[scanner] Error handling changes:', err);
+      console.error(`[scanner] [${projectId.slice(0, 8)}] Error handling changes:`, err);
     }
   }
 
-  private async groupChunksByFile(chunks: CodeChunk[]): Promise<ScanPushFile[]> {
+  private startRetryLoop(project: WatchedProject): void {
+    if (project.retryTimer) return;
+    project.retryTimer = setInterval(async () => {
+      if (project.pendingRetries.size === 0) {
+        if (project.retryTimer) clearInterval(project.retryTimer);
+        project.retryTimer = null;
+        return;
+      }
+      const files = [...project.pendingRetries];
+      project.pendingRetries.clear();
+      await this.handleChanges(project.projectId, project.path, files);
+    }, 10_000);
+    project.retryTimer.unref();
+  }
+
+  private async groupChunksByFile(
+    chunks: CodeChunk[],
+    projectRoot: string
+  ): Promise<ScanPushFile[]> {
     const fileMap = new Map<string, ScanPushFile>();
 
     for (const chunk of chunks) {
       const key = chunk.filePath;
       if (!fileMap.has(key)) {
-        fileMap.set(key, {
-          path: key,
-          hash: '',
-          chunks: [],
-        });
+        fileMap.set(key, { path: key, hash: '', chunks: [] });
       }
       const file = fileMap.get(key)!;
       file.chunks.push({
@@ -190,7 +305,7 @@ export class ScannerDaemon {
 
     for (const file of fileMap.values()) {
       try {
-        const absPath = join(this.config.projectRoot, file.path);
+        const absPath = join(projectRoot, file.path);
         const raw = await readFile(absPath, 'utf-8');
         file.hash = createHash('sha256').update(raw).digest('hex');
       } catch (err) {

package/src/discovery.ts

@@ -0,0 +1,125 @@
+import { readFile, readdir, stat } from 'fs/promises';
+import { join, resolve } from 'path';
+
+/** Directories to skip during discovery walk */
+const SKIP_DIRS = new Set([
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+  '.next',
+  '.nuxt',
+  '.output',
+  '__pycache__',
+  '.venv',
+  'venv',
+  '.tox',
+  'target',
+  '.cache',
+  'coverage',
+  '.turbo',
+  '.vercel',
+  '.svelte-kit',
+]);
+
+export interface DiscoveredProject {
+  projectId: string;
+  path: string;
+  projectName?: string;
+}
+
+export class ProjectDiscovery {
+  private maxDepth: number;
+  private timeoutMs: number;
+
+  constructor(
+    private roots: string[],
+    options?: { maxDepth?: number; timeoutMs?: number }
+  ) {
+    this.maxDepth = options?.maxDepth ?? 3;
+    this.timeoutMs = options?.timeoutMs ?? 30_000;
+  }
+
+  /**
+   * Discover all projects with .mnemonik.json files under the configured roots.
+   * Deduplicates by projectId (same project found at multiple paths = first wins).
+   */
+  async discover(): Promise<DiscoveredProject[]> {
+    const seen = new Map<string, DiscoveredProject>();
+
+    for (const root of this.roots) {
+      const absRoot = resolve(root.replace(/^~/, process.env.HOME || ''));
+      try {
+        await this.walkWithTimeout(absRoot, 0, seen);
+      } catch (err) {
+        if (err instanceof DiscoveryTimeoutError) {
+          console.warn(`[scanner] Discovery timeout for root: ${absRoot} (>${this.timeoutMs}ms)`);
+        } else {
+          console.warn(`[scanner] Error scanning root ${absRoot}:`, (err as Error).message);
+        }
+      }
+    }
+
+    return Array.from(seen.values());
+  }
+
+  private async walkWithTimeout(
+    dir: string,
+    depth: number,
+    seen: Map<string, DiscoveredProject>
+  ): Promise<void> {
+    const deadline = Date.now() + this.timeoutMs;
+    await this.walk(dir, depth, seen, deadline);
+  }
+
+  private async walk(
+    dir: string,
+    depth: number,
+    seen: Map<string, DiscoveredProject>,
+    deadline: number
+  ): Promise<void> {
+    if (depth > this.maxDepth) return;
+    if (Date.now() > deadline) throw new DiscoveryTimeoutError();
+
+    // Check for .mnemonik.json in this directory
+    const configPath = join(dir, '.mnemonik.json');
+    try {
+      const raw = await readFile(configPath, 'utf-8');
+      const parsed = JSON.parse(raw) as Record<string, unknown>;
+      if (typeof parsed.projectId === 'string' && parsed.projectId.length > 0) {
+        if (!seen.has(parsed.projectId)) {
+          seen.set(parsed.projectId, {
+            projectId: parsed.projectId,
+            path: dir,
+            projectName:
+              typeof parsed.projectName === 'string' ? parsed.projectName : undefined,
+          });
+        }
+      }
+      // Don't recurse into subdirectories of a project — the project owns this tree
+      return;
+    } catch {
+      // No .mnemonik.json here, continue walking
+    }
+
+    // Recurse into subdirectories
+    try {
+      const entries = await readdir(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isDirectory()) continue;
+        if (SKIP_DIRS.has(entry.name)) continue;
+        if (entry.name.startsWith('.') && entry.name !== '.mnemonik') continue;
+        await this.walk(join(dir, entry.name), depth + 1, seen, deadline);
+      }
+    } catch {
+      // Permission denied or inaccessible directory
+    }
+  }
+}
+
+class DiscoveryTimeoutError extends Error {
+  constructor() {
+    super('Discovery walk timed out');
+    this.name = 'DiscoveryTimeoutError';
+  }
+}