@nomos-arc/arc 0.1.0

package/src/search/chunk-extractor.ts

@@ -0,0 +1,167 @@
+import crypto from 'node:crypto';
+import type { Logger } from 'winston';
+import type { FileNode, ProjectMap, TextChunk } from '../types/index.js';
+
+// ─── Constants ────────────────────────────────────────────────────────────────
+
+const MIN_TEXT_LENGTH = 20;
+
+// ─── ChunkExtractor ───────────────────────────────────────────────────────────
+
+/**
+ * Extracts file-level and symbol-level TextChunks from a ProjectMap.
+ *
+ * Design notes:
+ * - Uses inline semantic data from ProjectMap.files[path].semantic — NO .semantic.md reads [TRAP-5].
+ * - Content hash computed from raw inputs, not composed text [S-6].
+ * - Chunks shorter than 20 chars are skipped (empty text guard).
+ */
+export class ChunkExtractor {
+  constructor(
+    private readonly projectRoot: string,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Extract file-level and symbol-level TextChunks from a ProjectMap.
+   * No filesystem reads are performed — all data comes from the in-memory map.
+   */
+  extract(map: ProjectMap): TextChunk[] {
+    const chunks: TextChunk[] = [];
+
+    for (const [file_path, fileNode] of Object.entries(map.files)) {
+      const fileChunk = this.extractFileChunk(file_path, fileNode);
+      if (fileChunk !== null) {
+        chunks.push(fileChunk);
+      }
+
+      const symbolChunks = this.extractSymbolChunks(file_path, fileNode);
+      chunks.push(...symbolChunks);
+    }
+
+    return chunks;
+  }
+
+  // ─── Private ───────────────────────────────────────────────────────────────
+
+  private extractFileChunk(file_path: string, fileNode: FileNode): TextChunk | null {
+    let text: string;
+
+    if (fileNode.semantic !== null) {
+      const exportedNames = fileNode.symbols
+        .filter(s => s.exported)
+        .map(s => s.name)
+        .join(', ');
+
+      text = [
+        `File: ${file_path}`,
+        `Purpose: ${fileNode.semantic.purpose}`,
+        `Overview: ${fileNode.semantic.overview}`,
+        `Key Logic: ${fileNode.semantic.key_logic.join('; ')}`,
+        `Usage Context: ${fileNode.semantic.usage_context.join('; ')}`,
+        `Exports: ${exportedNames}`,
+        `Dependencies: ${fileNode.dependencies.join(', ')}`,
+      ].join('\n');
+    } else {
+      // Fallback: file path + symbol names + import sources [TRAP-5: no .semantic.md read]
+      this.logger.warn(
+        `[nomos:search:warn] No semantic data for ${file_path} — using fallback text`,
+      );
+
+      const symbolNames = fileNode.symbols.map(s => s.name).join(', ');
+      const importSources = fileNode.imports.map(i => i.source).join(', ');
+
+      const lines = [
+        `File: ${file_path}`,
+      ];
+      if (symbolNames) lines.push(`Symbols: ${symbolNames}`);
+      if (importSources) lines.push(`Imports: ${importSources}`);
+
+      text = lines.join('\n');
+    }
+
+    if (text.length < MIN_TEXT_LENGTH) return null;
+
+    const content_hash = this.computeFileHash(file_path, fileNode);
+
+    return {
+      id: file_path,
+      type: 'file',
+      file_path,
+      text,
+      symbol_name: null,
+      symbol_type: null,
+      line_start: null,
+      line_end: null,
+      parent_file_id: null,
+      content_hash,
+    };
+  }
+
+  private extractSymbolChunks(file_path: string, fileNode: FileNode): TextChunk[] {
+    const chunks: TextChunk[] = [];
+    const fileContentHash = this.computeFileHash(file_path, fileNode);
+
+    for (const symbol of fileNode.symbols) {
+      if (!symbol.exported && symbol.kind !== 'class' && symbol.kind !== 'function') {
+        continue;
+      }
+
+      const purposeText = fileNode.semantic?.purpose ?? file_path;
+
+      const text = [
+        `Symbol: ${symbol.name} (${symbol.kind})`,
+        `File: ${file_path}`,
+        `Signature: ${symbol.signature ?? 'N/A'}`,
+        `Lines: ${symbol.line}-${symbol.end_line ?? '?'}`,
+        `File Purpose: ${purposeText}`,
+      ].join('\n');
+
+      if (text.length < MIN_TEXT_LENGTH) continue;
+
+      // Hash from raw symbol data + parent file content hash [S-6]
+      const symbolHashInput = JSON.stringify({
+        name: symbol.name,
+        kind: symbol.kind,
+        signature: symbol.signature,
+        line: symbol.line,
+        end_line: symbol.end_line,
+        parent_file_hash: fileContentHash,
+      });
+      const content_hash = crypto.createHash('sha256').update(symbolHashInput).digest('hex');
+
+      chunks.push({
+        id: `${file_path}::${symbol.name}`,
+        type: 'symbol',
+        file_path,
+        text,
+        symbol_name: symbol.name,
+        symbol_type: symbol.kind,
+        line_start: symbol.line,
+        line_end: symbol.end_line,
+        parent_file_id: file_path,
+        content_hash,
+      });
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Compute content hash from raw inputs — NOT composed text.
+   * Decouples hash stability from text composition format changes [S-6].
+   */
+  private computeFileHash(file_path: string, fileNode: FileNode): string {
+    const hashInput = JSON.stringify({
+      file_path,
+      semantic: fileNode.semantic,           // null-safe
+      symbols: fileNode.symbols.map(s => ({
+        name: s.name,
+        kind: s.kind,
+        signature: s.signature,
+      })),
+      dependencies: fileNode.dependencies,
+    });
+    return crypto.createHash('sha256').update(hashInput).digest('hex');
+  }
+}

package/src/search/embedder.ts

@@ -0,0 +1,209 @@
+import { GoogleGenerativeAI } from '@google/generative-ai';
+import type { RequestOptions } from '@google/generative-ai';
+import type { Logger } from 'winston';
+import { NomosError } from '../core/errors.js';
+import type { NomosConfig } from '../types/index.js';
+import { AuthManager } from '../core/auth/manager.js';
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+function chunk<T>(arr: T[], size: number): T[][] {
+  const result: T[][] = [];
+  for (let i = 0; i < arr.length; i += size) {
+    result.push(arr.slice(i, i + size));
+  }
+  return result;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+// ─── Embedder ─────────────────────────────────────────────────────────────────
+
+export class Embedder {
+  private readonly client: GoogleGenerativeAI;
+  private readonly requestOptions: RequestOptions | undefined;
+
+  constructor(
+    private readonly config: NomosConfig['search'],
+    private readonly logger: Logger,
+    apiKey?: string,
+    quotaProjectId?: string,
+  ) {
+    const key = apiKey ?? process.env['GEMINI_API_KEY'];
+    if (!key) {
+      throw new NomosError(
+        'search_api_key_missing',
+        'No credentials found. Set GEMINI_API_KEY or run: arc auth login',
+      );
+    }
+    this.client = new GoogleGenerativeAI(key);
+    if (quotaProjectId) {
+      this.requestOptions = {
+        customHeaders: { 'x-goog-user-project': quotaProjectId },
+      };
+    }
+  }
+
+  static async create(
+    config: NomosConfig['search'],
+    logger: Logger,
+    authManager?: AuthManager | null,
+  ): Promise<Embedder> {
+    // Priority 1: GEMINI_API_KEY (no quota header — billing is tied to the API key's project)
+    const envKey = process.env['GEMINI_API_KEY'];
+    if (envKey) {
+      return new Embedder(config, logger, envKey);
+    }
+
+    // Priority 2: OAuth credentials + quota project
+    if (authManager?.isLoggedIn()) {
+      const token = await authManager.getAccessToken();
+      const creds = authManager.loadCredentials();
+      return new Embedder(config, logger, token, creds?.quota_project_id);
+    }
+
+    // Priority 3: Neither — throw
+    throw new NomosError(
+      'search_api_key_missing',
+      'No credentials found. Set GEMINI_API_KEY or run: arc auth login',
+    );
+  }
+
+  /** Return the vector dimensions for the configured model. */
+  get dimensions(): number {
+    return this.config.embedding_dimensions;
+  }
+
+  /**
+   * Embed a single text string. Returns a Float32Array vector.
+   * Subject to request_timeout_ms [GAP-4].
+   */
+  async embedOne(text: string): Promise<Float32Array> {
+    return this.withRetry(() =>
+      this.withTimeout(this.embedOneRaw(text), 'embedOne'),
+    );
+  }
+
+  /**
+   * Embed a batch of text strings. Returns Float32Array[] in the same order.
+   * Processes batches SEQUENTIALLY with rate-limit delay [TRAP-2].
+   * Each API call subject to request_timeout_ms [GAP-4].
+   * Logs rate-limit delays [S-8].
+   */
+  async embedBatch(
+    texts: string[],
+    onBatchComplete?: (batchIndex: number, totalBatches: number) => void,
+  ): Promise<Float32Array[]> {
+    const results: Float32Array[] = [];
+    const batches = chunk(texts, this.config.batch_size);
+    const delayMs = Math.ceil(60_000 / this.config.embedding_requests_per_minute);
+
+    for (let i = 0; i < batches.length; i++) {
+      if (i > 0) {
+        this.logger.warn(
+          `[nomos:search:warn] Rate limiting. Waiting ${delayMs}ms before batch ${i + 1}/${batches.length}...`,
+        );
+        await sleep(delayMs);
+      }
+      const batch = batches[i]!;
+      const vectors = await this.withRetry(() =>
+        this.withTimeout(
+          this.embedBatchRaw(batch),
+          `batch ${i + 1}/${batches.length}`,
+        ),
+      );
+      results.push(...vectors);
+      onBatchComplete?.(i, batches.length);
+    }
+    return results;
+  }
+
+  // ─── Private ────────────────────────────────────────────────────────────────
+
+  private async embedOneRaw(text: string): Promise<Float32Array> {
+    const model = this.client.getGenerativeModel({ model: this.config.embedding_model }, this.requestOptions);
+    const result = await model.embedContent({
+      content: { role: 'user', parts: [{ text }] },
+      // outputDimensionality is a supported API param not yet reflected in SDK types
+      ...(this.config.embedding_dimensions && {
+        outputDimensionality: this.config.embedding_dimensions,
+      }),
+    } as Parameters<typeof model.embedContent>[0]);
+    return new Float32Array(result.embedding.values);
+  }
+
+  private async embedBatchRaw(texts: string[]): Promise<Float32Array[]> {
+    const model = this.client.getGenerativeModel({ model: this.config.embedding_model }, this.requestOptions);
+    const requests = texts.map(text => ({
+      content: { role: 'user', parts: [{ text }] },
+      ...(this.config.embedding_dimensions && {
+        outputDimensionality: this.config.embedding_dimensions,
+      }),
+    }));
+    // Cast to any: outputDimensionality not yet in SDK typings
+    const result = await model.batchEmbedContents({ requests } as Parameters<typeof model.batchEmbedContents>[0]);
+    return result.embeddings.map(e => new Float32Array(e.values));
+  }
+
+  private async withTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.config.request_timeout_ms);
+
+    const abortPromise = new Promise<never>((_, reject) => {
+      controller.signal.addEventListener('abort', () => {
+        reject(
+          new NomosError(
+            'search_embedding_failed',
+            `Embedding request timed out after ${this.config.request_timeout_ms}ms (${label})`,
+          ),
+        );
+      });
+    });
+    // Suppress unhandled-rejection warning when `promise` wins the race and
+    // the abort timer fires afterwards with no consumer for the rejection.
+    abortPromise.catch(() => {});
+
+    try {
+      return await Promise.race([promise, abortPromise]);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
+    const MAX_RETRIES = 3;
+    const BASE_DELAY_MS = 2_000;
+
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        return await fn();
+      } catch (err) {
+        // NomosErrors (timeout, missing key) are not retryable
+        if (err instanceof NomosError) throw err;
+
+        if (!this.isRetryable(err) || attempt === MAX_RETRIES) {
+          throw new NomosError(
+            'search_embedding_failed',
+            `Embedding failed after ${attempt + 1} attempt(s): ${(err as Error).message ?? String(err)}`,
+          );
+        }
+
+        const jitterMs = Math.random() * 1_000;
+        await sleep(BASE_DELAY_MS * Math.pow(2, attempt) + jitterMs);
+      }
+    }
+
+    // Unreachable — TypeScript requires exhaustive return
+    throw new NomosError('search_embedding_failed', 'Unexpected exit from retry loop');
+  }
+
+  private isRetryable(err: unknown): boolean {
+    if (err instanceof Error) {
+      const msg = err.message;
+      return /429|rate.?limit|500|503|internal.?server/i.test(msg);
+    }
+    return false;
+  }
+}

package/src/search/graph-enricher.ts

@@ -0,0 +1,95 @@
+import fs from 'node:fs/promises';
+import type { Logger } from 'winston';
+import type { ProjectMap, SearchResult, ChunkType } from '../types/index.js';
+
+// ─── RawSearchResult ──────────────────────────────────────────────────────────
+
+export interface RawSearchResult {
+  id: string;
+  type: ChunkType;
+  file_path: string;
+  symbol_name: string | null;
+  symbol_type: string | null;
+  line_start: number | null;
+  line_end: number | null;
+  purpose: string;
+  similarity_score: number;
+}
+
+// ─── GraphEnricher ────────────────────────────────────────────────────────────
+
+export class GraphEnricher {
+  private projectMap: ProjectMap | null = null;
+
+  constructor(
+    private readonly projectMapPath: string,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Lazy-load and cache project_map.json [GAP-5].
+   * Parsed once per instance lifetime. Subsequent calls return cached data.
+   */
+  private async loadMap(): Promise<ProjectMap> {
+    if (this.projectMap) return this.projectMap;
+    const raw = await fs.readFile(this.projectMapPath, 'utf-8');
+    this.projectMap = JSON.parse(raw) as ProjectMap;
+    return this.projectMap;
+  }
+
+  /**
+   * Enrich raw search results with dependency graph metadata.
+   * Stale results (file deleted since index) get is_stale = true [TRAP-4].
+   * Results sorted by similarity_score descending (preserves embedding rank).
+   */
+  async enrich(results: RawSearchResult[]): Promise<SearchResult[]> {
+    const map = await this.loadMap();
+    const coreModuleSet = new Set(map.stats.core_modules);
+
+    const enriched: SearchResult[] = results.map((r) => {
+      const fileNode = map.files[r.file_path];
+
+      if (fileNode) {
+        return {
+          id: r.id,
+          type: r.type,
+          file_path: r.file_path,
+          symbol_name: r.symbol_name,
+          symbol_type: r.symbol_type,
+          line_start: r.line_start,
+          line_end: r.line_end,
+          purpose: r.purpose,
+          similarity_score: r.similarity_score,
+          graph_depth: fileNode.depth,
+          dependents_count: fileNode.dependents.length,
+          is_core_module: coreModuleSet.has(r.file_path),
+          is_stale: false,
+        };
+      }
+
+      // [TRAP-4] File not found in project map — stale result (deleted since last index).
+      // graph_depth = -1 is the internal sentinel. is_stale = true is the public signal.
+      this.logger.warn(
+        `[nomos:search:warn] Result for "${r.file_path}" not found in project map — marked as stale.`,
+      );
+      return {
+        id: r.id,
+        type: r.type,
+        file_path: r.file_path,
+        symbol_name: r.symbol_name,
+        symbol_type: r.symbol_type,
+        line_start: r.line_start,
+        line_end: r.line_end,
+        purpose: r.purpose,
+        similarity_score: r.similarity_score,
+        graph_depth: -1,
+        dependents_count: 0,
+        is_core_module: false,
+        is_stale: true,
+      };
+    });
+
+    // Sort by similarity_score descending — preserve embedding rank
+    return enriched.sort((a, b) => b.similarity_score - a.similarity_score);
+  }
+}