@monoes/monograph 1.1.0 → 1.2.0

package/src/search/http-embedder.ts

@@ -0,0 +1,78 @@
+export interface HttpEmbedderConfig {
+  endpoint: string;
+  model?: string;
+  apiKey?: string;
+  timeoutMs?: number;
+  batchSize?: number;
+}
+
+export interface EmbedResponse {
+  embedding: number[];
+}
+
+export class HttpEmbedder {
+  private config: Required<HttpEmbedderConfig>;
+
+  constructor(config: HttpEmbedderConfig) {
+    this.config = {
+      model: '',
+      apiKey: '',
+      timeoutMs: 30_000,
+      batchSize: 32,
+      ...config,
+    };
+  }
+
+  async embedOne(text: string): Promise<number[]> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.config.timeoutMs);
+
+    try {
+      const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+      };
+      if (this.config.apiKey) {
+        headers['Authorization'] = `Bearer ${this.config.apiKey}`;
+      }
+
+      const body: Record<string, unknown> = { prompt: text };
+      if (this.config.model) {
+        body['model'] = this.config.model;
+      }
+
+      const response = await fetch(this.config.endpoint, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify(body),
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP embedder: ${response.status} ${response.statusText}`);
+      }
+
+      const data = (await response.json()) as Partial<EmbedResponse>;
+      if (!data.embedding) {
+        throw new Error('HTTP embedder: no embedding in response');
+      }
+
+      return data.embedding;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    const results: number[][] = [];
+    const batchSize = this.config.batchSize;
+
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      for (const text of batch) {
+        results.push(await this.embedOne(text));
+      }
+    }
+
+    return results;
+  }
+}

package/src/search/hybrid-query.ts

@@ -0,0 +1,162 @@
+/**
+ * Hybrid BM25 + vector search with Reciprocal Rank Fusion.
+ *
+ * When MONOGRAPH_EMBEDDINGS=true (or an embedder is provided), the query is
+ * embedded and cosine similarity is computed in JavaScript against all stored
+ * vectors.  Both result lists are merged via RRF(K=60).
+ *
+ * If embeddings are unavailable (table empty, env not set, no embedder),
+ * the function falls back to BM25 only — same behaviour as before.
+ */
+
+import type Database from 'better-sqlite3';
+import type { EmbedderFn } from './embedder.js';
+import { mergeRanks, type RankedResult } from './rrf.js';
+import { getAllEmbeddings, countEmbeddings } from '../storage/embedding-store.js';
+import { embedText } from './embedder.js';
+import { ftsSearch } from '../storage/fts-store.js';
+import { normalizeSearchTerm } from './diacritic.js';
+
+export interface HybridQueryOptions {
+  limit?: number;
+  label?: string;
+  /** Explicit embedder — overrides MONOGRAPH_EMBEDDINGS env check */
+  embedder?: EmbedderFn;
+}
+
+export interface HybridResult extends RankedResult {
+  id: string;
+  name: string;
+  normLabel: string;
+  filePath: string | null;
+  label: string;
+  score: number;
+}
+
+/**
+ * Run a hybrid BM25 + cosine search.
+ *
+ * Falls back to BM25-only when:
+ *  - MONOGRAPH_EMBEDDINGS env var is not 'true' AND no explicit embedder is given
+ *  - The embeddings table is empty
+ *  - Embedding the query string fails
+ */
+export async function hybridQuery(
+  db: Database.Database,
+  query: string,
+  options: HybridQueryOptions = {},
+): Promise<HybridResult[]> {
+  const { limit = 20, label, embedder } = options;
+
+  // Normalize the query for text-based lookups (strip diacritics, lowercase, trim)
+  const normalizedQuery = normalizeSearchTerm(query);
+
+  // ── BM25 via FTS5 ──────────────────────────────────────────────────────────
+  const bm25Limit = 50;
+  const bm25Raw = ftsSearch(db, normalizedQuery, bm25Limit, label);
+  const bm25Results: RankedResult[] = bm25Raw.map((r) => ({
+    id: r.id,
+    name: r.name,
+    normLabel: r.normLabel,
+    filePath: r.filePath,
+    label: r.label,
+    score: r.rank,
+  }));
+
+  // ── Decide whether to add vector results ──────────────────────────────────
+  const embeddingsEnabled =
+    embedder !== undefined || process.env['MONOGRAPH_EMBEDDINGS'] === 'true';
+
+  if (!embeddingsEnabled) return sliceHybrid(bm25Results, limit);
+
+  const embeddingCount = countEmbeddings(db);
+  if (embeddingCount === 0) return sliceHybrid(bm25Results, limit);
+
+  // ── Embed the query ────────────────────────────────────────────────────────
+  let queryVec: Float32Array;
+  try {
+    const fn: import('./embedder.js').EmbedderFn = embedder ?? (await import('./embedder.js').then((m) => m.getEmbedder()));
+    queryVec = await embedText(normalizedQuery, fn);
+  } catch {
+    // Embedding failed — degrade gracefully to BM25
+    return sliceHybrid(bm25Results, limit);
+  }
+
+  // ── Cosine similarity against stored vectors ──────────────────────────────
+  const allVecs = getAllEmbeddings(db);
+  const vectorScored: { id: string; cosine: number }[] = [];
+
+  for (const [nodeId, vec] of allVecs) {
+    const sim = dotProduct(queryVec, vec); // valid because vectors are L2-normalised
+    vectorScored.push({ id: nodeId, cosine: sim });
+  }
+
+  // Sort descending by cosine similarity
+  vectorScored.sort((a, b) => b.cosine - a.cosine);
+
+  // Build vector RankedResult list with node metadata joined from BM25 cache
+  const bm25ById = new Map(bm25Results.map((r) => [r.id, r]));
+  const vectorResults: RankedResult[] = vectorScored.map((vs) => {
+    const meta = bm25ById.get(vs.id);
+    return {
+      id: vs.id,
+      name: (meta?.name as string | undefined) ?? '',
+      normLabel: (meta?.normLabel as string | undefined) ?? '',
+      filePath: (meta?.filePath as string | null | undefined) ?? null,
+      label: (meta?.label as string | undefined) ?? '',
+      score: vs.cosine,
+    };
+  });
+
+  // ── Merge via RRF ──────────────────────────────────────────────────────────
+  const merged = mergeRanks(bm25Results, vectorResults);
+
+  // Enrich nodes not in bm25 cache with a DB lookup
+  const enrichable = merged as Partial<HybridResult>[];
+  const unknownIds = enrichable
+    .filter((r) => !r.name)
+    .map((r) => r.id as string);
+
+  if (unknownIds.length > 0) {
+    const placeholders = unknownIds.map(() => '?').join(',');
+    const rows = db
+      .prepare(
+        `SELECT id, name, norm_label, file_path, label FROM nodes WHERE id IN (${placeholders})`,
+      )
+      .all(...unknownIds) as {
+      id: string;
+      name: string;
+      norm_label: string;
+      file_path: string | null;
+      label: string;
+    }[];
+
+    const rowMap = new Map(rows.map((r) => [r.id, r]));
+    for (const item of enrichable) {
+      if (!item.name) {
+        const row = rowMap.get(item.id as string);
+        if (row) {
+          item.name = row.name;
+          item.normLabel = row.norm_label;
+          item.filePath = row.file_path;
+          item.label = row.label;
+        }
+      }
+    }
+  }
+
+  return sliceHybrid(merged, limit);
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function sliceHybrid(results: RankedResult[], limit: number): HybridResult[] {
+  return results.slice(0, limit) as HybridResult[];
+}
+
+function dotProduct(a: Float32Array, b: Float32Array): number {
+  let sum = 0;
+  const len = Math.min(a.length, b.length);
+  for (let i = 0; i < len; i++) sum += a[i] * b[i];
+  return sum;
+}

package/src/search/rrf.ts

@@ -0,0 +1,51 @@
+/**
+ * Reciprocal Rank Fusion (RRF)
+ *
+ * Merges two ranked result lists using the formula:
+ *   score(d) = sum over each list of: 1 / (K + rank_of_d_in_list)
+ *
+ * where rank is 1-based.
+ *
+ * Reference: Cormack, Clarke & Buettcher (SIGIR 2009).
+ */
+
+export interface RankedResult {
+  id: string;
+  score: number;
+  [key: string]: unknown;
+}
+
+/**
+ * Merge two ranked lists via RRF and return results sorted by fused score descending.
+ *
+ * @param list1 - First ranked list (ordered by relevance, best first)
+ * @param list2 - Second ranked list (ordered by relevance, best first)
+ * @param K     - RRF constant (default 60, per the original paper)
+ */
+export function mergeRanks(
+  list1: RankedResult[],
+  list2: RankedResult[],
+  K = 60,
+): RankedResult[] {
+  const scoreMap = new Map<string, { rrf: number; payload: RankedResult }>();
+
+  const addList = (list: RankedResult[]) => {
+    list.forEach((item, idx) => {
+      const rank = idx + 1; // 1-based
+      const contribution = 1 / (K + rank);
+      const existing = scoreMap.get(item.id);
+      if (existing) {
+        existing.rrf += contribution;
+      } else {
+        scoreMap.set(item.id, { rrf: contribution, payload: item });
+      }
+    });
+  };
+
+  addList(list1);
+  addList(list2);
+
+  return [...scoreMap.values()]
+    .sort((a, b) => b.rrf - a.rrf)
+    .map(({ rrf, payload }) => ({ ...payload, score: rrf }));
+}

package/src/security/safe-fetch.ts

@@ -0,0 +1,113 @@
+const PRIVATE_IP_PATTERNS: RegExp[] = [
+  /^127\./,
+  /^10\./,
+  /^192\.168\./,
+  /^172\.(1[6-9]|2\d|3[01])\./,
+  /^169\.254\./,
+  /^::1$|\[::1\]/,
+  /^fc00:|^\[fc00:/i,
+  /^fe80:|^\[fe80:/i,
+  // IPv4-mapped IPv6 (e.g. ::ffff:192.168.1.1) — covers private ranges via mapped form
+  /^::ffff:/i,
+  /^\[::ffff:/i,
+];
+
+const CLOUD_METADATA_HOSTS = new Set([
+  'metadata.google.internal',
+  'metadata.goog',
+  '169.254.169.254',
+]);
+
+/**
+ * Returns true if the given URL resolves to a private, loopback, or cloud
+ * metadata address that should be blocked to prevent SSRF attacks.
+ */
+export function isPrivateUrl(rawUrl: string): boolean {
+  let url: URL;
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    return false;
+  }
+  const host = url.hostname;
+  if (host === 'localhost') return true;
+  if (CLOUD_METADATA_HOSTS.has(host)) return true;
+  return PRIVATE_IP_PATTERNS.some(p => p.test(host));
+}
+
+/**
+ * Validates that a URL is safe to fetch: must use http/https and must not
+ * resolve to a private or reserved address.
+ * @throws {Error} if the URL is invalid, uses an unsupported scheme, or is private
+ */
+export function validateUrl(rawUrl: string): void {
+  let url: URL;
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    throw new Error(`Invalid URL: ${rawUrl}`);
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) {
+    throw new Error(`Unsupported scheme: ${url.protocol}`);
+  }
+  if (isPrivateUrl(rawUrl)) {
+    throw new Error(`URL resolves to private/reserved address: ${url.hostname}`);
+  }
+}
+
+/**
+ * Fetches the content of a URL after validating it is safe (not private/internal).
+ * Enforces a maximum response size and request timeout.
+ *
+ * @param rawUrl - The URL to fetch
+ * @param opts.maxBytes - Maximum allowed response size in bytes (default 10 MB)
+ * @param opts.timeoutMs - Request timeout in milliseconds (default 10 000)
+ * @returns The response body as a string
+ * @throws {Error} if the URL is unsafe, the request fails, or the response is too large
+ */
+export async function safeFetch(
+  rawUrl: string,
+  opts: { maxBytes?: number; timeoutMs?: number } = {},
+): Promise<string> {
+  validateUrl(rawUrl);
+  const { maxBytes = 10 * 1024 * 1024, timeoutMs = 10_000 } = opts;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(rawUrl, { signal: controller.signal });
+    if (!res.ok) throw new Error(`HTTP ${res.status}: ${rawUrl}`);
+
+    // Stream the body incrementally so we can enforce maxBytes before
+    // the full response is loaded into memory (prevents memory exhaustion
+    // on large/malicious payloads).
+    if (!res.body) {
+      throw new Error(`Response body is missing: ${rawUrl}`);
+    }
+    const reader = (res.body as ReadableStream<Uint8Array>).getReader();
+    const decoder = new TextDecoder();
+    const chunks: string[] = [];
+    let totalBytes = 0;
+
+    try {
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        if (value) {
+          totalBytes += value.byteLength;
+          if (totalBytes > maxBytes) {
+            throw new Error(`Response exceeds ${maxBytes} bytes: ${rawUrl}`);
+          }
+          chunks.push(decoder.decode(value, { stream: true }));
+        }
+      }
+      // Flush any remaining bytes held in the decoder
+      chunks.push(decoder.decode());
+    } finally {
+      reader.releaseLock();
+    }
+
+    return chunks.join('');
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/security/sensitive-files.ts

@@ -0,0 +1,24 @@
+import { basename as pathBasename } from 'path';
+
+const SENSITIVE_PATTERNS: RegExp[] = [
+  /^\.env(\.|$)/i,
+  /^\.netrc$/i,
+  /\baws_credentials\b/i,
+  /\bcredentials?\b.*\.(json|yaml|yml|toml|ini|cfg)$/i,
+  /\bsecrets?\b.*\.(json|yaml|yml|toml|ini|cfg)$/i,
+  /\bsecret[_-]?key\b/i,
+  /\bapi[_-]?key\b.*\.(json|txt)$/i,
+  /\bpasswd\b/i,
+  /\bpassword[s]?\b.*\.(txt|json|yaml|yml)$/i,
+  /\btoken\b.*\.(txt|json|env)$/i,
+  /\b(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$/i,
+  /\.(pem|key|pfx|p12|p8|pkcs8)$/i,
+  /\bservice[_-]?account\b.*\.json$/i,
+  /\bkeystore\b/i,
+  /\bvault[_-]?token\b/i,
+];
+
+export function isSensitiveFile(filePath: string): boolean {
+  const base = pathBasename(filePath);
+  return SENSITIVE_PATTERNS.some(p => p.test(base) || p.test(filePath));
+}

package/src/skills/platform-skills.ts

@@ -0,0 +1,233 @@
+/**
+ * Multi-platform IDE Skill File Installer
+ *
+ * Generates and installs per-community skill files for various IDE/platform
+ * targets (Claude, Cursor, VS Code, Zed, Codex, Gemini, Aider, Copilot, Kiro).
+ * Each platform gets one file per community describing exported symbols from
+ * that community.
+ */
+
+import fs from 'fs/promises';
+import fsSync from 'fs';
+import path from 'path';
+
+// ============================================================================
+// TYPES
+// ============================================================================
+
+export type SkillPlatform = 'claude' | 'cursor' | 'vscode' | 'zed' | 'codex' | 'gemini' | 'aider' | 'copilot' | 'kiro';
+
+export interface PlatformSkillConfig {
+  platform: SkillPlatform;
+  /** Override output directory. Default: platform-specific. */
+  outputDir?: string;
+}
+
+export interface SkillInstallResult {
+  platform: SkillPlatform;
+  filesWritten: string[];
+  outputDir: string;
+}
+
+export interface SyncSkillInstallResult {
+  filesWritten: string[];
+}
+
+// ============================================================================
+// SUPPORTED PLATFORMS
+// ============================================================================
+
+export const SUPPORTED_PLATFORMS: string[] = [
+  'claude',
+  'cursor',
+  'vscode',
+  'zed',
+  'codex',
+  'gemini',
+  'aider',
+  'copilot',
+  'kiro',
+];
+
+// ============================================================================
+// PLATFORM CONFIG
+// ============================================================================
+
+const PLATFORM_DEFAULTS: Record<SkillPlatform, { subPath: string[]; ext: string }> = {
+  claude:  { subPath: ['.claude', 'skills'], ext: '.md' },
+  cursor:  { subPath: ['.cursor', 'rules'], ext: '.md' },
+  vscode:  { subPath: ['.vscode'], ext: '.json' },
+  zed:     { subPath: ['.zed'], ext: '.md' },
+  codex:   { subPath: [], ext: '.md' },
+  gemini:  { subPath: [], ext: '.md' },
+  aider:   { subPath: [], ext: '.md' },
+  copilot: { subPath: ['.github'], ext: '.md' },
+  kiro:    { subPath: ['.kiro', 'steering'], ext: '.md' },
+};
+
+/**
+ * Fixed file names for single-file platforms (no per-community files).
+ * When defined, a single skill file is written regardless of communities.
+ */
+const PLATFORM_FIXED_FILE: Partial<Record<SkillPlatform, string>> = {
+  codex:   'AGENTS.md',
+  gemini:  'GEMINI.md',
+  aider:   'AGENTS.md',
+  copilot: 'copilot-instructions.md',
+  kiro:    'monograph.md',
+};
+
+// ============================================================================
+// SYNC INSTALLER (for new platforms + compatibility)
+// ============================================================================
+
+/**
+ * Synchronously install a skill file for a specific platform into the repo.
+ * Returns { filesWritten } where filesWritten contains absolute paths to written files.
+ *
+ * New single-file platforms (codex, gemini, aider, copilot, kiro) always write
+ * one file regardless of communities. Existing multi-file platforms write one
+ * file per community entry.
+ */
+export function installPlatformSkill(
+  repoPath: string,
+  platform: string,
+  communities: Array<{ name: string; symbols: string[] } | string>,
+): SyncSkillInstallResult {
+  const platformConfig = PLATFORM_DEFAULTS[platform as SkillPlatform];
+  if (!platformConfig) {
+    return { filesWritten: [] };
+  }
+
+  const outputDir = platformConfig.subPath.length > 0
+    ? path.join(repoPath, ...platformConfig.subPath)
+    : repoPath;
+
+  fsSync.mkdirSync(outputDir, { recursive: true });
+
+  const filesWritten: string[] = [];
+  const fixedFileName = PLATFORM_FIXED_FILE[platform as SkillPlatform];
+
+  if (fixedFileName) {
+    // Single-file platform: write one skill overview file
+    const filePath = path.join(outputDir, fixedFileName);
+    const content = renderMonographSkillDoc(platform);
+    fsSync.writeFileSync(filePath, content, 'utf-8');
+    filesWritten.push(filePath);
+  } else {
+    // Multi-file platform: one file per community (existing behavior)
+    const ext = platformConfig.ext;
+    const normalised = communities.map(c =>
+      typeof c === 'string' ? { name: c, symbols: [] } : c,
+    );
+    for (const community of normalised) {
+      const fileName = `${community.name}-skills${ext}`;
+      const filePath = path.join(outputDir, fileName);
+      let content: string;
+      if (ext === '.json') {
+        content = renderVsCodeSnippet(community);
+      } else {
+        content = renderMarkdown(community);
+      }
+      fsSync.writeFileSync(filePath, content, 'utf-8');
+      filesWritten.push(filePath);
+    }
+  }
+
+  return { filesWritten };
+}
+
+// ============================================================================
+// MAIN EXPORT
+// ============================================================================
+
+/**
+ * Install skill files for a specific platform into the repo.
+ *
+ * Platform output directories (relative to repoPath):
+ *   claude  → .claude/skills/
+ *   cursor  → .cursor/rules/
+ *   vscode  → .vscode/
+ *   zed     → .zed/
+ *
+ * Each platform gets one file per community. File names use the community name.
+ * Format:
+ *   claude/cursor/zed → <communityName>-skills.md
+ *   vscode            → <communityName>-skills.json (VS Code snippets format)
+ */
+export async function installSkillsForPlatform(
+  repoPath: string,
+  communities: Array<{ name: string; symbols: string[] }>,
+  config: PlatformSkillConfig,
+): Promise<SkillInstallResult> {
+  const platformConfig = PLATFORM_DEFAULTS[config.platform];
+  const outputDir = config.outputDir ?? path.join(repoPath, ...platformConfig.subPath);
+
+  if (communities.length === 0) {
+    return { platform: config.platform, filesWritten: [], outputDir };
+  }
+
+  await fs.mkdir(outputDir, { recursive: true });
+
+  const filesWritten: string[] = [];
+  const ext = platformConfig.ext;
+
+  for (const community of communities) {
+    const fileName = `${community.name}-skills${ext}`;
+    const filePath = path.join(outputDir, fileName);
+
+    let content: string;
+    if (ext === '.json') {
+      content = renderVsCodeSnippet(community);
+    } else {
+      content = renderMarkdown(community);
+    }
+
+    await fs.writeFile(filePath, content, 'utf-8');
+    filesWritten.push(filePath);
+  }
+
+  return { platform: config.platform, filesWritten, outputDir };
+}
+
+// ============================================================================
+// RENDERERS
+// ============================================================================
+
+function renderMonographSkillDoc(platform: string): string {
+  const lines: string[] = [];
+  lines.push(`# Monograph Skills — ${platform.charAt(0).toUpperCase() + platform.slice(1)}`);
+  lines.push('');
+  lines.push('This file was generated by Monograph to document available skills for this platform.');
+  lines.push('');
+  lines.push('## Usage');
+  lines.push('');
+  lines.push('Monograph indexes your codebase and exposes symbol-level knowledge to AI assistants.');
+  lines.push('Run `monograph build` to update the knowledge graph and regenerate skill files.');
+  lines.push('');
+  return lines.join('\n');
+}
+
+function renderMarkdown(community: { name: string; symbols: string[] }): string {
+  const lines: string[] = [];
+  lines.push(`# ${community.name} Skills`);
+  lines.push('');
+  for (const symbol of community.symbols) {
+    lines.push(`- ${symbol}`);
+  }
+  lines.push('');
+  return lines.join('\n');
+}
+
+function renderVsCodeSnippet(community: { name: string; symbols: string[] }): string {
+  const snippetKey = community.name;
+  const body = community.symbols;
+  const snippet = {
+    [snippetKey]: {
+      prefix: community.name,
+      body,
+      description: `${community.name} monograph skill`,
+    },
+  };
+  return JSON.stringify(snippet, null, 2);
+}