@monoes/monograph 1.5.6 → 1.5.8

package/src/pipeline/phases/communities.ts

@@ -1,183 +0,0 @@
-import louvain from 'graphology-communities-louvain';
-import { leiden } from './leiden.js';
-import type { PipelinePhase, PipelineContext } from '../types.js';
-import type { MonographEdge } from '../../types.js';
-import { loadGraphFromEdges } from '../../graph/loader.js';
-import type { CrossFileOutput } from './cross-file.js';
-import type { ParseOutput } from './parse.js';
-
-export interface CommunitiesOutput {
-  memberships: Map<string, number>;
-  communityLabels: Map<number, string>;
-  cohesionScores: Map<number, number>;
-}
-
-export function computeCohesion(
-  communityId: number,
-  memberships: Map<string, number>,
-  edges: Array<{ sourceId: string; targetId: string }>,
-): number {
-  const members = [...memberships.entries()].filter(([, c]) => c === communityId).map(([id]) => id);
-  const n = members.length;
-  if (n <= 1) return 1;
-
-  const memberSet = new Set(members);
-  const maxEdges = (n * (n - 1)) / 2; // undirected: n*(n-1)/2
-  if (maxEdges === 0) return 1;
-
-  // Count unique undirected internal edges
-  const seen = new Set<string>();
-  let internalCount = 0;
-  for (const e of edges) {
-    if (memberSet.has(e.sourceId) && memberSet.has(e.targetId)) {
-      const key = [e.sourceId, e.targetId].sort().join('\0');
-      if (!seen.has(key)) {
-        seen.add(key);
-        internalCount++;
-      }
-    }
-  }
-  return internalCount / maxEdges;
-}
-
-/**
- * Compute cohesion scores for all communities in a single O(N+E) pass.
- *
- * This replaces calling `computeCohesion` inside a loop, which was O(K*(N+E))
- * because each call re-scanned all memberships (O(N)) and all edges (O(E)).
- *
- * @param memberships - nodeId → communityId map from the clustering step
- * @param edges - all edges used for the graph (IMPORTS + resolved)
- * @returns Map of communityId → cohesion score ∈ [0, 1]
- */
-export function computeAllCohesionScores(
-  memberships: Map<string, number>,
-  edges: Array<{ sourceId: string; targetId: string }>,
-): Map<number, number> {
-  // Single pass over memberships: build communityId → Set<nodeId>
-  const memberSets = new Map<number, Set<string>>();
-  for (const [nodeId, commId] of memberships) {
-    let s = memberSets.get(commId);
-    if (!s) {
-      s = new Set();
-      memberSets.set(commId, s);
-    }
-    s.add(nodeId);
-  }
-
-  // Single pass over edges: count unique undirected internal edges per community
-  const internalEdgeCounts = new Map<number, number>();
-  const seenEdgeKeys = new Map<number, Set<string>>();
-
-  for (const e of edges) {
-    const commSrc = memberships.get(e.sourceId);
-    const commTgt = memberships.get(e.targetId);
-    if (commSrc === undefined || commTgt === undefined || commSrc !== commTgt) continue;
-
-    let seen = seenEdgeKeys.get(commSrc);
-    if (!seen) {
-      seen = new Set();
-      seenEdgeKeys.set(commSrc, seen);
-    }
-    const key = e.sourceId < e.targetId
-      ? `${e.sourceId}\0${e.targetId}`
-      : `${e.targetId}\0${e.sourceId}`;
-    if (!seen.has(key)) {
-      seen.add(key);
-      internalEdgeCounts.set(commSrc, (internalEdgeCounts.get(commSrc) ?? 0) + 1);
-    }
-  }
-
-  // Compute final scores
-  const scores = new Map<number, number>();
-  for (const [commId, members] of memberSets) {
-    const n = members.size;
-    if (n <= 1) {
-      scores.set(commId, 1);
-      continue;
-    }
-    const maxEdges = (n * (n - 1)) / 2;
-    const internalCount = internalEdgeCounts.get(commId) ?? 0;
-    scores.set(commId, maxEdges > 0 ? internalCount / maxEdges : 1);
-  }
-
-  return scores;
-}
-
-export const communitiesPhase: PipelinePhase<CommunitiesOutput> = {
-  name: 'communities',
-  deps: ['parse', 'cross-file', 'mro'],
-  async execute(ctx, deps) {
-    if (ctx.allFilesCached) {
-      return { memberships: new Map(), communityLabels: new Map(), cohesionScores: new Map() };
-    }
-    const { resolvedEdges } = deps.get('cross-file') as CrossFileOutput;
-    const { allEdges } = deps.get('parse') as ParseOutput;
-    const allUsedEdges: MonographEdge[] = [...allEdges, ...resolvedEdges];
-
-    const graph = loadGraphFromEdges(allUsedEdges);
-    let communities: Record<string, number> = {};
-    try {
-      communities = leiden(graph, { seed: 42 });
-    } catch (e) {
-      console.warn('[monograph] Leiden failed, falling back to Louvain:', e);
-      try {
-        communities = louvain(graph, { randomWalk: false });
-      } catch {
-        // Empty or disconnected graph
-      }
-    }
-
-    const memberships = new Map<string, number>(Object.entries(communities).map(([k, v]) => [k, v]));
-    const communityLabels = new Map<number, string>();
-
-    const communityDegrees = new Map<number, Map<string, number>>();
-    for (const [nodeId, commId] of memberships) {
-      if (!communityDegrees.has(commId)) communityDegrees.set(commId, new Map());
-      const deg = graph.degree(nodeId) ?? 0;
-      communityDegrees.get(commId)!.set(nodeId, deg);
-    }
-    for (const [commId, nodeDegs] of communityDegrees) {
-      const topNode = [...nodeDegs.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? '';
-      communityLabels.set(commId, `community-${commId}(${topNode.slice(0, 20)})`);
-    }
-
-    // Compute all cohesion scores in one O(N+E) pass instead of O(K*(N+E))
-    const cohesionScores = computeAllCohesionScores(memberships, allUsedEdges);
-
-    // Persist community assignments to the DB
-    if (memberships.size > 0) {
-      const commSizes = new Map<number, number>();
-      for (const commId of memberships.values()) {
-        commSizes.set(commId, (commSizes.get(commId) ?? 0) + 1);
-      }
-      const updateNode = ctx.db.prepare('UPDATE nodes SET community_id = ? WHERE id = ?');
-      const upsertComm = ctx.db.prepare(
-        'INSERT OR REPLACE INTO communities (id, label, size, cohesion_score) VALUES (?, ?, ?, ?)',
-      );
-      ctx.db.transaction(() => {
-        for (const [nodeId, commId] of memberships) {
-          updateNode.run(commId, nodeId);
-        }
-        for (const [commId, label] of communityLabels) {
-          upsertComm.run(commId, label, commSizes.get(commId) ?? 0, cohesionScores.get(commId) ?? 0);
-        }
-      })();
-    }
-
-    return { memberships, communityLabels, cohesionScores };
-  },
-};
-
-/**
- * Split a community that is too large into smaller sub-groups.
- * By graphify convention, communities >25% of total graph nodes are split.
- */
-export function splitOversizedCommunity(memberIds: string[], maxGroupSize: number): string[][] {
-  if (memberIds.length <= maxGroupSize) return [memberIds];
-  const groups: string[][] = [];
-  for (let i = 0; i < memberIds.length; i += maxGroupSize) {
-    groups.push(memberIds.slice(i, i + maxGroupSize));
-  }
-  return groups;
-}

package/src/pipeline/phases/contextual-proximity.ts

@@ -1,88 +0,0 @@
-import type { PipelinePhase, PipelineContext } from '../types.js';
-import type { MonographEdge } from '../../types.js';
-import { makeId, CONFIDENCE_SCORE } from '../../types.js';
-import { insertEdges } from '../../storage/edge-store.js';
-
-export interface ContextualProximityOutput {
-  coOccursEdges: number;
-  conceptsScored: number;
-}
-
-export const contextualProximityPhase: PipelinePhase<ContextualProximityOutput> = {
-  name: 'contextual-proximity',
-  deps: ['docs-parse', 'pdf-parse'],
-
-  async execute(ctx: PipelineContext) {
-    if (ctx.options.codeOnly) return { coOccursEdges: 0, conceptsScored: 0 };
-
-    // Fetch all Section → Concept links from TAGGED_AS edges
-    const rows = ctx.db
-      .prepare(`SELECT source_id, target_id FROM edges WHERE relation = 'TAGGED_AS'`)
-      .all() as { source_id: string; target_id: string }[];
-
-    if (rows.length === 0) return { coOccursEdges: 0, conceptsScored: 0 };
-
-    // Group concepts by section using ??= to avoid has+get pattern
-    const sectionConcepts = new Map<string, string[]>();
-    for (const { source_id, target_id } of rows) {
-      let list = sectionConcepts.get(source_id);
-      if (!list) { list = []; sectionConcepts.set(source_id, list); }
-      list.push(target_id);
-    }
-
-    // Count concept co-occurrences across all sections.
-    // Accumulate conceptDegree in the same pass to avoid a second loop over sectionConcepts.
-    // Use a ternary comparison instead of [a,b].sort() to avoid per-pair array allocation.
-    const coOccur = new Map<string, number>();
-    const conceptDegree = new Map<string, number>();
-    for (const concepts of sectionConcepts.values()) {
-      for (let i = 0; i < concepts.length; i++) {
-        conceptDegree.set(concepts[i], (conceptDegree.get(concepts[i]) ?? 0) + 1);
-        for (let j = i + 1; j < concepts.length; j++) {
-          // Avoid allocating a 2-element sort array — ternary comparison is equivalent
-          const a = concepts[i] <= concepts[j] ? concepts[i] : concepts[j];
-          const b = concepts[i] <= concepts[j] ? concepts[j] : concepts[i];
-          const key = `${a}::${b}`;
-          coOccur.set(key, (coOccur.get(key) ?? 0) + 1);
-        }
-      }
-    }
-
-    // Build CO_OCCURS edges (weight = number of sections where both concepts appear)
-    const edges: MonographEdge[] = [];
-    for (const [key, weight] of coOccur) {
-      const sep = key.indexOf('::');
-      const sourceId = key.slice(0, sep);
-      const targetId = key.slice(sep + 2);
-      edges.push({
-        id: makeId(sourceId, targetId, 'co_occurs'),
-        sourceId,
-        targetId,
-        relation: 'CO_OCCURS',
-        confidence: 'INFERRED',
-        confidenceScore: CONFIDENCE_SCORE.INFERRED,
-        weight,
-      });
-    }
-
-    if (conceptDegree.size > 0) {
-      // Compute maxDeg with a loop to avoid spread array allocation from [...values()]
-      let maxDeg = 0;
-      for (const deg of conceptDegree.values()) { if (deg > maxDeg) maxDeg = deg; }
-      const update = ctx.db.prepare(
-        `UPDATE nodes SET properties = json_set(COALESCE(properties, '{}'), '$.importance', ?) WHERE id = ?`,
-      );
-      ctx.db.transaction(() => {
-        for (const [cid, deg] of conceptDegree) {
-          const importance = Math.max(1, Math.ceil((deg / maxDeg) * 5));
-          update.run(importance, cid);
-        }
-      })();
-    }
-
-    if (edges.length > 0) insertEdges(ctx.db, edges);
-
-    ctx.onProgress?.({ phase: 'contextual-proximity', message: `${edges.length} CO_OCCURS edges, ${conceptDegree.size} concepts scored` });
-    return { coOccursEdges: edges.length, conceptsScored: conceptDegree.size };
-  },
-};

package/src/pipeline/phases/cross-file.ts

@@ -1,116 +0,0 @@
-import { readFileSync } from 'fs';
-import { join } from 'path';
-import type { PipelinePhase } from '../types.js';
-import type { MonographEdge } from '../../types.js';
-import { makeId, CONFIDENCE_SCORE } from '../../types.js';
-import { insertEdges } from '../../storage/edge-store.js';
-import type { ParseOutput } from './parse.js';
-import type { StructureOutput } from './structure.js';
-import { resolveModuleSpecifier, buildWorkspacePackageMap } from './scope-resolution.js';
-
-export interface CrossFileOutput {
-  resolvedEdges: MonographEdge[];
-}
-
-// Mirrors parsers/extractor.ts's re-export detection regex — used here to recover
-// the ORIGINAL specifier string behind a RE_EXPORTS edge's mangled placeholder id
-// (makeId('import', specifier) is lossy: '/' and '.' are collapsed to '_', so the
-// placeholder can't be reversed directly — we re-derive it from source instead).
-const RE_EXPORT_RE = /export\s+(?:\*|\{[^}]*\})\s+from\s+['"]([^'"]+)['"]/g;
-
-// monolean: IMPORTS name-matching removed — it matched import names against a
-// global symbol index (e.g. `import fs` → random Variable named `fs` in wrong file).
-// scope-resolution.ts now handles cross-file CALLS via source-parsed import maps.
-export const crossFilePhase: PipelinePhase<CrossFileOutput> = {
-  name: 'cross-file',
-  deps: ['parse', 'structure'],
-  async execute(ctx, deps) {
-    const { allEdges } = deps.get('parse') as ParseOutput;
-    const { fileNodes } = deps.get('structure') as StructureOutput;
-
-    const reExportEdges = allEdges.filter((e) => e.relation === 'RE_EXPORTS');
-    const resolvedEdges: MonographEdge[] = [];
-
-    if (reExportEdges.length > 0) {
-      // fileId <-> filePath maps for resolving edge endpoints to real file nodes.
-      const fileIdToPath = new Map<string, string>();
-      const knownFiles = new Set<string>();
-      for (const fileNode of fileNodes) {
-        if (fileNode.filePath) {
-          fileIdToPath.set(fileNode.id, fileNode.filePath);
-          knownFiles.add(fileNode.filePath);
-        }
-      }
-      const filePathToId = new Map<string, string>();
-      for (const [id, path] of fileIdToPath) filePathToId.set(path, id);
-
-      const workspaceMap = buildWorkspacePackageMap(ctx.repoPath);
-
-      // Group by importing file so each source file is read at most once.
-      const edgesBySource = new Map<string, MonographEdge[]>();
-      for (const edge of reExportEdges) {
-        const arr = edgesBySource.get(edge.sourceId);
-        if (arr) arr.push(edge);
-        else edgesBySource.set(edge.sourceId, [edge]);
-      }
-
-      for (const [sourceId, edges] of edgesBySource) {
-        const filePath = fileIdToPath.get(sourceId);
-        if (!filePath) continue;
-
-        let source: string;
-        try {
-          source = readFileSync(join(ctx.repoPath, filePath), 'utf-8');
-        } catch {
-          continue;
-        }
-
-        // Recover each `export * / { ... } from '<specifier>'` string, keyed by the
-        // same mangled placeholder id the extractor used (makeId('import', specifier))
-        // so it can be matched back to the corresponding edge.
-        const specifierByPlaceholder = new Map<string, string>();
-        RE_EXPORT_RE.lastIndex = 0;
-        let m: RegExpExecArray | null;
-        while ((m = RE_EXPORT_RE.exec(source)) !== null) {
-          const specifier = m[1]!;
-          specifierByPlaceholder.set(makeId('import', specifier), specifier);
-        }
-
-        for (const edge of edges) {
-          const specifier = specifierByPlaceholder.get(edge.targetId);
-          if (!specifier) continue;
-
-          // Resolve relative to THIS importing file's own directory (reusing the
-          // same resolver scope-resolution.ts uses for IMPORTS) instead of an
-          // ambiguous global basename lookup that collides on repeated basenames
-          // like index.ts / utils.ts across the repo.
-          const resolvedPath = resolveModuleSpecifier(
-            filePath,
-            specifier,
-            ctx.repoPath,
-            knownFiles,
-            workspaceMap,
-          );
-          if (!resolvedPath) continue;
-
-          const resolvedId = filePathToId.get(resolvedPath);
-          if (!resolvedId || resolvedId === edge.targetId) continue;
-
-          resolvedEdges.push({
-            ...edge,
-            id: makeId(edge.sourceId, resolvedId, 'reexports_resolved'),
-            targetId: resolvedId,
-            confidence: 'INFERRED',
-            confidenceScore: CONFIDENCE_SCORE.INFERRED,
-          });
-        }
-      }
-    }
-
-    if (ctx.db && resolvedEdges.length > 0) {
-      insertEdges(ctx.db, resolvedEdges);
-    }
-
-    return { resolvedEdges };
-  },
-};