@aiready/context-analyzer 0.9.4 → 0.9.6

package/src/__tests__/fragmentation-coupling.test.ts

@@ -0,0 +1,62 @@
+import { describe, it, expect } from 'vitest';
+import {
+  buildDependencyGraph,
+  detectModuleClusters,
+  calculateFragmentation,
+} from '../analyzer';
+
+describe('fragmentation coupling discount', () => {
+  it('does not apply discount when files have no shared imports', () => {
+    const files = [
+      {
+        file: 'src/billing/a.ts',
+        content: `export const getBillingA = 1;`,
+      },
+      {
+        file: 'src/api/billing/b.ts',
+        content: `export const getBillingB = 2;`,
+      },
+      {
+        file: 'lib/billing/c.ts',
+        content: `export const getBillingC = 3;`,
+      },
+    ];
+
+    const graph = buildDependencyGraph(files);
+    const clusters = detectModuleClusters(graph);
+    const cluster = clusters.find((c) => c.domain === 'billing');
+    expect(cluster).toBeDefined();
+
+    const base = calculateFragmentation(files.map(f => f.file), 'billing');
+    // With no import similarity the coupling discount should be 0 -> fragmentation unchanged
+    expect(cluster!.fragmentationScore).toBeCloseTo(base, 6);
+  });
+
+  it('applies up-to-20% discount when files share identical imports', () => {
+    const files = [
+      {
+        file: 'src/billing/a.ts',
+        content: `import { shared } from 'shared/module';\nexport const getBillingA = 1;`,
+      },
+      {
+        file: 'src/api/billing/b.ts',
+        content: `import { shared } from 'shared/module';\nexport const getBillingB = 2;`,
+      },
+      {
+        file: 'lib/billing/c.ts',
+        content: `import { shared } from 'shared/module';\nexport const getBillingC = 3;`,
+      },
+    ];
+
+    const graph = buildDependencyGraph(files);
+    const clusters = detectModuleClusters(graph);
+    const cluster = clusters.find((c) => c.domain === 'billing');
+    expect(cluster).toBeDefined();
+
+    const base = calculateFragmentation(files.map(f => f.file), 'billing');
+    const expected = base * 0.8; // full cohesion => 20% discount
+
+    // Allow small FP tolerance
+    expect(cluster!.fragmentationScore).toBeCloseTo(expected, 6);
+  });
+});

package/src/__tests__/fragmentation-log.test.ts

@@ -0,0 +1,38 @@
+import { describe, it, expect } from 'vitest';
+import { calculateFragmentation } from '../analyzer';
+
+describe('calculateFragmentation (log scale option)', () => {
+  it('returns 0 for single file regardless of option', () => {
+    const files = ['src/user/user.ts'];
+    expect(calculateFragmentation(files, 'user')).toBe(0);
+    expect(calculateFragmentation(files, 'user', { useLogScale: true })).toBe(0);
+  });
+
+  it('matches linear formula when not using log scale', () => {
+    const files = [
+      'a/one.ts',
+      'b/two.ts',
+      'c/three.ts',
+      'd/four.ts',
+    ];
+
+    const uniqueDirs = 4;
+    const linear = (uniqueDirs - 1) / (files.length - 1);
+    expect(calculateFragmentation(files, 'domain')).toBeCloseTo(linear);
+  });
+
+  it('computes normalized log-based fragmentation when requested', () => {
+    const files = [
+      'src/group/a.ts',
+      'src/group/b.ts',
+      'src/group/c.ts',
+      'lib/other/d.ts',
+      'tools/x/e.ts',
+    ];
+
+    const dirs = new Set(files.map((f) => f.split('/').slice(0, -1).join('/'))).size;
+    const expected = Math.log(dirs) / Math.log(files.length);
+
+    expect(calculateFragmentation(files, 'domain', { useLogScale: true })).toBeCloseTo(expected, 6);
+  });
+});

package/src/__tests__/structural-cohesion.test.ts

@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest'
+import { calculateStructuralCohesionFromCoUsage } from '../analyzer'
+
+describe('calculateStructuralCohesionFromCoUsage', () => {
+  it('returns 1 when no co-usage data present', () => {
+    const score = calculateStructuralCohesionFromCoUsage('missing', undefined)
+    expect(score).toBe(1)
+  })
+
+  it('returns 1 when co-usage only with a single file', () => {
+    const coUsage = new Map<string, Map<string, number>>()
+    coUsage.set('a', new Map([['b', 10]]))
+    const score = calculateStructuralCohesionFromCoUsage('a', coUsage)
+    expect(score).toBe(1)
+  })
+
+  it('returns ~0 when co-usage is perfectly balanced across two files', () => {
+    const coUsage = new Map<string, Map<string, number>>()
+    coUsage.set('a', new Map([['b', 5], ['c', 5]]))
+    const score = calculateStructuralCohesionFromCoUsage('a', coUsage)
+    // Balanced distribution => entropy == 1 (for 2 items) => cohesion ~= 0
+    expect(score).toBeCloseTo(0, 3)
+  })
+
+  it('returns intermediate value for skewed distribution', () => {
+    const coUsage = new Map<string, Map<string, number>>()
+    coUsage.set('a', new Map([['b', 8], ['c', 2]]))
+    const score = calculateStructuralCohesionFromCoUsage('a', coUsage)
+    // Expected approx 0.279
+    expect(score).toBeCloseTo(0.279, 2)
+  })
+})

package/src/analyzer.ts

@@ -297,8 +297,12 @@ export function detectCircularDependencies(
  * @param exports - Array of export information
  * @param filePath - Optional file path for context-aware scoring
  */
-export function calculateCohesion(exports: ExportInfo[], filePath?: string): number {
-  return calculateEnhancedCohesion(exports, filePath);
+export function calculateCohesion(
+  exports: ExportInfo[],
+  filePath?: string,
+  options?: { coUsageMatrix?: Map<string, Map<string, number>>; weights?: { importBased?: number; structural?: number; domainBased?: number } }
+): number {
+  return calculateEnhancedCohesion(exports, filePath, options);
 }
 
 /**
@@ -322,23 +326,105 @@ function isTestFile(filePath: string): boolean {
  */
 export function calculateFragmentation(
   files: string[],
-  domain: string
+  domain: string,
+  options?: { useLogScale?: boolean; logBase?: number }
 ): number {
   if (files.length <= 1) return 0; // Single file = no fragmentation
 
   // Calculate how many different directories contain these files
   const directories = new Set(files.map((f) => f.split('/').slice(0, -1).join('/')));
+  const uniqueDirs = directories.size;
+
+  // If log-scaling requested, normalize using logarithms so that
+  // going from 1 -> 2 directories shows a large jump while 10 -> 11
+  // is relatively small. Normalized value is log(uniqueDirs)/log(totalFiles).
+  if (options?.useLogScale) {
+    if (uniqueDirs <= 1) return 0;
+    const total = files.length;
+    const base = options.logBase || Math.E;
+    const num = Math.log(uniqueDirs) / Math.log(base);
+    const den = Math.log(total) / Math.log(base);
+    return den > 0 ? num / den : 0;
+  }
+
+  // Default (linear) Fragmentation = (unique directories - 1) / (total files - 1)
+  return (uniqueDirs - 1) / (files.length - 1);
+}
+
+/**
+ * Calculate path entropy for a set of files.
+ * Returns a normalized entropy in [0,1], where 0 = all files in one directory,
+ * and 1 = files are evenly distributed across directories.
+ */
+export function calculatePathEntropy(files: string[]): number {
+  if (!files || files.length === 0) return 0;
 
-  // Fragmentation = unique directories / total files
-  // 0 = all in same dir, 1 = all in different dirs
-  return (directories.size - 1) / (files.length - 1);
+  const dirCounts = new Map<string, number>();
+  for (const f of files) {
+    const dir = f.split('/').slice(0, -1).join('/') || '.';
+    dirCounts.set(dir, (dirCounts.get(dir) || 0) + 1);
+  }
+
+  const counts = Array.from(dirCounts.values());
+  if (counts.length <= 1) return 0; // single directory -> zero entropy
+
+  const total = counts.reduce((s, v) => s + v, 0);
+  let entropy = 0;
+  for (const c of counts) {
+    const p = c / total;
+    entropy -= p * Math.log2(p);
+  }
+
+  const maxEntropy = Math.log2(counts.length);
+  return maxEntropy > 0 ? entropy / maxEntropy : 0;
+}
+
+/**
+ * Calculate directory-distance metric based on common ancestor depth.
+ * For each file pair compute depth(commonAncestor) and normalize by the
+ * maximum path depth between the two files. Returns value in [0,1] where
+ * 0 means all pairs share a deep common ancestor (low fragmentation) and
+ * 1 means they share only the root (high fragmentation).
+ */
+export function calculateDirectoryDistance(files: string[]): number {
+  if (!files || files.length <= 1) return 0;
+
+  function pathSegments(p: string) {
+    return p.split('/').filter(Boolean);
+  }
+
+  function commonAncestorDepth(a: string[], b: string[]) {
+    const minLen = Math.min(a.length, b.length);
+    let i = 0;
+    while (i < minLen && a[i] === b[i]) i++;
+    return i; // number of shared segments from root
+  }
+
+  let totalNormalized = 0;
+  let comparisons = 0;
+
+  for (let i = 0; i < files.length; i++) {
+    for (let j = i + 1; j < files.length; j++) {
+      const segA = pathSegments(files[i]);
+      const segB = pathSegments(files[j]);
+      const shared = commonAncestorDepth(segA, segB);
+      const maxDepth = Math.max(segA.length, segB.length);
+      const normalizedShared = maxDepth > 0 ? shared / maxDepth : 0;
+      // distance is inverse of normalized shared depth
+      totalNormalized += 1 - normalizedShared;
+      comparisons++;
+    }
+  }
+
+  return comparisons > 0 ? totalNormalized / comparisons : 0;
 }
 
 /**
  * Group files by domain to detect module clusters
  */
 export function detectModuleClusters(
-  graph: DependencyGraph
+  graph: DependencyGraph,
+  options?: { useLogScale?: boolean }
 ): ModuleCluster[] {
   const domainMap = new Map<string, string[]>();
 
@@ -363,12 +449,47 @@ export function detectModuleClusters(
       return sum + (node?.tokenCost || 0);
     }, 0);
 
-    const fragmentationScore = calculateFragmentation(files, domain);
+    const baseFragmentation = calculateFragmentation(files, domain, { useLogScale: !!options?.useLogScale });
+
+    // Compute import-based cohesion across files in this domain cluster.
+    // This measures how much the files actually "talk" to each other.
+    // We'll compute average pairwise Jaccard similarity between each file's import lists.
+    let importSimilarityTotal = 0;
+    let importComparisons = 0;
+
+    for (let i = 0; i < files.length; i++) {
+      for (let j = i + 1; j < files.length; j++) {
+        const f1 = files[i];
+        const f2 = files[j];
+        const n1 = graph.nodes.get(f1)?.imports || [];
+        const n2 = graph.nodes.get(f2)?.imports || [];
+
+        // Treat two empty import lists as not coupled (similarity 0)
+        const similarity = (n1.length === 0 && n2.length === 0)
+          ? 0
+          : calculateJaccardSimilarity(n1, n2);
+
+        importSimilarityTotal += similarity;
+        importComparisons++;
+      }
+    }
+
+    const importCohesion = importComparisons > 0 ? importSimilarityTotal / importComparisons : 0;
+
+    // Coupling discount: if files are heavily importing each other, reduce fragmentation penalty.
+    // Following recommendation: up to 20% discount proportional to import cohesion.
+    const couplingDiscountFactor = 1 - 0.2 * importCohesion;
+
+    const fragmentationScore = baseFragmentation * couplingDiscountFactor;
+
+    // Additional metrics for richer reporting
+    const pathEntropy = calculatePathEntropy(files);
+    const directoryDistance = calculateDirectoryDistance(files);
 
     const avgCohesion =
       files.reduce((sum, file) => {
         const node = graph.nodes.get(file);
-        return sum + (node ? calculateCohesion(node.exports, file) : 0);
+        return sum + (node ? calculateCohesion(node.exports, file, { coUsageMatrix: graph.coUsageMatrix }) : 0);
       }, 0) / files.length;
 
     // Generate consolidation plan
@@ -384,6 +505,9 @@ export function detectModuleClusters(
       files,
       totalTokens,
       fragmentationScore,
+      pathEntropy,
+      directoryDistance,
+      importCohesion,
       avgCohesion,
       suggestedStructure: {
         targetFiles,
@@ -631,7 +755,8 @@ export function extractExportsWithAST(
  */
 export function calculateEnhancedCohesion(
   exports: ExportInfo[],
-  filePath?: string
+  filePath?: string,
+  options?: { coUsageMatrix?: Map<string, Map<string, number>>; weights?: { importBased?: number; structural?: number; domainBased?: number } }
 ): number {
   if (exports.length === 0) return 1;
   if (exports.length === 1) return 1;
@@ -645,17 +770,67 @@ export function calculateEnhancedCohesion(
   const domainCohesion = calculateDomainCohesion(exports);
 
   // Calculate import-based cohesion if imports are available
-  const hasImportData = exports.some(e => e.imports && e.imports.length > 0);
-  
-  if (!hasImportData) {
-    // No import data available, use domain-based only
-    return domainCohesion;
+  const hasImportData = exports.some((e) => e.imports && e.imports.length > 0);
+  const importCohesion = hasImportData ? calculateImportBasedCohesion(exports) : undefined;
+
+  // Calculate structural cohesion (co-usage) if coUsageMatrix and filePath available
+  const coUsageMatrix = options?.coUsageMatrix;
+  const structuralCohesion = filePath && coUsageMatrix ? calculateStructuralCohesionFromCoUsage(filePath, coUsageMatrix) : undefined;
+
+  // Default weights (can be overridden via options)
+  const defaultWeights = { importBased: 0.5, structural: 0.3, domainBased: 0.2 };
+  const weights = { ...defaultWeights, ...(options?.weights || {}) };
+
+  // Collect available signals and normalize weights
+  const signals: Array<{ score: number; weight: number }> = [];
+  if (importCohesion !== undefined) signals.push({ score: importCohesion, weight: weights.importBased });
+  if (structuralCohesion !== undefined) signals.push({ score: structuralCohesion, weight: weights.structural });
+  // domain cohesion is always available
+  signals.push({ score: domainCohesion, weight: weights.domainBased });
+
+  const totalWeight = signals.reduce((s, el) => s + el.weight, 0);
+  if (totalWeight === 0) return domainCohesion;
+
+  const combined = signals.reduce((sum, el) => sum + el.score * (el.weight / totalWeight), 0);
+  return combined;
+}
+
+/**
+ * Calculate structural cohesion for a file based on co-usage patterns.
+ * Uses the co-usage distribution (files commonly imported alongside this file)
+ * and computes an entropy-based cohesion score in [0,1].
+ * - 1 => highly cohesive (imports mostly appear together with a small set)
+ * - 0 => maximally fragmented (imports appear uniformly across many partners)
+ */
+export function calculateStructuralCohesionFromCoUsage(
+  file: string,
+  coUsageMatrix?: Map<string, Map<string, number>>
+): number {
+  if (!coUsageMatrix) return 1;
+
+  const coUsages = coUsageMatrix.get(file);
+  if (!coUsages || coUsages.size === 0) return 1;
+
+  // Build probability distribution over co-imported files
+  let total = 0;
+  for (const count of coUsages.values()) total += count;
+  if (total === 0) return 1;
+
+  const probs: number[] = [];
+  for (const count of coUsages.values()) {
+    if (count > 0) probs.push(count / total);
   }
 
-  const importCohesion = calculateImportBasedCohesion(exports);
+  if (probs.length <= 1) return 1;
 
-  // Weighted combination: 60% import-based, 40% domain-based
-  return importCohesion * 0.6 + domainCohesion * 0.4;
+  // Calculate entropy
+  let entropy = 0;
+  for (const p of probs) {
+    entropy -= p * Math.log2(p);
+  }
+
+  const maxEntropy = Math.log2(probs.length);
+  return maxEntropy > 0 ? 1 - entropy / maxEntropy : 1;
 }
 
 /**

package/src/index.ts

@@ -9,6 +9,8 @@ import {
   calculateCohesion,
   calculateFragmentation,
   detectModuleClusters,
+  calculatePathEntropy,
+  calculateDirectoryDistance,
 } from './analyzer';
 import { calculateContextScore } from './scoring';
 import type {
@@ -206,7 +208,9 @@ export async function analyzeContext(
   const circularDeps = detectCircularDependencies(graph);
 
   // Detect module clusters for fragmentation analysis
-  const clusters = detectModuleClusters(graph);
+  // Enable log-scaling for fragmentation by default on medium+ repos
+  const useLogScale = files.length >= 500; // medium and larger projects
+  const clusters = detectModuleClusters(graph, { useLogScale });
   const fragmentationMap = new Map<string, number>();
   for (const cluster of clusters) {
     for (const file of cluster.files) {
@@ -374,7 +378,6 @@ export function generateSummary(
   const fragmentedModules: ModuleCluster[] = [];
   for (const [domain, files] of moduleMap.entries()) {
     if (files.length < 2) continue;
-
     const fragmentationScore =
       files.reduce((sum, f) => sum + f.fragmentationScore, 0) / files.length;
     if (fragmentationScore < 0.3) continue; // Skip well-organized modules
@@ -384,11 +387,40 @@ export function generateSummary(
       files.reduce((sum, f) => sum + f.cohesionScore, 0) / files.length;
     const targetFiles = Math.max(1, Math.ceil(files.length / 3));
 
+    // Compute path entropy and directory distance for reporting
+    const filePaths = files.map((f) => f.file);
+    const pathEntropy = calculatePathEntropy(filePaths);
+    const directoryDistance = calculateDirectoryDistance(filePaths);
+
+    // Compute import cohesion based on dependency lists (Jaccard similarity)
+    function jaccard(a: string[], b: string[]) {
+      const s1 = new Set(a || []);
+      const s2 = new Set(b || []);
+      if (s1.size === 0 && s2.size === 0) return 0;
+      const inter = new Set([...s1].filter((x) => s2.has(x)));
+      const uni = new Set([...s1, ...s2]);
+      return uni.size === 0 ? 0 : inter.size / uni.size;
+    }
+
+    let importSimTotal = 0;
+    let importPairs = 0;
+    for (let i = 0; i < files.length; i++) {
+      for (let j = i + 1; j < files.length; j++) {
+        importSimTotal += jaccard(files[i].dependencyList || [], files[j].dependencyList || []);
+        importPairs++;
+      }
+    }
+
+    const importCohesion = importPairs > 0 ? importSimTotal / importPairs : 0;
+
     fragmentedModules.push({
       domain,
       files: files.map((f) => f.file),
       totalTokens,
       fragmentationScore,
+      pathEntropy,
+      directoryDistance,
+      importCohesion,
       avgCohesion,
       suggestedStructure: {
         targetFiles,

package/src/types.ts

@@ -44,6 +44,9 @@ export interface ModuleCluster {
   files: string[];
   totalTokens: number;
   fragmentationScore: number; // 0-1, higher = more scattered
+  pathEntropy?: number; // normalized [0-1] Shannon entropy of directory distribution
+  directoryDistance?: number; // normalized [0-1] based on common ancestor depth
+  importCohesion?: number; // 0-1 average pairwise Jaccard similarity of imports
   avgCohesion: number; // Average cohesion across files in cluster
   suggestedStructure: {
     targetFiles: number; // Recommended number of files