@monoes/monograph 1.1.0 → 1.2.0

package/src/duplicates/detect/extraction.ts

@@ -0,0 +1,107 @@
+export interface RawInstance {
+  fileId: number;
+  offset: number;
+}
+
+export interface RawGroup {
+  instances: RawInstance[];
+  lcpLength: number;
+}
+
+export function extractCloneGroups(
+  sa: number[],
+  lcp: number[],
+  fileOf: number[],
+  fileOffsets: number[],
+  minTokens: number,
+  focusFileIds?: Set<number>,
+): RawGroup[] {
+  const n = sa.length;
+  if (n < 2) return [];
+
+  const focusPrefix = focusFileIds !== undefined ? buildFocusPrefix(sa, fileOf, focusFileIds) : null;
+
+  const stack: Array<{ lcpVal: number; start: number }> = [];
+  const groups: RawGroup[] = [];
+
+  for (let i = 1; i <= n; i++) {
+    const curLcp = i < n ? lcp[i] : 0;
+    let start = i;
+
+    while (stack.length > 0) {
+      const top = stack[stack.length - 1];
+      if (top.lcpVal <= curLcp) break;
+      stack.pop();
+      start = top.start;
+
+      if (top.lcpVal >= minTokens) {
+        const intervalBegin = start - 1;
+        const intervalEnd = i;
+
+        if (focusPrefix !== null && !intervalHasFocus(focusPrefix, intervalBegin, intervalEnd)) {
+          continue;
+        }
+
+        const group = buildRawGroup(sa, fileOf, fileOffsets, intervalBegin, intervalEnd, top.lcpVal);
+        if (group !== null) groups.push(group);
+      }
+    }
+
+    if (i < n) {
+      stack.push({ lcpVal: curLcp, start });
+    }
+  }
+
+  return groups;
+}
+
+function buildFocusPrefix(sa: number[], fileOf: number[], focusFileIds: Set<number>): number[] {
+  const prefix = new Array<number>(sa.length + 1).fill(0);
+  for (let i = 0; i < sa.length; i++) {
+    const pos = sa[i];
+    const fid = fileOf[pos];
+    const focused = fid !== -1 && focusFileIds.has(fid) ? 1 : 0;
+    prefix[i + 1] = prefix[i] + focused;
+  }
+  return prefix;
+}
+
+function intervalHasFocus(focusPrefix: number[], begin: number, end: number): boolean {
+  return focusPrefix[end] > focusPrefix[begin];
+}
+
+function buildRawGroup(
+  sa: number[],
+  fileOf: number[],
+  fileOffsets: number[],
+  intervalBegin: number,
+  intervalEnd: number,
+  length: number,
+): RawGroup | null {
+  const instances: Array<{ fileId: number; offset: number }> = [];
+
+  for (let idx = intervalBegin; idx < intervalEnd; idx++) {
+    const pos = sa[idx];
+    const fid = fileOf[pos];
+    if (fid === -1) continue;
+    const offset = pos - fileOffsets[fid];
+    instances.push({ fileId: fid, offset });
+  }
+
+  if (instances.length < 2) return null;
+
+  instances.sort((a, b) => a.fileId !== b.fileId ? a.fileId - b.fileId : a.offset - b.offset);
+
+  const deduped: RawInstance[] = [];
+  for (const inst of instances) {
+    const last = deduped[deduped.length - 1];
+    if (last !== undefined && inst.fileId === last.fileId && inst.offset < last.offset + length) {
+      continue;
+    }
+    deduped.push(inst);
+  }
+
+  if (deduped.length < 2) return null;
+
+  return { instances: deduped, lcpLength: length };
+}

package/src/duplicates/detect/filtering.ts

@@ -0,0 +1,112 @@
+import type { RawGroup } from './extraction.js';
+
+export interface Interval {
+  start: number;
+  end: number;
+}
+
+export class IntervalIndex {
+  private intervals: Array<{ start: number; end: number }> = [];
+
+  insert(interval: Interval): void {
+    const { start, end } = interval;
+    const lo = this.partitionByEnd(start);
+    const hi = this.partitionByStart(end);
+
+    if (lo === hi) {
+      this.intervals.splice(lo, 0, { start, end });
+    } else {
+      const mergedStart = Math.min(this.intervals[lo].start, start);
+      const mergedEnd = Math.max(this.intervals[hi - 1].end, end);
+      this.intervals.splice(lo, hi - lo, { start: mergedStart, end: mergedEnd });
+    }
+  }
+
+  contains(interval: Interval): boolean {
+    const { start, end } = interval;
+    const idx = this.partitionByStart(start + 1) - 1;
+    if (idx < 0) return false;
+    const iv = this.intervals[idx];
+    return iv.start <= start && end <= iv.end;
+  }
+
+  overlaps(interval: Interval): boolean {
+    const { start, end } = interval;
+    const lo = this.partitionByEnd(start);
+    if (lo >= this.intervals.length) return false;
+    return this.intervals[lo].start < end;
+  }
+
+  private partitionByEnd(val: number): number {
+    let lo = 0;
+    let hi = this.intervals.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1;
+      if (this.intervals[mid].end < val) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo;
+  }
+
+  private partitionByStart(val: number): number {
+    let lo = 0;
+    let hi = this.intervals.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1;
+      if (this.intervals[mid].start <= val) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo;
+  }
+}
+
+export function filterCloneGroups(
+  groups: RawGroup[],
+  fileOffsets: number[],
+  fileOf: number[],
+  minLines: number,
+  lineCount: (fileId: number, offset: number, length: number) => number,
+): RawGroup[] {
+  const sorted = [...groups].sort((a, b) => b.lcpLength - a.lcpLength);
+
+  const numFiles = fileOffsets.length;
+  const covered = new Map<number, IntervalIndex>();
+
+  const getIndex = (fid: number): IntervalIndex => {
+    let idx = covered.get(fid);
+    if (!idx) {
+      idx = new IntervalIndex();
+      covered.set(fid, idx);
+    }
+    return idx;
+  };
+
+  const result: RawGroup[] = [];
+
+  for (const group of sorted) {
+    const len = group.lcpLength;
+
+    const allCovered = group.instances.every((inst) => {
+      const idx = covered.get(inst.fileId);
+      if (!idx) return false;
+      return idx.contains({ start: inst.offset, end: inst.offset + len });
+    });
+
+    if (allCovered) continue;
+
+    const lines = Math.max(
+      ...group.instances.map((inst) => lineCount(inst.fileId, inst.offset, len)),
+    );
+    if (lines < minLines) continue;
+
+    for (const inst of group.instances) {
+      getIndex(inst.fileId).insert({ start: inst.offset, end: inst.offset + len });
+    }
+
+    result.push(group);
+  }
+
+  void numFiles;
+  void fileOf;
+  return result;
+}

package/src/duplicates/detect/lcp.ts

@@ -0,0 +1,27 @@
+export function buildLcp(text: number[], sa: number[]): number[] {
+  const n = sa.length;
+  if (n === 0) return [];
+
+  const rank = new Int32Array(n);
+  for (let i = 0; i < n; i++) rank[sa[i]] = i;
+
+  const lcp = new Array<number>(n).fill(0);
+  let k = 0;
+
+  for (let i = 0; i < n; i++) {
+    if (rank[i] === 0) {
+      k = 0;
+      continue;
+    }
+    const j = sa[rank[i] - 1];
+    while (i + k < n && j + k < n) {
+      if (text[i + k] < 0 || text[j + k] < 0) break;
+      if (text[i + k] !== text[j + k]) break;
+      k++;
+    }
+    lcp[rank[i]] = k;
+    k = k > 0 ? k - 1 : 0;
+  }
+
+  return lcp;
+}

package/src/duplicates/detect/ranking.ts

@@ -0,0 +1,16 @@
+export function rankReduce(tokens: number[]): { ranked: number[]; maxRank: number } {
+  const hashToRank = new Map<number, number>();
+  let nextRank = 0;
+
+  const ranked = tokens.map((token) => {
+    let rank = hashToRank.get(token);
+    if (rank === undefined) {
+      rank = nextRank++;
+      hashToRank.set(token, rank);
+    }
+    return rank;
+  });
+
+  const maxRank = nextRank === 0 ? 0 : nextRank - 1;
+  return { ranked, maxRank };
+}

package/src/duplicates/detect/statistics.ts

@@ -0,0 +1,69 @@
+import type { RawGroup } from './extraction.js';
+
+export interface PipelineDuplicationStats {
+  totalFiles: number;
+  filesWithClones: number;
+  totalTokens: number;
+  duplicatedTokens: number;
+  totalLines: number;
+  duplicatedLines: number;
+  cloneGroups: number;
+  cloneInstances: number;
+  duplicationPct: number;
+}
+
+export function computePipelineStats(
+  groups: RawGroup[],
+  allFileIds: number[],
+  totalTokens: number,
+  totalLines: number,
+  fileLineCount: (fileId: number, offset: number, length: number) => number,
+): PipelineDuplicationStats {
+  const filesWithClones = new Set<number>();
+  const fileDupLines = new Map<number, Set<number>>();
+  let duplicatedTokens = 0;
+  let cloneInstances = 0;
+
+  for (const group of groups) {
+    for (const inst of group.instances) {
+      filesWithClones.add(inst.fileId);
+      cloneInstances++;
+
+      let lineSet = fileDupLines.get(inst.fileId);
+      if (!lineSet) {
+        lineSet = new Set<number>();
+        fileDupLines.set(inst.fileId, lineSet);
+      }
+
+      const lines = fileLineCount(inst.fileId, inst.offset, group.lcpLength);
+      for (let l = 0; l < lines; l++) {
+        lineSet.add(inst.offset + l);
+      }
+    }
+
+    if (group.instances.length > 1) {
+      duplicatedTokens += group.lcpLength * (group.instances.length - 1);
+    }
+  }
+
+  let duplicatedLines = 0;
+  for (const lineSet of fileDupLines.values()) {
+    duplicatedLines += lineSet.size;
+  }
+
+  duplicatedTokens = Math.min(duplicatedTokens, totalTokens);
+
+  const duplicationPct = totalLines > 0 ? (duplicatedLines / totalLines) * 100 : 0;
+
+  return {
+    totalFiles: allFileIds.length,
+    filesWithClones: filesWithClones.size,
+    totalTokens,
+    duplicatedTokens,
+    totalLines,
+    duplicatedLines,
+    cloneGroups: groups.length,
+    cloneInstances,
+    duplicationPct,
+  };
+}

package/src/duplicates/detect/suffix-array.ts

@@ -0,0 +1,88 @@
+export function buildSuffixArray(text: number[], _maxRank: number): number[] {
+  const n = text.length;
+  if (n === 0) return [];
+
+  const minVal = text.reduce((m, v) => (v < m ? v : m), text[0]);
+
+  let rank = new Int32Array(n);
+  for (let i = 0; i < n; i++) rank[i] = text[i] - minVal;
+
+  let sa = new Int32Array(n);
+  for (let i = 0; i < n; i++) sa[i] = i;
+
+  let tmp = new Int32Array(n);
+  const saTmp = new Int32Array(n);
+  let counts = new Int32Array(0);
+
+  let curMaxRank = 0;
+  for (let i = 0; i < n; i++) {
+    if (rank[i] > curMaxRank) curMaxRank = rank[i];
+  }
+
+  let k = 1;
+
+  while (k < n) {
+    const bucketCount = curMaxRank + 2;
+    if (counts.length < bucketCount + 1) {
+      counts = new Int32Array(bucketCount + 1);
+    } else {
+      counts.fill(0, 0, bucketCount + 1);
+    }
+
+    for (let idx = 0; idx < n; idx++) {
+      const i = sa[idx];
+      const r2 = i + k < n ? rank[i + k] + 1 : 0;
+      counts[r2]++;
+    }
+    let sum = 0;
+    for (let c = 0; c <= bucketCount; c++) {
+      const v = counts[c];
+      counts[c] = sum;
+      sum += v;
+    }
+    for (let idx = 0; idx < n; idx++) {
+      const i = sa[idx];
+      const r2 = i + k < n ? rank[i + k] + 1 : 0;
+      saTmp[counts[r2]++] = i;
+    }
+
+    counts.fill(0, 0, bucketCount + 1);
+    for (let idx = 0; idx < n; idx++) {
+      const i = saTmp[idx];
+      counts[rank[i]]++;
+    }
+    sum = 0;
+    for (let c = 0; c <= bucketCount; c++) {
+      const v = counts[c];
+      counts[c] = sum;
+      sum += v;
+    }
+    for (let idx = 0; idx < n; idx++) {
+      const i = saTmp[idx];
+      sa[counts[rank[i]]++] = i;
+    }
+
+    tmp[sa[0]] = 0;
+    for (let i = 1; i < n; i++) {
+      const prev = sa[i - 1];
+      const curr = sa[i];
+      const rp2 = prev + k < n ? rank[prev + k] : -1;
+      const rc2 = curr + k < n ? rank[curr + k] : -1;
+      const same = rank[prev] === rank[curr] && rp2 === rc2;
+      tmp[curr] = tmp[prev] + (same ? 0 : 1);
+    }
+
+    const newMaxRank = tmp[sa[n - 1]];
+    const swapRank = rank;
+    rank = tmp;
+    tmp = swapRank;
+
+    if (newMaxRank === n - 1) break;
+    curMaxRank = newMaxRank;
+    k *= 2;
+  }
+
+  const result: number[] = new Array(n);
+  for (let i = 0; i < n; i++) result[i] = sa[i];
+  return result;
+}

package/src/duplicates/families.ts

@@ -0,0 +1,95 @@
+export const MODULE_EXTRACTION_THRESHOLD = 50;
+
+export interface RefactoringSuggestion {
+  kind: 'ExtractFunction' | 'ExtractModule' | 'MergeDirectories';
+  description: string;
+  estimatedLines: number;
+  files: string[];
+}
+
+export interface CloneFamily {
+  files: string[];
+  groupCount: number;
+  totalDuplicatedLines: number;
+  suggestions: RefactoringSuggestion[];
+}
+
+interface RawGroupInput {
+  instances: Array<{ filePath: string; startLine: number; endLine: number }>;
+  duplicatedLines: number;
+}
+
+function fileSetKey(files: string[]): string {
+  return [...files].sort().join('\0');
+}
+
+function dirOf(filePath: string): string {
+  const normalized = filePath.replace(/\\/g, '/');
+  const idx = normalized.lastIndexOf('/');
+  return idx >= 0 ? normalized.slice(0, idx) : '';
+}
+
+function generateSuggestions(
+  files: string[],
+  groups: RawGroupInput[],
+  totalDuplicatedLines: number,
+): RefactoringSuggestion[] {
+  if (totalDuplicatedLines < MODULE_EXTRACTION_THRESHOLD) {
+    return groups.map((g) => ({
+      kind: 'ExtractFunction' as const,
+      description: `Extract shared function (${g.duplicatedLines} lines) from ${g.instances.map((i) => i.filePath.split('/').pop() ?? i.filePath).join(', ')}`,
+      estimatedLines: g.duplicatedLines,
+      files,
+    }));
+  }
+
+  const dirs = [...new Set(files.map(dirOf))];
+  if (dirs.length > 1 && files.length >= 3) {
+    return [
+      {
+        kind: 'MergeDirectories' as const,
+        description: `Consider merging ${dirs.length} directories with ${totalDuplicatedLines} shared lines`,
+        estimatedLines: totalDuplicatedLines,
+        files,
+      },
+    ];
+  }
+
+  return [
+    {
+      kind: 'ExtractModule' as const,
+      description: `Extract ${totalDuplicatedLines} duplicated lines into a shared module`,
+      estimatedLines: totalDuplicatedLines,
+      files,
+    },
+  ];
+}
+
+export function groupRawGroupsIntoFamilies(groups: RawGroupInput[]): CloneFamily[] {
+  const familyMap = new Map<string, { files: Set<string>; groups: RawGroupInput[] }>();
+
+  for (const group of groups) {
+    const files = group.instances.map((i) => i.filePath);
+    const key = fileSetKey(files);
+    if (!familyMap.has(key)) {
+      familyMap.set(key, { files: new Set(files), groups: [] });
+    }
+    familyMap.get(key)!.groups.push(group);
+  }
+
+  const families: CloneFamily[] = [];
+
+  for (const { files, groups: fg } of familyMap.values()) {
+    const fileList = [...files].sort();
+    const totalDuplicatedLines = fg.reduce((s, g) => s + g.duplicatedLines, 0);
+    families.push({
+      files: fileList,
+      groupCount: fg.length,
+      totalDuplicatedLines,
+      suggestions: generateSuggestions(fileList, fg, totalDuplicatedLines),
+    });
+  }
+
+  families.sort((a, b) => b.totalDuplicatedLines - a.totalDuplicatedLines);
+  return families;
+}

package/src/duplicates/normalize.ts

@@ -0,0 +1,58 @@
+import type { SourceToken, TokenKind } from './token-types.js';
+
+export type DetectionMode = 'Exact' | 'NormalizeIdentifiers' | 'NormalizeLiterals' | 'NormalizeAll';
+
+export interface ResolvedNormalization {
+  normalizeIdentifiers: boolean;
+  normalizeLiterals: boolean;
+  normalizeKeywords: boolean;
+}
+
+export interface HashedToken {
+  hash: number;
+  originalIndex: number;
+}
+
+export function resolveNormalization(mode: DetectionMode): ResolvedNormalization {
+  switch (mode) {
+    case 'Exact':
+      return { normalizeIdentifiers: false, normalizeLiterals: false, normalizeKeywords: false };
+    case 'NormalizeIdentifiers':
+      return { normalizeIdentifiers: true, normalizeLiterals: false, normalizeKeywords: false };
+    case 'NormalizeLiterals':
+      return { normalizeIdentifiers: false, normalizeLiterals: true, normalizeKeywords: false };
+    case 'NormalizeAll':
+      return { normalizeIdentifiers: true, normalizeLiterals: true, normalizeKeywords: true };
+  }
+}
+
+export function normalizeAndHash(tokens: SourceToken[], mode: DetectionMode): HashedToken[] {
+  return normalizeAndHashResolved(tokens, resolveNormalization(mode));
+}
+
+export function normalizeAndHashResolved(tokens: SourceToken[], norm: ResolvedNormalization): HashedToken[] {
+  return tokens.map((tok, i) => ({
+    hash: hashTokenResolved(tok.kind, norm),
+    originalIndex: i,
+  }));
+}
+
+export function hashTokenResolved(kind: TokenKind, norm: ResolvedNormalization): number {
+  if (norm.normalizeIdentifiers && kind.kind === 'Identifier') return djb2('__ID__');
+  if (norm.normalizeLiterals) {
+    if (kind.kind === 'StringLiteral') return djb2('__STR__');
+    if (kind.kind === 'NumericLiteral') return djb2('__NUM__');
+    if (kind.kind === 'TemplateLiteral') return djb2('__TPL__');
+  }
+  if (norm.normalizeKeywords && kind.kind === 'Keyword') return djb2('__KW__');
+  return djb2(JSON.stringify(kind));
+}
+
+function djb2(s: string): number {
+  let h = 5381;
+  for (let i = 0; i < s.length; i++) {
+    h = ((h << 5) + h) ^ s.charCodeAt(i);
+    h = h >>> 0;
+  }
+  return h;
+}

package/src/duplicates/shingle-filter.ts

@@ -0,0 +1,37 @@
+export const SHINGLE_SIZE = 7;
+
+export function buildShingleSet(tokens: number[], k: number = SHINGLE_SIZE): Set<string> {
+  const result = new Set<string>();
+  if (tokens.length < k) return result;
+  for (let i = 0; i <= tokens.length - k; i++) {
+    result.add(tokens.slice(i, i + k).join(','));
+  }
+  return result;
+}
+
+export function filterToFocusCandidates(
+  focusFileTokens: Map<number, number[]>,
+  allFileTokens: Map<number, number[]>,
+  k: number = SHINGLE_SIZE,
+): Set<number> {
+  const focusShingles = new Set<string>();
+  for (const tokens of focusFileTokens.values()) {
+    for (const shingle of buildShingleSet(tokens, k)) {
+      focusShingles.add(shingle);
+    }
+  }
+
+  const candidates = new Set<number>();
+  for (const [fileId, tokens] of allFileTokens) {
+    if (focusFileTokens.has(fileId)) continue;
+    const fileShingles = buildShingleSet(tokens, k);
+    for (const s of fileShingles) {
+      if (focusShingles.has(s)) {
+        candidates.add(fileId);
+        break;
+      }
+    }
+  }
+
+  return candidates;
+}

package/src/duplicates/token-types.ts

@@ -0,0 +1,62 @@
+export type KeywordType =
+  | 'Var' | 'Let' | 'Const' | 'Function' | 'Return' | 'If' | 'Else'
+  | 'For' | 'While' | 'Do' | 'Switch' | 'Case' | 'Break' | 'Continue'
+  | 'Default' | 'Throw' | 'Try' | 'Catch' | 'Finally' | 'New' | 'Delete'
+  | 'Typeof' | 'Instanceof' | 'In' | 'Of' | 'Void' | 'This' | 'Super'
+  | 'Class' | 'Extends' | 'Import' | 'Export' | 'From' | 'As' | 'Async'
+  | 'Await' | 'Yield' | 'Static' | 'Get' | 'Set' | 'Type' | 'Interface'
+  | 'Enum' | 'Implements' | 'Abstract' | 'Declare' | 'Readonly' | 'Keyof'
+  | 'Satisfies';
+
+export type OperatorType =
+  | 'Assign' | 'Add' | 'Sub' | 'Mul' | 'Div' | 'Mod' | 'Exp'
+  | 'Eq' | 'NEq' | 'StrictEq' | 'StrictNEq' | 'Lt' | 'Gt' | 'LtEq' | 'GtEq'
+  | 'And' | 'Or' | 'Not' | 'BitwiseAnd' | 'BitwiseOr' | 'BitwiseXor' | 'BitwiseNot'
+  | 'ShiftLeft' | 'ShiftRight' | 'UnsignedShiftRight'
+  | 'NullishCoalescing' | 'OptionalChaining' | 'Spread' | 'Ternary' | 'Arrow' | 'Comma'
+  | 'AddAssign' | 'SubAssign' | 'MulAssign' | 'DivAssign' | 'ModAssign' | 'ExpAssign'
+  | 'AndAssign' | 'OrAssign' | 'NullishAssign'
+  | 'BitwiseAndAssign' | 'BitwiseOrAssign' | 'BitwiseXorAssign'
+  | 'ShiftLeftAssign' | 'ShiftRightAssign' | 'UnsignedShiftRightAssign'
+  | 'Increment' | 'Decrement';
+
+export type PunctuationType =
+  | 'OpenParen' | 'CloseParen' | 'OpenBrace' | 'CloseBrace'
+  | 'OpenBracket' | 'CloseBracket' | 'Semicolon' | 'Colon' | 'Dot';
+
+export type TokenKind =
+  | { kind: 'Keyword'; kwType: KeywordType }
+  | { kind: 'Identifier'; name: string }
+  | { kind: 'StringLiteral' }
+  | { kind: 'NumericLiteral' }
+  | { kind: 'BooleanLiteral'; value: boolean }
+  | { kind: 'NullLiteral' }
+  | { kind: 'TemplateLiteral' }
+  | { kind: 'RegExpLiteral' }
+  | { kind: 'Operator'; opType: OperatorType }
+  | { kind: 'Punctuation'; punctType: PunctuationType };
+
+export interface Span {
+  start: number;
+  end: number;
+}
+
+export interface SourceToken {
+  kind: TokenKind;
+  span: Span;
+}
+
+export interface FileTokens {
+  tokens: SourceToken[];
+  source: string;
+  lineCount: number;
+}
+
+export function pointSpan(pos: number): Span {
+  return { start: pos, end: pos + 1 };
+}
+
+export function emptyTokens(source: string): FileTokens {
+  const lineCount = (source.match(/\n/g)?.length ?? 0) + 1;
+  return { tokens: [], source, lineCount };
+}