distyll 0.1.0

package/src/rules/unused-imports.ts

@@ -0,0 +1,115 @@
+import type Parser from 'tree-sitter';
+import { findNodes, walkTree } from '../parser';
+import type { Finding, Rule } from '../types';
+
+function getImportedNames(importNode: Parser.SyntaxNode): Array<{ name: string; node: Parser.SyntaxNode }> {
+  const names: Array<{ name: string; node: Parser.SyntaxNode }> = [];
+
+  walkTree(importNode, (node) => {
+    // import Foo from '...' (default import)
+    if (node.type === 'identifier' && node.parent?.type === 'import_clause') {
+      names.push({ name: node.text, node });
+    }
+    // import { Foo } from '...' or import { Foo as Bar } from '...'
+    if (node.type === 'import_specifier') {
+      const alias = node.childForFieldName('alias');
+      const nameNode = alias ?? node.childForFieldName('name');
+      if (nameNode) {
+        names.push({ name: nameNode.text, node: nameNode });
+      }
+    }
+    // import * as Foo from '...'
+    if (node.type === 'namespace_import') {
+      const nameNode = node.namedChildren.find((c) => c.type === 'identifier');
+      if (nameNode) {
+        names.push({ name: nameNode.text, node: nameNode });
+      }
+    }
+  });
+
+  return names;
+}
+
+function collectAllIdentifiers(root: Parser.SyntaxNode, excludeImports: Parser.SyntaxNode[]): Set<string> {
+  const excludeSet = new Set(excludeImports.map((n) => n.id));
+  const identifiers = new Set<string>();
+
+  walkTree(root, (node) => {
+    if (node.type === 'identifier' || node.type === 'shorthand_property_identifier' ||
+        node.type === 'shorthand_property_identifier_pattern') {
+      // Skip nodes that are part of import declarations
+      let parent: Parser.SyntaxNode | null = node;
+      let isInImport = false;
+      while (parent) {
+        if (parent.type === 'import_statement' || parent.type === 'import_declaration') {
+          isInImport = true;
+          break;
+        }
+        parent = parent.parent;
+      }
+      if (!isInImport) {
+        identifiers.add(node.text);
+      }
+    }
+    // Also check type references for TypeScript
+    if (node.type === 'type_identifier') {
+      let parent: Parser.SyntaxNode | null = node;
+      let isInImport = false;
+      while (parent) {
+        if (parent.type === 'import_statement' || parent.type === 'import_declaration') {
+          isInImport = true;
+          break;
+        }
+        parent = parent.parent;
+      }
+      if (!isInImport) {
+        identifiers.add(node.text);
+      }
+    }
+  });
+
+  return identifiers;
+}
+
+export const unusedImports: Rule = {
+  name: 'unused-imports',
+  description: 'Flags imported identifiers that are never used in the file',
+  severity: 'warning',
+
+  check(tree: Parser.Tree, source: string, filePath: string): Finding[] {
+    const findings: Finding[] = [];
+    const importStatements = findNodes(tree.rootNode, 'import_statement');
+
+    if (importStatements.length === 0) return findings;
+
+    // Collect all identifiers used outside of imports
+    const usedIdentifiers = collectAllIdentifiers(tree.rootNode, importStatements);
+
+    for (const importStmt of importStatements) {
+      // Skip side-effect imports: import './foo' or import 'module'
+      const hasClause = importStmt.children.some(
+        (c) => c.type === 'import_clause'
+      );
+      if (!hasClause) continue;
+
+      const importedNames = getImportedNames(importStmt);
+
+      for (const { name, node } of importedNames) {
+        if (!usedIdentifiers.has(name)) {
+          findings.push({
+            file: filePath,
+            line: node.startPosition.row + 1,
+            column: node.startPosition.column + 1,
+            endLine: node.endPosition.row + 1,
+            endColumn: node.endPosition.column + 1,
+            rule: 'unused-imports',
+            severity: 'warning',
+            message: `'${name}' is imported but never used`,
+          });
+        }
+      }
+    }
+
+    return findings;
+  },
+};

package/src/rules/verbose-comments.ts

@@ -0,0 +1,105 @@
+import type Parser from 'tree-sitter';
+import { findNodes } from '../parser';
+import type { Finding, Rule } from '../types';
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9_$]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter((t) => t.length > 1);
+}
+
+function jaccardSimilarity(a: string[], b: string[]): number {
+  if (a.length === 0 || b.length === 0) return 0;
+  const setA = new Set(a);
+  const setB = new Set(b);
+  let intersection = 0;
+  for (const item of setA) {
+    if (setB.has(item)) intersection++;
+  }
+  const union = new Set([...setA, ...setB]).size;
+  return union === 0 ? 0 : intersection / union;
+}
+
+function stripCommentSyntax(text: string): string {
+  return text
+    .replace(/^\/\/\s*/, '')
+    .replace(/^\/\*+\s*/, '')
+    .replace(/\s*\*+\/$/, '')
+    .replace(/^\s*\*\s?/gm, '')
+    .trim();
+}
+
+function getNextSiblingCode(node: Parser.SyntaxNode): string | null {
+  let sibling = node.nextNamedSibling;
+  // Skip over other comments
+  while (sibling && sibling.type === 'comment') {
+    sibling = sibling.nextNamedSibling;
+  }
+  if (!sibling) return null;
+  // Get just the first line of the sibling code for comparison
+  const text = sibling.text;
+  const firstLine = text.split('\n')[0];
+  return firstLine || null;
+}
+
+const TRIVIAL_COMMENTS = new Set([
+  'todo', 'fixme', 'hack', 'note', 'bug', 'xxx',
+  'eslint-disable', 'eslint-enable', 'ts-ignore', 'ts-expect-error',
+  'prettier-ignore', 'istanbul ignore', 'c8 ignore',
+  '@ts-ignore', '@ts-expect-error', '@ts-nocheck',
+]);
+
+function isTrivialOrDirective(commentText: string): boolean {
+  const lower = commentText.toLowerCase().trim();
+  for (const directive of TRIVIAL_COMMENTS) {
+    if (lower.startsWith(directive)) return true;
+  }
+  // Skip very short comments (likely meaningful labels)
+  if (lower.length < 10) return true;
+  return false;
+}
+
+export const verboseComments: Rule = {
+  name: 'verbose-comments',
+  description: 'Flags comments that restate the code they describe',
+  severity: 'info',
+
+  check(tree: Parser.Tree, source: string, filePath: string): Finding[] {
+    const findings: Finding[] = [];
+    const comments = findNodes(tree.rootNode, 'comment');
+
+    for (const comment of comments) {
+      const raw = stripCommentSyntax(comment.text);
+      if (isTrivialOrDirective(raw)) continue;
+
+      const nextCode = getNextSiblingCode(comment);
+      if (!nextCode) continue;
+
+      const commentTokens = tokenize(raw);
+      const codeTokens = tokenize(nextCode);
+
+      if (commentTokens.length < 3) continue;
+
+      const similarity = jaccardSimilarity(commentTokens, codeTokens);
+
+      // Only flag if >80% similar — very conservative threshold
+      if (similarity >= 0.8) {
+        findings.push({
+          file: filePath,
+          line: comment.startPosition.row + 1,
+          column: comment.startPosition.column + 1,
+          endLine: comment.endPosition.row + 1,
+          endColumn: comment.endPosition.column + 1,
+          rule: 'verbose-comments',
+          severity: 'info',
+          message: `Comment restates the code (${Math.round(similarity * 100)}% similar) — consider removing it`,
+        });
+      }
+    }
+
+    return findings;
+  },
+};

package/src/scanner.ts

@@ -0,0 +1,184 @@
+import * as fs from 'fs';
+import fg from 'fast-glob';
+import { parse, detectLanguage } from './parser';
+import { allRules } from './rules';
+import { computeScore } from './scorer';
+import { loadConfig } from './config';
+import { getSupportedExtensions } from './languages';
+import { attachFixes } from './fixes';
+import { loadProfile } from './fingerprint/profile';
+import { compareFileToProfile } from './fingerprint/comparator';
+import { safeReadFile } from './errors';
+import type { ScanResult, ScanSummary, Rule, StyleProfile } from './types';
+
+function countLines(source: string): number {
+  if (source.length === 0) return 0;
+  let count = 1;
+  for (let i = 0; i < source.length; i++) {
+    if (source[i] === '\n') count++;
+  }
+  return count;
+}
+
+export interface ScanPathsOptions {
+  rules?: Rule[];
+  configDir?: string;
+  style?: boolean;
+  verbose?: boolean;
+}
+
+export async function scanPaths(paths: string[], options?: Rule[] | ScanPathsOptions): Promise<ScanSummary> {
+  let activeRules: Rule[] | undefined;
+  let configDir: string | undefined;
+  let useStyle = false;
+  let verbose = false;
+
+  if (Array.isArray(options)) {
+    activeRules = options;
+  } else if (options && !Array.isArray(options)) {
+    activeRules = options.rules;
+    configDir = options.configDir;
+    useStyle = options.style ?? false;
+    verbose = options.verbose ?? false;
+  }
+
+  const config = loadConfig(configDir ?? (paths.length > 0 ? paths[0] : process.cwd()));
+  let rules = activeRules ?? allRules;
+
+  if (config.rules) {
+    rules = rules.filter((rule) => {
+      const setting = config.rules?.[rule.name];
+      return setting !== 'off';
+    });
+    rules = rules.map((rule) => {
+      const setting = config.rules?.[rule.name];
+      if (setting === 'warn' || setting === 'error') {
+        return { ...rule, severity: setting === 'warn' ? 'warning' : 'error' };
+      }
+      return rule;
+    });
+  }
+
+  let profile: StyleProfile | null = null;
+  if (useStyle) {
+    const profileDir = configDir ?? (paths.length > 0 ? paths[0] : process.cwd());
+    profile = loadProfile(profileDir);
+    if (!profile) {
+      profile = loadProfile(process.cwd());
+    }
+  }
+
+  const files = await resolveFiles(paths, config.ignore);
+
+  if (verbose) {
+    console.error(`Scanning ${files.length} file(s) with ${rules.length} rule(s)...`);
+  }
+
+  const results: ScanResult[] = [];
+
+  for (const filePath of files) {
+    const startTime = verbose ? performance.now() : 0;
+    const result = scanFile(filePath, rules, profile, verbose);
+
+    if (verbose && result) {
+      const elapsed = (performance.now() - startTime).toFixed(1);
+      console.error(`  ${filePath}: ${result.findings.length} finding(s) in ${elapsed}ms`);
+    }
+
+    if (result) {
+      results.push(result);
+    }
+  }
+
+  return computeScore(results);
+}
+
+export async function resolveFiles(paths: string[], extraIgnore?: string[]): Promise<string[]> {
+  const files: string[] = [];
+  const supportedExts = getSupportedExtensions();
+
+  const ignorePatterns = [
+    '**/node_modules/**', '**/dist/**', '**/build/**', '**/.git/**', '**/vendor/**',
+    ...(extraIgnore ?? []),
+  ];
+
+  for (const p of paths) {
+    let stat: fs.Stats;
+    try {
+      stat = fs.statSync(p);
+    } catch {
+      continue;
+    }
+
+    if (stat.isFile()) {
+      const lang = detectLanguage(p);
+      if (lang) files.push(p);
+    } else if (stat.isDirectory()) {
+      const pattern = `**/*.{${supportedExts.join(',')}}`;
+      const found = await fg(pattern, {
+        cwd: p,
+        absolute: true,
+        ignore: ignorePatterns,
+      });
+      files.push(...found);
+    }
+  }
+
+  return files.sort();
+}
+
+export function scanFile(
+  filePath: string,
+  rules?: Rule[],
+  profile?: StyleProfile | null,
+  verbose = false
+): ScanResult | null {
+  const activeRules = rules ?? allRules;
+  const language = detectLanguage(filePath);
+
+  if (!language) {
+    if (verbose) console.error(`  [skip] ${filePath}: unsupported language`);
+    return null;
+  }
+
+  const fileData = safeReadFile(filePath, verbose);
+  if (!fileData) return null;
+
+  const { source } = fileData;
+  const loc = countLines(source);
+
+  let tree;
+  try {
+    tree = parse(source, language);
+  } catch (err) {
+    if (verbose) {
+      console.error(`  [skip] ${filePath}: parse error — ${(err as Error).message}`);
+    }
+    return null;
+  }
+
+  let findings = activeRules.flatMap((rule) => {
+    try {
+      const ruleStart = verbose ? performance.now() : 0;
+      const ruleFindings = rule.check(tree, source, filePath);
+      if (verbose) {
+        const elapsed = (performance.now() - ruleStart).toFixed(1);
+        if (ruleFindings.length > 0) {
+          console.error(`    [rule] ${rule.name}: ${ruleFindings.length} finding(s) (${elapsed}ms)`);
+        }
+      }
+      return ruleFindings;
+    } catch {
+      return [];
+    }
+  });
+
+  findings = attachFixes(findings, tree, source);
+
+  if (profile) {
+    const styleFindings = compareFileToProfile(filePath, source, tree, profile);
+    findings.push(...styleFindings);
+  }
+
+  return { file: filePath, findings, loc };
+}

package/src/scorer.ts

@@ -0,0 +1,26 @@
+import type { ScanResult, ScanSummary, Severity } from './types';
+
+const SEVERITY_WEIGHTS: Record<Severity, number> = {
+  info: 1,
+  warning: 3,
+  error: 5,
+};
+
+export function computeScore(results: ScanResult[]): ScanSummary {
+  const totalLoc = results.reduce((sum, r) => sum + r.loc, 0);
+  const totalFindings = results.reduce((sum, r) => sum + r.findings.length, 0);
+
+  if (totalLoc === 0 || totalFindings === 0) {
+    return { results, totalFindings: 0, score: 0 };
+  }
+
+  const weightedSum = results.reduce((sum, r) => {
+    return sum + r.findings.reduce((fSum, f) => fSum + SEVERITY_WEIGHTS[f.severity], 0);
+  }, 0);
+
+  // Normalize: weighted findings per 100 lines of code, capped at 100
+  const rawScore = (weightedSum / totalLoc) * 100;
+  const score = Math.min(100, Math.round(rawScore));
+
+  return { results, totalFindings, score };
+}

package/src/types.ts

@@ -0,0 +1,70 @@
+import type Parser from 'tree-sitter';
+
+export type Severity = 'info' | 'warning' | 'error';
+
+export interface FixSuggestion {
+  description: string;
+  replacement: string;
+}
+
+export interface Finding {
+  file: string;
+  line: number;
+  column: number;
+  endLine: number;
+  endColumn: number;
+  rule: string;
+  severity: Severity;
+  message: string;
+  snippet?: string;
+  fix?: FixSuggestion;
+}
+
+export interface Rule {
+  name: string;
+  description: string;
+  severity: Severity;
+  check(tree: Parser.Tree, source: string, filePath: string): Finding[];
+}
+
+export interface ScanResult {
+  file: string;
+  findings: Finding[];
+  loc: number;
+}
+
+export interface ScanSummary {
+  results: ScanResult[];
+  totalFindings: number;
+  score: number;
+}
+
+export type Language = 'javascript' | 'typescript' | 'tsx' | 'python';
+
+export interface ScanOptions {
+  paths: string[];
+  format?: 'text' | 'json';
+  threshold?: number;
+  style?: boolean;
+}
+
+export interface StyleProfile {
+  generatedAt: string;
+  fileCount: number;
+  totalLoc: number;
+  metrics: {
+    medianFunctionLength: number;
+    averageFunctionLength: number;
+    maxFunctionLength: number;
+    namingConventions: {
+      camelCase: number;
+      snake_case: number;
+      PascalCase: number;
+      other: number;
+    };
+    commentToCodeRatio: number;
+    averageNestingDepth: number;
+    tryCatchDensity: number;
+    averageImportsPerFile: number;
+  };
+}

package/tests/commands/diff.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { execSync } from 'child_process';
+
+describe('diff command (integration)', () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'distyll-diff-test-'));
+    execSync('git init', { cwd: tmpDir, stdio: 'pipe' });
+    execSync('git config user.email "test@test.com"', { cwd: tmpDir, stdio: 'pipe' });
+    execSync('git config user.name "Test"', { cwd: tmpDir, stdio: 'pipe' });
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('should detect slop in changed files via CLI', () => {
+    // Create initial commit
+    const filePath = path.join(tmpDir, 'test.js');
+    fs.writeFileSync(filePath, 'const x = 1;\n');
+    execSync('git add . && git commit -m "init"', { cwd: tmpDir, stdio: 'pipe' });
+
+    // Add sloppy code
+    fs.writeFileSync(
+      filePath,
+      `const x = 1;
+function wrapper(y) { return y; }
+`
+    );
+
+    // Run distyll diff (unstaged changes)
+    const distyllBin = path.resolve(__dirname, '..', '..', 'dist', 'cli.js');
+    let output: string;
+    try {
+      output = execSync(`node ${distyllBin} diff`, {
+        cwd: tmpDir,
+        encoding: 'utf-8',
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+    } catch (err: any) {
+      // May exit with non-zero if threshold exceeded
+      output = err.stdout || '';
+    }
+
+    // Should produce some output (either findings or "No changes")
+    expect(typeof output).toBe('string');
+  });
+
+  it('should handle repo with no changes gracefully', () => {
+    const filePath = path.join(tmpDir, 'test.js');
+    fs.writeFileSync(filePath, 'const x = 1;\n');
+    execSync('git add . && git commit -m "init"', { cwd: tmpDir, stdio: 'pipe' });
+
+    const distyllBin = path.resolve(__dirname, '..', '..', 'dist', 'cli.js');
+    const output = execSync(`node ${distyllBin} diff`, {
+      cwd: tmpDir,
+      encoding: 'utf-8',
+    });
+
+    expect(output).toContain('No changes');
+  });
+
+  it('should scan staged changes with --staged flag', () => {
+    const filePath = path.join(tmpDir, 'test.js');
+    fs.writeFileSync(filePath, 'const x = 1;\n');
+    execSync('git add . && git commit -m "init"', { cwd: tmpDir, stdio: 'pipe' });
+
+    // Add and stage new content
+    fs.writeFileSync(filePath, 'const x = 1;\nfunction wrapper(y) { return y; }\n');
+    execSync('git add .', { cwd: tmpDir, stdio: 'pipe' });
+
+    const distyllBin = path.resolve(__dirname, '..', '..', 'dist', 'cli.js');
+    let output: string;
+    try {
+      output = execSync(`node ${distyllBin} diff --staged`, {
+        cwd: tmpDir,
+        encoding: 'utf-8',
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+    } catch (err: any) {
+      output = err.stdout || '';
+    }
+
+    expect(typeof output).toBe('string');
+  });
+
+  it('should support --format json', () => {
+    const filePath = path.join(tmpDir, 'test.js');
+    fs.writeFileSync(filePath, 'const x = 1;\n');
+    execSync('git add . && git commit -m "init"', { cwd: tmpDir, stdio: 'pipe' });
+
+    // No changes — should get empty JSON
+    const distyllBin = path.resolve(__dirname, '..', '..', 'dist', 'cli.js');
+    const output = execSync(`node ${distyllBin} diff --format json`, {
+      cwd: tmpDir,
+      encoding: 'utf-8',
+    });
+
+    const parsed = JSON.parse(output);
+    expect(parsed).toHaveProperty('score', 0);
+    expect(parsed).toHaveProperty('totalFindings', 0);
+  });
+});

package/tests/config.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { loadConfig } from '../src/config';
+
+function createTempConfig(config: object): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'distyll-config-'));
+  fs.writeFileSync(path.join(dir, '.distyll.json'), JSON.stringify(config));
+  return dir;
+}
+
+describe('config', () => {
+  it('returns empty config when no file exists', () => {
+    const config = loadConfig('/tmp/nonexistent-dir-12345');
+    expect(config).toEqual({});
+  });
+
+  it('loads rules from .distyll.json', () => {
+    const dir = createTempConfig({
+      rules: {
+        'verbose-comments': 'off',
+        'unused-imports': 'error',
+      },
+    });
+    const config = loadConfig(dir);
+    expect(config.rules?.['verbose-comments']).toBe('off');
+    expect(config.rules?.['unused-imports']).toBe('error');
+  });
+
+  it('loads threshold from config', () => {
+    const dir = createTempConfig({ threshold: 42 });
+    const config = loadConfig(dir);
+    expect(config.threshold).toBe(42);
+  });
+
+  it('loads ignore patterns from config', () => {
+    const dir = createTempConfig({ ignore: ['**/generated/**', '**/*.min.js'] });
+    const config = loadConfig(dir);
+    expect(config.ignore).toEqual(['**/generated/**', '**/*.min.js']);
+  });
+
+  it('ignores invalid rule values', () => {
+    const dir = createTempConfig({
+      rules: {
+        'good-rule': 'warn',
+        'bad-rule': 'invalid-value',
+      },
+    });
+    const config = loadConfig(dir);
+    expect(config.rules?.['good-rule']).toBe('warn');
+    expect(config.rules?.['bad-rule']).toBeUndefined();
+  });
+
+  it('ignores invalid JSON gracefully', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'distyll-config-'));
+    fs.writeFileSync(path.join(dir, '.distyll.json'), 'not json {{{');
+    const config = loadConfig(dir);
+    expect(config).toEqual({});
+  });
+
+  it('walks up directory tree to find config', () => {
+    const parentDir = createTempConfig({ threshold: 75 });
+    const childDir = path.join(parentDir, 'src', 'lib');
+    fs.mkdirSync(childDir, { recursive: true });
+    const config = loadConfig(childDir);
+    expect(config.threshold).toBe(75);
+  });
+});