agentic-compaction 0.0.1

package/src/parsers/python.js

@@ -0,0 +1,181 @@
+const PYTHON_EXTENSIONS = ['.py'];
+
+export const isPythonParseable = (path) => {
+  return PYTHON_EXTENSIONS.some(ext => path.endsWith(ext));
+};
+
+/**
+ * Extract skeleton from Python source code using regex-based line-by-line parsing.
+ * Only parses top-level statements (no indentation).
+ * @param {string} code - Source code to parse
+ * @param {string} filePath - File path for error reporting
+ * @returns {Object|null} Skeleton data
+ */
+export const extractSkeleton = (code, filePath = '') => {
+  const lines = code.split('\n');
+  const skeleton = {
+    imports: [],
+    functions: [],
+    classes: [],
+    constants: 0,
+  };
+
+  // Collect decorators as we scan
+  let pendingDecorators = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const lineNum = i + 1;
+
+    // Skip empty lines and comments
+    if (/^\s*$/.test(line) || /^\s*#/.test(line)) continue;
+
+    // Only process top-level (no indentation)
+    if (/^\s/.test(line) && !/^@/.test(line)) {
+      pendingDecorators = [];
+      continue;
+    }
+
+    // Decorators
+    const decoratorMatch = line.match(/^@(\w[\w.]*)/);
+    if (decoratorMatch) {
+      pendingDecorators.push(decoratorMatch[1]);
+      continue;
+    }
+
+    // import x / import x, y
+    const importMatch = line.match(/^import\s+(.+)/);
+    if (importMatch) {
+      const modules = importMatch[1].split(',').map(s => s.trim().split(/\s+as\s+/)[0]);
+      for (const mod of modules) {
+        skeleton.imports.push({ module: mod, names: [] });
+      }
+      pendingDecorators = [];
+      continue;
+    }
+
+    // from x import y, z
+    const fromImportMatch = line.match(/^from\s+([\w.]+)\s+import\s+(.+)/);
+    if (fromImportMatch) {
+      const module = fromImportMatch[1];
+      let namesStr = fromImportMatch[2].trim();
+
+      // Handle multi-line imports with parentheses
+      if (namesStr.startsWith('(')) {
+        namesStr = namesStr.slice(1);
+        while (i + 1 < lines.length && !namesStr.includes(')')) {
+          i++;
+          namesStr += ' ' + lines[i].trim();
+        }
+        namesStr = namesStr.replace(')', '');
+      }
+
+      const names = namesStr
+        .split(',')
+        .map(s => s.trim().split(/\s+as\s+/)[0])
+        .filter(Boolean);
+
+      skeleton.imports.push({ module, names });
+      pendingDecorators = [];
+      continue;
+    }
+
+    // async def / def
+    const funcMatch = line.match(/^(?:async\s+)?def\s+(\w+)\s*\(([^)]*)\)/);
+    if (funcMatch) {
+      // For multi-line params, grab until closing paren
+      let params = funcMatch[2];
+      if (!line.includes(')')) {
+        let j = i + 1;
+        while (j < lines.length && !lines[j].includes(')')) {
+          params += ' ' + lines[j].trim();
+          j++;
+        }
+        if (j < lines.length) {
+          params += ' ' + lines[j].split(')')[0].trim();
+        }
+      }
+      // Clean up params: remove type annotations and defaults for brevity
+      params = params.replace(/\s+/g, ' ').trim();
+
+      skeleton.functions.push({
+        name: funcMatch[1],
+        line: lineNum,
+        decorators: [...pendingDecorators],
+        params,
+      });
+      pendingDecorators = [];
+      continue;
+    }
+
+    // class Name(bases):
+    const classMatch = line.match(/^class\s+(\w+)\s*(?:\(([^)]*)\))?\s*:/);
+    if (classMatch) {
+      const bases = classMatch[2]
+        ? classMatch[2].split(',').map(s => s.trim()).filter(Boolean)
+        : [];
+
+      skeleton.classes.push({
+        name: classMatch[1],
+        line: lineNum,
+        decorators: [...pendingDecorators],
+        bases,
+      });
+      pendingDecorators = [];
+      continue;
+    }
+
+    // Top-level assignments (constants)
+    const assignMatch = line.match(/^[A-Za-z_]\w*\s*[=:]/) || line.match(/^[A-Za-z_]\w*\s*:/);
+    if (assignMatch) {
+      skeleton.constants++;
+      pendingDecorators = [];
+      continue;
+    }
+
+    // Reset decorators for any other top-level statement
+    pendingDecorators = [];
+  }
+
+  return skeleton;
+};
+
+/**
+ * Format Python skeleton for prompt output
+ * @param {Object} skeleton - Skeleton data object
+ * @returns {string}
+ */
+export const formatSkeletonForPrompt = (skeleton) => {
+  if (!skeleton) return '';
+
+  const lines = [];
+
+  if (skeleton.imports.length > 0) {
+    const local = skeleton.imports.filter(i => i.module.startsWith('.'));
+    const extCount = skeleton.imports.length - local.length;
+    const parts = [];
+    if (extCount > 0) parts.push(`${extCount} ext`);
+    parts.push(...local.map(i => i.module));
+    lines.push(`imports: ${parts.join(', ')}`);
+  }
+
+  if (skeleton.classes.length > 0) {
+    const classList = skeleton.classes.map(c => {
+      const parts = [c.name];
+      if (c.decorators.length > 0) parts.push(`@${c.decorators[0]}`);
+      if (c.bases.length > 0) parts.push(`(${c.bases.join(',')})`);
+      return `${parts.join(' ')}:${c.line}`;
+    }).join(', ');
+    lines.push(`classes: ${classList}`);
+  }
+
+  if (skeleton.functions.length > 0) {
+    const funcList = skeleton.functions.map(f => {
+      const deco = f.decorators.length > 0 ? `@${f.decorators[0]} ` : '';
+      return `${deco}${f.name}:${f.line}`;
+    }).join(', ');
+    lines.push(`fn: ${funcList}`);
+  }
+
+  return lines.join('\n');
+};

package/src/walker.js

@@ -0,0 +1,54 @@
+import { readdirSync } from 'fs';
+import { join } from 'path';
+import { isBabelParseable } from './parsers/babel.js';
+import { isPythonParseable } from './parsers/python.js';
+
+const SKIP_DIRECTORIES = new Set([
+  'node_modules',
+  'dist',
+  '.git',
+  'target',
+  'build',
+  '.next',
+  '.turbo',
+  'out',
+  'coverage',
+  '.cache',
+  '__pycache__',
+  '.venv',
+  'venv',
+  '.idea',
+  '.vscode',
+]);
+
+export function collectFiles(dir, rootDir = dir, files = []) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return files;
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      if (!SKIP_DIRS_OR_DOT(entry.name)) {
+        collectFiles(fullPath, rootDir, files);
+      }
+    } else if (entry.isFile() && isParseable(fullPath)) {
+      const relativePath = fullPath.slice(rootDir.length + 1);
+      files.push({ path: fullPath, relativePath });
+    }
+  }
+
+  return files;
+}
+
+function SKIP_DIRS_OR_DOT(name) {
+  return SKIP_DIRECTORIES.has(name) || name.startsWith('.');
+}
+
+function isParseable(path) {
+  return isBabelParseable(path) || isPythonParseable(path);
+}

package/test/fixtures/sample.js

@@ -0,0 +1,18 @@
+import React, { useState, useEffect } from 'react';
+import { fetchData } from './api';
+
+const API_URL = 'https://example.com';
+
+export function MyComponent({ id }) {
+  const [data, setData] = useState(null);
+
+  useEffect(() => {
+    fetchData(id).then(setData);
+  }, [id]);
+
+  return <div>{data}</div>;
+}
+
+const helper = (x) => x * 2;
+
+export default MyComponent;

package/test/fixtures/sample.py

@@ -0,0 +1,25 @@
+import os
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+MAX_RETRIES = 3
+DEFAULT_TIMEOUT = 30
+
+class BaseProcessor:
+    pass
+
+@dataclass
+class Config(BaseProcessor):
+    name: str
+    value: int
+
+def process_data(items, timeout=30):
+    pass
+
+async def fetch_remote(url, **kwargs):
+    pass
+
+@app.route('/api')
+def api_handler(request):
+    pass

package/test/test.js

@@ -0,0 +1,102 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import { readFileSync } from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+import { compactFile, compactProject } from '../src/index.js';
+import { extractSkeleton as extractBabelSkeleton } from '../src/parsers/babel.js';
+import { extractSkeleton as extractPythonSkeleton, formatSkeletonForPrompt as formatPythonSkeleton } from '../src/parsers/python.js';
+import { collectFiles } from '../src/walker.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const fixturesDir = resolve(__dirname, 'fixtures');
+
+describe('babel parser', () => {
+  it('extracts skeleton from JS file', () => {
+    const code = readFileSync(resolve(fixturesDir, 'sample.js'), 'utf-8');
+    const skeleton = extractBabelSkeleton(code, 'sample.js');
+
+    assert.ok(skeleton);
+    assert.ok(skeleton.imports.length > 0);
+    assert.ok(skeleton.components.length > 0);
+    assert.ok(skeleton.functions.length > 0);
+    assert.strictEqual(skeleton.hooks.useState, 1);
+    assert.strictEqual(skeleton.hooks.useEffect.length, 1);
+  });
+});
+
+describe('python parser', () => {
+  it('extracts skeleton from Python file', () => {
+    const code = readFileSync(resolve(fixturesDir, 'sample.py'), 'utf-8');
+    const skeleton = extractPythonSkeleton(code, 'sample.py');
+
+    assert.ok(skeleton);
+    assert.strictEqual(skeleton.imports.length, 4);
+    assert.strictEqual(skeleton.functions.length, 3);
+    assert.strictEqual(skeleton.classes.length, 2);
+    assert.ok(skeleton.constants >= 2);
+  });
+
+  it('parses decorators', () => {
+    const code = readFileSync(resolve(fixturesDir, 'sample.py'), 'utf-8');
+    const skeleton = extractPythonSkeleton(code, 'sample.py');
+
+    const config = skeleton.classes.find(c => c.name === 'Config');
+    assert.ok(config);
+    assert.deepStrictEqual(config.decorators, ['dataclass']);
+    assert.deepStrictEqual(config.bases, ['BaseProcessor']);
+
+    const apiHandler = skeleton.functions.find(f => f.name === 'api_handler');
+    assert.ok(apiHandler);
+    assert.deepStrictEqual(apiHandler.decorators, ['app.route']);
+  });
+
+  it('formats skeleton', () => {
+    const code = readFileSync(resolve(fixturesDir, 'sample.py'), 'utf-8');
+    const skeleton = extractPythonSkeleton(code, 'sample.py');
+    const output = formatPythonSkeleton(skeleton);
+
+    assert.ok(output.includes('imports:'));
+    assert.ok(output.includes('classes:'));
+    assert.ok(output.includes('fn:'));
+  });
+});
+
+describe('walker', () => {
+  it('collects files from fixtures', () => {
+    const files = collectFiles(fixturesDir);
+    assert.ok(files.length >= 2);
+    assert.ok(files.some(f => f.relativePath.endsWith('.js')));
+    assert.ok(files.some(f => f.relativePath.endsWith('.py')));
+  });
+});
+
+describe('compactFile', () => {
+  it('compacts a JS file', () => {
+    const code = readFileSync(resolve(fixturesDir, 'sample.js'), 'utf-8');
+    const result = compactFile('sample.js', code);
+    assert.ok(result.skeleton);
+    assert.ok(result.formatted.length > 0);
+  });
+
+  it('compacts a Python file', () => {
+    const code = readFileSync(resolve(fixturesDir, 'sample.py'), 'utf-8');
+    const result = compactFile('sample.py', code);
+    assert.ok(result.skeleton);
+    assert.ok(result.formatted.length > 0);
+  });
+});
+
+describe('compactProject', () => {
+  it('compacts the fixtures directory', () => {
+    const result = compactProject(fixturesDir);
+    assert.ok(result.output.length > 0);
+    assert.ok(result.output.includes('## sample.js'));
+    assert.ok(result.output.includes('## sample.py'));
+    assert.ok(result.stats.files >= 2);
+    assert.ok(result.stats.rawTokens > 0);
+    assert.ok(result.stats.compactedTokens > 0);
+    assert.ok(result.stats.compactedTokens < result.stats.rawTokens);
+  });
+});