catport 1.0.0

package/src/config/options.js

@@ -0,0 +1,181 @@
+export const OPTIONS = {
+  help: {
+    short: 'h',
+    type: 'boolean',
+    desc: 'Display this help message and exit.',
+    category: 'General'
+  },
+  version: {
+    short: 'V',
+    type: 'boolean',
+    desc: 'Display version information and exit.',
+    category: 'General'
+  },
+  verbose: {
+    short: 'v',
+    type: 'boolean',
+    desc: 'Enable verbose logging to stderr.',
+    category: 'General'
+  },
+  output: {
+    short: 'o',
+    type: 'string',
+    desc: 'Write output to <FILE> instead of stdout.',
+    metavar: 'FILE',
+    category: 'General'
+  },
+  format: {
+    short: 'f',
+    type: 'string',
+    desc: 'Output format: "md", "xml", "json", "yaml", or "multipart".',
+    default: 'md',
+    metavar: 'FMT',
+    category: 'Bundling'
+  },
+  replyFormat: {
+    short: 'R',
+    type: 'string',
+    desc: 'Instruct LLM to reply in this format (default: same as output).',
+    metavar: 'FMT',
+    category: 'Bundling'
+  },
+  context: {
+    short: 'C',
+    type: 'string',
+    desc: 'Prepend a custom context block to the header.',
+    metavar: 'TEXT',
+    category: 'Bundling'
+  },
+  task: {
+    short: 'T',
+    type: 'string',
+    desc: 'Append a specific task instruction to the footer.',
+    metavar: 'TEXT',
+    category: 'Bundling'
+  },
+  noInstruct: {
+    short: 'I',
+    type: 'boolean',
+    desc: 'Disable automatic "how to use" instructions.',
+    category: 'Bundling'
+  },
+  noStructure: {
+    short: 'n',
+    type: 'boolean',
+    desc: 'Disable directory structure generation.',
+    category: 'Bundling'
+  },
+  listDirs: {
+    short: 'l',
+    type: 'boolean',
+    desc: 'Include directories in the structure listing.',
+    category: 'Bundling'
+  },
+  skeleton: {
+    short: 'k',
+    type: 'boolean',
+    desc: 'Output directory structure only, omitting content.',
+    category: 'Bundling'
+  },
+  extensions: {
+    short: 'e',
+    type: 'string',
+    desc: 'Filter by comma-separated extensions (e.g. "js,ts").',
+    metavar: 'LIST',
+    category: 'Bundling'
+  },
+  ignore: {
+    short: 'i',
+    type: 'array',
+    desc: 'Add a glob pattern to the ignore list.',
+    metavar: 'GLOB',
+    category: 'Bundling'
+  },
+  noIgnore: {
+    short: 'u',
+    type: 'boolean',
+    desc: 'Unrestricted mode. Ignore .gitignore and defaults.',
+    category: 'Bundling'
+  },
+  gitDiff: {
+    short: 'g',
+    type: 'string',
+    desc: 'Bundle only files changed relative to <REF>.',
+    metavar: 'REF',
+    category: 'Bundling',
+    optional: true
+  },
+  budget: {
+    short: 'b',
+    type: 'number',
+    desc: 'Stop processing after <INT> tokens.',
+    metavar: 'INT',
+    category: 'Bundling'
+  },
+  priority: {
+    short: 'p',
+    type: 'array',
+    desc: 'Set priority rules (e.g. "README.md:100").',
+    metavar: 'RULE',
+    category: 'Bundling'
+  },
+  optimize: {
+    short: 'O',
+    type: 'string',
+    desc: 'Mode ("whitespace", "comments", "minify") OR shell command like ("terser" or "wc -l {}").',
+    default: 'none',
+    metavar: 'MODE|CMD',
+    category: 'Bundling'
+  },
+  maxSize: {
+    short: 'S',
+    type: 'string',
+    desc: 'Max file size to process (e.g. "1MB", "500KB").',
+    default: '10MB',
+    metavar: 'SIZE',
+    category: 'Bundling'
+  },
+  charsPerToken: {
+    short: 'c',
+    type: 'number',
+    desc: 'Ratio for token estimation (default: 4.2).',
+    default: 4.2,
+    metavar: 'NUM',
+    category: 'Bundling'
+  },
+  concurrency: {
+    short: 'P',
+    type: 'number',
+    desc: 'Maximum concurrent file reads.',
+    default: 32,
+    metavar: 'INT',
+    category: 'Bundling'
+  },
+  xmlMode: {
+    short: 'X',
+    type: 'string',
+    desc: 'XML strategy: "auto", "cdata", or "escape".',
+    metavar: 'MODE',
+    category: 'Bundling'
+  },
+  extract: {
+    short: 'x',
+    type: 'boolean',
+    desc: 'Switch to extraction mode.',
+    category: 'Extraction'
+  },
+  extractDir: {
+    short: 'd',
+    type: 'string',
+    desc: 'Target directory for extracted files.',
+    default: '.',
+    metavar: 'DIR',
+    category: 'Extraction'
+  },
+  unsafe: {
+    short: 'U',
+    type: 'boolean',
+    desc: 'Disable path traversal protection.',
+    category: 'Extraction'
+  }
+};

package/src/core/analyzer.js

@@ -0,0 +1,23 @@
+export const Analyzer = {
+  isBinary: (buf) => {
+    return buf.indexOf(0) !== -1;
+  },
+
+  countTokens: (content, charsPerToken) => {
+    // Fallback to 1 to avoid division by zero or negative tokens
+    const cpt = charsPerToken || 4.2;
+    return Math.max(1, Math.ceil(content.length / cpt));
+  },
+
+  getPriority: (path, rules = []) => {
+    if (!rules || rules.length === 0) {
+      return 1; // Default low priority
+    }
+    for (const { regex, score } of rules) {
+      if (regex && regex.test(path)) {
+        return score;
+      }
+    }
+    return 1;
+  }
+};

package/src/core/bundler.js

@@ -0,0 +1,165 @@
+import { basename, resolve, dirname } from 'node:path';
+import { Logger as DefaultLoggerFactory } from '../utils/logger.js';
+import { Scanner as DefaultScanner } from './scanner.js';
+import { Analyzer as DefaultAnalyzer } from './analyzer.js';
+import { Processor as DefaultProcessor } from './processor.js';
+import { Formatter as DefaultFormatter } from '../formatters/index.js';
+import { Optimizer as DefaultOptimizer } from '../optimizers/index.js';
+import { Path } from '../utils/path.js';
+import { FORMAT, LOG } from '../config/constants.js';
+
+export const Bundler = {
+  async run(config, io, services = {}) {
+    const {
+      scanner = DefaultScanner,
+      analyzer = DefaultAnalyzer,
+      processor = DefaultProcessor,
+      formatter = DefaultFormatter,
+      optimizer = DefaultOptimizer,
+      logger = null
+    } = services;
+
+    const log = logger || DefaultLoggerFactory(config.logLevel || LOG.WARN, io);
+    const fmt = formatter.get(config.format);
+
+    const priorityRules = (config.priorityRules || []).map(r => ({
+      regex: Path.toRegex(r.pattern)?.regex,
+      score: r.score
+    })).filter(r => r.regex);
+
+    const candidates = [];
+    const structure = [];
+
+    for await (const item of scanner.scan(config, io)) {
+      if (item.isDir) {
+        if (config.listDirs) {
+          structure.push(`${item.rel}/`);
+        }
+      } else {
+        item.priority = analyzer.getPriority(item.rel, priorityRules);
+        candidates.push(item);
+        structure.push(item.rel);
+      }
+    }
+
+    if (candidates.length === 0 && structure.length === 0) {
+      log.warn('No files matched.');
+      return { files: 0, tokens: 0 };
+    }
+
+    candidates.sort((a, b) => {
+      const pDiff = b.priority - a.priority;
+      if (pDiff !== 0) {
+        return pDiff;
+      }
+      return a.rel.localeCompare(b.rel);
+    });
+
+    const BATCH_SIZE = config.concurrency || 32;
+    const hasBudget = config.budget > 0;
+
+    const processBatchItem = async (item) => {
+      const result = await processor.run(item, config, io, { analyzer, optimizer });
+      if (result.error) {
+        log.warn(`Failed to read ${item.rel}: ${result.error.message}`);
+        return null;
+      }
+      return result;
+    };
+
+    const rootName = config.paths?.[0] ? basename(resolve(config.paths[0])) : 'project';
+
+    if (config.output) {
+      const outDir = dirname(resolve(config.output));
+      await io.mkdir(outDir);
+    }
+
+    const outStream = config.output ? io.createWriteStream(config.output) : {
+      write: io.writeStdout,
+      end: () => {}
+    };
+
+    let instructionText = '';
+    if (config.instruct) {
+      const replyFmt = formatter.get(config.replyFormat || config.format);
+      if (replyFmt.getInstruction) {
+        instructionText = replyFmt.getInstruction();
+      }
+    }
+
+    const head = fmt.header({
+      name: rootName,
+      tree: config.structure ? structure.join('\n') : null,
+      context: config.context,
+      task: config.task
+    });
+
+    const foot = fmt.footer({
+      task: config.task,
+      instructionText
+    });
+
+    let usedTokens = analyzer.countTokens(head + foot, config.charsPerToken);
+    if (hasBudget && usedTokens >= config.budget) {
+      log.warn(`Budget exceeded by directory tree and metadata alone (${usedTokens} > ${config.budget}). Outputting skeleton only.`);
+    }
+
+    if ((hasBudget && usedTokens >= config.budget) || config.skeleton) {
+      const skeletonHead = fmt.header({
+        name: rootName,
+        tree: structure.join('\n'),
+        context: config.context,
+        task: config.task
+      });
+      outStream.write(skeletonHead);
+      outStream.write(foot);
+      if (config.output) {
+        outStream.end();
+      }
+      return { files: 0, tokens: usedTokens };
+    }
+
+    outStream.write(head);
+
+    const isJson = config.format === FORMAT.JSON;
+    let writtenCount = 0;
+
+    for (let i = 0; i < candidates.length; i += BATCH_SIZE) {
+      if (hasBudget && usedTokens >= config.budget) {
+        break;
+      }
+
+      const batch = candidates.slice(i, i + BATCH_SIZE);
+      const results = await Promise.all(batch.map(processBatchItem));
+
+      for (const res of results) {
+        if (!res) {
+          continue;
+        }
+
+        const formatted = fmt.file(res, config);
+        const prefix = (isJson && writtenCount > 0) ? ',\n' : '';
+        const fullBlock = prefix + formatted;
+        
+        const blockTokens = analyzer.countTokens(fullBlock, config.charsPerToken);
+
+        if (hasBudget && (usedTokens + blockTokens > config.budget)) {
+          log.debug(`Skipping ${res.rel} (budget exceeded)`);
+          continue;
+        }
+
+        usedTokens += blockTokens;
+        outStream.write(fullBlock);
+        writtenCount++;
+      }
+    }
+
+    outStream.write(foot);
+
+    if (config.output) {
+      outStream.end();
+    }
+
+    return { files: writtenCount, tokens: usedTokens };
+  }
+};

package/src/core/extractor.js

@@ -0,0 +1,76 @@
+import { resolve, join, dirname, relative, isAbsolute } from 'node:path';
+import { Path } from '../utils/path.js';
+import { Logger as DefaultLoggerFactory } from '../utils/logger.js';
+import { Formatter as DefaultFormatter } from '../formatters/index.js';
+import { LOG } from '../config/constants.js';
+
+export const Extractor = {
+  async run(config, io, services = {}) {
+    const {
+      formatter = DefaultFormatter,
+      logger = null
+    } = services;
+
+    if (config.safeMode === undefined) {
+      config.safeMode = true;
+    }
+
+    const log = logger || DefaultLoggerFactory(config.logLevel || LOG.WARN, io);
+    let content = '';
+
+    if (config.paths && config.paths.length > 0) {
+      content = await io.readText(config.paths[0]);
+    } else {
+      log.info('Reading stdin...');
+      for await (const chunk of io.readStdin()) {
+        content += chunk;
+      }
+    }
+
+    if (!content.trim()) {
+      throw new Error('Empty input');
+    }
+
+    const fmt = formatter.detect(content);
+    const files = fmt.parse(content, log);
+
+    if (files.length === 0) {
+      log.warn('No files found.');
+      return;
+    }
+
+    const outDir = resolve(config.extractDir || '.');
+    if (config.extractDir) {
+      await io.mkdir(outDir);
+    }
+
+    log.info(`Extracting ${files.length} files to ${outDir}`);
+
+    let count = 0;
+    for (const f of files) {
+      const safeRel = Path.sanitize(f.path);
+
+      if (config.safeMode) {
+        if (isAbsolute(safeRel) || safeRel.includes('../') || safeRel.startsWith('..')) {
+          log.warn(`[SECURITY] Skipping traversal attempt: ${f.path}`);
+          continue;
+        }
+
+        const abs = join(outDir, safeRel);
+        const resolved = resolve(abs);
+        const rel = relative(outDir, resolved);
+
+        if (isAbsolute(rel) || rel.startsWith('..')) {
+          log.warn(`[SECURITY] Skipping traversal attempt: ${f.path}`);
+          continue;
+        }
+      }
+
+      const abs = join(outDir, safeRel);
+      await io.mkdir(dirname(abs));
+      await io.writeFile(abs, f.content);
+      count++;
+    }
+    log.info(`Extracted ${count} files.`);
+  }
+};

package/src/core/ignore.js

@@ -0,0 +1,65 @@
+
+
+import { relative } from 'node:path';
+import { Path } from '../utils/path.js';
+
+const createMatcher = (existingRules, newPatterns) => {
+  const newRules = newPatterns.map(Path.toRegex).filter(Boolean);
+  const rules = existingRules ? existingRules.concat(newRules) : newRules;
+
+  return {
+    test: (rel) => {
+      let ignored = false;
+      for (const { regex, isNegated } of rules) {
+        if (regex.test(rel)) {
+          ignored = !isNegated;
+        }
+      }
+      return ignored;
+    },
+    
+    extend: (morePatterns) => {
+      return createMatcher(rules, morePatterns);
+    }
+  };
+};
+
+export const Ignore = {
+  parse: (content, dir, root) => {
+    if (!content) {
+      return [];
+    }
+    
+    const lines = content.split(/\r?\n/)
+      .map(l => l.trim())
+      .filter(l => l && !l.startsWith('#'));
+
+    return lines.map(l => {
+      let p = l;
+      const isNeg = p.startsWith('!');
+      if (isNeg) {
+        p = p.slice(1);
+      }
+
+      // Standard git behavior: "foo/bar" is anchored to dir, "foo" is recursive.
+      // "foo/" is recursive directory.
+      // Only anchor if it starts with slash OR has a slash in the middle
+      const hasInternalSlash = p.slice(0, -1).includes('/');
+      
+      if (p.startsWith('/') || hasInternalSlash) {
+        const relDir = relative(root, dir).replace(/\\/g, '/');
+        if (relDir) {
+          const cleanP = p.startsWith('/') ? p : '/' + p;
+          p = `/${relDir}${cleanP}`;
+        } else if (!p.startsWith('/')) {
+          p = '/' + p;
+        }
+      }
+      return isNeg ? '!' + p : p;
+    });
+  },
+
+  create: (patterns) => {
+    return createMatcher([], patterns);
+  }
+};

package/src/core/processor.js

@@ -0,0 +1,59 @@
+const quotePath = (path) => {
+  if (process.platform === 'win32') {
+    return `"${path.replace(/"/g, '""')}"`;
+  }
+  return `'${path.replace(/'/g, "'\\''")}'`;
+};
+
+export const Processor = {
+  run: async (item, config, io, services) => {
+    const { analyzer, optimizer } = services;
+
+    if (config.skeleton) {
+      return { ...item, content: '' };
+    }
+
+    try {
+      const SAMPLE_SIZE = 1024 * 8;
+      const sample = await io.readSample(item.path, SAMPLE_SIZE);
+
+      if (analyzer.isBinary(sample)) {
+        return { ...item, content: '(binary omitted)' };
+      }
+
+      let raw;
+      if (sample.length < SAMPLE_SIZE) {
+        raw = sample.toString('utf8');
+      } else {
+        const maxBytes = config.maxSize !== undefined ? config.maxSize : 0;
+        const stats = await io.stat(item.path);
+        
+        if (maxBytes > 0 && stats.size > maxBytes && !config.optimizeCmd) {
+          return { ...item, content: `(file too large for processing: ${stats.size} bytes)` };
+        }
+
+        raw = await io.readText(item.path);
+      }
+      
+      let content;
+      if (config.optimizeCmd) {
+        // Standard Unix convention: {} represents the file path
+        if (config.optimizeCmd.includes('{}')) {
+          const safePath = quotePath(item.path);
+          const cmd = config.optimizeCmd.split('{}').join(safePath);
+          content = await io.exec(cmd); 
+        } else {
+          // Pipe content to stdin
+          content = await io.execPipe(config.optimizeCmd, raw);
+        }
+      } else {
+        const ext = item.rel.split('.').pop();
+        content = optimizer.run(raw, ext, config.optimize);
+      }
+      
+      return { ...item, content };
+    } catch (err) {
+      return { error: err };
+    }
+  }
+};