catport 1.0.0

package/src/core/scanner.js

@@ -0,0 +1,184 @@
+import { join, relative, resolve, isAbsolute, dirname } from 'node:path';
+import { Ignore } from './ignore.js';
+
+export const Scanner = {
+  async *scan(config, io) {
+    const cwd = io.cwd ? io.cwd() : process.cwd();
+    const baseIgnore = Ignore.create(config.ignore || []);
+
+    if (config.gitFiles) {
+      yield* Scanner._scanDirect(config.gitFiles, config, io, cwd, baseIgnore);
+      return;
+    }
+
+    const roots = (config.paths && config.paths.length) ? config.paths : ['.'];
+    const visited = new Set();
+
+    for (const root of roots) {
+      const full = resolve(cwd, root);
+      let stats;
+      try {
+        stats = await io.stat(full);
+      } catch {
+        continue;
+      }
+
+      if (stats.isDirectory()) {
+        yield* Scanner._walk(full, baseIgnore, config, io, full, cwd, visited);
+      } else {
+        const rel = relative(dirname(full), full).replace(/\\/g, '/');
+        
+        if (baseIgnore.test(rel)) {
+          continue;
+        }
+
+        if (config.extSet && config.extSet.size > 0) {
+          const ext = full.split('.').pop().toLowerCase();
+          if (!config.extSet.has(ext)) {
+            continue;
+          }
+        }
+
+        yield {
+          path: full,
+          rel: rel,
+          isDir: false
+        };
+      }
+    }
+  },
+
+  async *_scanDirect(files, config, io, cwd, baseIgnore) {
+    // If specific paths were requested (e.g. "catport src/ -g"), we must intersect
+    // the gitFiles list with those path scopes.
+    const roots = (config.paths && config.paths.length) 
+      ? config.paths.map(p => resolve(cwd, p)) 
+      : [cwd];
+
+    for (const full of files) {
+      // relative() returns a path starting with '..' if outside, or an absolute path on different drives (Win32)
+      const inScope = roots.some(root => {
+        const rel = relative(root, full);
+        return !rel.startsWith('..') && !isAbsolute(rel);
+      });
+      if (!inScope) {
+        continue;
+      }
+
+      // Git reports deleted files, we must skip them.
+      try {
+        const stats = await io.stat(full);
+        if (!stats.isFile()) {
+          continue;
+        }
+      } catch {
+        continue;
+      }
+
+      const fileRoot = roots.find(root => {
+        const rel = relative(root, full);
+        return !rel.startsWith('..') && !isAbsolute(rel);
+      }) || cwd;
+
+      const rel = relative(fileRoot, full).replace(/\\/g, '/');
+
+      if (baseIgnore.test(rel)) {
+        continue;
+      }
+
+      if (config.extSet && config.extSet.size > 0) {
+        const ext = full.split('.').pop().toLowerCase();
+        if (!config.extSet.has(ext)) {
+          continue;
+        }
+      }
+
+      yield {
+        path: full,
+        rel: rel,
+        isDir: false
+      };
+    }
+  },
+
+  async *_walk(dir, ignore, config, io, root, cwd, visited) {
+    // Cycle detection
+    try {
+      const stats = await io.stat(dir);
+      if (stats.dev !== undefined && stats.ino !== undefined) {
+        const key = `${stats.dev}:${stats.ino}`;
+        if (visited.has(key)) {
+          return;
+        }
+        visited.add(key);
+      }
+    } catch {
+      return;
+    }
+
+    let currentIgnore = ignore;
+
+    if (!config.noIgnore) {
+      try {
+        const gitignore = await io.readText(join(dir, '.gitignore'));
+        const scopedPatterns = Ignore.parse(gitignore, dir, cwd);
+        currentIgnore = ignore.extend(scopedPatterns);
+      } catch {
+        // No .gitignore found, proceed
+      }
+    }
+
+    let entries = [];
+    try {
+      entries = await io.readdir(dir);
+    } catch {
+      return;
+    }
+
+    for (const e of entries) {
+      const full = join(dir, e.name);
+      const rel = relative(root, full).replace(/\\/g, '/');
+
+      if (e.name === '.git') {
+        continue;
+      }
+
+      if (currentIgnore.test(rel)) {
+        continue;
+      }
+
+      let isDir = e.isDirectory();
+      if (e.isSymbolicLink()) {
+        try {
+          const stats = await io.stat(full);
+          isDir = stats.isDirectory();
+        } catch {
+          // Broken link or permission error
+          isDir = false;
+        }
+      }
+
+      if (isDir) {
+        yield {
+          path: full,
+          rel: rel,
+          isDir: true
+        };
+        yield* Scanner._walk(full, currentIgnore, config, io, root, cwd, visited);
+      } else {
+        if (config.extSet && config.extSet.size > 0) {
+          const ext = e.name.split('.').pop().toLowerCase();
+          if (!config.extSet.has(ext)) {
+            continue;
+          }
+        }
+
+        yield {
+          path: full,
+          rel: rel,
+          isDir: false
+        };
+      }
+    }
+  }
+};

package/src/formatters/index.js

@@ -0,0 +1,78 @@
+import { Markdown } from './markdown.js';
+import { Xml } from './xml.js';
+import { Json } from './json.js';
+import { Yaml } from './yaml.js';
+import { Multipart } from './multipart.js';
+import { FORMAT } from '../config/constants.js';
+
+const REGISTRY = {
+  [FORMAT.MD]: Markdown,
+  [FORMAT.XML]: Xml,
+  [FORMAT.JSON]: Json,
+  [FORMAT.YAML]: Yaml,
+  [FORMAT.MULTIPART]: Multipart
+};
+
+export const Formatter = {
+  register: (key, impl) => {
+    REGISTRY[key] = impl;
+  },
+  get: (type) => {
+    return REGISTRY[type] || Markdown;
+  },
+  detect: (content) => {
+    const t = content.trim();
+
+    if (t.startsWith('{')) {
+      return Json;
+    }
+    if (t.startsWith('<')) {
+      return Xml;
+    }
+    if (t.startsWith('meta:')) {
+      return Yaml;
+    }
+    if (t.startsWith('MIME-Version: 1.0')) {
+      return Multipart;
+    }
+
+    const codeBlockMatch = t.match(/```(\w*)\n([\s\S]*?)```/);
+    if (codeBlockMatch) {
+      const lang = codeBlockMatch[1].toLowerCase();
+      const body = codeBlockMatch[2].trim();
+
+      if (lang === 'json') {
+        return Json;
+      }
+      if (lang === 'xml') {
+        return Xml;
+      }
+      if (lang === 'yaml' || lang === 'yml') {
+        return Yaml;
+      }
+
+      if (body.startsWith('{')) {
+        return Json;
+      }
+      if (body.startsWith('<')) {
+        return Xml;
+      }
+      if (body.startsWith('meta:')) {
+        return Yaml;
+      }
+      if (body.startsWith('files:')) {
+        return Yaml;
+      }
+    }
+
+    const sample = t.slice(0, 1024);
+    if (sample.includes('<?xml') || sample.includes('<project name=')) {
+      return Xml;
+    }
+    if (sample.includes('"files": [')) {
+      return Json;
+    }
+
+    return Markdown;
+  }
+};

package/src/formatters/json.js

@@ -0,0 +1,284 @@
+const ESCAPE_MAP = Object.freeze({
+  '"': '\\"', '\\': '\\\\', '\b': '\\b', '\f': '\\f',
+  '\n': '\\n', '\r': '\\r', '\t': '\\t'
+});
+
+const normalizeJsonLike = (() => {
+  const R_DQ = /"(?:\\[\s\S]|[^\\"])*"/;
+  const R_SQ = /'(?:\\[\s\S]|[^\\'])*'/;
+  const R_CMT = /\/\/[^\n]*|\/\*[\s\S]*?\*\//;
+
+  // We explicitly exclude control chars, quotes, and JSON structure chars from keys.
+  const R_KEY_GENERIC = /(\s*)([^\s"':,{}[\]]+)\s*(:)/;
+
+  // Trailing Comma
+  const R_TRL = /(,)\s*([}\]])/;
+
+  // Order is critical: strings -> comments -> keys -> trailing commas
+  const MASTER = new RegExp(
+    `(${R_DQ.source})|(${R_SQ.source})|(${R_CMT.source})|` +
+    `${R_KEY_GENERIC.source}|${R_TRL.source}`,
+    'g'
+  );
+
+  const SQ_ESCAPE_REGEX = /["\b\f\n\r\t]/g;
+  const esc = (c) => ESCAPE_MAP[c];
+
+  return (src) => {
+    if (!src || typeof src !== 'string') {
+      return src;
+    }
+
+    return src.replace(MASTER, (match, ...args) => {
+      const [
+        doubleQuotedString, singleQuotedString, comment,
+        keyPre, keyName, keyCol,       // R_KEY_GENERIC (3 groups)
+        trailingComma, trailingBracket // R_TRL (2 groups)
+      ] = args;
+
+      if (doubleQuotedString) {
+        return doubleQuotedString;
+      }
+      if (comment) {
+        return '';
+      }
+
+      if (singleQuotedString) {
+        let content = singleQuotedString.slice(1, -1);
+        content = content.replace(/\\'/g, "'");
+        content = content.replace(SQ_ESCAPE_REGEX, esc);
+        return `"${content}"`;
+      }
+
+      // Quote bare keys
+      if (keyName !== undefined) {
+        return `${keyPre}"${keyName}"${keyCol}`;
+      }
+
+      if (trailingComma) {
+        return trailingBracket;
+      }
+
+      return match;
+    });
+  };
+})();
+
+/**
+ * Extracts balanced JSON-like blocks from a string.
+ */
+const extractBalancedObjects = (text) => {
+  const results = [];
+  let depth = 0;
+  let inString = false;
+  let quoteChar = '';
+  let start = -1;
+  let escape = false;
+
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+
+    if (inString) {
+      if (escape) {
+        escape = false;
+      } else if (c === '\\') {
+        escape = true;
+      } else if (c === quoteChar) {
+        inString = false;
+      }
+    } else {
+      if (c === '"' || c === "'") {
+        inString = true;
+        quoteChar = c;
+      } else if (c === '{') {
+        if (depth === 0) {
+          start = i;
+        }
+        depth++;
+      } else if (c === '}') {
+        if (depth > 0) {
+          depth--;
+          if (depth === 0) {
+            results.push(text.slice(start, i + 1));
+          }
+        }
+      }
+    }
+  }
+  return results;
+};
+
+export const Json = {
+  getInstruction: () => `**CRITICAL:** Rules for every file (follow strictly and EXACTLY):
+- Return valid JSON.
+- The root MUST be an object containing a "files" array.
+- Each item in the array MUST have "path" and "content" fields.
+- Properly escape strings (e.g., quotes, newlines).
+
+## Expected Schema
+
+\`\`\`json
+{
+  "type": "object",
+  "required": ["files"],
+  "properties": {
+    "files": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["path", "content"],
+        "properties": {
+          "path": { "type": "string", "description": "Relative file path" },
+          "content": { "type": "string", "description": "File content" }
+        }
+      }
+    }
+  }
+}
+\`\`\`
+
+## Examples
+
+## Correct:
+
+{
+  "files": [
+    {
+      "path": "src/main.js",
+      "content": "console.log(\\"Hello\\");"
+    },
+    {
+      "path": "config.json",
+      "content": "{\\n  \\"debug\\": true\\n}"
+    }
+  ]
+}
+
+## Wrong:
+
+\`\`\`json
+[ { "path": ... } ]   ✗ (Root must be object with "files" key)
+\`\`\`
+
+{ "path": ... }       ✗ (Must be inside "files" array)
+
+Only output valid JSON. No explanations, no markdown, no extra text.`,
+
+  header: (m) => {
+    const meta = {
+      name: m.name,
+      context: m.context
+    };
+    if (m.tree) {
+      meta.tree = m.tree;
+    }
+    return `{\n  "meta": ${JSON.stringify(meta)},\n  "files": [\n`;
+  },
+
+  file: (f) => {
+    return `    ${JSON.stringify({
+      path: f.rel,
+      content: f.content
+    })}`;
+  },
+
+  footer: (m) => {
+    let out = '\n  ]';
+    if (m.task) {
+      out += `,\n  "task": ${JSON.stringify(m.task)}`;
+    }
+    if (m.instructionText) {
+      out += `,\n  "instruction": ${JSON.stringify(m.instructionText)}`;
+    }
+    return out + '\n}';
+  },
+
+  parse: (txt, logger) => {
+    if (!txt) {
+      return [];
+    }
+    const clean = txt.trim();
+    if (!clean) {
+      return [];
+    }
+
+    const tryParse = (src) => {
+      try {
+        const obj = JSON.parse(src);
+        if (!obj || typeof obj !== 'object' || obj === null) {
+          return null;
+        }
+        if (!Array.isArray(obj.files)) {
+          return null;
+        }
+        const valid = obj.files.every((f) => {
+          return f &&
+            typeof f === 'object' &&
+            typeof f.path === 'string' &&
+            f.path.trim() !== '' &&
+            Object.hasOwn(f, 'content') &&
+            typeof f.content === 'string';
+        });
+        if (!valid) {
+          return null;
+        }
+        return obj.files;
+      } catch {
+        return null;
+      }
+    };
+
+    const candidates = [];
+
+    candidates.push(clean);
+    {
+      const normalized = normalizeJsonLike(clean);
+      if (normalized !== clean) {
+        candidates.push(normalized);
+      }
+    }
+
+    const mdBlocks = [...clean.matchAll(/(?:^|\n)[ \t]*```(?:json)?\s*\n?([\s\S]*?)\n?[ \t]*```/g)];
+    for (const m of mdBlocks) {
+      const block = m[1].trim();
+      if (block) {
+        candidates.push(block);
+        {
+          const norm = normalizeJsonLike(block);
+          if (norm !== block) {
+            candidates.push(norm);
+          }
+        }
+      }
+    }
+
+    const rawObjects = extractBalancedObjects(clean);
+    // Sort by length descending to prefer larger wrapping objects
+    rawObjects.sort((a, b) => b.length - a.length);
+
+    const NOISE_TRESHOLD = 20; // arbitrary min length to avoid noise
+    for (const objStr of rawObjects) {
+      if (objStr.length > NOISE_TRESHOLD) {
+        candidates.push(objStr);
+        {
+          const norm = normalizeJsonLike(objStr);
+          if (norm !== objStr) {
+            candidates.push(norm);
+          }
+        }
+      }
+    }
+
+    for (const candidate of candidates) {
+      const files = tryParse(candidate);
+      if (files !== null) {
+        return files;
+      }
+    }
+
+    if (logger) {
+      logger.warn('Failed to extract a files array from the JSON');
+    }
+    return [];
+  }
+};

package/src/formatters/markdown.js

@@ -0,0 +1,164 @@
+import { Path } from '../utils/path.js';
+
+export const FILE_MARKER = '### ◼◼◼ FILE:';
+export const INSTRUCTION_MARKER = '### ◼◼◼ END OF FILES - INSTRUCTIONS FOLLOW';
+const ESCAPED_MARKER = FILE_MARKER.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const FILE_MARKER_RE = new RegExp(`^\\s*${ESCAPED_MARKER}\\s*(.+)$`, 'gim');
+
+export const Markdown = {
+  getInstruction: () => `**CRITICAL:** Rules for every file (follow strictly and EXACTLY):
+- Output only blocks starting with "${FILE_MARKER} <path>" followed by a fenced code block.
+- Code block with language tag must follow immediately. Nothing else.
+- No bold, no "(updated / fixed)", no explanations, no extra text.
+
+## Examples
+
+## Correct:
+
+${FILE_MARKER} src/components/Button.tsx
+\`\`\`tsx
+import React from 'react';
+export const Button = () => <button>Click</button>;
+\`\`\`
+
+${FILE_MARKER} utils/helpers.ts
+\`\`\`ts
+export const format = (n: number) => n.toFixed(2);
+\`\`\`
+
+## Wrong:
+
+\`\`\`
+**src/app.ts**          ✗
+File: config.json       ✗
+Here is the update:     ✗
+\`\`\`
+
+Only output "${FILE_MARKER} <path>" followed by a fenced code block. Nothing else.`,
+
+  header: (m) => {
+    let out = `# ${m.name}\n`;
+    if (m.context) {
+      out += `> **Context**: ${m.context}\n`;
+    }
+    if (m.tree) {
+      out += `\n## Structure\n\`\`\`text\n${m.tree}\n\`\`\`\n\n`;
+    }
+    return out + '---\n\n';
+  },
+
+  file: (f) => {
+    const content = f.content || '';
+    const ext = f.rel.split('.').pop() ?? 'txt';
+    const existingFences = content.match(/^(`{3,}|~{3,})/gm) ?? [];
+    const longest = existingFences.reduce((len, m) => Math.max(len, m.length), 0);
+    const fenceLen = Math.max(3, longest + 1);
+    const fence = '`'.repeat(fenceLen);
+
+    return `${FILE_MARKER} ${f.rel}\n${fence}${ext}\n${content}\n${fence}\n\n`;
+  },
+
+  footer: (m) => {
+    let out = m.task ? `\n---\n> **Task**: ${m.task}\n` : '';
+    if (m.instructionText) {
+      out += `\n${INSTRUCTION_MARKER}\n${m.instructionText}\n`;
+    }
+    return out;
+  },
+
+  parse: (text, logger) => {
+    // Find the last occurrence of INSTRUCTION_MARKER to avoid splitting on markers inside file content
+    const lastInstructionIdx = text.lastIndexOf(INSTRUCTION_MARKER);
+    const processText = lastInstructionIdx !== -1 ? text.slice(0, lastInstructionIdx) : text;
+
+    const files = [];
+    let pos = 0;
+
+    while (true) {
+      FILE_MARKER_RE.lastIndex = pos;
+      const markerMatch = FILE_MARKER_RE.exec(processText);
+      if (!markerMatch) {
+        break;
+      }
+
+      const rawPath = markerMatch[1].trim()
+        .replace(/[:'"`]*$/, '')          // trailing : or quotes
+        .replace(/\s*\(.*\)$/, '')        // (updated), (copy), …
+        .replace(/\s*\[.*\]$/, '')        // [modified], …
+        .trim();
+
+      const path = Path.clean(rawPath);
+      if (!path) {
+        pos = markerMatch.index + markerMatch[0].length;
+        if (logger) {
+          logger.warn(`Skipping invalid path in marker at index ${markerMatch.index}: "${rawPath}"`);
+        }
+        continue;
+      }
+
+      const afterMarkerPos = markerMatch.index + markerMatch[0].length;
+      const rest = processText.slice(afterMarkerPos);
+
+      const openFenceMatch = rest.match(/^([ \t]*)(`{3,}|~{3,})/m);
+      if (!openFenceMatch) {
+        pos = afterMarkerPos;
+        if (logger) {
+          logger.warn(`Skipping file "${path}": Marker found but no fenced code block follows immediately.`);
+        }
+        continue;
+      }
+
+      const indent = openFenceMatch[1];
+      const fencePart = openFenceMatch[2];
+      const fenceChar = fencePart[0];
+      const fenceLen = fencePart.length;
+      const fenceRE = new RegExp(`^${indent}${fenceChar}{${fenceLen},}(?:[ \t]*|[ \t]+\\S.*)$`, 'm');
+
+      const fenceStartPos = afterMarkerPos + openFenceMatch.index + openFenceMatch[0].length;
+      const afterFence = rest.slice(openFenceMatch.index + openFenceMatch[0].length);
+      const langLineMatch = afterFence.match(/^(.*\r?\n)/);
+      const codeBlockStart = fenceStartPos + (langLineMatch?.[0].length ?? 0);
+
+      const closeMatch = fenceRE.exec(processText.slice(codeBlockStart));
+      if (!closeMatch) {
+        if (logger) {
+          logger.warn(`Skipping file "${path}": Code block not closed (reached end of input).`);
+        }
+        // Skip this file and continue with the next one
+        pos = afterMarkerPos;
+        continue;
+      }
+
+      const closePosAbs = codeBlockStart + closeMatch.index + closeMatch[0].length;
+
+      let content = processText.slice(codeBlockStart, codeBlockStart + closeMatch.index);
+
+      if (indent) {
+        const indentRE = new RegExp(`^${indent}`, 'gm');
+        content = content.replace(indentRE, '');
+      }
+
+      content = content
+        .replace(/\r\n/g, '\n')
+        .replace(/\r/g, '\n')
+        .trimEnd();
+
+      if (content.length > 0 || processText.slice(codeBlockStart, codeBlockStart + closeMatch.index).trim().length > 0) {
+        files.push({ path, content });
+      } else {
+        if (logger) {
+          logger.warn(`Skipping file "${path}": Empty code block extracted.`);
+        }
+      }
+
+      pos = closePosAbs;
+
+      const trailing = processText.slice(pos).match(/^\r?\n/);
+      if (trailing) {
+        pos += trailing[0].length;
+      }
+    }
+
+    return files;
+  }
+};