ctxman 3.0.0 → 3.1.0

package/lib/pack/renderers/xml.js

@@ -0,0 +1,85 @@
+/**
+ * XML style: Anthropic's long-context document format
+ *
+ *   <documents>
+ *   <document index="1">
+ *   <source>path</source>
+ *   <document_content>
+ *   ...raw file content...
+ *   </document_content>
+ *   </document>
+ *   </documents>
+ *
+ * File content is not XML-escaped (Claude reads raw code, and escaping every `<` and `&`
+ * costs tokens). Only a closing tag of the wrapper itself (`</document_content>`,
+ * `</document>`, `</documents>`) inside a file is neutralised by writing its `<` as `&lt;`,
+ * so a file cannot end its own document early. Paths and the tree are escaped.
+ */
+
+import {
+  joinSections,
+  omittedLines,
+  omittedSummary,
+  sourceLabel,
+  withoutFinalNewline,
+} from './common.js';
+
+const escapeXml = (text) => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+// `</document_content>`, `</document>` and `</documents>`, in any case, spaces before `>`
+const WRAPPER_CLOSING_TAG = /<(?=\/(?:document_content|documents?)\s*>)/gi;
+
+/**
+ * Neutralise wrapper closing tags inside file content
+ * @param {string} content
+ * @returns {string}
+ */
+export function protectContent(content) {
+  return content.replace(WRAPPER_CLOSING_TAG, '&lt;');
+}
+
+/**
+ * One `<document>` element
+ * @param {object} file - Document model file
+ * @param {number} index - Document number
+ * @returns {string}
+ */
+export function renderFile(file, index) {
+  return [
+    `<document index="${index}">`,
+    `<source>${escapeXml(sourceLabel(file))}</source>`,
+    '<document_content>',
+    protectContent(withoutFinalNewline(file.content)),
+    '</document_content>',
+    '</document>',
+  ].join('\n');
+}
+
+/**
+ * @param {object} doc - Document model (see common.js)
+ * @returns {string}
+ */
+export function render(doc) {
+  const documents = [
+    '<documents>',
+    ...doc.files.map((file, i) => renderFile(file, doc.firstIndex + i)),
+    '</documents>',
+  ].join('\n');
+
+  return joinSections([
+    doc.header,
+    doc.partLabel,
+    doc.tree !== null && `<directory_structure>\n${escapeXml(doc.tree)}\n</directory_structure>`,
+    documents,
+    doc.omitted &&
+      [
+        `<omitted_files count="${doc.omitted.count}" tokens="${doc.omitted.tokens}">`,
+        `${omittedSummary(doc.omitted)}:`,
+        ...omittedLines(doc.omitted).map(escapeXml),
+        '</omitted_files>',
+      ].join('\n'),
+    doc.instructions && `<instructions>\n${doc.instructions}\n</instructions>`,
+  ]);
+}
+
+export default { name: 'xml', extension: '.xml', render, renderFile };

package/lib/pack/select.js

@@ -0,0 +1,177 @@
+/**
+ * File selection for `pack`: the project scan (with .gitignore, .contextignore and
+ * .contextinclude) or an explicit list, narrowed by --include/--exclude globs
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { Scanner } from '../core/Scanner.js';
+import GitIgnoreParser from '../parsers/gitignore-parser.js';
+import FileUtils from '../utils/file-utils.js';
+
+/** Error with a machine-readable code (PATH_OUTSIDE_ROOT, ROOT_NOT_FOUND, ...) */
+export class PackError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.name = 'PackError';
+    this.code = code;
+  }
+}
+
+const toPosix = (relativePath) => relativePath.split(path.sep).join('/');
+
+/**
+ * Path of `target` relative to `root` when it lies strictly below it, else null
+ * (`..config` is a file name, `../config` is outside)
+ * @param {string} root - Absolute directory
+ * @param {string} target - Absolute path
+ * @returns {string|null}
+ */
+function relativeInside(root, target) {
+  const relative = path.relative(root, target);
+  if (relative === '' || path.isAbsolute(relative)) return null;
+  if (relative === '..' || relative.startsWith(`..${path.sep}`)) return null;
+  return relative;
+}
+
+/**
+ * Compile gitignore-style globs with the same rules as .contextinclude/.contextignore
+ * @param {string[]} globs
+ * @param {boolean} includeMode - true: a `!glob` excludes (as in .contextinclude)
+ * @returns {{test: function(string): boolean}|null} null when there are no globs
+ */
+function globMatcher(globs, includeMode) {
+  const patterns = globs.map((glob) => glob.trim()).filter(Boolean);
+  if (!patterns.length) return null;
+  // Only the pattern helpers are used: missing ignore files load nothing
+  const parser = new GitIgnoreParser('', '', '');
+  const compiled = patterns.map((pattern) => parser.convertToRegex(pattern));
+  return {
+    test: (relativePath) =>
+      includeMode
+        ? parser.testPatternsWithNegation(compiled, relativePath)
+        : parser.testPatterns(compiled, relativePath, 'exclude'),
+  };
+}
+
+/**
+ * Pick the files to pack
+ * @param {string} root - Project root
+ * @param {object} options
+ * @param {string[]} [options.files] - Explicit list (replaces the scan); relative entries
+ *   are resolved against `options.cwd`. Every entry must stay inside the root, and the
+ *   project's ignore files still apply to it.
+ * @param {string} [options.cwd] - Base for relative `files` entries (default: root)
+ * @param {string[]} [options.include] - Keep only matching files
+ * @param {string[]} [options.exclude] - Leave matching files out
+ * @param {string[]} [options.excludeFiles] - Absolute paths to leave out (the output file)
+ * @param {number} [options.maxFileSize] - Larger files are skipped
+ * @returns {{root: string, files: Array<{absolutePath: string, path: string, size: number,
+ *   modified: Date}>, skipped: Array<{path: string, reason: string}>}}
+ */
+export function selectFiles(root, options = {}) {
+  const rootPath = path.resolve(root);
+  if (!fs.statSync(rootPath, { throwIfNoEntry: false })?.isDirectory()) {
+    throw new PackError('ROOT_NOT_FOUND', `Not a directory: ${root}`);
+  }
+  const skipped = [];
+  const candidates = options.files
+    ? listedFiles(rootPath, options.files, options.cwd ?? rootPath, skipped)
+    : scannedFiles(rootPath);
+
+  const include = globMatcher(options.include ?? [], true);
+  const exclude = globMatcher(options.exclude ?? [], false);
+  const excluded = new Set((options.excludeFiles ?? []).map((file) => path.resolve(file)));
+  const maxFileSize = options.maxFileSize ?? Infinity;
+
+  const files = [];
+  for (const file of candidates) {
+    if (include && !include.test(file.path)) continue;
+    if (exclude?.test(file.path)) continue;
+    if (excluded.has(file.absolutePath)) continue;
+    if (file.size > maxFileSize) {
+      skipped.push({ path: file.path, reason: 'too-large' });
+      continue;
+    }
+    files.push(file);
+  }
+  return { root: rootPath, files, skipped };
+}
+
+/** Files found by the Scanner, in path order */
+function scannedFiles(rootPath) {
+  return new Scanner(rootPath)
+    .scan()
+    .map((file) => ({
+      absolutePath: file.path,
+      path: toPosix(file.relativePath),
+      size: file.size,
+      modified: file.modified,
+    }))
+    .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+}
+
+/**
+ * Whether the scan would leave a listed file out: a skipped directory (node_modules, .git,
+ * ...) on its path, or a .gitignore/.contextignore/.contextinclude rule on the file or one
+ * of its directories, checked the way the Scanner walks down to it
+ */
+function isIgnored(scanner, rootPath, relativePath) {
+  const segments = relativePath.split(path.sep);
+  for (let depth = 1; depth < segments.length; depth++) {
+    const directory = segments.slice(0, depth).join(path.sep);
+    if (FileUtils.isSkippedDirectory(segments[depth - 1])) return true;
+    if (scanner.shouldIgnore(directory, path.join(rootPath, directory))) return true;
+  }
+  return scanner.shouldIgnore(relativePath, path.join(rootPath, relativePath));
+}
+
+/**
+ * An explicit list, in the given order. A path that leaves the root (`..`, an absolute path
+ * elsewhere, a symlink pointing out) is an error; files the ignore rules exclude are skipped.
+ */
+function listedFiles(rootPath, entries, cwd, skipped) {
+  const realRoot = fs.realpathSync(rootPath);
+  const scanner = new Scanner(rootPath);
+  const files = [];
+  const seen = new Set();
+  for (const entry of entries) {
+    if (typeof entry !== 'string' || entry.trim() === '') continue;
+    const absolute = path.resolve(cwd, entry);
+    // The root as given, or its real path (`find "$(pwd -P)"` under a symlinked root)
+    let relative = relativeInside(rootPath, absolute);
+    if (relative === null && realRoot !== rootPath) relative = relativeInside(realRoot, absolute);
+    if (relative === null) {
+      throw new PackError('PATH_OUTSIDE_ROOT', `Path is outside the project root: ${entry}`);
+    }
+    const absolutePath = path.join(rootPath, relative);
+    const relativePath = toPosix(relative);
+    if (seen.has(relativePath)) continue;
+    seen.add(relativePath);
+
+    let stats;
+    try {
+      // Symlinks are followed, but must not lead out of the root
+      if (relativeInside(realRoot, fs.realpathSync(absolutePath)) === null) {
+        throw new PackError('PATH_OUTSIDE_ROOT', `Path is outside the project root: ${entry}`);
+      }
+      stats = fs.statSync(absolutePath);
+    } catch (error) {
+      if (error instanceof PackError) throw error;
+      skipped.push({ path: relativePath, reason: 'missing' });
+      continue;
+    }
+    if (!stats.isFile()) {
+      skipped.push({ path: relativePath, reason: 'not-a-file' });
+      continue;
+    }
+    if (isIgnored(scanner, rootPath, relative)) {
+      skipped.push({ path: relativePath, reason: 'ignored' });
+      continue;
+    }
+    files.push({ absolutePath, path: relativePath, size: stats.size, modified: stats.mtime });
+  }
+  return files;
+}
+
+export default { selectFiles, PackError };

package/lib/pack/split.js

@@ -0,0 +1,182 @@
+/**
+ * `pack --split`: cut the packed files into numbered parts of at most `limit` tokens
+ *
+ * Every part is a complete document in the chosen style. Part 1 carries the header and the
+ * directory tree, the last part the omitted-files list and the instructions, and document
+ * numbers continue across parts. Files are never cut, except a file that does not fit in a
+ * part on its own: it is split by lines into pieces labelled "(lines a-b of n)".
+ */
+
+import { PackError } from './select.js';
+
+// Label of a middle part when estimating: as long as any real label can be
+const PLACEHOLDER_PART = { index: 9999, count: 9999 };
+
+/**
+ * @param {Array} entries - Packed entries in output order
+ * @param {number} limit - Token limit per part
+ * @param {object} context
+ * @param {object} context.renderer - Style renderer
+ * @param {function(string): number} context.count - Token counter
+ * @param {object} context.doc - Whole-document fields: header, tree, omitted, instructions
+ * @returns {{parts: Array<{index: number, text: string, tokens: number, files: string[]}>,
+ *   splitFiles: Array<{path: string, pieces: number}>}}
+ */
+export function splitIntoParts(entries, limit, { renderer, count, doc }) {
+  const hasFirstExtras = Boolean(doc.header) || doc.tree !== null;
+  const hasLastExtras = Boolean(doc.omitted) || Boolean(doc.instructions);
+
+  const partDoc = (units, index, total, firstIndex) => ({
+    header: index === 0 ? doc.header : null,
+    partLabel: `Part ${index + 1} of ${total}`,
+    part: { index: index + 1, count: total },
+    tree: index === 0 ? doc.tree : null,
+    files: units,
+    firstIndex,
+    omitted: index === total - 1 ? doc.omitted : null,
+    instructions: index === total - 1 ? doc.instructions : null,
+  });
+  const middleDoc = (units) => ({
+    ...partDoc(units, 1, 3, 99999),
+    part: PLACEHOLDER_PART,
+    partLabel: 'Part 9999 of 9999',
+  });
+  const middleBase = count(renderer.render(middleDoc([])));
+  const unitCost = (unit) => count(renderer.renderFile(unit, 1)) + 1;
+
+  // Files too large for a part of their own are cut into pieces by lines
+  const units = [];
+  const splitFiles = [];
+  for (const entry of entries) {
+    // Estimates near the limit are checked exactly
+    const estimate = middleBase + unitCost(entry);
+    if (estimate <= limit * 0.9 || count(renderer.render(middleDoc([entry]))) <= limit) {
+      units.push(entry);
+      continue;
+    }
+    const pieces = splitByLines(entry, limit, { renderer, count, middleDoc });
+    splitFiles.push({ path: entry.path, pieces: pieces.length });
+    units.push(...pieces);
+  }
+
+  // Fill parts greedily with the estimates
+  let parts = [[]];
+  let used = count(renderer.render({ ...middleDoc([]), header: doc.header, tree: doc.tree }));
+  for (const unit of units) {
+    const cost = unitCost(unit);
+    if (parts[parts.length - 1].length && used + cost > limit) {
+      parts.push([]);
+      used = middleBase;
+    }
+    parts[parts.length - 1].push(unit);
+    used += cost;
+  }
+
+  // Then count each part exactly and move files forward until every part fits. A move
+  // changes two parts (all of them when the number of parts changes): the others are
+  // counted once
+  const counted = new Map();
+  const countOnce = (text) => {
+    if (!counted.has(text)) counted.set(text, count(text));
+    return counted.get(text);
+  };
+  const maxIterations = (units.length + 5) * 4;
+  for (let iteration = 0; ; iteration++) {
+    if (iteration > maxIterations) {
+      throw new PackError('SPLIT_TOO_SMALL', `Could not split into parts of ${limit} tokens`);
+    }
+    const rendered = renderParts(parts, partDoc, renderer, countOnce);
+    const over = rendered.findIndex((part) => part.tokens > limit);
+    if (over === -1) {
+      return {
+        parts: rendered.map((part, i) => ({ ...part, files: parts[i].map(labelOf) })),
+        splitFiles,
+      };
+    }
+
+    const isLast = over === parts.length - 1;
+    if (isLast && hasLastExtras && parts[over].length) {
+      parts.push([]); // the omitted list and instructions get a part of their own
+    } else if (parts[over].length > 1 || (over === 0 && hasFirstExtras && parts[over].length)) {
+      if (isLast) parts.push([]);
+      parts[over + 1].unshift(parts[over].pop());
+    } else {
+      throw new PackError(
+        'SPLIT_TOO_SMALL',
+        `Part ${over + 1} needs ${rendered[over].tokens} tokens; raise --split` +
+          (over === 0 && doc.tree !== null ? ' or use --no-tree' : '')
+      );
+    }
+    parts = parts.filter(
+      (units, i) =>
+        units.length ||
+        (i === 0 && hasFirstExtras) ||
+        (i === parts.length - 1 && hasLastExtras && parts.length > 1)
+    );
+    if (!parts.length) parts = [[]];
+  }
+}
+
+function renderParts(parts, partDoc, renderer, count) {
+  let firstIndex = 1;
+  return parts.map((units, i) => {
+    const text = renderer.render(partDoc(units, i, parts.length, firstIndex));
+    firstIndex += units.length;
+    return { index: i + 1, text, tokens: count(text) };
+  });
+}
+
+const labelOf = (unit) =>
+  unit.lines ? `${unit.path} (lines ${unit.lines[0]}-${unit.lines[1]})` : unit.path;
+
+/**
+ * Cut one entry into pieces of whole lines, each fitting in a part on its own
+ * @returns {Array} Entries with `lines: [first, last]` and `totalLines`
+ */
+function splitByLines(entry, limit, { renderer, count, middleDoc }) {
+  const endsWithNewline = entry.content.endsWith('\n');
+  const lines = (endsWithNewline ? entry.content.slice(0, -1) : entry.content).split('\n');
+  const numberOf = (i) => (entry.lineNumbers ? entry.lineNumbers[i] : i + 1);
+  const lineTokens = lines.map((line) => count(`${line}\n`));
+  const fits = (piece) => count(renderer.render(middleDoc([piece]))) <= limit;
+
+  const piece = (from, to) => {
+    const content = lines.slice(from, to).join('\n') + (endsWithNewline ? '\n' : '');
+    return {
+      ...entry,
+      content,
+      tokens: count(content),
+      lines: [numberOf(from), numberOf(to - 1)],
+      totalLines: entry.totalLines,
+    };
+  };
+
+  const empty = count(renderer.render(middleDoc([piece(0, 1)]))) - lineTokens[0];
+  let margin = Math.max(16, Math.ceil(limit * 0.02));
+  for (let attempt = 0; attempt < 6; attempt++, margin *= 2) {
+    const room = limit - empty - margin;
+    const pieces = [];
+    let start = 0;
+    let used = 0;
+    for (let i = 0; i < lines.length; i++) {
+      if (lineTokens[i] > room) {
+        throw new PackError(
+          'SPLIT_TOO_SMALL',
+          `${entry.path}: line ${numberOf(i)} alone has ${lineTokens[i]} tokens, more than ` +
+            `a ${limit}-token part can hold`
+        );
+      }
+      if (used + lineTokens[i] > room && i > start) {
+        pieces.push(piece(start, i));
+        start = i;
+        used = 0;
+      }
+      used += lineTokens[i];
+    }
+    pieces.push(piece(start, lines.length));
+    if (pieces.every(fits)) return pieces;
+  }
+  throw new PackError('SPLIT_TOO_SMALL', `Could not split ${entry.path} into ${limit}-token parts`);
+}
+
+export default { splitIntoParts };

package/lib/pack/tree.js

@@ -0,0 +1,35 @@
+/**
+ * Directory tree for the top of a packed document
+ */
+
+/**
+ * Render project-relative paths as an indented tree, directories first
+ * @param {string[]} paths - Paths with `/` separators
+ * @returns {string} One entry per line, two spaces per level, directories end with `/`
+ */
+export function buildTree(paths) {
+  const root = { dirs: new Map(), files: [] };
+  for (const filePath of paths) {
+    const parts = filePath.split('/');
+    let node = root;
+    for (const dir of parts.slice(0, -1)) {
+      if (!node.dirs.has(dir)) node.dirs.set(dir, { dirs: new Map(), files: [] });
+      node = node.dirs.get(dir);
+    }
+    node.files.push(parts[parts.length - 1]);
+  }
+
+  const lines = [];
+  const walk = (node, depth) => {
+    const indent = '  '.repeat(depth);
+    for (const name of [...node.dirs.keys()].sort()) {
+      lines.push(`${indent}${name}/`);
+      walk(node.dirs.get(name), depth + 1);
+    }
+    for (const name of [...node.files].sort()) lines.push(`${indent}${name}`);
+  };
+  walk(root, 0);
+  return lines.join('\n');
+}
+
+export default buildTree;