overleaf-forge 2.9.1 → 2.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "overleaf-forge",
-  "version": "2.9.1",
+  "version": "2.11.0",
   "description": "MCP server to read, edit, compile, and verify Overleaf/LaTeX projects over git: conflict-safe edits, figure upload, clean-build verification, citation and voice linting.",
   "type": "module",
   "main": "overleaf-mcp-server.js",
@@ -12,7 +12,12 @@
     "templates/",
     "examples/",
     "writing-guidelines.md",
-    "projects.example.json"
+    "projects.example.json",
+    "efficiency.js",
+    "dependency-index.js",
+    "render-cache.js",
+    "transactions.js",
+    "runtime-observability.js"
   ],
   "scripts": {
     "start": "node overleaf-mcp-server.js",

package/render-cache.js

@@ -0,0 +1,161 @@
+import { createHash } from 'node:crypto';
+import { execFile as childExecFile } from 'node:child_process';
+import { lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, stat, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { promisify } from 'node:util';
+
+const execFile = promisify(childExecFile);
+const DEFAULT_CACHE_DIR = path.join(os.tmpdir(), 'overleaf-mcp-render');
+const MAX_PAGES = 20;
+const MIN_DPI = 72;
+const MAX_DPI = 200;
+const snapshotRetries = 3;
+
+// One promise per key keeps concurrent callers from both invoking pdftoppm and
+// replacing the same destination. The rename below still makes each write
+// safe if another process is using the same cache directory.
+const pending = new Map();
+
+const digest = bytes => createHash('sha256').update(bytes).digest('hex');
+const inside = (parent, child) => child === parent || child.startsWith(`${parent}${path.sep}`);
+
+async function containedSource(root, filePath) {
+  const rootReal = await pathReal(root);
+  const candidate = path.resolve(root, filePath);
+  if (!inside(path.resolve(root), candidate)) throw new Error('PDF path must be inside project');
+  const sourceReal = await pathReal(candidate);
+  if (!inside(rootReal, sourceReal)) throw new Error('PDF symlink escapes project');
+  return sourceReal;
+}
+
+async function pathReal(file) {
+  try {
+    return await realpath(file);
+  } catch (error) {
+    if (error.code === 'ENOENT') throw new Error(`Path does not exist: ${file}`);
+    throw error;
+  }
+}
+
+async function cacheRoot(cacheDir) {
+  const requested = path.resolve(cacheDir ?? DEFAULT_CACHE_DIR);
+  await mkdir(requested, { recursive: true });
+  const entry = await lstat(requested);
+  if (entry.isSymbolicLink()) throw new Error('Cache directory must not be a symlink');
+  return await pathReal(requested);
+}
+
+function validatePages(pages) {
+  if (!Array.isArray(pages) || pages.length === 0 || pages.length > MAX_PAGES) {
+    throw new Error(`pages must contain 1-${MAX_PAGES} page numbers`);
+  }
+  if (!pages.every(page => Number.isInteger(page) && page > 0)) {
+    throw new Error('pages must contain positive integers');
+  }
+  return [...new Set(pages)];
+}
+
+function validateDpi(dpi) {
+  if (!Number.isInteger(dpi) || dpi < MIN_DPI || dpi > MAX_DPI) {
+    throw new Error(`dpi must be an integer from ${MIN_DPI} to ${MAX_DPI}`);
+  }
+  return dpi;
+}
+
+async function toolVersion() {
+  try {
+    const result = await execFile('pdftoppm', ['-v'], { encoding: 'utf8' });
+    return `${result.stdout}\n${result.stderr}`.trim();
+  } catch (error) {
+    // pdftoppm writes its version to stderr and exits successfully on common
+    // builds, but retain a stable failure if the executable cannot be run.
+    if (error.stdout || error.stderr) return `${error.stdout ?? ''}\n${error.stderr ?? ''}`.trim();
+    throw new Error(`Unable to run pdftoppm: ${error.message}`);
+  }
+}
+
+async function pageCount(pdf) {
+  let result;
+  try {
+    result = await execFile('pdfinfo', [pdf], { encoding: 'utf8' });
+  } catch (error) {
+    throw new Error(`Unable to inspect PDF: ${error.stderr?.trim() || error.message}`);
+  }
+  const match = result.stdout.match(/^Pages:\s+(\d+)\s*$/m);
+  const count = match ? Number(match[1]) : NaN;
+  if (!Number.isInteger(count) || count < 1) throw new Error('pdfinfo returned no valid page count');
+  return count;
+}
+
+async function snapshot(source, root) {
+  for (let attempt = 0; attempt < snapshotRetries; attempt += 1) {
+    const before = await stat(source);
+    const bytes = await readFile(source);
+    const after = await stat(source);
+    if (before.size !== after.size || before.mtimeNs !== after.mtimeNs) continue;
+    const hash = digest(bytes);
+    const dir = await mkdtemp(path.join(root, '.snapshot-'));
+    const file = path.join(dir, 'input.pdf');
+    await writeFile(file, bytes, { flag: 'wx' });
+    if (digest(await readFile(file)) !== hash) {
+      await rm(dir, { recursive: true, force: true });
+      throw new Error('PDF snapshot verification failed');
+    }
+    return { dir, file, hash };
+  }
+  throw new Error('PDF changed while it was being read; retry the render');
+}
+
+async function existingFile(file, root) {
+  if (!inside(root, path.resolve(file))) throw new Error('Cache path escapes cache directory');
+  try {
+    const info = await lstat(file);
+    return info.isFile() && !info.isSymbolicLink() && info.size > 0;
+  } catch (error) {
+    if (error.code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
+async function renderOne(snapshotFile, root, key, page, dpi) {
+  const finalFile = path.join(root, `${key}.png`);
+  if (await existingFile(finalFile, root)) return { file: finalFile, hit: true };
+  const work = await mkdtemp(path.join(root, '.render-'));
+  try {
+    const prefix = path.join(work, 'page');
+    await execFile('pdftoppm', ['-png', '-r', String(dpi), '-f', String(page), '-l', String(page), snapshotFile, prefix], { encoding: 'utf8' });
+    const generated = path.join(work, `page-${page}.png`);
+    if (!(await existingFile(generated, work))) throw new Error(`pdftoppm produced no output for page ${page}`);
+    await rename(generated, finalFile);
+    return { file: finalFile, hit: false };
+  } finally {
+    await rm(work, { recursive: true, force: true });
+  }
+}
+
+export async function renderPages(root, { filePath, pages, dpi = 110, cacheDir } = {}) {
+  if (typeof root !== 'string' || typeof filePath !== 'string') throw new Error('root and filePath are required');
+  const wanted = validatePages(pages);
+  const resolution = validateDpi(dpi);
+  const cache = await cacheRoot(cacheDir);
+  const source = await containedSource(root, filePath);
+  const input = await snapshot(source, cache);
+  try {
+    const [version, count] = await Promise.all([toolVersion(), pageCount(input.file)]);
+    if (wanted.some(page => page > count)) throw new Error(`Requested page exceeds PDF page count (${count})`);
+    const results = [];
+    for (const page of wanted) {
+      const key = digest(JSON.stringify([input.hash, page, resolution, version]));
+      let work = pending.get(key);
+      if (!work) {
+        work = renderOne(input.file, cache, key, page, resolution).finally(() => pending.delete(key));
+        pending.set(key, work);
+      }
+      results.push(await work);
+    }
+    return { files: results.map(result => result.file), cacheHits: results.filter(result => result.hit).length, requestedPages: wanted, pageCount: count };
+  } finally {
+    await rm(input.dir, { recursive: true, force: true });
+  }
+}

package/runtime-observability.js

@@ -0,0 +1,109 @@
+const totals = {
+  callCount: 0,
+  durationMs: 0,
+  responseBytes: 0,
+  cacheHits: 0,
+  failures: 0,
+  byTool: {},
+};
+
+const ERROR_CODES = new Map([
+  ['LOCAL_CLONE', { retryable: false, nextAction: 'Check the local clone path and synchronize the project explicitly.' }],
+  ['CONFLICT', { retryable: false, nextAction: 'Resolve the repository conflict, then retry the requested operation.' }],
+  ['STALE_SOURCE', { retryable: false, nextAction: 'Read the latest revision and affected files before preparing a new batch.' }],
+  ['BUILD_FAILED', { retryable: false, nextAction: 'Inspect the build log and correct the reported failure before rebuilding.' }],
+  ['NETWORK', { retryable: true, nextAction: 'Check the network connection and retry when the service is reachable.' }],
+  ['MISSING_PACKAGE', { retryable: false, nextAction: 'Install the missing package or dependency, then retry.' }],
+  ['INVALID_INPUT', { retryable: false, nextAction: 'Correct the input arguments and retry.' }],
+  ['INTERNAL', { retryable: false, nextAction: 'Inspect the server logs for the underlying failure.' }],
+]);
+
+const scrub = value => String(value)
+  .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+(?::[^\s/@]*)?@/gi, '$1***@')
+  .replace(/\bgit:[^\s/@]+@/gi, 'git:***@')
+  .slice(0, 1500);
+
+function classify(error) {
+  const rawCode = String(error?.code ?? '').toUpperCase();
+  if (ERROR_CODES.has(rawCode)) return rawCode;
+  if (/^STALE_/.test(rawCode)) return 'STALE_SOURCE';
+  if (/^VERIFICATION_/.test(rawCode)) return 'BUILD_FAILED';
+  if (/^INVALID_|SYMLINK_PATH|DUPLICATE_PATH/.test(rawCode)) return 'INVALID_INPUT';
+  if (rawCode === 'DIRTY_CHECKOUT') return 'CONFLICT';
+  const text = `${rawCode} ${error?.message ?? error}`.toLowerCase();
+  if (/local.?clone|clone failed|clone path|repository clone/.test(text)) return 'LOCAL_CLONE';
+  if (/conflict|non-fast-forward|merge conflict/.test(text)) return 'CONFLICT';
+  if (/network|econn|etimedout|enotfound|enetunreach|ehostunreach|fetch failed|socket/.test(text)) return 'NETWORK';
+  if (/missing package|module_not_found|cannot find module|package .*not found/.test(text)) return 'MISSING_PACKAGE';
+  if (/invalid input|invalid argument|bad argument|validation|must be|expected .* but/.test(text)) return 'INVALID_INPUT';
+  return 'INTERNAL';
+}
+
+const responseBytes = result => {
+  try {
+    return Buffer.byteLength(JSON.stringify(result) ?? String(result), 'utf8');
+  } catch {
+    return Buffer.byteLength(String(result), 'utf8');
+  }
+};
+
+function blankStats() {
+  return { callCount: 0, durationMs: 0, responseBytes: 0, cacheHits: 0, failures: 0, byTool: {} };
+}
+
+export function usageStats({ reset = false } = {}) {
+  const snapshot = structuredClone(totals);
+  if (reset) {
+    Object.assign(totals, blankStats());
+  }
+  return snapshot;
+}
+
+function record(name, elapsed, result, failed) {
+  const tool = totals.byTool[name] ??= { callCount: 0, durationMs: 0, responseBytes: 0, cacheHits: 0, failures: 0 };
+  const bytes = responseBytes(result);
+  const hits = failed ? 0 : Number(result?.cacheHits ?? result?.structuredContent?.cacheHits ?? (result?.structuredContent?.reused || result?.structuredContent?.unchanged ? 1 : 0));
+  const safeHits = Number.isFinite(hits) && hits > 0 ? hits : 0;
+  for (const target of [totals, tool]) {
+    target.callCount += 1;
+    target.durationMs += elapsed;
+    target.responseBytes += bytes;
+    target.cacheHits += safeHits;
+    if (failed) target.failures += 1;
+  }
+}
+
+export function observeTool(name, asyncFn) {
+  if (typeof name !== 'string' || !name) throw new TypeError('tool name is required');
+  if (typeof asyncFn !== 'function') throw new TypeError('asyncFn must be a function');
+  return async function observedTool(...args) {
+    const started = performance.now();
+    try {
+      const result = await asyncFn.apply(this, args);
+      record(name, performance.now() - started, result, result?.isError === true || result?.structuredContent?.pass === false);
+      return result;
+    } catch (error) {
+      record(name, performance.now() - started, null, true);
+      throw error;
+    }
+  };
+}
+
+export function toolError(error) {
+  const code = classify(error);
+  const policy = ERROR_CODES.get(code);
+  const message = scrub(error?.message ?? error);
+  return {
+    isError: true,
+    content: [{ type: 'text', text: `Error: ${message}` }],
+    structuredContent: {
+      error: {
+        code,
+        message,
+        retryable: policy.retryable,
+        nextAction: policy.nextAction,
+        maxAutomaticRetries: 0,
+      },
+    },
+  };
+}

package/transactions.js

@@ -0,0 +1,219 @@
+import { createHash } from 'node:crypto';
+import { execFile as execFileCallback } from 'node:child_process';
+import { promisify } from 'node:util';
+import { lstat, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { tmpdir } from 'node:os';
+
+const execFile = promisify(execFileCallback);
+
+class TransactionError extends Error {
+  constructor(code, message, cause) {
+    super(message, cause ? { cause } : undefined);
+    this.name = 'TransactionError';
+    this.code = code;
+  }
+}
+
+const git = (cwd, args) => execFile('git', ['-C', cwd, ...args], { maxBuffer: 10 * 1024 * 1024 });
+
+async function gitText(cwd, args) {
+  const { stdout } = await git(cwd, args);
+  return stdout.trim();
+}
+
+function fail(code, message, cause) {
+  throw new TransactionError(code, message, cause);
+}
+
+async function status(root) {
+  // Verification tools commonly leave generated, untracked artifacts behind.
+  // Tracked edits still make the checkout dirty and are rejected below.
+  return gitText(root, ['status', '--porcelain=v1', '--untracked-files=no']);
+}
+
+async function head(root) {
+  return gitText(root, ['rev-parse', 'HEAD']);
+}
+
+function sha256(bytes) {
+  return createHash('sha256').update(bytes).digest('hex');
+}
+
+async function ensureNoSymlink(root, filePath) {
+  const absolute = path.resolve(root, filePath);
+  const relative = path.relative(root, absolute);
+  if (!relative || relative.startsWith('..' + path.sep) || path.isAbsolute(relative)) fail('invalid_path', `Path escapes repository: ${filePath}`);
+  let current = root;
+  for (const part of relative.split(path.sep)) {
+    current = path.join(current, part);
+    try {
+      if ((await lstat(current)).isSymbolicLink()) fail('symlink_path', `Symlink path is not allowed: ${filePath}`);
+    } catch (err) {
+      if (err.code !== 'ENOENT') throw err;
+      break;
+    }
+  }
+  return absolute;
+}
+
+function validateUtf8(content) {
+  if (typeof content !== 'string') fail('invalid_content', 'Change content must be a UTF-8 string');
+  const bytes = Buffer.from(content, 'utf8');
+  if (bytes.toString('utf8') !== content) fail('invalid_content', 'Change content is not valid UTF-8');
+  return bytes;
+}
+
+async function validateChanges(root, changes) {
+  if (!Array.isArray(changes)) fail('invalid_changes', 'changes must be an array');
+  const seen = new Set();
+  const checked = [];
+  for (const change of changes) {
+    if (!change || typeof change.filePath !== 'string' || change.filePath.includes('\0')) fail('invalid_path', 'Each change needs a valid filePath');
+    const normalized = path.normalize(change.filePath);
+    if (path.isAbsolute(change.filePath) || normalized === '.' || normalized.startsWith('..' + path.sep) || normalized === '..' || normalized === '.git' || normalized.startsWith(`.git${path.sep}`)) fail('invalid_path', `Invalid file path: ${change.filePath}`);
+    if (seen.has(normalized)) fail('duplicate_path', `Duplicate file path: ${change.filePath}`);
+    seen.add(normalized);
+    if (change.baseHash !== null && (typeof change.baseHash !== 'string' || !/^[0-9a-f]{64}$/i.test(change.baseHash))) fail('invalid_hash', `baseHash must be a SHA-256 hex digest or null: ${change.filePath}`);
+    const bytes = validateUtf8(change.content);
+    const absolute = await ensureNoSymlink(root, normalized);
+    let current = null;
+    try {
+      const st = await lstat(absolute);
+      if (st.isSymbolicLink()) fail('symlink_path', `Symlink target is not allowed: ${change.filePath}`);
+      if (!st.isFile()) fail('invalid_path', `Change target is not a regular file: ${change.filePath}`);
+      current = await readFile(absolute);
+    } catch (err) {
+      if (err.code !== 'ENOENT') throw err;
+    }
+    const actualHash = current === null ? null : sha256(current);
+    if (actualHash !== change.baseHash) fail('stale_file', `Base hash does not match ${change.filePath}`);
+    if (change.baseHash === null && current !== null) fail('stale_file', `New file already exists: ${change.filePath}`);
+    checked.push({ filePath: normalized, bytes, baseHash: change.baseHash });
+  }
+  return checked;
+}
+
+async function assertCleanAt(root, expectedHead, checked) {
+  if (await head(root) !== expectedHead) fail('stale_revision', 'Repository HEAD changed during the transaction');
+  if (await status(root) !== '') fail('dirty_checkout', 'Repository working tree changed during the transaction');
+  for (const change of checked) {
+    const absolute = await ensureNoSymlink(root, change.filePath);
+    let bytes = null;
+    try { bytes = await readFile(absolute); } catch (err) { if (err.code !== 'ENOENT') throw err; }
+    if ((bytes === null ? null : sha256(bytes)) !== change.baseHash) fail('stale_file', `Base hash changed for ${change.filePath}`);
+  }
+}
+
+function verificationPassed(value) {
+  return value && (value.passed === true || value.pass === true);
+}
+
+async function runVerification(verify, stageRoot) {
+  if (typeof verify !== 'function') fail('invalid_verify', 'verify must be a callback');
+  let result;
+  try { result = await verify(stageRoot); } catch (err) { fail('verification_error', err.message || 'Verification callback failed', err); }
+  if (!verificationPassed(result)) fail('verification_failed', 'Verification did not pass');
+  return result;
+}
+
+function compactVerification(value) {
+  if (!value || typeof value !== 'object') return value;
+  const result = { ...value };
+  delete result.tail;
+  delete result.logPath;
+  delete result.pdfPath;
+  return result;
+}
+
+async function removeWorktree(root, stageRoot) {
+  try { await git(root, ['worktree', 'remove', '--force', stageRoot]); } catch { /* best effort cleanup */ }
+  await rm(stageRoot, { recursive: true, force: true });
+}
+
+async function applyChangesUnlocked(root, { baseRevision, changes }, verify) {
+  const repo = path.resolve(root);
+  let stageRoot;
+  try {
+    if (typeof baseRevision !== 'string' || !/^[0-9a-f]{40}$/i.test(baseRevision)) fail('invalid_revision', 'baseRevision must be a full 40-hex commit SHA');
+    if (await status(repo) !== '') fail('dirty_checkout', 'Repository working tree must be clean');
+    const currentHead = await head(repo);
+    const requestedHead = await gitText(repo, ['rev-parse', `${baseRevision}^{commit}`]);
+    if (requestedHead !== currentHead) fail('stale_revision', 'baseRevision does not match HEAD');
+    const checked = await validateChanges(repo, changes);
+    stageRoot = await mkdtemp(path.join(tmpdir(), 'overleaf-transaction-'));
+    await git(repo, ['worktree', 'add', '--detach', stageRoot, currentHead]);
+    for (const change of checked) {
+      const target = await ensureNoSymlink(stageRoot, change.filePath);
+      await mkdir(path.dirname(target), { recursive: true });
+      await writeFile(target, change.bytes);
+    }
+    const verification = await runVerification(verify, stageRoot);
+    for (const change of checked) {
+      const bytes = await readFile(await ensureNoSymlink(stageRoot, change.filePath));
+      if (!bytes.equals(change.bytes)) fail('verification_mutation', `Verification changed ${change.filePath}`);
+    }
+    const changed = await gitText(stageRoot, ['diff', '--name-only']);
+    const allowed = new Set(checked.map(c => c.filePath));
+    if (changed && changed.split('\n').some(file => !allowed.has(file))) fail('unexpected_change', 'Verification changed files outside the batch');
+    let revision = currentHead;
+    if (checked.length) {
+      await git(stageRoot, ['add', '--', ...checked.map(c => c.filePath)]);
+      const staged = await gitText(stageRoot, ['diff', '--cached', '--name-only']);
+      if (staged && staged.split('\n').some(file => !allowed.has(file))) fail('unexpected_change', 'Unexpected file staged');
+      if (staged) {
+        await git(stageRoot, ['-c', 'user.name=Overleaf Forge', '-c', 'user.email=overleaf-forge@localhost', 'commit', '-m', 'Apply verified transaction']);
+        revision = await head(stageRoot);
+      }
+    }
+    await assertCleanAt(repo, currentHead, checked);
+    await git(repo, ['merge', '--ff-only', revision]);
+    return { revision: await head(repo), verification: compactVerification(verification), files: checked.map(c => c.filePath) };
+  } catch (err) {
+    if (err instanceof TransactionError) throw err;
+    throw new TransactionError('git_error', err.message || 'Git transaction failed', err);
+  } finally {
+    if (stageRoot) await removeWorktree(repo, stageRoot);
+  }
+}
+
+async function publishChangesUnlocked(root, { revision }, verify, push) {
+  const repo = path.resolve(root);
+  try {
+    if (typeof revision !== 'string' || !/^[0-9a-f]{40}$/i.test(revision)) fail('invalid_revision', 'revision must be a full 40-hex commit SHA');
+    if (await status(repo) !== '') fail('dirty_checkout', 'Repository working tree must be clean');
+    if (await head(repo) !== await gitText(repo, ['rev-parse', `${revision}^{commit}`])) fail('stale_revision', 'revision does not match HEAD');
+    const verification = await runVerification(verify, repo);
+    if (await status(repo) !== '') fail('verification_mutation', 'Verification changed the repository');
+    if (await head(repo) !== revision) fail('stale_revision', 'Repository HEAD changed during verification');
+    const branch = await gitText(repo, ['symbolic-ref', '--quiet', '--short', 'HEAD']);
+    if (!branch) fail('detached_head', 'Cannot publish from a detached HEAD');
+    if (push !== undefined && typeof push !== 'function') fail('invalid_push', 'push must be a callback');
+    if (push) await push(repo, revision, branch);
+    else await git(repo, ['push', 'HEAD:refs/heads/' + branch]);
+    const files = await gitText(repo, ['diff-tree', '--no-commit-id', '--name-only', '-r', revision]);
+    return { revision: await head(repo), verification: compactVerification(verification), files: files ? files.split('\n') : [] };
+  } catch (err) {
+    if (err instanceof TransactionError) throw err;
+    throw new TransactionError('push_failed', err.message || 'Git push failed', err);
+  }
+}
+
+const locks = new Map();
+function enqueue(root, operation) {
+  const key = path.resolve(root);
+  const prior = locks.get(key) || Promise.resolve();
+  const current = prior.catch(() => {}).then(operation);
+  locks.set(key, current);
+  return current.finally(() => { if (locks.get(key) === current) locks.delete(key); });
+}
+
+export function applyChanges(root, input, verify) {
+  return enqueue(root, () => applyChangesUnlocked(root, input, verify));
+}
+
+export function publishChanges(root, input, verify, push) {
+  return enqueue(root, () => publishChangesUnlocked(root, input, verify, push));
+}
+
+export { TransactionError };