@devmarketplacenpm/devmp 0.1.1-beta.5

package/lib/workspace.js

@@ -0,0 +1,608 @@
+"use strict";
+
+const fs = require("fs/promises");
+const path = require("path");
+
+// Always skipped, even without a .gitignore — junk that should never be sent as
+// context or treated as source.
+// Files this client will not read, send, or write, whatever .gitignore says.
+//
+// The agent runs on a server, so anything read here leaves the machine. A fresh
+// project usually has no .gitignore yet, and that is exactly when a `.env` full
+// of live keys is sitting in the working directory — leaving this to gitignore
+// means the default is "upload the secrets" and the safe case is the one the
+// user has to remember to configure.
+//
+// Sample files are the deliberate exception: `.env.example` exists to be read.
+const SECRET_FILE_PATTERNS = [
+  /^\.env$/i,
+  /^\.env\.(?!example$|sample$|template$|dist$)[^/]+$/i,
+  /^\.npmrc$/i,
+  /^\.netrc$/i,
+  /^\.pgpass$/i,
+  /^id_(rsa|dsa|ecdsa|ed25519)$/i,
+  /^credentials(\.json)?$/i,
+  /\.(pem|key|p12|pfx|keystore|jks)$/i,
+  /^\.ssh\//i,
+  /^\.aws\/credentials$/i,
+  /^\.gnupg\//i,
+  /^service-account.*\.json$/i,
+];
+
+/**
+ * Is this path one the client refuses to touch?
+ *
+ * Matched on the POSIX-relative path and on the basename, so a secret nested in
+ * a subdirectory is caught as well as one at the root.
+ */
+function isSecretPath(relPath) {
+  const rel = String(relPath).replace(/\\/g, "/").replace(/^\.\//, "");
+  const base = rel.split("/").pop() || rel;
+  return SECRET_FILE_PATTERNS.some((re) => re.test(rel) || re.test(base));
+}
+
+const IGNORE_DIRS = new Set([
+  "node_modules",
+  ".git",
+  ".next",
+  "dist",
+  "build",
+  "out",
+  "coverage",
+  ".cache",
+  ".turbo",
+  ".devmarketplace",
+  "vendor",
+  ".venv",
+  "__pycache__",
+]);
+
+const MAX_SNAPSHOT_FILES = 60;
+const MAX_FILE_BYTES = 100 * 1024; // 100 KB per file
+const MAX_TOTAL_BYTES = 400 * 1024; // 400 KB total snapshot
+
+function looksBinary(buffer) {
+  const len = Math.min(buffer.length, 4096);
+  for (let i = 0; i < len; i += 1) if (buffer[i] === 0) return true;
+  return false;
+}
+
+// ── minimal .gitignore support ──────────────────────────────────────────────
+// A pragmatic subset of gitignore semantics: comments/blanks, `!` negation,
+// trailing `/` (dir-only), leading `/` or an embedded `/` (anchored to root),
+// and `*` / `?` / `**` globs. Nested .gitignore files are not read (root only).
+
+function globToRegex(glob) {
+  let re = "";
+  for (let i = 0; i < glob.length; i += 1) {
+    const c = glob[i];
+    if (c === "*") {
+      if (glob[i + 1] === "*") {
+        re += ".*";
+        i += 1;
+      } else {
+        re += "[^/]*";
+      }
+    } else if (c === "?") {
+      re += "[^/]";
+    } else if ("\\^$.|+()[]{}".includes(c)) {
+      re += `\\${c}`;
+    } else {
+      re += c;
+    }
+  }
+  return new RegExp(`^${re}$`);
+}
+
+function compilePattern(raw) {
+  let pat = raw.replace(/\r$/, "").trim();
+  if (!pat || pat.startsWith("#")) return null;
+  let negate = false;
+  if (pat.startsWith("!")) {
+    negate = true;
+    pat = pat.slice(1);
+  }
+  let dirOnly = false;
+  if (pat.endsWith("/")) {
+    dirOnly = true;
+    pat = pat.slice(0, -1);
+  }
+  const rootAnchored = pat.startsWith("/");
+  if (rootAnchored) pat = pat.slice(1);
+  const anchored = rootAnchored || pat.includes("/");
+  if (!pat) return null;
+  return { negate, dirOnly, anchored, regex: globToRegex(pat) };
+}
+
+async function loadGitignore(rootDir) {
+  let text;
+  try {
+    text = await fs.readFile(path.join(rootDir, ".gitignore"), "utf8");
+  } catch {
+    return () => false;
+  }
+  const patterns = text.split("\n").map(compilePattern).filter(Boolean);
+
+  return function isIgnored(relPath, isDir) {
+    const base = relPath.split("/").pop();
+    let ignored = false;
+    for (const p of patterns) {
+      if (p.dirOnly && !isDir) continue;
+      const target = p.anchored ? relPath : base;
+      if (p.regex.test(target)) ignored = !p.negate;
+    }
+    return ignored;
+  };
+}
+
+/**
+ * Read the caller's current text files (the "snapshot") so the agent can read
+ * and extend real code. Skips the built-in junk dirs, anything matched by the
+ * project's .gitignore, binary/large files, and caps the total so we never ship
+ * a whole node_modules. Returns [{ path, content }] with POSIX-relative paths.
+ */
+async function readSnapshot(rootDir) {
+  const isIgnored = await loadGitignore(rootDir);
+  const files = [];
+  let totalBytes = 0;
+  let truncated = false;
+  let ignoredCount = 0;
+  // Counted, not silent: a file the agent never sees is the likeliest reason
+  // for it to answer as though the file does not exist.
+  let oversizedCount = 0;
+
+  async function walk(dir) {
+    if (truncated) return;
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (truncated) return;
+      const full = path.join(dir, entry.name);
+      const rel = path.relative(rootDir, full).split(path.sep).join("/");
+
+      if (entry.isDirectory()) {
+        if (IGNORE_DIRS.has(entry.name)) continue;
+        if (isIgnored(rel, true)) {
+          ignoredCount += 1;
+          continue;
+        }
+        await walk(full);
+      } else if (entry.isFile()) {
+        // Never leaves the machine, regardless of .gitignore.
+        if (isSecretPath(rel)) continue;
+        if (isIgnored(rel, false)) {
+          ignoredCount += 1;
+          continue;
+        }
+        let buf;
+        try {
+          buf = await fs.readFile(full);
+        } catch {
+          continue;
+        }
+        if (looksBinary(buf)) continue;
+        if (buf.length > MAX_FILE_BYTES) {
+          oversizedCount += 1;
+          continue;
+        }
+        if (
+          files.length >= MAX_SNAPSHOT_FILES ||
+          totalBytes + buf.length > MAX_TOTAL_BYTES
+        ) {
+          truncated = true;
+          return;
+        }
+        totalBytes += buf.length;
+        files.push({ path: rel, content: buf.toString("utf8") });
+      }
+    }
+  }
+
+  await walk(rootDir);
+  return { files, truncated, ignoredCount, oversizedCount };
+}
+
+/** Resolve a client-relative path to an absolute one, or null if it escapes
+ * the workspace root (path-traversal guard; defense in depth). */
+function resolveInside(rootDir, relPath) {
+  const normalized = String(relPath).replace(/^[/\\]+/, "");
+  const target = path.resolve(rootDir, normalized);
+  const rootWithSep = path.resolve(rootDir) + path.sep;
+  if (target !== path.resolve(rootDir) && !target.startsWith(rootWithSep)) {
+    return null;
+  }
+  return target;
+}
+
+/**
+ * Write a file the agent produced to local disk, sandboxed to rootDir. Returns
+ * 'created' or 'updated', or null when refused (outside the workspace).
+ */
+async function applyFile(rootDir, relPath, content) {
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await isUnsafeFileTarget(rootDir, target))) return null;
+  let existed = true;
+  try {
+    await fs.access(target);
+  } catch {
+    existed = false;
+  }
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.writeFile(target, content, "utf8");
+  return existed ? "updated" : "created";
+}
+
+// ── on-demand disk ops for the live tunnel (all sandboxed to rootDir) ────────
+
+async function readFileRaw(rootDir, relPath) {
+  if (isSecretPath(relPath)) return null;
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await isUnsafeFileTarget(rootDir, target))) return null;
+  try {
+    return await fs.readFile(target, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+async function writeFileRaw(rootDir, relPath, content) {
+  if (isSecretPath(relPath)) return null;
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await isUnsafeFileTarget(rootDir, target))) return false;
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.writeFile(target, content, "utf8");
+  return true;
+}
+
+async function appendFileRaw(rootDir, relPath, content) {
+  if (isSecretPath(relPath)) return null;
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await isUnsafeFileTarget(rootDir, target))) return false;
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.appendFile(target, content, "utf8");
+  return true;
+}
+
+async function editFileRaw(rootDir, relPath, oldString, newString) {
+  if (isSecretPath(relPath)) return null;
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await isUnsafeFileTarget(rootDir, target))) {
+    return { refused: true, replaced: false };
+  }
+  let current;
+  try {
+    current = await fs.readFile(target, "utf8");
+  } catch {
+    return { replaced: false };
+  }
+  const idx = current.indexOf(oldString);
+  if (idx < 0) return { replaced: false };
+  const next =
+    current.slice(0, idx) + newString + current.slice(idx + oldString.length);
+  await fs.writeFile(target, next, "utf8");
+  return { replaced: true };
+}
+
+async function hasSymlinkAncestor(rootDir, target) {
+  const root = path.resolve(rootDir);
+  let current = path.dirname(target);
+  while (current !== root && current.startsWith(`${root}${path.sep}`)) {
+    try {
+      if ((await fs.lstat(current)).isSymbolicLink()) return true;
+    } catch (error) {
+      if (error?.code !== "ENOENT") throw error;
+    }
+    current = path.dirname(current);
+  }
+  return false;
+}
+
+async function isUnsafeFileTarget(rootDir, target) {
+  if (await hasSymlinkAncestor(rootDir, target)) return true;
+  try {
+    const stat = await fs.lstat(target);
+    return stat.isSymbolicLink() || !stat.isFile();
+  } catch (error) {
+    if (error?.code === "ENOENT") return false;
+    throw error;
+  }
+}
+
+/** Destructive operations are stricter than ordinary writes: only regular
+ * files are accepted and symlink traversal is refused. */
+async function deleteFileRaw(rootDir, relPath) {
+  if (isSecretPath(relPath)) return null;
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await hasSymlinkAncestor(rootDir, target))) {
+    return { refused: true, deleted: false };
+  }
+  try {
+    const stat = await fs.lstat(target);
+    if (!stat.isFile() || stat.isSymbolicLink()) {
+      return { refused: true, deleted: false };
+    }
+    await fs.unlink(target);
+    return { deleted: true };
+  } catch (error) {
+    if (error?.code === "ENOENT") return { deleted: false };
+    throw error;
+  }
+}
+
+async function moveFileRaw(rootDir, fromPath, toPath) {
+  if (isSecretPath(fromPath) || isSecretPath(toPath)) return null;
+  const source = resolveInside(rootDir, fromPath);
+  const target = resolveInside(rootDir, toPath);
+  if (
+    !source ||
+    !target ||
+    source === target ||
+    (await hasSymlinkAncestor(rootDir, source)) ||
+    (await hasSymlinkAncestor(rootDir, target))
+  ) {
+    return { refused: true, moved: false };
+  }
+  try {
+    const sourceStat = await fs.lstat(source);
+    if (!sourceStat.isFile() || sourceStat.isSymbolicLink()) {
+      return { refused: true, moved: false };
+    }
+  } catch (error) {
+    if (error?.code === "ENOENT") return { moved: false };
+    throw error;
+  }
+  try {
+    const targetStat = await fs.lstat(target);
+    if (!targetStat.isFile() || targetStat.isSymbolicLink()) {
+      return { refused: true, moved: false };
+    }
+  } catch (error) {
+    if (error?.code !== "ENOENT") throw error;
+  }
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.rename(source, target);
+  return { moved: true };
+}
+
+/** List local file paths for the agent (gitignore-aware, junk-dir pruned,
+ * size-capped). Names only — the agent reads what it needs via `read`. */
+async function listFiles(rootDir) {
+  const isIgnored = await loadGitignore(rootDir);
+  const out = [];
+  const MAX = 500;
+
+  async function walk(dir) {
+    if (out.length >= MAX) return;
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (out.length >= MAX) return;
+      const full = path.join(dir, entry.name);
+      const rel = path.relative(rootDir, full).split(path.sep).join("/");
+      if (entry.isDirectory()) {
+        if (IGNORE_DIRS.has(entry.name)) continue;
+        if (isIgnored(rel, true)) continue;
+        await walk(full);
+      } else if (entry.isFile()) {
+        // Not merely unreadable — invisible. A path the agent can see is a path
+        // it will ask for, and the refusal would read as a bug rather than a
+        // boundary.
+        if (isSecretPath(rel)) continue;
+        if (isIgnored(rel, false)) continue;
+        let st;
+        try {
+          st = await fs.stat(full);
+        } catch {
+          continue;
+        }
+        if (st.size > MAX_FILE_BYTES) continue;
+        out.push(rel);
+      }
+    }
+  }
+
+  await walk(rootDir);
+  return out;
+}
+
+// ── search ─────────────────────────────────────────────────────────────────
+// `list_files` caps at 500 paths and `read_file` opens one file at a time, so
+// without this the agent is effectively blind on a real repo: it has to guess
+// which files matter. Searching happens HERE, on the user's machine — only the
+// matching lines cross the socket, never the repo.
+
+const MAX_SEARCH_FILES = 5000;
+const MAX_SEARCH_FILE_BYTES = 2 * 1024 * 1024;
+const MAX_MATCH_LINE_CHARS = 400;
+
+/** Translate a glob (`*.ts`, `src/**' + '/*.tsx`) into an anchored RegExp. */
+function globToRegExp(glob) {
+  let out = "";
+  for (let i = 0; i < glob.length; i += 1) {
+    const char = glob[i];
+    if (char === "*") {
+      if (glob[i + 1] === "*") {
+        // `**/` may match zero directories, so the slash is optional.
+        if (glob[i + 2] === "/") {
+          out += "(?:.*/)?";
+          i += 2;
+        } else {
+          out += ".*";
+          i += 1;
+        }
+      } else {
+        out += "[^/]*";
+      }
+      continue;
+    }
+    if (char === "?") {
+      out += "[^/]";
+      continue;
+    }
+    out += char.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+  }
+  return new RegExp(`^${out}$`);
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Walk the workspace, gitignore-aware, collecting relative file paths. */
+async function collectFiles(rootDir, max) {
+  const isIgnored = await loadGitignore(rootDir);
+  const out = [];
+
+  async function walk(dir) {
+    if (out.length >= max) return;
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (out.length >= max) return;
+      const full = path.join(dir, entry.name);
+      const rel = path.relative(rootDir, full).split(path.sep).join("/");
+      if (entry.isDirectory()) {
+        if (IGNORE_DIRS.has(entry.name)) continue;
+        if (isIgnored(rel, true)) continue;
+        await walk(full);
+      } else if (entry.isFile()) {
+        if (isSecretPath(rel)) continue;
+        if (isIgnored(rel, false)) continue;
+        out.push(rel);
+      }
+    }
+  }
+
+  await walk(rootDir);
+  return out;
+}
+
+/**
+ * Grep the workspace. With `filesOnly` it behaves as a glob (find files by
+ * name); otherwise it returns matching lines with their line numbers.
+ */
+async function searchFiles(
+  rootDir,
+  {
+    pattern,
+    include,
+    filesOnly = false,
+    caseSensitive = false,
+    literal = false,
+    maxResults = 100,
+  } = {}
+) {
+  const limit = Math.min(Math.max(1, Number(maxResults) || 100), 500);
+  const includeRe = include ? globToRegExp(include) : null;
+  let candidates = await collectFiles(rootDir, MAX_SEARCH_FILES);
+  const scannedAll = candidates.length < MAX_SEARCH_FILES;
+  if (includeRe) candidates = candidates.filter((rel) => includeRe.test(rel));
+
+  if (filesOnly && !pattern) {
+    return {
+      matches: candidates.slice(0, limit).map((rel) => ({ path: rel })),
+      truncated: candidates.length > limit || !scannedAll,
+      scannedFiles: candidates.length,
+    };
+  }
+
+  let regex;
+  try {
+    regex = new RegExp(
+      literal ? escapeRegExp(String(pattern)) : String(pattern),
+      caseSensitive ? "" : "i"
+    );
+  } catch (error) {
+    throw new Error(`Invalid search pattern: ${error.message}`);
+  }
+
+  const matches = [];
+  let scanned = 0;
+  for (const rel of candidates) {
+    if (matches.length >= limit) break;
+    const target = resolveInside(rootDir, rel);
+    if (!target || (await isUnsafeFileTarget(rootDir, target))) continue;
+
+    let buffer;
+    try {
+      const stat = await fs.stat(target);
+      if (stat.size > MAX_SEARCH_FILE_BYTES) continue;
+      buffer = await fs.readFile(target);
+    } catch {
+      continue;
+    }
+    // Binary files produce noise, not answers.
+    if (looksBinary(buffer)) continue;
+    scanned += 1;
+
+    if (filesOnly) {
+      if (regex.test(rel)) matches.push({ path: rel });
+      continue;
+    }
+
+    const lines = buffer.toString("utf8").split("\n");
+    for (let index = 0; index < lines.length; index += 1) {
+      if (matches.length >= limit) break;
+      if (!regex.test(lines[index])) continue;
+      const text = lines[index].trim();
+      matches.push({
+        path: rel,
+        line: index + 1,
+        text:
+          text.length > MAX_MATCH_LINE_CHARS
+            ? `${text.slice(0, MAX_MATCH_LINE_CHARS)}…`
+            : text,
+      });
+    }
+  }
+
+  return {
+    matches,
+    truncated: matches.length >= limit || !scannedAll,
+    scannedFiles: scanned,
+  };
+}
+
+/** Does a file already exist on disk under rootDir? Used to decide overwrites. */
+async function fileExists(rootDir, relPath) {
+  const target = resolveInside(rootDir, relPath);
+  if (!target || (await isUnsafeFileTarget(rootDir, target))) return false;
+  try {
+    await fs.access(target);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+module.exports = {
+  IGNORE_DIRS,
+  readSnapshot,
+  applyFile,
+  fileExists,
+  loadGitignore,
+  resolveInside,
+  isSecretPath,
+  isUnsafeFileTarget,
+  readFileRaw,
+  writeFileRaw,
+  appendFileRaw,
+  editFileRaw,
+  deleteFileRaw,
+  moveFileRaw,
+  listFiles,
+  searchFiles,
+  globToRegExp,
+};