getprismo 0.1.21 → 0.1.23

package/lib/prismo-dev/scan.js

@@ -27,197 +27,13 @@ const ASSUMED_TURNS_PER_AI_SESSION = 40;
 const ASSUMED_INPUT_COST_PER_1K_TOKENS = 0.003;
 const ASSUMED_SESSIONS_PER_MONTH = 30;
 
-function normalizeRel(value) {
-  return value.split(path.sep).join("/");
-}
-
-function readIgnoreFile(root, fileName) {
-  const filePath = path.join(root, fileName);
-  if (!fs.existsSync(filePath)) return [];
-  const text = fs.readFileSync(filePath, "utf8");
-  return text
-    .split(/\r?\n/)
-    .map((line) => line.trim())
-    .filter((line) => line && !line.startsWith("#"))
-    .map((line) => line.replace(/^!/, ""));
-}
-
-function patternMatches(pattern, relPath, isDir = false) {
-  const normalizedPattern = pattern.replace(/\\/g, "/").replace(/^\//, "");
-  const normalizedRel = normalizeRel(relPath);
-  const dirRel = isDir && !normalizedRel.endsWith("/") ? `${normalizedRel}/` : normalizedRel;
-
-  if (!normalizedPattern) return false;
-  if (normalizedPattern.endsWith("/")) {
-    const base = normalizedPattern.slice(0, -1);
-    return (
-      normalizedRel === base ||
-      normalizedRel.startsWith(`${base}/`) ||
-      normalizedRel.endsWith(`/${base}`) ||
-      normalizedRel.includes(`/${base}/`) ||
-      dirRel.includes(`/${base}/`)
-    );
-  }
-  if (normalizedPattern.startsWith("*.")) {
-    return normalizedRel.endsWith(normalizedPattern.slice(1));
-  }
-  if (normalizedPattern.includes("*")) {
-    const escaped = normalizedPattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-    return new RegExp(`(^|/)${escaped}$`).test(normalizedRel);
-  }
-  return (
-    normalizedRel === normalizedPattern ||
-    dirRel === normalizedPattern ||
-    normalizedRel.startsWith(`${normalizedPattern}/`) ||
-    normalizedRel.endsWith(`/${normalizedPattern}`)
-  );
-}
-
-function isIgnored(relPath, patterns, isDir = false) {
-  return patterns.some((pattern) => patternMatches(pattern, relPath, isDir));
-}
-
-function ignoreSuggestionCovered(pattern, existingPatterns) {
-  if (!pattern) return true;
-  if (existingPatterns.includes(pattern)) return true;
-  const sample = pattern
-    .replace(/^\*\//, "")
-    .replace(/^\*\*/, "sample")
-    .replace(/\*/g, "sample")
-    .replace(/\/$/, "");
-  const isDir = pattern.endsWith("/") || pattern.endsWith("/**");
-  return existingPatterns.some((existing) => {
-    if (existing === pattern) return true;
-    if (existing.endsWith("/") && pattern.startsWith(existing)) return true;
-    return patternMatches(existing, sample, isDir);
-  });
-}
-
-function missingIgnoreSuggestions(recommended, existingPatterns) {
-  return recommended.filter((pattern) => !ignoreSuggestionCovered(pattern, existingPatterns));
-}
-
-const SESSION_NOISE_DIRS = new Set([
-  ".next",
-  ".nuxt",
-  ".prismo",
-  ".pytest_cache",
-  ".turbo",
-  "__pycache__",
-  "build",
-  "calendar-dumps",
-  "coverage",
-  "dist",
-  "event-dumps",
-  "events",
-  "exports",
-  "htmlcov",
-  "inbox-dumps",
-  "logs",
-  "models",
-  "node_modules",
-  "out",
-  "playwright-report",
-  "session-dumps",
-  "source-streams",
-  "state-backups",
-  "test-results",
-  "tmp",
-  "temp",
-]);
-
-const SESSION_NOISE_FILE_NAMES = new Set([
-  "package-lock.json",
-  "pnpm-lock.yaml",
-  "yarn.lock",
-  "bun.lockb",
-  "coverage-final.json",
-  "lcov.info",
-]);
-
-const SESSION_NOISE_EXTENSIONS = new Set([
-  ".db",
-  ".jsonl",
-  ".lock",
-  ".log",
-  ".sqlite",
-  ".sqlite3",
-]);
-
-function cleanSessionPath(value) {
-  const text = String(value || "").trim().replace(/\\/g, "/");
-  if (!text || /^https?:\/\//.test(text)) return null;
-  const withoutQuotes = text.replace(/^["'`]+|["'`.,:;)\]]+$/g, "");
-  if (!withoutQuotes || withoutQuotes.includes("\n")) return null;
-  const markerIndex = withoutQuotes.indexOf("/Users/");
-  if (markerIndex > 0) return withoutQuotes.slice(markerIndex);
-  return withoutQuotes;
-}
-
-function sessionIgnorePatternForPath(value, root) {
-  const cleaned = cleanSessionPath(value);
-  if (!cleaned) return null;
-  const rootNormalized = normalizeRel(root);
-  let rel = cleaned;
-  if (path.isAbsolute(cleaned)) {
-    const normalized = normalizeRel(cleaned);
-    if (!normalized.startsWith(`${rootNormalized}/`)) return null;
-    rel = normalizeRel(path.relative(root, cleaned));
-  }
-  rel = normalizeRel(rel).replace(/^\.\//, "");
-  if (!rel || rel === "." || rel.startsWith("../") || rel.includes("..")) return null;
-
-  const segments = rel.split("/").filter(Boolean);
-  if (!segments.length) return null;
-  for (let index = 0; index < segments.length; index += 1) {
-    const segment = segments[index];
-    if (SESSION_NOISE_DIRS.has(segment)) {
-      return `${segments.slice(0, index + 1).join("/")}/`;
-    }
-  }
-
-  const fileName = segments[segments.length - 1];
-  const lowerName = fileName.toLowerCase();
-  const ext = path.extname(lowerName);
-  if (SESSION_NOISE_FILE_NAMES.has(lowerName)) return fileName;
-  if (SESSION_NOISE_EXTENSIONS.has(ext)) return rel;
-  if (/_state\.json$/i.test(fileName)) return "*_state.json";
-  if (/_tokens\.json$/i.test(fileName)) return "*_tokens.json";
-  if (/_export\.json$/i.test(fileName)) return "*_export.json";
-  if (/secret|credential|token/i.test(fileName) && /\.json$/i.test(fileName)) return rel;
-  return null;
-}
-
-function buildSessionIgnoreSuggestions(realUsage, root) {
-  if (!realUsage || !Array.isArray(realUsage.sessions)) return [];
-  const byPattern = new Map();
-  const add = (pattern, item, source, reason) => {
-    if (!pattern) return;
-    const existing = byPattern.get(pattern) || {
-      pattern,
-      source,
-      reason,
-      count: 0,
-      examples: [],
-    };
-    existing.count += Number(item?.count || 1);
-    const example = item?.value || item?.path || pattern;
-    if (example && !existing.examples.includes(example) && existing.examples.length < 3) existing.examples.push(example);
-    byPattern.set(pattern, existing);
-  };
-
-  for (const session of realUsage.sessions) {
-    for (const item of session.generatedArtifacts || []) {
-      add(sessionIgnorePatternForPath(item.value, root), item, session.tool || "session", "Generated artifact entered local session context.");
-    }
-    for (const item of session.repeatedPathMentions || []) {
-      add(sessionIgnorePatternForPath(item.value, root), item, session.tool || "session", "Noisy path appeared repeatedly in local session context.");
-    }
-  }
-  return Array.from(byPattern.values())
-    .sort((a, b) => b.count - a.count || a.pattern.localeCompare(b.pattern))
-    .slice(0, 25);
-}
+const {
+  buildSessionIgnoreSuggestions,
+  isIgnored,
+  missingIgnoreSuggestions,
+  normalizeRel,
+  readIgnoreFile,
+} = require("./scan-path-utils")({ fs, path });
 
 function getFileKind(filePath) {
   const ext = path.extname(filePath).toLowerCase();
@@ -390,12 +206,21 @@ function detectOptimizationStack(root, claudeConfig, codexConfig) {
   const projectHeadroom = fs.existsSync(path.join(root, ".headroom")) || fs.existsSync(path.join(os.homedir(), ".headroom"));
   const projectDistill = fs.existsSync(path.join(os.homedir(), ".config", "distill")) || commandExists("distill");
   const projectRtk = fs.existsSync(path.join(root, ".rtk")) || commandExists("rtk");
+  const packageText = readIfText(path.join(root, "package.json"), 512 * 1024) || "";
+  const readmeText = readIfText(path.join(root, "README.md"), 512 * 1024) || "";
+  const projectText = `${packageText}\n${readmeText}`.toLowerCase();
+  const hasText = (pattern) => pattern.test(projectText);
 
   const tools = {
     rtk: { detected: projectRtk, source: projectRtk ? "binary-or-project-config" : "not-detected" },
     headroom: { detected: projectHeadroom || commandExists("headroom"), source: projectHeadroom ? "local-config" : commandExists("headroom") ? "binary" : "not-detected" },
     distill: { detected: projectDistill, source: projectDistill ? "binary-or-user-config" : "not-detected" },
     mana: { detected: projectMana || commandExists("mana"), source: projectMana ? "local-config" : commandExists("mana") ? "binary" : "not-detected" },
+    contextMode: { detected: commandExists("context-mode") || hasText(/context-mode/), source: commandExists("context-mode") ? "binary" : hasText(/context-mode/) ? "project-reference" : "not-detected" },
+    leanCtx: { detected: commandExists("lean-ctx") || hasText(/lean-ctx|lean ctx/), source: commandExists("lean-ctx") ? "binary" : hasText(/lean-ctx|lean ctx/) ? "project-reference" : "not-detected" },
+    repomix: { detected: commandExists("repomix") || hasText(/repomix/), source: commandExists("repomix") ? "binary" : hasText(/repomix/) ? "project-reference" : "not-detected" },
+    codegraph: { detected: commandExists("codegraph") || hasText(/codegraph|codebase-memory-mcp|jcodemunch|sigmap/), source: commandExists("codegraph") ? "binary" : hasText(/codegraph|codebase-memory-mcp|jcodemunch|sigmap/) ? "project-reference" : "not-detected" },
+    tokf: { detected: commandExists("tokf") || hasText(/tokf/), source: commandExists("tokf") ? "binary" : hasText(/tokf/) ? "project-reference" : "not-detected" },
   };
 
   return {
@@ -908,6 +733,13 @@ function buildOptimizerFit(result) {
   const mcpEvidence = [];
   let mcpScore = result.optimizationStack.mcpServerTotal >= 10 ? 70 : result.optimizationStack.mcpServerTotal >= 5 ? 45 : 10;
   if (result.optimizationStack.mcpServerTotal) addEvidence(mcpEvidence, `${result.optimizationStack.mcpServerTotal} MCP/tool servers detected`);
+  const totalToolCalls = realUsage ? realUsage.sessions.reduce((sum, session) => sum + Number(session.toolCalls || 0), 0) : 0;
+  const repeatedCommands = realUsage ? realUsage.sessions.reduce((sum, session) => sum + (session.repeatedCommands || []).reduce((inner, item) => inner + Number(item.count || 0), 0), 0) : 0;
+  if (totalToolCalls >= 500) mcpScore += 30;
+  else if (totalToolCalls >= 100) mcpScore += 15;
+  if (repeatedCommands >= 20) mcpScore += 20;
+  if (totalToolCalls) addEvidence(mcpEvidence, `${totalToolCalls} tool calls in recent local sessions`);
+  if (repeatedCommands) addEvidence(mcpEvidence, `${repeatedCommands} repeated command/tool mentions`);
   bottlenecks.push({
     id: "tool-surface",
     label: "Tool/MCP surface overhead",
@@ -1006,6 +838,17 @@ function buildOptimizerFit(result) {
         reason: "Best for one-shot repo handoff, less ideal for long live coding sessions.",
       },
     ],
+    roundTripContext: {
+      level: levelFromScore(Math.max(mcpScore, repeatedSourceReads >= 50 ? 60 : repeatedSourceReads >= 12 ? 35 : 0)),
+      toolCalls: totalToolCalls,
+      repeatedCommandMentions: repeatedCommands,
+      repeatedSourceReads,
+      mcpServers: result.optimizationStack.mcpServerTotal,
+      summary: totalToolCalls || repeatedCommands || repeatedSourceReads || result.optimizationStack.mcpServerTotal
+        ? "Round-trip context risk includes tool calls, repeated commands, repeated source reads, and MCP/tool surface."
+        : "No strong round-trip context signal found in local logs.",
+      recommendation: "Measure workflow-level savings, not only compressed payload size. Fewer tool round trips can beat smaller individual responses.",
+    },
     caveats: [
       "Do not stack optimizers blindly; measure one real workflow before and after.",
       "Payload reduction is not the same as workflow savings; repeated tool calls can erase compression wins.",

package/lib/prismo-dev/usage-log-utils.js

@@ -0,0 +1,251 @@
+module.exports = function createUsageLogUtils(deps) {
+  const {
+    fs,
+    path,
+    GENERATED_ARTIFACT_PATTERNS,
+    readIfText,
+  } = deps;
+
+  function listFilesRecursive(root, predicate = () => true, limit = 300) {
+    const files = [];
+    if (!fs.existsSync(root)) return files;
+    const stack = [root];
+    while (stack.length && files.length < limit) {
+      const current = stack.pop();
+      let entries;
+      try {
+        entries = fs.readdirSync(current, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const entry of entries) {
+        const fullPath = path.join(current, entry.name);
+        if (entry.isDirectory()) {
+          stack.push(fullPath);
+        } else if (entry.isFile() && predicate(fullPath)) {
+          files.push(fullPath);
+        }
+      }
+    }
+    return files.sort((a, b) => {
+      try {
+        return fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs;
+      } catch {
+        return 0;
+      }
+    });
+  }
+
+  function parseJsonl(filePath, maxLines = 20000) {
+    const text = readIfText(filePath, 30 * 1024 * 1024);
+    if (!text) return [];
+    const rows = [];
+    const lines = text.split(/\r?\n/).filter(Boolean);
+    for (const line of lines.slice(Math.max(0, lines.length - maxLines))) {
+      try {
+        rows.push(JSON.parse(line));
+      } catch {
+        // Local tool logs can contain partial writes while a session is active.
+      }
+    }
+    return rows;
+  }
+
+  function collectText(value, options = {}, depth = 0) {
+    if (value == null || depth > 8) return "";
+    if (typeof value === "string") return value;
+    if (typeof value === "number" || typeof value === "boolean") return String(value);
+    if (Array.isArray(value)) return value.map((item) => collectText(item, options, depth + 1)).join("\n");
+    if (typeof value !== "object") return "";
+
+    const skipKeys = new Set(["signature", "encrypted_content", "image_url", "data", "auth", "api_key", "token"]);
+    const parts = [];
+    for (const [key, child] of Object.entries(value)) {
+      if (skipKeys.has(key)) continue;
+      parts.push(collectText(child, options, depth + 1));
+    }
+    return parts.filter(Boolean).join("\n");
+  }
+
+  function addUsage(target, usage) {
+    if (!usage || typeof usage !== "object") return;
+    target.inputTokens += Number(usage.input_tokens || usage.prompt_tokens || 0);
+    target.outputTokens += Number(usage.output_tokens || usage.completion_tokens || 0);
+    target.cacheReadTokens += Number(usage.cache_read_input_tokens || 0);
+    target.cacheCreationTokens += Number(usage.cache_creation_input_tokens || 0);
+  }
+
+  function totalUsageTokens(usage) {
+    if (!usage) return 0;
+    return (
+      Number(usage.input_tokens || usage.prompt_tokens || 0) +
+      Number(usage.output_tokens || usage.completion_tokens || 0) +
+      Number(usage.cache_read_input_tokens || 0) +
+      Number(usage.cache_creation_input_tokens || 0)
+    );
+  }
+
+  function incrementMap(map, key, amount = 1) {
+    if (!key) return;
+    map[key] = (map[key] || 0) + amount;
+  }
+
+  function normalizeMentionedPath(value, cwd = "") {
+    let normalized = String(value || "")
+      .replace(/\\/g, "/")
+      .replace(/^[`'"]+|[`'",:;)\]}]+$/g, "")
+      .trim();
+    normalized = normalized.replace(/^[ MADRCU?!]{1,4}\s+(?=\/|Users\/|home\/)/, "");
+    const normalizedCwd = String(cwd || "").replace(/\\/g, "/");
+    const wasAbsolute = normalized.startsWith("/");
+    if (wasAbsolute && normalizedCwd && !normalized.startsWith(`${normalizedCwd}/`) && normalized !== normalizedCwd) {
+      return "";
+    }
+    if (normalizedCwd && normalized.startsWith(normalizedCwd)) {
+      normalized = normalized.slice(normalizedCwd.length);
+    }
+    normalized = normalized.replace(/^\.?\//, "");
+    if (normalizedCwd) {
+      const repoName = path.basename(normalizedCwd);
+      const repoIndex = normalized.indexOf(`${repoName}/`);
+      if (repoIndex >= 0) normalized = normalized.slice(repoIndex + repoName.length + 1);
+    }
+    return normalized;
+  }
+
+  function isGeneratedArtifactPath(relPath) {
+    const normalized = normalizeMentionedPath(relPath);
+    return GENERATED_ARTIFACT_PATTERNS.some((pattern) => pattern.test(normalized));
+  }
+
+  function looksLikeUsefulPath(relPath) {
+    const normalized = normalizeMentionedPath(relPath);
+    if (!normalized || normalized.startsWith("http") || normalized.includes("://")) return false;
+    if (normalized.length < 3 || normalized.split("/").some((part) => !part || part.length > 120)) return false;
+    if (/^(Users|home|var|tmp|private|Volumes)\//i.test(normalized)) return false;
+    if (/^(Users|home|var|tmp|Downloads|Code|Projects)$/i.test(normalized)) return false;
+    if (isGeneratedArtifactPath(normalized)) return true;
+    if (/\.[A-Za-z0-9]{1,12}$/.test(normalized)) return true;
+    return /(^|\/)(src|app|lib|backend|frontend|tests|docs|scripts|components|pages|routes|api)\//.test(normalized);
+  }
+
+  function extractMentionedPaths(text, cwd = "") {
+    const found = new Set();
+    const source = String(text || "");
+    const pathPattern = /(?:^|[\s"'`])((?:\.{0,2}\/)?(?:[\w.@-]+\/)+[\w.@+-]+\.[A-Za-z0-9]{1,12})/g;
+    const filePattern = /(?:^|[\s"'`])((?:package-lock\.json|pnpm-lock\.yaml|yarn\.lock|coverage-final\.json|tsconfig\.json|pyproject\.toml|requirements\.txt|README\.md|CLAUDE\.md|AGENTS\.md))/g;
+    for (const pattern of [pathPattern, filePattern]) {
+      let match;
+      while ((match = pattern.exec(source))) {
+        const rel = normalizeMentionedPath(match[1], cwd);
+        if (!looksLikeUsefulPath(rel)) continue;
+        if (cwd && !isGeneratedArtifactPath(rel) && !fs.existsSync(path.join(cwd, rel))) continue;
+        found.add(rel);
+      }
+    }
+    return Array.from(found);
+  }
+
+  function normalizeCommand(value) {
+    return String(value || "")
+      .replace(/\s+/g, " ")
+      .replace(/[;|&]+$/g, "")
+      .trim()
+      .slice(0, 160);
+  }
+
+  function isShellCommand(value) {
+    return /^(npm|pnpm|yarn|bun|pytest|python3?|node|npx|uv|ruff|cargo|go|make|git|cd|rm|cp|mv|sed|rg|grep|find|cat)\b/.test(String(value || "").trim());
+  }
+
+  function extractCommandCandidates(row, text) {
+    const commands = [];
+    const directInputs = [
+      row.payload?.input,
+      row.payload?.arguments,
+      row.message?.input,
+      row.message?.arguments,
+    ];
+    for (const input of directInputs) {
+      if (typeof input === "string") commands.push(input);
+      else if (input && typeof input === "object") {
+        for (const value of Object.values(input)) {
+          if (typeof value === "string") commands.push(value);
+        }
+      }
+    }
+    const toolItems = Array.isArray(row.message?.content) ? row.message.content : [];
+    for (const item of toolItems) {
+      if (!item || typeof item !== "object") continue;
+      if (typeof item.input === "string") commands.push(item.input);
+      if (item.input && typeof item.input === "object") {
+        for (const value of Object.values(item.input)) {
+          if (typeof value === "string") commands.push(value);
+        }
+      }
+    }
+    if (/tool_use|function_call/i.test(row.type || row.payload?.type || "")) {
+      const commandPattern = /\b(?:npm|pnpm|yarn|bun|pytest|python3?|node|npx|uv|ruff|cargo|go test|make|git)\b[^\n\r"`']{0,140}/g;
+      for (const match of String(text || "").matchAll(commandPattern)) {
+        commands.push(match[0]);
+      }
+    }
+    return Array.from(new Set(commands.map(normalizeCommand).filter((cmd) => cmd.length >= 3 && /\s/.test(cmd) && isShellCommand(cmd))));
+  }
+
+  function topCountEntries(map, limit = 5, minCount = 2) {
+    return Object.entries(map || {})
+      .filter(([, count]) => count >= minCount)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, limit)
+      .map(([value, count]) => ({ value, count }));
+  }
+
+  function isExpectedRepeatedPath(value) {
+    const normalized = normalizeMentionedPath(value).toLowerCase();
+    return ["claude.md", "agents.md", "readme.md"].includes(normalized) || normalized.endsWith("/readme.md");
+  }
+
+  function getActionableRepeatedPaths(session, limit = 3) {
+    return (session.repeatedPathMentions || [])
+      .filter((item) => !isExpectedRepeatedPath(item.value))
+      .filter((item) => !isGeneratedArtifactPath(item.value))
+      .slice(0, limit);
+  }
+
+  function summarizeGeneratedArtifacts(items = [], limit = 4) {
+    const groups = new Map();
+    for (const item of items) {
+      const value = normalizeMentionedPath(item.value);
+      let key = "generated files";
+      if (value.includes("__pycache__/") || value.endsWith(".pyc")) key = "__pycache__";
+      else if (value.includes("node_modules/")) key = "node_modules";
+      else if (/package-lock\.json|pnpm-lock\.yaml|yarn\.lock$/i.test(value)) key = "lockfiles";
+      else if (value.includes("/dist/") || value.startsWith("dist/")) key = "dist";
+      else if (value.includes("/build/") || value.startsWith("build/")) key = "build";
+      else if (value.includes("/coverage/") || value.startsWith("coverage/")) key = "coverage";
+      else if (/(^|\/)assets\/[^/]+-[A-Za-z0-9_-]{6,}\.(js|css|map)$/i.test(value)) key = "hashed assets";
+      const current = groups.get(key) || { type: key, count: 0, examples: [] };
+      current.count += Number(item.count || 1);
+      if (current.examples.length < 2) current.examples.push(value);
+      groups.set(key, current);
+    }
+    return Array.from(groups.values()).sort((a, b) => b.count - a.count).slice(0, limit);
+  }
+
+  return {
+    addUsage,
+    collectText,
+    extractCommandCandidates,
+    extractMentionedPaths,
+    getActionableRepeatedPaths,
+    incrementMap,
+    isGeneratedArtifactPath,
+    listFilesRecursive,
+    normalizeMentionedPath,
+    parseJsonl,
+    summarizeGeneratedArtifacts,
+    topCountEntries,
+    totalUsageTokens,
+  };
+};