codex-token-saver 1.0.0

package/src/core/config.js

@@ -0,0 +1,250 @@
+import path from "node:path";
+
+export const OLD_START = "<!-- CODEX-CONTEXT-INIT:START -->";
+export const OLD_END = "<!-- CODEX-CONTEXT-INIT:END -->";
+export const PROJECT_START = "<!-- CODEX-CONTEXT-INIT:PROJECT:START -->";
+export const PROJECT_END = "<!-- CODEX-CONTEXT-INIT:PROJECT:END -->";
+export const GLOBAL_START = "<!-- CODEX-CONTEXT-INIT:GLOBAL:START -->";
+export const GLOBAL_END = "<!-- CODEX-CONTEXT-INIT:GLOBAL:END -->";
+
+export const DEFAULT_MAX_FILE_SIZE_KB = 300;
+export const SCHEMA_VERSION = 2;
+export const GENERATOR_VERSION = "0.1.0";
+export const HEAVY_DIRS = new Set(["node_modules", ".git", "dist", "build", "out", "coverage", ".next", ".nuxt", "target", "vendor", ".venv", "__pycache__"]);
+export const CONTEXT_FILES = ["index.json", "summary.md", "symbols.md", "files.md", "routes.md", "dependencies.md", "recent_changes.md"];
+export const RELEVANT_CONTEXT_FILE = "relevant.md";
+export const SECRET_FILE_NAMES = new Set([".env", "id_rsa", "id_ed25519"]);
+export const SECRET_PREFIXES = [".env.", "secrets.", "credentials."];
+export const SECRET_SUFFIXES = [".pem", ".key"];
+export const DEPENDENCY_FILES = ["package.json", "requirements.txt", "pyproject.toml", "Cargo.toml", "go.mod", "pom.xml", "build.gradle"];
+export const RELEVANT_EXTENSIONS = new Set([".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".py", ".rs", ".go", ".java", ".cs", ".json", ".md", ".yml", ".yaml", ".toml", ".gradle", ".xml"]);
+export const RELEVANT_FILE_NAMES = new Set([...DEPENDENCY_FILES.map((file) => file.toLowerCase()), "dockerfile"]);
+
+export const requiredFiles = [
+  path.join(".codex", "AGENTS.md"),
+  path.join(".codex", "templates", "project_context.template.md"),
+  path.join(".codex", "templates", "architecture.template.md"),
+  path.join(".codex", "templates", "task.template.md"),
+  path.join(".codex", "templates", "decision_log.template.md"),
+  "project_context.md",
+  "architecture.md",
+  "task.md",
+  "decision_log.md"
+];
+
+export const contextFiles = CONTEXT_FILES.map((file) => path.join(".codex", "context", file));
+
+export const globalManagedBlock = `${GLOBAL_START}
+# Global Codex Token Optimization Rules
+
+You are operating in AGGRESSIVE TOKEN OPTIMIZATION MODE.
+
+## Global Behavior
+
+- Complete coding tasks with the fewest tokens, fewest file reads, smallest diff, and shortest useful response.
+- Do not explain unless explicitly asked.
+- Do not teach.
+- Do not summarize the repository.
+- Do not restate the user request.
+- Do not provide alternatives unless blocked.
+- Do not create long plans.
+- Ask clarifying questions only when the task is impossible or unsafe without one.
+
+## Context Usage
+
+- Read the minimum files required.
+- Prefer targeted search over broad exploration.
+- Never scan the whole repository unless explicitly requested.
+- Stop reading once enough context is found.
+- Use open files, visible errors, and user-provided context first.
+
+## Editing
+
+- Make the smallest correct change.
+- Prefer local fixes over refactors.
+- Reuse existing patterns.
+- Do not rename, reformat, or reorganize unrelated code.
+- Avoid dependency changes unless essential.
+
+## Validation
+
+- Run only the narrowest relevant check.
+- Prefer targeted tests over full test suites.
+- If validation is skipped, say why in one short sentence.
+
+## Response Format
+
+Return only:
+
+CHANGED
+- path/to/file
+
+VALIDATION
+- command or "not run"
+
+DONE
+${GLOBAL_END}`;
+
+export const projectManagedBlock = `${PROJECT_START}
+# Project Codex Context Rules
+
+## Precomputed Context Engine
+
+Before broad repository search, read these generated context files if present:
+
+1. .codex/context/relevant.md
+2. .codex/context/summary.md
+3. .codex/context/dependencies.md
+4. .codex/context/files.md
+5. .codex/context/symbols.md
+6. .codex/context/routes.md
+7. .codex/context/recent_changes.md
+8. .codex/context/index.json
+
+If \`.codex/context/relevant.md\` exists, treat it as the task-specific context shortlist generated from the user's latest query.
+
+Use these as pre-indexed repository context.
+
+Rules:
+- Prefer these files before scanning directories.
+- Use importance scores to identify likely relevant files.
+- Use them to identify the smallest relevant file set.
+- Do not treat them as always complete.
+- If generated context conflicts with source code, source code wins.
+- After meaningful code changes, update the context index by running:
+  \`codex-context-init index\`
+
+## Context Source Priority
+
+Then read these project-maintained files when present:
+
+1. task.md
+2. architecture.md
+3. decision_log.md
+4. project_context.md
+
+Do not scan the repository until these context files have been checked.
+
+## Project Documentation Rules
+
+- Update task.md after meaningful progress.
+- Update decision_log.md only when a technical decision is made.
+- Update architecture.md only when structure, dependencies, boundaries, or data flow change.
+- Update project_context.md only when product goals, constraints, or scope change.
+- Do not generate extra documentation unless requested.
+
+## Project Search Rules
+
+- Prefer context files before broad repository search.
+- Search only files directly related to the current task.
+- Stop searching once sufficient context is found.
+${PROJECT_END}`;
+
+export const templates = {
+  "project_context.md": `# Project Context
+
+## Product / Project Name
+
+TODO
+
+## Goal
+
+TODO
+
+## Users
+
+TODO
+
+## Core Features
+
+TODO
+
+## Non-Goals
+
+TODO
+
+## Constraints
+
+- Prefer small, maintainable changes.
+- Prefer existing patterns.
+- Avoid unnecessary dependencies.
+- Optimize Codex token usage.
+
+## Current Scope
+
+TODO
+`,
+  "architecture.md": `# Architecture
+
+## Overview
+
+TODO
+
+## Tech Stack
+
+TODO
+
+## Main Components
+
+TODO
+
+## Data Flow
+
+TODO
+
+## Important Directories
+
+TODO
+
+## Integration Points
+
+TODO
+
+## Constraints
+
+- Keep architecture simple.
+- Avoid premature abstractions.
+- Prefer modular boundaries.
+`,
+  "task.md": `# Task
+
+## Current Task
+
+TODO
+
+## Status
+
+Not started.
+
+## Relevant Files
+
+TODO
+
+## Acceptance Criteria
+
+TODO
+
+## Notes for Codex
+
+- Read this file first.
+- Only inspect files directly related to the current task.
+- Keep changes minimal.
+`,
+  "decision_log.md": `# Decision Log
+
+Record only meaningful technical decisions.
+
+## Format
+
+### YYYY-MM-DD - Decision Title
+
+Decision:
+TODO
+
+Reason:
+TODO
+
+Impact:
+TODO
+`
+};

package/src/core/engine/doctor.js

@@ -0,0 +1 @@
+export { runContextDoctor, runDebug, runDoctor, runProjectDoctor } from "../../core.js";

package/src/core/engine/global.js

@@ -0,0 +1 @@
+export { getGlobalAgentsPath, runGlobalDoctor, runGlobalSetup } from "../../core.js";

package/src/core/engine/index.js

@@ -0,0 +1,19 @@
+export {
+  collectFiles,
+  countEligibleFiles,
+  detectDependencies,
+  extractExports,
+  extractFileMetadata,
+  extractImports,
+  extractRouteHints,
+  extractSymbols,
+  generateSummary,
+  getWatchDirs,
+  isIgnoredPath,
+  isIgnoredWorkspacePath,
+  isLikelyBinary,
+  runContextClean,
+  runContextIndex,
+  runIndex,
+  writeContextArtifacts
+} from "../../core.js";

package/src/core/engine/query.js

@@ -0,0 +1 @@
+export { runQuery } from "../../core.js";

package/src/core/engine/sync.js

@@ -0,0 +1 @@
+export { runNew, runSync } from "../../core.js";

package/src/core/engine/upgrade.js

@@ -0,0 +1 @@
+export { runProjectUpgrade, runUpgrade, upsertManagedBlock } from "../../core.js";

package/src/core/logger.js

@@ -0,0 +1 @@
+export * from "./utils/logger.js";

package/src/core/parsers/genericParser.js

@@ -0,0 +1,16 @@
+export function emptyParseResult() {
+  return { imports: [], exports: [], symbols: [], routes: [], headings: [], dependencies: [] };
+}
+
+export function safeParse(parse, content, context = {}) {
+  try {
+    return parse(content, context);
+  } catch (error) {
+    context.logger?.warn?.(`Parser failed for ${context.relativePath || "unknown file"}: ${error.message}`);
+    return emptyParseResult();
+  }
+}
+
+export function parseGeneric() {
+  return emptyParseResult();
+}

package/src/core/parsers/index.js

@@ -0,0 +1,19 @@
+import path from "node:path";
+import { parseGeneric } from "./genericParser.js";
+import { parseJavaScript } from "./javascriptParser.js";
+import { parseJson } from "./jsonParser.js";
+import { parseMarkdown } from "./markdownParser.js";
+import { parsePython } from "./pythonParser.js";
+import { parseTypeScript } from "./typescriptParser.js";
+
+export function parseFile(content, context = {}) {
+  const ext = context.ext || path.extname(context.relativePath || "").toLowerCase();
+  const fileName = context.fileName || path.basename(context.relativePath || "");
+  const parserContext = { ...context, ext, fileName };
+  if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) return parseJavaScript(content, parserContext);
+  if ([".ts", ".tsx"].includes(ext)) return parseTypeScript(content, parserContext);
+  if (ext === ".py") return parsePython(content, parserContext);
+  if ([".md", ".mdx"].includes(ext)) return parseMarkdown(content, parserContext);
+  if (ext === ".json") return parseJson(content, parserContext);
+  return parseGeneric(content, parserContext);
+}

package/src/core/parsers/javascriptParser.js

@@ -0,0 +1,41 @@
+import { emptyParseResult, safeParse } from "./genericParser.js";
+
+const ROUTE_RE = /\b(app|router)\.(get|post|put|patch|delete|use)\s*\(\s*["'`]([^"'`]+)["'`]/i;
+const REACT_ROUTE_RE = /<(?:Route|Link|NavLink)\b[^>]*(?:path|to)=["'`]([^"'`]+)["'`]/i;
+
+export function parseJavaScript(content, context = {}) {
+  return safeParse((text) => {
+    const result = emptyParseResult();
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i += 1) {
+      const line = lines[i];
+      const importMatch = line.match(/^\s*import\s+.*?\s+from\s+["'`]([^"'`]+)|^\s*import\s+["'`]([^"'`]+)|require\(\s*["'`]([^"'`]+)["'`]\s*\)/);
+      const importName = importMatch?.[1] || importMatch?.[2] || importMatch?.[3];
+      if (importName) result.imports.push(importName);
+
+      const exportMatch = line.match(/^\s*export\s+(?:default\s+)?(?:(?:async\s+)?(?:function|class|const|let|var)\s+)?([A-Za-z_$][\w$]*)?/);
+      if (exportMatch) result.exports.push(exportMatch[1] || "default");
+
+      const fn = line.match(/^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)/);
+      if (fn) result.symbols.push({ type: /^[A-Z]/.test(fn[1]) ? "component" : "function", name: fn[1], line: i + 1 });
+
+      const arrow = line.match(/^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/);
+      if (arrow) result.symbols.push({ type: /^[A-Z]/.test(arrow[1]) ? "component" : "function", name: arrow[1], line: i + 1 });
+
+      const cls = line.match(/^\s*(?:export\s+)?class\s+([A-Za-z_$][\w$]*)/);
+      if (cls) result.symbols.push({ type: /^[A-Z].*(Component|Page|Form|View)$/.test(cls[1]) ? "component" : "class", name: cls[1], line: i + 1 });
+
+      const exportedConst = line.match(/^\s*export\s+const\s+([A-Za-z_$][\w$]*)/);
+      if (exportedConst && !result.symbols.some((item) => item.name === exportedConst[1])) {
+        result.symbols.push({ type: "exported constant", name: exportedConst[1], line: i + 1 });
+      }
+
+      const route = line.match(ROUTE_RE);
+      if (route) result.routes.push({ method: route[2].toUpperCase(), path: route[3], source: context.relativePath, line: i + 1, kind: "api" });
+
+      const uiRoute = line.match(REACT_ROUTE_RE);
+      if (uiRoute) result.routes.push({ method: "", path: uiRoute[1], source: context.relativePath, line: i + 1, kind: "ui" });
+    }
+    return result;
+  }, content, context);
+}

package/src/core/parsers/jsonParser.js

@@ -0,0 +1,17 @@
+import { emptyParseResult, safeParse } from "./genericParser.js";
+
+export function parseJson(content, context = {}) {
+  return safeParse((text) => {
+    const result = emptyParseResult();
+    if (context.fileName !== "package.json") return result;
+    const pkg = JSON.parse(text);
+    result.dependencies.push({
+      file: context.relativePath,
+      packageName: pkg.name || "",
+      scripts: Object.keys(pkg.scripts || {}),
+      dependencies: Object.keys(pkg.dependencies || {}),
+      devDependencies: Object.keys(pkg.devDependencies || {})
+    });
+    return result;
+  }, content, context);
+}

package/src/core/parsers/markdownParser.js

@@ -0,0 +1,13 @@
+import { emptyParseResult, safeParse } from "./genericParser.js";
+
+export function parseMarkdown(content, context = {}) {
+  return safeParse((text) => {
+    const result = emptyParseResult();
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i += 1) {
+      const heading = lines[i].match(/^(#{1,6})\s+(.+)/);
+      if (heading) result.headings.push({ level: heading[1].length, text: heading[2].trim(), line: i + 1 });
+    }
+    return result;
+  }, content, context);
+}

package/src/core/parsers/pythonParser.js

@@ -0,0 +1,24 @@
+import { emptyParseResult, safeParse } from "./genericParser.js";
+
+export function parsePython(content, context = {}) {
+  return safeParse((text) => {
+    const result = emptyParseResult();
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i += 1) {
+      const line = lines[i];
+      const imported = line.match(/^\s*import\s+([A-Za-z0-9_.,\s]+)|^\s*from\s+([A-Za-z0-9_.]+)\s+import/);
+      const importName = imported?.[1] || imported?.[2];
+      if (importName) result.imports.push(importName.trim());
+
+      const fn = line.match(/^\s*def\s+([A-Za-z_]\w*)/);
+      if (fn) result.symbols.push({ type: "function", name: fn[1], line: i + 1 });
+
+      const cls = line.match(/^\s*class\s+([A-Za-z_]\w*)/);
+      if (cls) result.symbols.push({ type: "class", name: cls[1], line: i + 1 });
+
+      const route = line.match(/^\s*@(app|router)\.(get|post|put|patch|delete|route)\s*\(\s*["']([^"']+)["']/i);
+      if (route) result.routes.push({ method: route[2].toUpperCase() === "ROUTE" ? "" : route[2].toUpperCase(), path: route[3], source: context.relativePath, line: i + 1, kind: "api" });
+    }
+    return result;
+  }, content, context);
+}

package/src/core/parsers/typescriptParser.js

@@ -0,0 +1,5 @@
+import { parseJavaScript } from "./javascriptParser.js";
+
+export function parseTypeScript(content, context = {}) {
+  return parseJavaScript(content, context);
+}

package/src/core/scoring/queryScorer.js

@@ -0,0 +1,104 @@
+import path from "node:path";
+
+const STOP_WORDS = new Set(["what", "which", "where", "how", "do", "does", "the", "a", "an", "to", "for", "of", "in", "on", "with", "and", "or", "is", "are"]);
+
+const SYNONYMS = {
+  auth: ["auth", "authentication", "authorize", "authorization", "login", "logout", "session", "token", "jwt", "user"],
+  database: ["db", "database", "model", "schema", "migration", "repository", "query"],
+  api: ["api", "route", "controller", "endpoint", "handler", "request", "response"],
+  ui: ["ui", "component", "page", "view", "screen", "form"],
+  config: ["config", "settings", "env", "environment"],
+  test: ["test", "spec", "mock", "fixture"]
+};
+
+function normalizeTerm(term) {
+  const cleaned = term.toLowerCase().replace(/[^a-z0-9_-]/g, "");
+  if (cleaned.length > 3 && cleaned.endsWith("s")) return cleaned.slice(0, -1);
+  return cleaned;
+}
+
+export function queryTerms(question) {
+  const base = question.split(/\s+/).map(normalizeTerm).filter((term) => term && !STOP_WORDS.has(term));
+  const expanded = new Set(base);
+  for (const term of base) {
+    for (const values of Object.values(SYNONYMS)) {
+      if (values.includes(term)) values.forEach((value) => expanded.add(value));
+    }
+  }
+  return [...expanded];
+}
+
+function includesTerm(value, term) {
+  return String(value || "").toLowerCase().includes(term);
+}
+
+function add(reasons, reason) {
+  if (!reasons.includes(reason)) reasons.push(reason);
+}
+
+export function scoreFileForQuery(file, terms, recentPaths = new Set()) {
+  let score = 0;
+  const reasons = [];
+  const filePath = file.path || "";
+  const filename = path.basename(filePath);
+  const ext = file.ext || path.extname(filePath);
+  const generated = /(^|\/)(dist|build|out|coverage|generated)(\/|$)|(\.min\.)/i.test(filePath);
+
+  for (const term of terms) {
+    if (includesTerm(filePath, term)) {
+      score += 40;
+      add(reasons, `path matched ${term}`);
+    }
+    if (includesTerm(filename, term)) {
+      score += 30;
+      add(reasons, `filename matched ${term}`);
+    }
+    for (const symbol of file.symbols || []) {
+      if (includesTerm(symbol.name, term)) {
+        score += 25;
+        add(reasons, `symbol matched ${symbol.name}`);
+      }
+    }
+    for (const route of file.routes || file.routeHints || []) {
+      const routePath = route.path || route.route || "";
+      if (includesTerm(routePath, term)) {
+        score += 25;
+        add(reasons, `route matched ${routePath}`);
+      }
+    }
+    for (const value of [...(file.imports || []), ...(file.exports || [])]) {
+      if (includesTerm(value, term)) {
+        score += 20;
+        add(reasons, `import/export matched ${value}`);
+      }
+    }
+    for (const heading of file.headings || []) {
+      if (includesTerm(heading.text, term)) {
+        score += 15;
+        add(reasons, `heading matched ${heading.text}`);
+      }
+    }
+  }
+
+  if (terms.some((term) => includesTerm(ext, term))) {
+    score += 10;
+    add(reasons, `extension matched ${ext}`);
+  }
+  if (recentPaths.has(filePath)) {
+    score += 10;
+    add(reasons, "recently changed");
+  }
+  if (Number.isFinite(file.importanceScore)) {
+    const bonus = Math.max(0, Math.min(20, Math.round(file.importanceScore / 5)));
+    if (bonus) {
+      score += bonus;
+      add(reasons, `importance score bonus ${bonus}`);
+    }
+  }
+  if (generated) {
+    score -= 20;
+    add(reasons, "generated/build-like file penalty");
+  }
+
+  return { path: filePath, score, reasons };
+}

package/src/core/services/AgentService.js

@@ -0,0 +1,16 @@
+import { runGlobalDoctor, runGlobalSetup } from "../engine/global.js";
+import { runProjectUpgrade } from "../engine/upgrade.js";
+
+export class AgentService {
+  setupGlobal() {
+    return runGlobalSetup();
+  }
+
+  doctorGlobal() {
+    return runGlobalDoctor();
+  }
+
+  upgradeProject(root) {
+    return runProjectUpgrade(root);
+  }
+}

package/src/core/services/ContextService.js

@@ -0,0 +1,15 @@
+import { runContextClean, runContextDoctor, runContextIndex } from "../engine/index.js";
+
+export class ContextService {
+  index(root, options) {
+    return runContextIndex(root, options);
+  }
+
+  doctor(root) {
+    return runContextDoctor(root);
+  }
+
+  clean(root) {
+    return runContextClean(root);
+  }
+}

package/src/core/services/RepositoryService.js

@@ -0,0 +1,19 @@
+import { collectFiles, getWatchDirs, isIgnoredPath, isIgnoredWorkspacePath } from "../engine/index.js";
+
+export class RepositoryService {
+  collectFiles(root, options) {
+    return collectFiles(root, options);
+  }
+
+  getWatchDirs(root) {
+    return getWatchDirs(root);
+  }
+
+  isIgnoredPath(relativePath) {
+    return isIgnoredPath(relativePath);
+  }
+
+  isIgnoredWorkspacePath(root, file) {
+    return isIgnoredWorkspacePath(root, file);
+  }
+}

package/src/core/utils/config.js

@@ -0,0 +1 @@
+export * from "../config.js";

package/src/core/utils/fsSafe.js

@@ -0,0 +1,28 @@
+import fs from "node:fs";
+import path from "node:path";
+
+export function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+export function fileExists(file) {
+  return fs.existsSync(file);
+}
+
+export function writeFileAtomic(file, content) {
+  ensureDir(path.dirname(file));
+  const temp = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.${Date.now()}.tmp`);
+  fs.writeFileSync(temp, content, "utf8");
+  fs.renameSync(temp, file);
+}
+
+export function writeFileForce(file, content) {
+  ensureDir(path.dirname(file));
+  fs.writeFileSync(file, content, "utf8");
+}
+
+export function writeFileIfMissing(file, content) {
+  if (fileExists(file)) return false;
+  writeFileForce(file, content);
+  return true;
+}

package/src/core/utils/logger.js

@@ -0,0 +1,49 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ensureDir } from "./fsSafe.js";
+
+function format(level, message) {
+  return `${new Date().toISOString()} [${level}] ${message}`;
+}
+
+export function createLogger(options = {}) {
+  const verbose = Boolean(options.verbose);
+  const sink = options.sink ?? console;
+  const root = options.root ?? process.cwd();
+  const logFile = options.logFile ?? path.join(root, ".codex", "logs", "latest.log");
+
+  function write(level, message) {
+    const line = format(level, message);
+    try {
+      ensureDir(path.dirname(logFile));
+      fs.appendFileSync(logFile, `${line}\n`, "utf8");
+    } catch {
+      // Logging must never break the command being run.
+    }
+    return line;
+  }
+
+  return {
+    verbose,
+    logFile,
+    info(message) {
+      sink.log?.(message);
+      write("INFO", message);
+    },
+    warn(message) {
+      if (sink.warn) sink.warn(message);
+      else sink.log?.(message);
+      write("WARN", message);
+    },
+    error(error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const output = verbose && error instanceof Error ? error.stack : message;
+      sink.error?.(output);
+      write("ERROR", message);
+    },
+    debug(message) {
+      if (verbose) sink.log?.(message);
+      write("DEBUG", message);
+    }
+  };
+}