compose-agentsmd 4.0.0 → 6.0.0

package/README.md

@@ -115,11 +115,16 @@ Ruleset keys:
 - `global` (optional): write `rules/global` to user-global instruction files (defaults to true). Set `false` to skip global writes.
 - `domains` (optional): domain folders under `rules/domains/<domain>`.
 - `extra` (optional): additional local rule files to append.
+- `budget` (optional): global-rule budget thresholds in `o200k_base` tokens.
+- `budget.totalTokens` (optional): hard total token budget for the composed global instruction output (defaults to `8000`). Exceeding this is reported as a budget violation.
+- `budget.moduleTokens` (optional): per-module advisory threshold for each composed global rule section (defaults to `800`). Crossing this is **not** a violation; it triggers a review prompt to check whether the listed modules contain procedural content that should move to skills (procedures belong in skills, not rules).
 - `claude` (optional): repository companion settings for Claude Code.
 - `claude.enabled` (optional): enable/disable companion generation (defaults to `true`).
 - `claude.output` (optional): companion file path (defaults to `CLAUDE.md`).
 - `output` (optional): repository output file name (defaults to `AGENTS.md`).
 
+When the composed global instruction output exceeds the total budget, the CLI emits a `⚠ Global rules budget exceeded` warning to `stderr`. When any module crosses the per-module advisory threshold, the CLI emits a separate `ℹ Modules over per-module review threshold` advisory to `stderr`. Both can be suppressed with `--quiet`. The machine-readable `--json` output includes `budget.totalExceeded`, `budget.moduleReviewTriggered`, the tokenizer name, total token count, and any over-threshold modules.
+
 ### Ruleset schema validation
 
 `compose-agentsmd` validates rulesets against `agent-ruleset.schema.json` on every run. If the ruleset does not conform to the schema, the tool exits with a schema error.

package/agent-ruleset.schema.json

@@ -48,11 +48,11 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
-        "totalLines": {
+        "totalTokens": {
           "type": "integer",
           "minimum": 1
         },
-        "moduleLines": {
+        "moduleTokens": {
           "type": "integer",
           "minimum": 1
         }

package/dist/compose-agents.js

@@ -6,6 +6,8 @@ import { execFileSync } from "node:child_process";
 import readline from "node:readline";
 import { Ajv } from "ajv";
 import { createTwoFilesPatch } from "diff";
+import { countTokens } from "gpt-tokenizer";
+import { prepareGitFallbackDestination } from "./git-fallback.js";
 const DEFAULT_RULESET_NAME = "agent-ruleset.json";
 const DEFAULT_OUTPUT = "AGENTS.md";
 const DEFAULT_CLAUDE_OUTPUT = "CLAUDE.md";
@@ -22,8 +24,18 @@ const RULESET_SCHEMA_PATH = new URL("../agent-ruleset.schema.json", import.meta.
 const PACKAGE_JSON_PATH = new URL("../package.json", import.meta.url);
 const TOOL_RULES_PATH = new URL("../tools/tool-rules.md", import.meta.url);
 const USAGE_PATH = new URL("../tools/usage.txt", import.meta.url);
-const DEFAULT_TOTAL_BUDGET = 350;
-const DEFAULT_MODULE_BUDGET = 30;
+const BUDGET_TOKENIZER = "o200k_base";
+// Token budgets for the composed global rules.
+// - DEFAULT_TOTAL_BUDGET: hard budget for the always-loaded global rules.
+//   Sized to accommodate realistic invariant density (~80–120 invariants ×
+//   ~30–50 tokens each ≈ 5–6k tokens) plus structural margin and growth
+//   headroom, while staying a small fraction of the smallest target model's
+//   effective system-prompt window. Total exceedance is a budget violation.
+// - DEFAULT_MODULE_BUDGET: per-module advisory threshold, NOT a violation.
+//   Crossing it triggers a review prompt to check whether the module is
+//   leaking procedural content (procedures belong in skills, not rules).
+const DEFAULT_TOTAL_BUDGET = 8000;
+const DEFAULT_MODULE_BUDGET = 800;
 const LINT_HEADER = "<!-- markdownlint-disable MD025 -->";
 const readValueArg = (remaining, index, flag) => {
     const value = remaining[index + 1];
@@ -441,7 +453,7 @@ const cloneAtRef = (repoUrl, ref, destination) => {
     execGit(["clone", "--depth", "1", "--branch", ref, repoUrl, destination]);
 };
 const fetchCommit = (repoUrl, commitHash, destination) => {
-    ensureDir(destination);
+    prepareGitFallbackDestination(destination);
     execGit(["init"], destination);
     execGit(["remote", "add", "origin", repoUrl], destination);
     execGit(["fetch", "--depth", "1", "origin", commitHash], destination);
@@ -584,6 +596,12 @@ const buildInstructionContent = (parts, includeToolRules) => {
     }
     return `${LINT_HEADER}\n${sections.join("\n\n")}\n`;
 };
+const countBudgetTokens = (content) => {
+    if (content.length === 0) {
+        return 0;
+    }
+    return countTokens(content);
+};
 const buildScopeDiff = (scope, targetPaths, desiredContent, rootDir) => {
     if (targetPaths.length === 0) {
         return undefined;
@@ -634,32 +652,37 @@ const composeRuleset = (rulesetPath, rootDir, options) => {
     const extraRules = Array.isArray(projectRuleset.extra) ? projectRuleset.extra : [];
     const directRulePaths = extraRules.map((rulePath) => resolveFrom(rulesetDir, rulePath));
     addRulePaths(directRulePaths, resolvedRepositoryRules, seenRepositoryRules);
-    const totalBudget = projectRuleset.budget?.totalLines ?? DEFAULT_TOTAL_BUDGET;
-    const moduleBudget = projectRuleset.budget?.moduleLines ?? DEFAULT_MODULE_BUDGET;
-    const normalizedGlobalRoot = normalizePath(path.resolve(globalRoot));
-    const globalRuleFiles = resolvedGlobalRules.filter((p) => normalizePath(p).startsWith(`${normalizedGlobalRoot}/`));
-    const moduleLineCounts = globalRuleFiles.map((filePath) => {
-        const content = fs.readFileSync(filePath, "utf8");
-        return { name: path.basename(filePath), lines: content.split("\n").length };
-    });
-    const totalLines = moduleLineCounts.reduce((sum, m) => sum + m.lines, 0);
-    const overBudgetModules = moduleLineCounts.filter((m) => m.lines > moduleBudget);
-    const budgetResult = {
-        totalLines,
-        totalBudget,
-        moduleBudget,
-        overBudgetModules,
-        exceeded: totalLines > totalBudget || overBudgetModules.length > 0
-    };
+    const totalBudget = projectRuleset.budget?.totalTokens ?? DEFAULT_TOTAL_BUDGET;
+    const moduleBudget = projectRuleset.budget?.moduleTokens ?? DEFAULT_MODULE_BUDGET;
     const buildRuleParts = (rulePaths) => rulePaths.map((rulePath) => {
         const body = normalizeTrailingWhitespace(fs.readFileSync(rulePath, "utf8"));
         const sourcePath = formatRuleSourcePath(rulePath, rulesRoot, rulesetDir, projectRuleset.source, resolvedRef);
-        return `Source: ${sourcePath}\n\n${body}`;
+        return {
+            name: path.basename(rulePath),
+            content: `Source: ${sourcePath}\n\n${body}`
+        };
     });
     const repositoryParts = buildRuleParts(resolvedRepositoryRules);
     const globalParts = buildRuleParts(resolvedGlobalRules);
-    const primaryOutputContent = buildInstructionContent(repositoryParts, true);
-    const globalOutputContent = buildInstructionContent(globalParts, false);
+    const repositoryContentParts = repositoryParts.map((part) => part.content);
+    const globalContentParts = globalParts.map((part) => part.content);
+    const primaryOutputContent = buildInstructionContent(repositoryContentParts, true);
+    const globalOutputContent = buildInstructionContent(globalContentParts, false);
+    const moduleTokenCounts = globalParts.map((part) => ({
+        name: part.name,
+        tokens: countBudgetTokens(part.content)
+    }));
+    const totalTokens = countBudgetTokens(globalOutputContent);
+    const overBudgetModules = moduleTokenCounts.filter((module) => module.tokens > moduleBudget);
+    const budgetResult = {
+        tokenizer: BUDGET_TOKENIZER,
+        totalTokens,
+        totalBudget,
+        moduleBudget,
+        overBudgetModules,
+        totalExceeded: totalTokens > totalBudget,
+        moduleReviewTriggered: overBudgetModules.length > 0
+    };
     const repositoryOutputs = [toDisplayPath(rootDir, primaryOutputPath)];
     const globalOutputs = globalOutputPaths.map((filePath) => toDisplayPath(rootDir, filePath));
     const composedFiles = [
@@ -746,18 +769,20 @@ const formatComposedOutputs = (result) => {
     }
     return `${lines.join("\n")}\n`;
 };
-const formatBudgetWarning = (result) => {
-    const totalInfo = result.totalLines > result.totalBudget
-        ? `: ${result.totalLines}/${result.totalBudget} lines`
-        : "";
-    const lines = [`⚠ Global rules budget exceeded${totalInfo}`];
-    if (result.overBudgetModules.length > 0) {
-        lines.push(`  Over-budget modules (>${result.moduleBudget} lines):`);
+const formatBudgetReport = (result) => {
+    const lines = [];
+    if (result.totalExceeded) {
+        lines.push(`⚠ Global rules budget exceeded (${result.tokenizer}): ` +
+            `${result.totalTokens}/${result.totalBudget} tokens`);
+    }
+    if (result.moduleReviewTriggered) {
+        lines.push(`ℹ Modules over per-module review threshold (> ${result.moduleBudget} tokens, advisory):`);
         for (const mod of result.overBudgetModules) {
-            lines.push(`    ${mod.name}: ${mod.lines} lines`);
+            lines.push(`    ${mod.name}: ${mod.tokens} tokens`);
         }
+        lines.push("  Review whether listed modules contain procedural content that should move to skills.");
     }
-    return `${lines.join("\n")}\n`;
+    return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
 };
 const LOCAL_RULES_TEMPLATE = "# Local Rules\n\n- Add project-specific instructions here.\n";
 const buildInitRuleset = (args) => {
@@ -938,8 +963,9 @@ const initProject = async (args, rootDir, rulesetName) => {
         if (composedOutput) {
             process.stdout.write(formatComposedOutputs(composedOutput));
             printOutputDiffs(composedOutput);
-            if (composedOutput.budgetResult.exceeded) {
-                process.stderr.write(formatBudgetWarning(composedOutput.budgetResult));
+            if (composedOutput.budgetResult.totalExceeded ||
+                composedOutput.budgetResult.moduleReviewTriggered) {
+                process.stderr.write(formatBudgetReport(composedOutput.budgetResult));
             }
         }
     }
@@ -1038,8 +1064,8 @@ const main = async () => {
         else if (!args.quiet) {
             process.stdout.write(formatComposedOutputs(output));
             printOutputDiffs(output);
-            if (output.budgetResult.exceeded) {
-                process.stderr.write(formatBudgetWarning(output.budgetResult));
+            if (output.budgetResult.totalExceeded || output.budgetResult.moduleReviewTriggered) {
+                process.stderr.write(formatBudgetReport(output.budgetResult));
             }
         }
         return;
@@ -1075,8 +1101,8 @@ const main = async () => {
             printOutputDiffs(result);
         }
         for (const result of outputs) {
-            if (result.budgetResult.exceeded) {
-                process.stderr.write(formatBudgetWarning(result.budgetResult));
+            if (result.budgetResult.totalExceeded || result.budgetResult.moduleReviewTriggered) {
+                process.stderr.write(formatBudgetReport(result.budgetResult));
             }
         }
     }

package/dist/git-fallback.js

@@ -0,0 +1,7 @@
+import fs from "node:fs";
+export const prepareGitFallbackDestination = (destination) => {
+    if (fs.existsSync(destination)) {
+        fs.rmSync(destination, { recursive: true, force: true });
+    }
+    fs.mkdirSync(destination, { recursive: true });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "compose-agentsmd",
-  "version": "4.0.0",
+  "version": "6.0.0",
   "description": "CLI tools for composing repository-local and user-global agent instruction files from modular rule sets",
   "license": "MIT",
   "repository": {
@@ -62,7 +62,8 @@
   },
   "dependencies": {
     "ajv": "^8.17.1",
-    "diff": "^8.0.3"
+    "diff": "^8.0.3",
+    "gpt-tokenizer": "^3.4.0"
   },
   "overrides": {
     "minimatch": "^10.2.4",