compose-agentsmd 4.0.0 → 5.0.0

package/README.md

@@ -115,11 +115,16 @@ Ruleset keys:
 - `global` (optional): write `rules/global` to user-global instruction files (defaults to true). Set `false` to skip global writes.
 - `domains` (optional): domain folders under `rules/domains/<domain>`.
 - `extra` (optional): additional local rule files to append.
+- `budget` (optional): global-rule budget thresholds in `o200k_base` tokens.
+- `budget.totalTokens` (optional): total token budget for the composed global instruction output (defaults to `4500`).
+- `budget.moduleTokens` (optional): per-module token budget for each composed global rule section (defaults to `400`).
 - `claude` (optional): repository companion settings for Claude Code.
 - `claude.enabled` (optional): enable/disable companion generation (defaults to `true`).
 - `claude.output` (optional): companion file path (defaults to `CLAUDE.md`).
 - `output` (optional): repository output file name (defaults to `AGENTS.md`).
 
+When the composed global instruction output exceeds either budget, the CLI emits a warning to `stderr`. The machine-readable `--json` output includes the tokenizer name, total token count, and any over-budget modules.
+
 ### Ruleset schema validation
 
 `compose-agentsmd` validates rulesets against `agent-ruleset.schema.json` on every run. If the ruleset does not conform to the schema, the tool exits with a schema error.

package/agent-ruleset.schema.json

@@ -48,11 +48,11 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
-        "totalLines": {
+        "totalTokens": {
           "type": "integer",
           "minimum": 1
         },
-        "moduleLines": {
+        "moduleTokens": {
           "type": "integer",
           "minimum": 1
         }

package/dist/compose-agents.js

@@ -6,6 +6,8 @@ import { execFileSync } from "node:child_process";
 import readline from "node:readline";
 import { Ajv } from "ajv";
 import { createTwoFilesPatch } from "diff";
+import { countTokens } from "gpt-tokenizer";
+import { prepareGitFallbackDestination } from "./git-fallback.js";
 const DEFAULT_RULESET_NAME = "agent-ruleset.json";
 const DEFAULT_OUTPUT = "AGENTS.md";
 const DEFAULT_CLAUDE_OUTPUT = "CLAUDE.md";
@@ -22,8 +24,9 @@ const RULESET_SCHEMA_PATH = new URL("../agent-ruleset.schema.json", import.meta.
 const PACKAGE_JSON_PATH = new URL("../package.json", import.meta.url);
 const TOOL_RULES_PATH = new URL("../tools/tool-rules.md", import.meta.url);
 const USAGE_PATH = new URL("../tools/usage.txt", import.meta.url);
-const DEFAULT_TOTAL_BUDGET = 350;
-const DEFAULT_MODULE_BUDGET = 30;
+const BUDGET_TOKENIZER = "o200k_base";
+const DEFAULT_TOTAL_BUDGET = 4500;
+const DEFAULT_MODULE_BUDGET = 400;
 const LINT_HEADER = "<!-- markdownlint-disable MD025 -->";
 const readValueArg = (remaining, index, flag) => {
     const value = remaining[index + 1];
@@ -441,7 +444,7 @@ const cloneAtRef = (repoUrl, ref, destination) => {
     execGit(["clone", "--depth", "1", "--branch", ref, repoUrl, destination]);
 };
 const fetchCommit = (repoUrl, commitHash, destination) => {
-    ensureDir(destination);
+    prepareGitFallbackDestination(destination);
     execGit(["init"], destination);
     execGit(["remote", "add", "origin", repoUrl], destination);
     execGit(["fetch", "--depth", "1", "origin", commitHash], destination);
@@ -584,6 +587,12 @@ const buildInstructionContent = (parts, includeToolRules) => {
     }
     return `${LINT_HEADER}\n${sections.join("\n\n")}\n`;
 };
+const countBudgetTokens = (content) => {
+    if (content.length === 0) {
+        return 0;
+    }
+    return countTokens(content);
+};
 const buildScopeDiff = (scope, targetPaths, desiredContent, rootDir) => {
     if (targetPaths.length === 0) {
         return undefined;
@@ -634,32 +643,36 @@ const composeRuleset = (rulesetPath, rootDir, options) => {
     const extraRules = Array.isArray(projectRuleset.extra) ? projectRuleset.extra : [];
     const directRulePaths = extraRules.map((rulePath) => resolveFrom(rulesetDir, rulePath));
     addRulePaths(directRulePaths, resolvedRepositoryRules, seenRepositoryRules);
-    const totalBudget = projectRuleset.budget?.totalLines ?? DEFAULT_TOTAL_BUDGET;
-    const moduleBudget = projectRuleset.budget?.moduleLines ?? DEFAULT_MODULE_BUDGET;
-    const normalizedGlobalRoot = normalizePath(path.resolve(globalRoot));
-    const globalRuleFiles = resolvedGlobalRules.filter((p) => normalizePath(p).startsWith(`${normalizedGlobalRoot}/`));
-    const moduleLineCounts = globalRuleFiles.map((filePath) => {
-        const content = fs.readFileSync(filePath, "utf8");
-        return { name: path.basename(filePath), lines: content.split("\n").length };
-    });
-    const totalLines = moduleLineCounts.reduce((sum, m) => sum + m.lines, 0);
-    const overBudgetModules = moduleLineCounts.filter((m) => m.lines > moduleBudget);
-    const budgetResult = {
-        totalLines,
-        totalBudget,
-        moduleBudget,
-        overBudgetModules,
-        exceeded: totalLines > totalBudget || overBudgetModules.length > 0
-    };
+    const totalBudget = projectRuleset.budget?.totalTokens ?? DEFAULT_TOTAL_BUDGET;
+    const moduleBudget = projectRuleset.budget?.moduleTokens ?? DEFAULT_MODULE_BUDGET;
     const buildRuleParts = (rulePaths) => rulePaths.map((rulePath) => {
         const body = normalizeTrailingWhitespace(fs.readFileSync(rulePath, "utf8"));
         const sourcePath = formatRuleSourcePath(rulePath, rulesRoot, rulesetDir, projectRuleset.source, resolvedRef);
-        return `Source: ${sourcePath}\n\n${body}`;
+        return {
+            name: path.basename(rulePath),
+            content: `Source: ${sourcePath}\n\n${body}`
+        };
     });
     const repositoryParts = buildRuleParts(resolvedRepositoryRules);
     const globalParts = buildRuleParts(resolvedGlobalRules);
-    const primaryOutputContent = buildInstructionContent(repositoryParts, true);
-    const globalOutputContent = buildInstructionContent(globalParts, false);
+    const repositoryContentParts = repositoryParts.map((part) => part.content);
+    const globalContentParts = globalParts.map((part) => part.content);
+    const primaryOutputContent = buildInstructionContent(repositoryContentParts, true);
+    const globalOutputContent = buildInstructionContent(globalContentParts, false);
+    const moduleTokenCounts = globalParts.map((part) => ({
+        name: part.name,
+        tokens: countBudgetTokens(part.content)
+    }));
+    const totalTokens = countBudgetTokens(globalOutputContent);
+    const overBudgetModules = moduleTokenCounts.filter((module) => module.tokens > moduleBudget);
+    const budgetResult = {
+        tokenizer: BUDGET_TOKENIZER,
+        totalTokens,
+        totalBudget,
+        moduleBudget,
+        overBudgetModules,
+        exceeded: totalTokens > totalBudget || overBudgetModules.length > 0
+    };
     const repositoryOutputs = [toDisplayPath(rootDir, primaryOutputPath)];
     const globalOutputs = globalOutputPaths.map((filePath) => toDisplayPath(rootDir, filePath));
     const composedFiles = [
@@ -747,14 +760,14 @@ const formatComposedOutputs = (result) => {
     return `${lines.join("\n")}\n`;
 };
 const formatBudgetWarning = (result) => {
-    const totalInfo = result.totalLines > result.totalBudget
-        ? `: ${result.totalLines}/${result.totalBudget} lines`
+    const totalInfo = result.totalTokens > result.totalBudget
+        ? `: ${result.totalTokens}/${result.totalBudget} tokens`
         : "";
-    const lines = [`⚠ Global rules budget exceeded${totalInfo}`];
+    const lines = [`⚠ Global rules budget exceeded (${result.tokenizer})${totalInfo}`];
     if (result.overBudgetModules.length > 0) {
-        lines.push(`  Over-budget modules (>${result.moduleBudget} lines):`);
+        lines.push(`  Over-budget modules (> ${result.moduleBudget} tokens):`);
         for (const mod of result.overBudgetModules) {
-            lines.push(`    ${mod.name}: ${mod.lines} lines`);
+            lines.push(`    ${mod.name}: ${mod.tokens} tokens`);
         }
     }
     return `${lines.join("\n")}\n`;

package/dist/git-fallback.js

@@ -0,0 +1,7 @@
+import fs from "node:fs";
+export const prepareGitFallbackDestination = (destination) => {
+    if (fs.existsSync(destination)) {
+        fs.rmSync(destination, { recursive: true, force: true });
+    }
+    fs.mkdirSync(destination, { recursive: true });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "compose-agentsmd",
-  "version": "4.0.0",
+  "version": "5.0.0",
   "description": "CLI tools for composing repository-local and user-global agent instruction files from modular rule sets",
   "license": "MIT",
   "repository": {
@@ -62,7 +62,8 @@
   },
   "dependencies": {
     "ajv": "^8.17.1",
-    "diff": "^8.0.3"
+    "diff": "^8.0.3",
+    "gpt-tokenizer": "^3.4.0"
   },
   "overrides": {
     "minimatch": "^10.2.4",