@tekyzinc/gsd-t 3.10.16 → 3.11.11

package/scripts/context-meter/estimate-tokens.js

@@ -0,0 +1,96 @@
+/**
+ * estimate-tokens.js
+ *
+ * Local token estimator — replaces the Anthropic count_tokens API call.
+ * Uses byte-length heuristics to estimate token count from a parsed transcript.
+ *
+ * Claude's BPE tokenizer averages ~3.5 chars per token for English text/code
+ * (range: 3.0 for dense prose, 4.5 for simple ASCII). We use 3.5 as the
+ * divisor, which slightly overestimates token count — this is the safe
+ * direction for a context-window guard (triggers pause earlier, not later).
+ *
+ * The estimate includes JSON structural overhead from the messages array
+ * (keys, brackets, commas) since that's what the API would count too.
+ *
+ * Accuracy: within ~5-10% of the real count_tokens API. For threshold bands
+ * with 15-point gaps (normal < 70%, warn < 85%), this is more than sufficient.
+ *
+ * @module scripts/context-meter/estimate-tokens
+ */
+
+"use strict";
+
+const CHARS_PER_TOKEN = 3.5;
+
+/**
+ * Estimate token count from a parsed transcript.
+ *
+ * @param {object} opts
+ * @param {string} opts.system - system prompt text
+ * @param {Array}  opts.messages - messages array from transcript-parser.js
+ * @returns {{ inputTokens: number } | null}
+ */
+function estimateTokens(opts) {
+  try {
+    if (!opts || typeof opts !== "object") return null;
+
+    const { system, messages } = opts;
+    if (!Array.isArray(messages)) return null;
+
+    let totalChars = 0;
+
+    if (typeof system === "string") {
+      totalChars += system.length;
+    }
+
+    for (const msg of messages) {
+      if (!msg || typeof msg !== "object") continue;
+      totalChars += measureContent(msg.content);
+    }
+
+    const inputTokens = Math.ceil(totalChars / CHARS_PER_TOKEN);
+    return { inputTokens };
+  } catch (_) {
+    return null;
+  }
+}
+
+/**
+ * Recursively measure character length of a message content value.
+ * Handles strings, arrays of blocks, and nested tool_result content.
+ */
+function measureContent(content) {
+  if (typeof content === "string") return content.length;
+  if (!Array.isArray(content)) return 0;
+
+  let chars = 0;
+  for (const block of content) {
+    if (!block || typeof block !== "object") continue;
+
+    if (block.type === "text" && typeof block.text === "string") {
+      chars += block.text.length;
+    } else if (block.type === "tool_use") {
+      chars += (typeof block.name === "string" ? block.name.length : 0);
+      if (block.input != null) {
+        try {
+          chars += JSON.stringify(block.input).length;
+        } catch (_) {
+          // skip
+        }
+      }
+    } else if (block.type === "tool_result") {
+      chars += measureContent(block.content);
+    } else if (block.type === "image" && block.source) {
+      // base64 images: ~0.75 bytes per base64 char, tokenized differently
+      // but we count the source data length as a rough proxy
+      try {
+        chars += JSON.stringify(block.source).length;
+      } catch (_) {
+        // skip
+      }
+    }
+  }
+  return chars;
+}
+
+module.exports = { estimateTokens, CHARS_PER_TOKEN };

package/scripts/context-meter/estimate-tokens.test.js

@@ -0,0 +1,158 @@
+/**
+ * estimate-tokens.test.js — unit tests for the local token estimator.
+ *
+ * @module scripts/context-meter/estimate-tokens.test
+ */
+
+"use strict";
+
+const { test } = require("node:test");
+const assert = require("node:assert/strict");
+const { estimateTokens, CHARS_PER_TOKEN } = require("./estimate-tokens");
+
+test("null/undefined opts returns null", () => {
+  assert.equal(estimateTokens(null), null);
+  assert.equal(estimateTokens(undefined), null);
+});
+
+test("missing messages returns null", () => {
+  assert.equal(estimateTokens({ system: "hi" }), null);
+  assert.equal(estimateTokens({ system: "hi", messages: "not-array" }), null);
+});
+
+test("empty messages returns 0 tokens (system empty)", () => {
+  const r = estimateTokens({ system: "", messages: [] });
+  assert.ok(r);
+  assert.equal(r.inputTokens, 0);
+});
+
+test("system-only content counted", () => {
+  const sys = "a".repeat(350);
+  const r = estimateTokens({ system: sys, messages: [] });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(350 / CHARS_PER_TOKEN));
+});
+
+test("text message content counted", () => {
+  const r = estimateTokens({
+    system: "",
+    messages: [
+      { role: "user", content: [{ type: "text", text: "a".repeat(700) }] },
+    ],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(700 / CHARS_PER_TOKEN));
+});
+
+test("string content (user shorthand) counted", () => {
+  const r = estimateTokens({
+    system: "",
+    messages: [{ role: "user", content: "hello world" }],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(11 / CHARS_PER_TOKEN));
+});
+
+test("tool_use input JSON counted", () => {
+  const input = { file_path: "/some/long/path/to/file.js" };
+  const inputJson = JSON.stringify(input);
+  const toolName = "Read";
+  const r = estimateTokens({
+    system: "",
+    messages: [
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: "t1", name: toolName, input }],
+      },
+    ],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil((toolName.length + inputJson.length) / CHARS_PER_TOKEN));
+});
+
+test("tool_result content counted (string)", () => {
+  const resultText = "file contents here".repeat(10);
+  const r = estimateTokens({
+    system: "",
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "tool_result", tool_use_id: "t1", content: resultText },
+        ],
+      },
+    ],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(resultText.length / CHARS_PER_TOKEN));
+});
+
+test("tool_result content counted (array of text blocks)", () => {
+  const r = estimateTokens({
+    system: "",
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "t1",
+            content: [
+              { type: "text", text: "abc" },
+              { type: "text", text: "defgh" },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(8 / CHARS_PER_TOKEN));
+});
+
+test("multiple messages accumulate", () => {
+  const r = estimateTokens({
+    system: "sys".repeat(100),
+    messages: [
+      { role: "user", content: [{ type: "text", text: "a".repeat(200) }] },
+      { role: "assistant", content: [{ type: "text", text: "b".repeat(300) }] },
+    ],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil((300 + 200 + 300) / CHARS_PER_TOKEN));
+});
+
+test("skips blocks with missing type", () => {
+  const r = estimateTokens({
+    system: "",
+    messages: [
+      { role: "user", content: [{ text: "no type field" }, { type: "text", text: "ok" }] },
+    ],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(2 / CHARS_PER_TOKEN));
+});
+
+test("handles null/non-object messages gracefully", () => {
+  const r = estimateTokens({
+    system: "",
+    messages: [null, undefined, 42, { role: "user", content: [{ type: "text", text: "ok" }] }],
+  });
+  assert.ok(r);
+  assert.equal(r.inputTokens, Math.ceil(2 / CHARS_PER_TOKEN));
+});
+
+test("realistic conversation produces reasonable estimate", () => {
+  const msgs = [];
+  for (let i = 0; i < 20; i++) {
+    msgs.push({ role: "user", content: [{ type: "text", text: "Tell me about X. ".repeat(5) }] });
+    msgs.push({
+      role: "assistant",
+      content: [{ type: "text", text: "Here is info about X. ".repeat(20) }],
+    });
+  }
+  const r = estimateTokens({ system: "You are a helpful assistant.", messages: msgs });
+  assert.ok(r);
+  assert.ok(r.inputTokens > 500, `expected >500 tokens, got ${r.inputTokens}`);
+  assert.ok(r.inputTokens < 10000, `expected <10000 tokens, got ${r.inputTokens}`);
+});

package/scripts/context-meter/threshold.js

@@ -71,8 +71,9 @@ function bandFor(pct) {
  * Build the `additionalContext` string the hook emits, or null if the
  * measured percentage is below the configured thresholdPct.
  *
- * Exact format (from .gsd-t/contracts/context-meter-contract.md line 139):
- *   ⚠️ Context window at {pct.toFixed(1)}% of {modelWindowSize}. Run /user:gsd-t-pause to checkpoint and clear before continuing.
+ * M37 (v1.2.0): This is a MANDATORY STOP instruction, not a suggestion.
+ * The multi-line format is designed to be unignorable — same enforcement
+ * weight as the Destructive Action Guard.
  *
  * `modelWindowSize` is emitted as the raw integer — no commas, no "K" suffix.
  *
@@ -82,7 +83,14 @@ function bandFor(pct) {
 function buildAdditionalContext({ pct, modelWindowSize, thresholdPct } = {}) {
   if (!Number.isFinite(pct) || !Number.isFinite(thresholdPct)) return null;
   if (pct < thresholdPct) return null;
-  return `⚠️ Context window at ${pct.toFixed(1)}% of ${modelWindowSize}. Run /user:gsd-t-pause to checkpoint and clear before continuing.`;
+  return [
+    `🛑 MANDATORY STOP — Context window at ${pct.toFixed(1)}% of ${modelWindowSize} (threshold: ${thresholdPct}%).`,
+    `You MUST stop what you are doing RIGHT NOW and execute these steps in order:`,
+    `1. Run /user:gsd-t-pause to save your exact position`,
+    `2. Tell the user to run /clear to free the context window`,
+    `3. Tell the user to run /user:gsd-t-resume to continue from the saved position`,
+    `Do NOT continue working, spawn subagents, or start new tasks. This has the same enforcement weight as the Destructive Action Guard — violating it risks losing work to context overflow.`,
+  ].join("\n");
 }
 
 module.exports = { computePct, bandFor, buildAdditionalContext, BANDS };

package/scripts/context-meter/threshold.test.js

@@ -154,7 +154,7 @@ test("buildAdditionalContext — below threshold returns null", () => {
   );
 });
 
-test("buildAdditionalContext — at threshold returns string", () => {
+test("buildAdditionalContext — at threshold returns multi-line MANDATORY STOP", () => {
   const result = buildAdditionalContext({
     pct: 75,
     modelWindowSize: 200000,
@@ -162,18 +162,28 @@ test("buildAdditionalContext — at threshold returns string", () => {
   });
   assert.ok(typeof result === "string");
   assert.ok(result.includes("75.0%"));
+  assert.ok(result.includes("MANDATORY STOP"));
+  assert.ok(result.includes("/user:gsd-t-pause"));
+  assert.ok(result.includes("/clear"));
+  assert.ok(result.includes("/user:gsd-t-resume"));
+  assert.ok(result.includes("Destructive Action Guard"));
+  assert.ok(result.includes("\n"), "must be multi-line");
 });
 
-test("buildAdditionalContext — above threshold exact contract string", () => {
+test("buildAdditionalContext — above threshold exact contract string (M37 multi-line)", () => {
   const result = buildAdditionalContext({
     pct: 76.2,
     modelWindowSize: 200000,
     thresholdPct: 75,
   });
-  assert.equal(
-    result,
-    "⚠️ Context window at 76.2% of 200000. Run /user:gsd-t-pause to checkpoint and clear before continuing."
-  );
+  const lines = result.split("\n");
+  assert.equal(lines.length, 6, "must have exactly 6 lines");
+  assert.equal(lines[0], "🛑 MANDATORY STOP — Context window at 76.2% of 200000 (threshold: 75%).");
+  assert.equal(lines[1], "You MUST stop what you are doing RIGHT NOW and execute these steps in order:");
+  assert.equal(lines[2], "1. Run /user:gsd-t-pause to save your exact position");
+  assert.equal(lines[3], "2. Tell the user to run /clear to free the context window");
+  assert.equal(lines[4], "3. Tell the user to run /user:gsd-t-resume to continue from the saved position");
+  assert.ok(lines[5].includes("Destructive Action Guard"));
 });
 
 test("buildAdditionalContext — decimal formatting rounds via toFixed(1)", () => {
@@ -195,7 +205,7 @@ test("buildAdditionalContext — modelWindowSize emitted raw (no commas)", () =>
     modelWindowSize: 200000,
     thresholdPct: 75,
   });
-  assert.ok(result.includes("of 200000."));
+  assert.ok(result.includes("of 200000"));
   assert.ok(!result.includes("200,000"));
   assert.ok(!result.includes("200K"));
 });
@@ -228,6 +238,7 @@ test("buildAdditionalContext — zero pct vs zero threshold emits", () => {
   });
   assert.ok(typeof result === "string");
   assert.ok(result.includes("0.0%"));
+  assert.ok(result.includes("MANDATORY STOP"));
 });
 
 test("buildAdditionalContext — pct over 100% still formats correctly", () => {
@@ -236,10 +247,8 @@ test("buildAdditionalContext — pct over 100% still formats correctly", () => {
     modelWindowSize: 200000,
     thresholdPct: 75,
   });
-  assert.equal(
-    result,
-    "⚠️ Context window at 102.3% of 200000. Run /user:gsd-t-pause to checkpoint and clear before continuing."
-  );
+  assert.ok(result.startsWith("🛑 MANDATORY STOP — Context window at 102.3% of 200000"));
+  assert.ok(result.includes("MANDATORY STOP"));
 });
 
 test("buildAdditionalContext — different modelWindowSize (1M)", () => {
@@ -248,8 +257,6 @@ test("buildAdditionalContext — different modelWindowSize (1M)", () => {
     modelWindowSize: 1000000,
     thresholdPct: 75,
   });
-  assert.equal(
-    result,
-    "⚠️ Context window at 80.0% of 1000000. Run /user:gsd-t-pause to checkpoint and clear before continuing."
-  );
+  assert.ok(result.startsWith("🛑 MANDATORY STOP — Context window at 80.0% of 1000000"));
+  assert.ok(result.includes("threshold: 75%"));
 });