onbuzz 4.7.1 → 4.8.0

package/src/tools/parserHelpers.js

@@ -0,0 +1,296 @@
+/**
+ * @file src/tools/parserHelpers.js
+ * @description String-utility primitives that replace the regex-heavy
+ *   line classifiers in codeMapTool.js. These functions are intentionally
+ *   small + composable; their goal is "make a hand-written line classifier
+ *   readable" rather than to be a full parser.
+ *
+ * Design rules every helper follows:
+ *   - Word-boundary aware (so `class` never matches `classroom`)
+ *   - Position-indexed (idx in, idx-or-null out) — caller composes
+ *     by passing idx forward; no string slicing inside the hot loop
+ *   - Zero regex
+ *   - Pure: no globals, no surprise state, no I/O
+ *
+ * NO REGEX in this file. The whole point. If you find yourself reaching
+ * for `.test()` / `.match()` / `new RegExp()`, build a new helper instead.
+ *
+ * Tested at src/tools/__tests__/parserHelpers.test.js — keep that file
+ * green and the parsers downstream are free.
+ */
+
+// ── Character predicates ──────────────────────────────────────────────
+
+/** Word char per most-languages convention: ASCII letter, digit, underscore. */
+export function isWordChar(ch) {
+  if (!ch) return false;
+  const c = ch.charCodeAt(0);
+  // 0–9
+  if (c >= 48 && c <= 57)  return true;
+  // A–Z
+  if (c >= 65 && c <= 90)  return true;
+  // a–z
+  if (c >= 97 && c <= 122) return true;
+  // _
+  if (c === 95)            return true;
+  return false;
+}
+
+/** Identifier-start char: letter or underscore (not digit). */
+export function isIdentStart(ch) {
+  if (!ch) return false;
+  const c = ch.charCodeAt(0);
+  if (c >= 65 && c <= 90)  return true;
+  if (c >= 97 && c <= 122) return true;
+  if (c === 95)            return true;
+  return false;
+}
+
+/** ASCII whitespace: space, tab, vertical tab, form feed (not newline — line-based callers). */
+export function isWhitespace(ch) {
+  if (!ch) return false;
+  return ch === ' ' || ch === '\t' || ch === '\v' || ch === '\f';
+}
+
+// ── Index advancers ───────────────────────────────────────────────────
+
+/** Skip ASCII whitespace forward. Returns the new idx (may equal s.length). */
+export function skipWhitespace(s, idx = 0) {
+  let i = idx;
+  while (i < s.length && isWhitespace(s[i])) i += 1;
+  return i;
+}
+
+/** Skip whitespace + a single line comment if present. Returns new idx. */
+export function skipWhitespaceAndLineComment(s, idx = 0) {
+  let i = skipWhitespace(s, idx);
+  if (i < s.length - 1 && s[i] === '/' && s[i + 1] === '/') return s.length;
+  return i;
+}
+
+/**
+ * Read an identifier starting at idx.
+ * @returns {{ident: string, nextIdx: number} | null}  null if no identifier here.
+ */
+export function readIdent(s, idx = 0) {
+  if (!isIdentStart(s[idx])) return null;
+  let end = idx + 1;
+  while (end < s.length && isWordChar(s[end])) end += 1;
+  return { ident: s.slice(idx, end), nextIdx: end };
+}
+
+// ── Keyword matchers ──────────────────────────────────────────────────
+
+/**
+ * Returns true iff `s` (optionally from `idx`) begins with `keyword`
+ * AND the next char is a word boundary (or end-of-string).
+ *
+ * This is the safe replacement for `^keyword\b` regexes. Use it
+ * EVERYWHERE you'd reach for a startsWith on a keyword.
+ */
+export function startsWithKeyword(s, keyword, idx = 0) {
+  if (idx + keyword.length > s.length) return false;
+  // Loop instead of slice+startsWith to avoid an allocation in the hot path.
+  for (let k = 0; k < keyword.length; k += 1) {
+    if (s[idx + k] !== keyword[k]) return false;
+  }
+  const after = idx + keyword.length;
+  // word boundary: end-of-string OR next char is a non-word char
+  return after === s.length || !isWordChar(s[after]);
+}
+
+/**
+ * If `s` at `idx` starts with `keyword` and the next char is a word
+ * boundary, return the idx PAST the keyword. Else return -1.
+ */
+export function consumeKeyword(s, keyword, idx = 0) {
+  return startsWithKeyword(s, keyword, idx) ? idx + keyword.length : -1;
+}
+
+/**
+ * Try each keyword in order. Returns { keyword, nextIdx } for the first
+ * match, or null if none match.
+ */
+export function consumeAnyKeyword(s, keywords, idx = 0) {
+  for (const kw of keywords) {
+    const next = consumeKeyword(s, kw, idx);
+    if (next !== -1) return { keyword: kw, nextIdx: next };
+  }
+  return null;
+}
+
+/**
+ * Convenience: trimmed-start of `s` starts with one of the keywords,
+ * each followed by a word boundary. Allocation-free.
+ */
+export function trimmedStartsWithAny(s, keywords) {
+  const idx = skipWhitespace(s, 0);
+  for (const kw of keywords) if (startsWithKeyword(s, kw, idx)) return true;
+  return false;
+}
+
+// ── Pattern checks over a whole line ──────────────────────────────────
+
+/**
+ * Find the index of `needle` in `s` if and only if it appears at a
+ * word boundary on both sides (i.e. preceded by start-of-string or a
+ * non-word char, and followed by end-of-string or a non-word char).
+ * Returns -1 otherwise.
+ *
+ * Replaces `\bneedle\b` regex.
+ */
+export function indexOfWord(s, needle, fromIdx = 0) {
+  if (!needle) return -1;
+  let i = s.indexOf(needle, fromIdx);
+  while (i !== -1) {
+    const before = i === 0 ? null : s[i - 1];
+    const after = s[i + needle.length] /* may be undefined */;
+    const leftOk  = before === null   || !isWordChar(before);
+    const rightOk = after === undefined || !isWordChar(after);
+    if (leftOk && rightOk) return i;
+    i = s.indexOf(needle, i + 1);
+  }
+  return -1;
+}
+
+/** Does `s` contain `needle` as a whole word (word boundaries both sides)? */
+export function containsWord(s, needle, fromIdx = 0) {
+  return indexOfWord(s, needle, fromIdx) !== -1;
+}
+
+// ── Code-only stripping (strings + line comments → spaces) ───────────
+
+/**
+ * Replace string literals (', ", `) and line comments (// …) inside a
+ * line with spaces, preserving length and positions so callers (e.g.
+ * brace-counting in _parseJS) get accurate non-string brace counts.
+ *
+ * - Single quotes: `'…'` — handles `\'` escapes.
+ * - Double quotes: `"…"` — handles `\"` escapes.
+ * - Template literals: `` ` … ` `` — handles `\``; does NOT recurse into
+ *   `${…}` interpolation (the inner expression's braces are masked
+ *   along with everything else inside the backticks, which is fine for
+ *   our brace-counting use case since template literals are
+ *   syntactically a single expression).
+ * - Line comments: from `//` to end-of-line are masked.
+ *
+ * This replaces 4 chained `.replace(/…/g, '')` regexes in _parseJS.
+ *
+ * Returns a string of the same length as the input.
+ */
+export function stripStringsAndLineComments(line) {
+  const n = line.length;
+  if (n === 0) return line;
+  // Use an array of chars instead of a mutable string for O(n) writes.
+  const out = new Array(n);
+  let i = 0;
+  while (i < n) {
+    const ch = line[i];
+
+    // Line comment: from // to end-of-line → spaces
+    if (ch === '/' && line[i + 1] === '/') {
+      while (i < n) { out[i] = ' '; i += 1; }
+      break;
+    }
+
+    // String literal: quote char terminates the run; respect `\<char>` escapes.
+    if (ch === '"' || ch === "'" || ch === '`') {
+      const quote = ch;
+      out[i] = ch;          // keep the open quote in place to preserve length
+      i += 1;
+      while (i < n) {
+        if (line[i] === '\\' && i + 1 < n) {
+          out[i] = ' ';     // mask escape backslash
+          out[i + 1] = ' '; // mask the escaped char (handles \', \", \`, \\, \n …)
+          i += 2;
+          continue;
+        }
+        if (line[i] === quote) {
+          out[i] = quote;   // keep the closing quote
+          i += 1;
+          break;
+        }
+        out[i] = ' ';
+        i += 1;
+      }
+      continue;
+    }
+
+    out[i] = ch;
+    i += 1;
+  }
+  return out.join('');
+}
+
+// ── Counting (allocation-free) ────────────────────────────────────────
+
+/** Count occurrences of single-char `ch` in `s`. */
+export function countChar(s, ch) {
+  let n = 0;
+  for (let i = 0; i < s.length; i += 1) if (s[i] === ch) n += 1;
+  return n;
+}
+
+// ── String matchers used by the parsers ──────────────────────────────
+
+/**
+ * Does the trimmed-start of `s` look like an assignment LHS reaching a
+ * `=` after optional dotted property accesses? Returns the `=`'s
+ * starting index after `=`, or -1.
+ *
+ * Used by the JS parser for patterns like:
+ *   window.Foo = function () {}
+ *   MyApp.utils.bar = (x) => x
+ *
+ * The caller supplies the head matcher (e.g. "is this `window` or
+ * TitleCase?"). This helper only handles the `(\.\w+)*\s*=\s*` tail.
+ *
+ * @param {string} s
+ * @param {number} idx — caller has already consumed the LHS head
+ *                       (e.g. `window` or `MyApp`).
+ * @returns {number} idx past the `=`, or -1 if no assignment.
+ */
+export function consumeDottedAssignTail(s, idx) {
+  let i = idx;
+  // Walk over zero or more `.ident` segments.
+  while (i < s.length && s[i] === '.') {
+    i += 1;
+    const id = readIdent(s, i);
+    if (!id) return -1;
+    i = id.nextIdx;
+  }
+  i = skipWhitespace(s, i);
+  if (s[i] !== '=' || s[i + 1] === '=') return -1;     // not assignment (could be == or ===)
+  return skipWhitespace(s, i + 1);
+}
+
+/** Internal: does `s` end at `lastIdx` with `needle`? */
+function matchesAt(s, lastIdx, needle) {
+  const start = lastIdx - needle.length + 1;
+  if (start < 0) return false;
+  for (let k = 0; k < needle.length; k += 1) {
+    if (s[start + k] !== needle[k]) return false;
+  }
+  return true;
+}
+
+/**
+ * Returns true if the line ends with `endsWith` after trailing
+ * whitespace AND optionally a trailing `;`. The `;` skip only kicks in
+ * when `endsWith` itself doesn't already match at the trimmed tail —
+ * i.e. asking "does this end with `;`?" still works for `'return 1;'`.
+ *
+ * Replaces patterns like `/\)\s*\{$/.test(raw)` — "ends with `) {`".
+ */
+export function trimmedEndsWith(s, endsWith) {
+  let i = s.length - 1;
+  while (i >= 0 && isWhitespace(s[i])) i -= 1;
+  if (matchesAt(s, i, endsWith)) return true;
+  // Second try: allow a trailing `;` before the actual end-marker.
+  if (i >= 0 && s[i] === ';') {
+    i -= 1;
+    while (i >= 0 && isWhitespace(s[i])) i -= 1;
+    if (matchesAt(s, i, endsWith)) return true;
+  }
+  return false;
+}

package/src/utilities/__tests__/planInjection.test.js

@@ -0,0 +1,125 @@
+/**
+ * Tests for buildPlanBlock — the formatter that turns a list of
+ * plan/* memories into the AGENT WORKING PLAN block injected into
+ * the agent's system prompt every turn.
+ *
+ * What we pin:
+ *   - Empty input → empty block, no truncation
+ *   - Multiple plans render under a single header, newest-first
+ *     (the caller sorts; we just preserve order)
+ *   - Per-plan soft cap truncates with a clear marker + records the
+ *     plan in `truncated`
+ *   - Cross-plan hard cap drops later plans (records them too)
+ *   - Header / footer markers are stable (so downstream consumers
+ *     can rely on them for detection)
+ */
+import { describe, test, expect } from '@jest/globals';
+import { buildPlanBlock, _PLAN_INJECTION_INTERNALS } from '../planInjection.js';
+
+const { PER_PLAN_TOKEN_CAP, TOTAL_PLAN_TOKEN_CAP } = _PLAN_INJECTION_INTERNALS;
+
+function makePlan(title, content, updatedAt = '2026-05-16T12:00:00.000Z') {
+  return { title, content, updatedAt, createdAt: updatedAt };
+}
+
+describe('buildPlanBlock', () => {
+  test('empty input → empty block', () => {
+    const out = buildPlanBlock([]);
+    expect(out.block).toBe('');
+    expect(out.truncated).toEqual([]);
+  });
+
+  test('null/undefined input → empty block (no crash)', () => {
+    expect(buildPlanBlock(null).block).toBe('');
+    expect(buildPlanBlock(undefined).block).toBe('');
+  });
+
+  test('single plan: wraps in header + footer, includes title + content', () => {
+    const out = buildPlanBlock([makePlan('plan/research', 'Find ten papers.')]);
+    expect(out.block).toMatch(/AGENT WORKING PLAN/);
+    expect(out.block).toMatch(/END WORKING PLAN/);
+    expect(out.block).toMatch(/plan\/research/);
+    expect(out.block).toMatch(/Find ten papers\./);
+    expect(out.truncated).toEqual([]);
+  });
+
+  test('multiple plans render under a SINGLE header (not one each)', () => {
+    const out = buildPlanBlock([
+      makePlan('plan/a', 'one'),
+      makePlan('plan/b', 'two'),
+      makePlan('plan/c', 'three'),
+    ]);
+    // Exactly ONE header / footer
+    expect((out.block.match(/AGENT WORKING PLAN/g) || []).length).toBe(1);
+    expect((out.block.match(/END WORKING PLAN/g) || []).length).toBe(1);
+    // All three plans present
+    expect(out.block).toMatch(/plan\/a/);
+    expect(out.block).toMatch(/plan\/b/);
+    expect(out.block).toMatch(/plan\/c/);
+  });
+
+  test('REGRESSION: per-plan soft cap truncates over-budget content with a clear marker', () => {
+    const bigContent = 'x'.repeat(PER_PLAN_TOKEN_CAP * 4 * 2);  // 2× the cap in chars
+    const out = buildPlanBlock([makePlan('plan/big', bigContent)]);
+    expect(out.block).toMatch(/content truncated/);
+    expect(out.truncated.length).toBe(1);
+    expect(out.truncated[0].title).toBe('plan/big');
+    expect(out.truncated[0].reason).toMatch(/per-plan cap/);
+  });
+
+  test('REGRESSION: cross-plan hard cap drops later plans entirely', () => {
+    // Each plan is small (under per-plan cap) but in aggregate exceed total.
+    // Use 8 plans of ~1500 tokens each → ~12000 total, exceeds 8000 cap.
+    const planSize = 1500 * 4;
+    const many = [];
+    for (let i = 0; i < 8; i += 1) {
+      many.push(makePlan(`plan/p${i}`, 'y'.repeat(planSize)));
+    }
+    const out = buildPlanBlock(many);
+    // Some plans got DROPPED entirely (not just truncated).
+    const droppedReasons = out.truncated.filter(t => /cross-plan cap/.test(t.reason));
+    expect(droppedReasons.length).toBeGreaterThan(0);
+    // The dropped plans should be absent from the block.
+    for (const t of droppedReasons) {
+      expect(out.block).not.toMatch(new RegExp(t.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
+    }
+  });
+
+  test('missing content (null/undefined) is rendered as empty string, no crash', () => {
+    const out = buildPlanBlock([
+      { title: 'plan/empty', content: null, updatedAt: '2026-01-01T00:00:00.000Z' },
+      { title: 'plan/undef', content: undefined, updatedAt: '2026-01-01T00:00:00.000Z' },
+    ]);
+    expect(out.block).toMatch(/plan\/empty/);
+    expect(out.block).toMatch(/plan\/undef/);
+    expect(out.truncated).toEqual([]);
+  });
+
+  test('untitled plans get a fallback label, no crash', () => {
+    const out = buildPlanBlock([{ title: undefined, content: 'oops', updatedAt: '2026-01-01T00:00:00Z' }]);
+    expect(out.block).toMatch(/plan\/\(untitled\)/);
+  });
+
+  test('updatedAt is formatted as readable "YYYY-MM-DD HH:MM:SS"', () => {
+    const out = buildPlanBlock([makePlan('plan/r', 'x', '2026-05-16T14:32:00.000Z')]);
+    expect(out.block).toMatch(/2026-05-16 14:32:00/);
+  });
+
+  test('plans are emitted in input order (caller sorts; we preserve)', () => {
+    const out = buildPlanBlock([
+      makePlan('plan/first',  'A', '2026-05-16T12:00:00Z'),
+      makePlan('plan/second', 'B', '2026-05-15T12:00:00Z'),
+    ]);
+    expect(out.block.indexOf('plan/first')).toBeLessThan(out.block.indexOf('plan/second'));
+  });
+
+  test('all plans truncated below threshold → "Summarize one" hint', () => {
+    // Edge case: every plan exceeds the total cap on its own (impossible
+    // mathematically given our caps, but defensive — same handler covers
+    // "all dropped" path)
+    const huge = 'z'.repeat(TOTAL_PLAN_TOKEN_CAP * 4 * 10);
+    const out = buildPlanBlock([makePlan('plan/giant1', huge), makePlan('plan/giant2', huge)]);
+    // Header still present
+    expect(out.block).toMatch(/AGENT WORKING PLAN/);
+  });
+});