@dalzoubi/dev-agents-sync 2.0.16 → 2.0.18

package/tests/steering-agent-behavior.test.mjs

@@ -0,0 +1,491 @@
+/**
+ * tests/steering-agent-behavior.test.mjs
+ *
+ * Slice: S2 — Steering agent behavior
+ * Spec: docs/specs/260616-agent-steering-and-hooks/
+ *
+ * Asserts the content contract that every S2 behavior criterion introduces into
+ * the canonical agent prompts under .agents/agents/. These tests FAIL before
+ * the implement agent edits the prompts and PASS after.
+ *
+ * Behavior anchors (one named test per criterion — §7):
+ *
+ *  C1. Define reads steering — .agents/agents/define.md instructs reading
+ *      steering/*.md (the steering files) at session start.
+ *  C2. Define writes decisions — .agents/agents/define.md instructs appending
+ *      one dated, one-line decision to steering/decisions.md at spec finalize
+ *      (append-only, gated).
+ *  C3. Implement reads steering — .agents/agents/implement.md instructs reading
+ *      steering at session start.
+ *  C4. Implement writes gotchas — .agents/agents/implement.md instructs
+ *      appending dated gotchas/constraints discovered during build.
+ *  C5a. Analyze is read-only on steering — .agents/agents/analyze.md does NOT
+ *       instruct appending/writing to steering (may instruct reading).
+ *  C5b. Test is read-only on steering — .agents/agents/test.md does NOT instruct
+ *       appending/writing to steering (may instruct reading).
+ *  C5c. Validate is read-only on steering — .agents/agents/validate.md does NOT
+ *       instruct appending/writing to steering (may instruct reading).
+ *  C5d. Supervise is read-only on steering — .agents/agents/supervise.md does NOT
+ *       instruct appending/writing to steering (may instruct reading).
+ *  C6. Shared read mechanism exists — there is a canonical source for the
+ *      "read steering/*.md at session start" instruction (either a shared rule
+ *      under .agents/rules/ or inline text in define.md + implement.md that
+ *      references steering/ at session start).
+ *  C7a. Mirror parity (define) — .claude/agents/define.md contains the same
+ *       steering read instruction as the canonical source.
+ *  C7b. Mirror parity (implement) — .claude/agents/implement.md contains the
+ *       same steering write/gotcha instruction as the canonical source.
+ *
+ * Assertions match on stable substrings (e.g. "steering/decisions.md",
+ * "steering/", "session start", "append") rather than exact sentences, but are
+ * strict enough to fail when the instruction is absent.
+ *
+ * Test runner: node --test (Node built-in, >= 20)
+ * No third-party dependencies.
+ */
+
+import { describe, it, before } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync, existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+
+// ---------------------------------------------------------------------------
+// Resolve paths relative to this file's location.
+// Repo layout: packages/dev-agents-sync/tests/ -> repo root is 3 levels up.
+// ---------------------------------------------------------------------------
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(__dirname, '../../..');
+
+const AGENTS_DIR = path.join(REPO_ROOT, '.agents', 'agents');
+const RULES_DIR = path.join(REPO_ROOT, '.agents', 'rules');
+const MIRROR_DIR = path.join(REPO_ROOT, '.claude', 'agents');
+
+// ---------------------------------------------------------------------------
+// Load canonical agent files once.
+// ---------------------------------------------------------------------------
+
+let defineContent;
+let implementContent;
+let analyzeContent;
+let testContent;
+let validateContent;
+let superviseContent;
+
+// Mirror files — loaded lazily in tests that need parity checks.
+let mirrorDefineContent;
+let mirrorImplementContent;
+
+before(() => {
+  defineContent = readFileSync(path.join(AGENTS_DIR, 'define.md'), 'utf8');
+  implementContent = readFileSync(path.join(AGENTS_DIR, 'implement.md'), 'utf8');
+  analyzeContent = readFileSync(path.join(AGENTS_DIR, 'analyze.md'), 'utf8');
+  testContent = readFileSync(path.join(AGENTS_DIR, 'test.md'), 'utf8');
+  validateContent = readFileSync(path.join(AGENTS_DIR, 'validate.md'), 'utf8');
+  superviseContent = readFileSync(path.join(AGENTS_DIR, 'supervise.md'), 'utf8');
+
+  mirrorDefineContent = existsSync(path.join(MIRROR_DIR, 'define.md'))
+    ? readFileSync(path.join(MIRROR_DIR, 'define.md'), 'utf8')
+    : '';
+  mirrorImplementContent = existsSync(path.join(MIRROR_DIR, 'implement.md'))
+    ? readFileSync(path.join(MIRROR_DIR, 'implement.md'), 'utf8')
+    : '';
+});
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Returns true if the content contains an instruction to read steering files
+ * at session start. Matches on stable substrings: "steering/" and "session start"
+ * both appear in close textual proximity, OR explicit "steering/*.md" reference
+ * paired with "session start".
+ */
+function instructsReadSteeringAtStart(content) {
+  // Direct conjunction: both "steering/" and "session start" anywhere in the file.
+  // The spec says agents read steering/*.md at session start — both tokens must be present.
+  const hasSteeringRef = /steering\//i.test(content);
+  const hasSessionStart = /session\s+start/i.test(content);
+  return hasSteeringRef && hasSessionStart;
+}
+
+/**
+ * Returns true if content contains an instruction to APPEND to steering/decisions.md.
+ * Matches on "append" (or synonym) plus "steering/decisions.md" or "decisions.md".
+ */
+function instructsAppendToDecisions(content) {
+  const hasDecisionsRef = /steering\/decisions\.md/i.test(content);
+  const hasAppend = /\bappend\b/i.test(content);
+  return hasDecisionsRef && hasAppend;
+}
+
+/**
+ * Returns true if content contains an instruction to APPEND gotchas/constraints
+ * to a steering file. Matches on "append" plus "steering/" plus a gotcha/constraint signal.
+ */
+function instructsAppendGotchas(content) {
+  const hasSteeringRef = /steering\//i.test(content);
+  const hasAppend = /\bappend\b/i.test(content);
+  const hasGotchaSignal =
+    /gotcha/i.test(content) ||
+    /constraint/i.test(content) ||
+    /non[- ]obvious/i.test(content) ||
+    /discovered during build/i.test(content);
+  return hasSteeringRef && hasAppend && hasGotchaSignal;
+}
+
+/**
+ * Returns true if content contains an append-to-steering instruction.
+ * Used to assert ABSENCE in read-only agents.
+ */
+function containsAnySteeringWriteInstruction(content) {
+  // "append" + "steering/" is the signal for a write instruction.
+  // Pure read references (e.g. "read steering/") are acceptable.
+  return /\bappend\b.*steering\//i.test(content) || /steering\/.*\bappend\b/i.test(content);
+}
+
+// ---------------------------------------------------------------------------
+// C1 — Define reads steering at session start
+// ---------------------------------------------------------------------------
+
+describe('S2 — C1: define.md instructs reading steering/*.md at session start', () => {
+  it('define.md references steering/ AND session start (read-on-start instruction present)', () => {
+    assert.ok(
+      instructsReadSteeringAtStart(defineContent),
+      [
+        'define.md must instruct the agent to read steering/*.md at session start.',
+        'Expected both "steering/" and "session start" to be present in .agents/agents/define.md.',
+        'Neither token was found — the read-on-start instruction is absent.',
+      ].join(' '),
+    );
+  });
+
+  it('define.md explicitly references steering/decisions.md (the primary steering file)', () => {
+    assert.ok(
+      /steering\/decisions\.md/i.test(defineContent),
+      [
+        'define.md must reference "steering/decisions.md" so it is clear which',
+        'file the Define agent appends decisions to.',
+        'The reference is absent from .agents/agents/define.md.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C2 — Define appends one dated decision to steering/decisions.md at finalize
+// ---------------------------------------------------------------------------
+
+describe('S2 — C2: define.md instructs appending a dated decision to steering/decisions.md at spec finalize', () => {
+  it('define.md instructs appending to steering/decisions.md', () => {
+    assert.ok(
+      instructsAppendToDecisions(defineContent),
+      [
+        'define.md must instruct the Define agent to append a decision to',
+        '"steering/decisions.md".',
+        'Expected both "append" and "steering/decisions.md" in .agents/agents/define.md.',
+        'At least one is absent — the append-decision instruction is missing.',
+      ].join(' '),
+    );
+  });
+
+  it('define.md gates the steering/decisions.md append to spec finalize (not just any time)', () => {
+    // The spec says appending to steering/decisions.md happens "at spec finalize" — the finalize
+    // gate must appear in the context of the steering decisions instruction, not just anywhere
+    // in the file (define.md already contains "finalize" for other purposes).
+    // We require the substring "steering/decisions.md" to appear AND a finalize gate to appear
+    // within the same section (within 500 characters) — or a phrase that combines both concepts.
+    const decisionsIdx = defineContent.indexOf('steering/decisions.md');
+    if (decisionsIdx === -1) {
+      // steering/decisions.md is absent entirely; the test for C2 will catch this.
+      // This test adds the additional gate constraint, so also fail here.
+      assert.fail(
+        'define.md must reference "steering/decisions.md" before the finalize gate can be checked. ' +
+        'The reference is absent from .agents/agents/define.md.',
+      );
+    }
+
+    // Extract a window of 600 chars around the decisions reference to check for finalize signal.
+    const window = defineContent.slice(
+      Math.max(0, decisionsIdx - 200),
+      Math.min(defineContent.length, decisionsIdx + 400),
+    );
+
+    const hasFinalize =
+      /finali[sz]e/i.test(window) ||
+      /spec\s+(?:is\s+)?(?:approved|confirmed)/i.test(window) ||
+      /at\s+(?:spec\s+)?(?:approv|confirm|finali)/i.test(window);
+
+    assert.ok(
+      hasFinalize,
+      [
+        'define.md must gate the steering/decisions.md append to spec finalize.',
+        'Expected a "finalize" (or "approved"/"confirmed") signal within 600 characters',
+        'of the "steering/decisions.md" reference in .agents/agents/define.md.',
+        'The finalize gate is absent near the append instruction.',
+      ].join(' '),
+    );
+  });
+
+  it('define.md specifies a dated format for decisions (YYYY-MM-DD or "dated") in the steering context', () => {
+    // The spec says decisions are "date-stamped, one-line". The dated-format signal must
+    // appear in the context of the steering/decisions instruction, not just anywhere in the file
+    // (define.md already contains date-like strings in folder-naming examples unrelated to steering).
+    const decisionsIdx = defineContent.indexOf('steering/decisions.md');
+    if (decisionsIdx === -1) {
+      assert.fail(
+        'define.md must reference "steering/decisions.md" before a dated-format check can be applied. ' +
+        'The reference is absent from .agents/agents/define.md.',
+      );
+    }
+
+    const window = defineContent.slice(
+      Math.max(0, decisionsIdx - 200),
+      Math.min(defineContent.length, decisionsIdx + 400),
+    );
+
+    const hasDatedFormat =
+      /\bdated\b/i.test(window) ||
+      /YYYY[- ]MM[- ]DD/i.test(window) ||
+      /date[- ]stamp/i.test(window) ||
+      /date\s+stamp/i.test(window);
+
+    assert.ok(
+      hasDatedFormat,
+      [
+        'define.md must specify that appended decisions are dated',
+        '(e.g. "YYYY-MM-DD" format, "dated", or "date-stamped") in the steering context.',
+        'Expected the dated-format signal within 600 characters of "steering/decisions.md"',
+        'in .agents/agents/define.md. The dated-format requirement is absent.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C3 — Implement reads steering at session start
+// ---------------------------------------------------------------------------
+
+describe('S2 — C3: implement.md instructs reading steering at session start', () => {
+  it('implement.md references steering/ AND session start (read-on-start instruction present)', () => {
+    assert.ok(
+      instructsReadSteeringAtStart(implementContent),
+      [
+        'implement.md must instruct the agent to read steering files at session start.',
+        'Expected both "steering/" and "session start" in .agents/agents/implement.md.',
+        'At least one token is absent — the read-on-start instruction is missing.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C4 — Implement appends dated gotchas discovered during build
+// ---------------------------------------------------------------------------
+
+describe('S2 — C4: implement.md instructs appending dated gotchas/constraints to steering during build', () => {
+  it('implement.md instructs appending gotchas/constraints to a steering file', () => {
+    assert.ok(
+      instructsAppendGotchas(implementContent),
+      [
+        'implement.md must instruct the Implement agent to append gotchas or',
+        'non-obvious constraints discovered during build to a steering file.',
+        'Expected "append" + "steering/" + a gotcha signal ("gotcha", "constraint",',
+        '"non-obvious", or "discovered during build") in .agents/agents/implement.md.',
+        'The gotcha-append instruction is absent.',
+      ].join(' '),
+    );
+  });
+
+  it('implement.md specifies that gotcha appends are dated', () => {
+    // The spec says "date-stamped, one-line rationale" — must see dated signal near steering append.
+    const hasDatedSignal =
+      /dated/i.test(implementContent) ||
+      /date[- ]stamp/i.test(implementContent) ||
+      /YYYY[- ]MM[- ]DD/i.test(implementContent);
+
+    assert.ok(
+      hasDatedSignal,
+      [
+        'implement.md must specify that gotcha appends are dated',
+        '(e.g. "dated", "date-stamped", or "YYYY-MM-DD").',
+        'The dated-append requirement is absent from .agents/agents/implement.md.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C5a — Analyze is read-only on steering (no append instruction)
+// ---------------------------------------------------------------------------
+
+describe('S2 — C5a: analyze.md does NOT instruct appending/writing to steering', () => {
+  it('analyze.md contains no append-to-steering instruction', () => {
+    assert.ok(
+      !containsAnySteeringWriteInstruction(analyzeContent),
+      [
+        'analyze.md must NOT instruct the Analyze agent to append to steering files.',
+        'Analyze is read-only on steering.',
+        'Found an append-to-steering instruction in .agents/agents/analyze.md.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C5b — Test agent is read-only on steering (no append instruction)
+// ---------------------------------------------------------------------------
+
+describe('S2 — C5b: test.md does NOT instruct appending/writing to steering', () => {
+  it('test.md contains no append-to-steering instruction', () => {
+    assert.ok(
+      !containsAnySteeringWriteInstruction(testContent),
+      [
+        'test.md must NOT instruct the Test agent to append to steering files.',
+        'Test is read-only on steering.',
+        'Found an append-to-steering instruction in .agents/agents/test.md.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C5c — Validate is read-only on steering (no append instruction)
+// ---------------------------------------------------------------------------
+
+describe('S2 — C5c: validate.md does NOT instruct appending/writing to steering', () => {
+  it('validate.md contains no append-to-steering instruction', () => {
+    assert.ok(
+      !containsAnySteeringWriteInstruction(validateContent),
+      [
+        'validate.md must NOT instruct the Validate agent to append to steering files.',
+        'Validate is read-only on steering.',
+        'Found an append-to-steering instruction in .agents/agents/validate.md.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C5d — Supervise is read-only on steering (no append instruction)
+// ---------------------------------------------------------------------------
+
+describe('S2 — C5d: supervise.md does NOT instruct appending/writing to steering', () => {
+  it('supervise.md contains no append-to-steering instruction', () => {
+    assert.ok(
+      !containsAnySteeringWriteInstruction(superviseContent),
+      [
+        'supervise.md must NOT instruct the Supervise agent to append to steering files.',
+        'Supervise is read-only on steering.',
+        'Found an append-to-steering instruction in .agents/agents/supervise.md.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C6 — Shared read mechanism: "read steering/*.md at session start" exists in
+//      canonical sources (either a shared rule or inline in both define + implement)
+// ---------------------------------------------------------------------------
+
+describe('S2 — C6: shared read-steering mechanism present in canonical source', () => {
+  it('a steering-read rule exists in .agents/rules/ OR both define.md and implement.md instruct reading steering at session start', () => {
+    // Acceptable mechanism 1: a dedicated rule file under .agents/rules/ that
+    // mentions steering/ and session start. Check well-known candidate names.
+    const candidateRuleFiles = [
+      path.join(RULES_DIR, 'steering.md'),
+      path.join(RULES_DIR, 'steering-read.md'),
+      path.join(RULES_DIR, 'read-steering.md'),
+    ];
+
+    let sharedRuleExists = false;
+    for (const ruleFile of candidateRuleFiles) {
+      if (existsSync(ruleFile)) {
+        const ruleContent = readFileSync(ruleFile, 'utf8');
+        if (instructsReadSteeringAtStart(ruleContent)) {
+          sharedRuleExists = true;
+          break;
+        }
+      }
+    }
+
+    // Acceptable mechanism 2: both define.md and implement.md carry the
+    // read-on-start instruction inline (shared by repetition in both entry-point agents).
+    const bothAgentsHaveReadInstruction =
+      instructsReadSteeringAtStart(defineContent) &&
+      instructsReadSteeringAtStart(implementContent);
+
+    assert.ok(
+      sharedRuleExists || bothAgentsHaveReadInstruction,
+      [
+        'A shared read-steering mechanism must exist in canonical source.',
+        'Expected EITHER (a) a rule file under .agents/rules/ (e.g. steering.md or steering-read.md)',
+        'that instructs reading steering/*.md at session start,',
+        'OR (b) both .agents/agents/define.md and .agents/agents/implement.md',
+        'to each instruct reading steering files at session start.',
+        'Neither condition is satisfied — the shared read mechanism is absent.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C7a — Mirror parity: .claude/agents/define.md contains the steering read instruction
+// ---------------------------------------------------------------------------
+
+describe('S2 — C7a: mirror parity — .claude/agents/define.md reflects steering read instruction', () => {
+  it('.claude/agents/define.md contains a steering/ reference (mirrors have been rebuilt)', () => {
+    assert.ok(
+      /steering\//i.test(mirrorDefineContent),
+      [
+        '.claude/agents/define.md must contain a "steering/" reference,',
+        'proving the mirror was rebuilt after the canonical .agents/agents/define.md was edited.',
+        'The "steering/" reference is absent from the mirror.',
+        'Run `npm run build` to regenerate the committed mirror from the canonical source.',
+      ].join(' '),
+    );
+  });
+
+  it('.claude/agents/define.md references steering/decisions.md (append instruction mirrored)', () => {
+    assert.ok(
+      /steering\/decisions\.md/i.test(mirrorDefineContent),
+      [
+        '.claude/agents/define.md must contain "steering/decisions.md".',
+        'The append-to-decisions instruction is missing from the mirror.',
+        'Canonical source may have the instruction but the mirror has not been rebuilt.',
+        'Run `npm run build` to regenerate.',
+      ].join(' '),
+    );
+  });
+});
+
+// ---------------------------------------------------------------------------
+// C7b — Mirror parity: .claude/agents/implement.md contains the steering write instruction
+// ---------------------------------------------------------------------------
+
+describe('S2 — C7b: mirror parity — .claude/agents/implement.md reflects steering write instruction', () => {
+  it('.claude/agents/implement.md contains a steering/ reference (mirrors have been rebuilt)', () => {
+    assert.ok(
+      /steering\//i.test(mirrorImplementContent),
+      [
+        '.claude/agents/implement.md must contain a "steering/" reference,',
+        'proving the mirror was rebuilt after the canonical .agents/agents/implement.md was edited.',
+        'The "steering/" reference is absent from the mirror.',
+        'Run `npm run build` to regenerate the committed mirror from the canonical source.',
+      ].join(' '),
+    );
+  });
+
+  it('.claude/agents/implement.md contains an append instruction for steering (gotcha instruction mirrored)', () => {
+    assert.ok(
+      /\bappend\b/i.test(mirrorImplementContent) && /steering\//i.test(mirrorImplementContent),
+      [
+        '.claude/agents/implement.md must contain both "append" and "steering/".',
+        'The gotcha-append instruction is missing from the mirror.',
+        'Run `npm run build` after editing .agents/agents/implement.md.',
+      ].join(' '),
+    );
+  });
+});