@mthanh/vespercli 1.7.1 → 2.0.0

package/scripts/generate-claude-assets.strict.test.js

@@ -0,0 +1,303 @@
+/**
+ * Generator strictness, atomicity, whole-tree freshness and residue (epic (d) review).
+ *
+ * The review found the generator warned-skipped-exited-zero on per-entry failures,
+ * emitted a frontmatter-less agent verbatim (pickable), deleted its output before
+ * rewriting it, checked freshness only for top-level SKILL.md files, and never
+ * rendered skill reference files — so generated Copilot content delegated to
+ * `vesper:<name>` agents Copilot cannot resolve. Each describe below pins one fix.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, stat, cp } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, relative, sep } from 'node:path';
+
+import {
+  ASSETS_DIR,
+  COPILOT_DIR,
+  buildCopilotTree,
+  writeTreeAtomically,
+  serializeManifest,
+  findCopilotResidue,
+  COPILOT_RESIDUE_ALLOW_LIST,
+  COPILOT_RESIDUE_TOKENS,
+} from './generate-claude-assets.js';
+
+async function walk(root) {
+  const out = [];
+  const visit = async (dir) => {
+    for (const e of await readdir(dir, { withFileTypes: true })) {
+      const abs = join(dir, e.name);
+      if (e.isDirectory()) await visit(abs);
+      else out.push(relative(root, abs).split(sep).join('/'));
+    }
+  };
+  await visit(root);
+  return out.sort();
+}
+
+// ---------------------------------------------------------------------------
+// Fixture tree helpers
+// ---------------------------------------------------------------------------
+
+let fx;
+beforeEach(async () => { fx = await mkdtemp(join(tmpdir(), 'vesper-gen-fx-')); });
+afterEach(async () => { await rm(fx, { recursive: true, force: true }); });
+
+const GOOD_AGENT = '---\nname: a\ndescription: \'An agent.\'\nmodel: claude-sonnet-5\ntools: Read\n---\nBody.\n';
+const GOOD_SKILL = '---\nname: s\ndescription: \'A skill.\'\n---\nBody.\n';
+const GOOD_MAINT = '---\nname: m\ndescription: \'Maintain.\'\ndisable-model-invocation: true\n---\nRun `vesper:workspace-learner`.\n';
+
+async function fixture({ agents = { a: GOOD_AGENT }, skills = { s: { 'SKILL.md': GOOD_SKILL } }, committed = { m: GOOD_MAINT } } = {}) {
+  for (const [name, content] of Object.entries(agents)) {
+    await mkdir(join(fx, 'claude', 'agents'), { recursive: true });
+    await writeFile(join(fx, 'claude', 'agents', `${name}.md`), content);
+  }
+  await mkdir(join(fx, 'claude', 'skills'), { recursive: true });
+  for (const [name, files] of Object.entries(skills)) {
+    for (const [rel, content] of Object.entries(files)) {
+      await mkdir(join(fx, 'claude', 'skills', name, rel, '..'), { recursive: true });
+      await writeFile(join(fx, 'claude', 'skills', name, rel), content);
+    }
+  }
+  await mkdir(join(fx, 'claude-committed', 'skills'), { recursive: true });
+  for (const [name, content] of Object.entries(committed)) {
+    await mkdir(join(fx, 'claude-committed', 'skills', name), { recursive: true });
+    await writeFile(join(fx, 'claude-committed', 'skills', name, 'SKILL.md'), content);
+  }
+  return buildCopilotTree({ assetsDir: fx });
+}
+
+// ---------------------------------------------------------------------------
+// Strict: every former warn-and-skip is an error
+// ---------------------------------------------------------------------------
+
+describe('strict generation — per-entry failures are errors, never silent drops', () => {
+  it('a clean fixture builds with zero errors', async () => {
+    const { errors, files } = await fixture();
+    expect(errors).toEqual([]);
+    expect([...files.keys()].sort()).toEqual(['agents/a.agent.md', 'prompts/m.prompt.md', 'skills/s/SKILL.md']);
+  });
+
+  it('an agent with no parseable frontmatter is an error (it would ship pickable)', async () => {
+    const { errors, files } = await fixture({ agents: { rogue: '# no frontmatter\n' } });
+    expect(errors.join('\n')).toMatch(/claude\/agents\/rogue\.md: .*no parseable frontmatter/);
+    expect(files.has('agents/rogue.agent.md')).toBe(false);
+  });
+
+  it('a CRLF agent is converted (hidden), not rejected and not emitted verbatim', async () => {
+    const { errors, files } = await fixture({ agents: { crlf: GOOD_AGENT.replace(/\n/g, '\r\n') } });
+    expect(errors).toEqual([]);
+    expect(files.get('agents/crlf.agent.md').content.toString()).toContain('user-invocable: false');
+  });
+
+  it('a skill without frontmatter is an error', async () => {
+    const { errors } = await fixture({ skills: { bare: { 'SKILL.md': '# Bare\n' } } });
+    expect(errors.join('\n')).toMatch(/claude\/skills\/bare: SKILL\.md has no frontmatter/);
+  });
+
+  it('a nested bundle SKILL.md without frontmatter is an error too', async () => {
+    const { errors } = await fixture({ skills: { bundle: { 'part/SKILL.md': '# Part\n' } } });
+    expect(errors.join('\n')).toMatch(/claude\/skills\/bundle: part\/SKILL\.md has no frontmatter/);
+  });
+
+  it('a committed skill WITHOUT the maintenance marker is an error (was silently dropped)', async () => {
+    const { errors } = await fixture({ committed: { plain: GOOD_SKILL } });
+    expect(errors.join('\n')).toMatch(/claude-committed\/skills\/plain: .*lacks disable-model-invocation/);
+  });
+
+  it('a PLUGIN skill carrying the maintenance marker is an error (was emitted as ordinary)', async () => {
+    const { errors } = await fixture({ skills: { rogue: { 'SKILL.md': GOOD_MAINT } } });
+    expect(errors.join('\n')).toMatch(/claude\/skills\/rogue: .*belong in assets\/claude-committed/);
+  });
+
+  it('an unquoted description that is invalid YAML is an error', async () => {
+    const { errors } = await fixture({ skills: { y: { 'SKILL.md': '---\nname: y\ndescription: Reviews code: fast\n---\nBody\n' } } });
+    expect(errors.join('\n')).toMatch(/not valid YAML/);
+  });
+
+  it('copilot-apply-to without a tier is an error; a tier without copilot-apply-to is an error', async () => {
+    const noTier = '---\nname: t\ndescription: \'x\'\ncopilot-apply-to: \'**/*.ts\'\n---\nB\n';
+    const onlyTier = '---\nname: u\ndescription: \'x\'\ncopilot-instructions-tier: extended\n---\nB\n';
+    const { errors } = await fixture({ skills: { t: { 'SKILL.md': noTier }, u: { 'SKILL.md': onlyTier } } });
+    expect(errors.join('\n')).toMatch(/claude\/skills\/t: .*copilot-instructions-tier is missing/);
+    expect(errors.join('\n')).toMatch(/claude\/skills\/u: .*without copilot-apply-to/);
+  });
+
+  it('an unmapped tool name is an error (not a warning with exit 0)', async () => {
+    const { errors } = await fixture({ agents: { a: GOOD_AGENT.replace('tools: Read', 'tools: Read, Teleport') } });
+    expect(errors.join('\n')).toMatch(/unknown-claude-tool.*Teleport/);
+  });
+
+  it('routes tiers: core → instructions/, extended → instructions-extended/<name>/', async () => {
+    const core = '---\nname: c\ndescription: \'x\'\ncopilot-apply-to: \'**/*.go\'\ncopilot-instructions-tier: core\n---\nB\n';
+    const ext = '---\nname: e\ndescription: \'x\'\ncopilot-apply-to: \'**/*.rb\'\ncopilot-instructions-tier: extended\n---\nB\n';
+    const { errors, files, manifest } = await fixture({ skills: { c: { 'SKILL.md': core }, e: { 'SKILL.md': ext } } });
+    expect(errors).toEqual([]);
+    expect(files.has('instructions/c.instructions.md')).toBe(true);
+    expect(files.has('instructions-extended/e/e.instructions.md')).toBe(true);
+    expect(manifest.coreInstructions).toEqual(['c']);
+    expect(manifest.instructions).toEqual(['e']);
+    // Source-only keys never leak into the generated skill.
+    expect(files.get('skills/c/SKILL.md').content.toString()).not.toMatch(/copilot-(apply-to|instructions-tier)/);
+  });
+
+  it('renders EVERY markdown file in a skill folder, not only the top-level SKILL.md', async () => {
+    const ref = 'Delegate to `vesper:code-reviewer`; roster at `.claude/rules/vesper-orchestration.md`.\n';
+    const { errors, files } = await fixture({ skills: { s: { 'SKILL.md': GOOD_SKILL, 'references/pipeline.md': ref } } });
+    expect(errors).toEqual([]);
+    expect(files.get('skills/s/references/pipeline.md').content.toString())
+      .toBe('Delegate to `code-reviewer`; roster at `.vesper/instructions/vesper-orchestration.instructions.md`.\n');
+  });
+
+  it('the maintenance prompt delegates by plain name', async () => {
+    const { files } = await fixture();
+    expect(files.get('prompts/m.prompt.md').content.toString()).toContain('Run `workspace-learner`.');
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Atomic write
+// ---------------------------------------------------------------------------
+
+describe('atomic write — an interrupted run never leaves a partial tree', () => {
+  it('swaps a complete new tree in and leaves no temp/old siblings behind', async () => {
+    const out = join(fx, 'copilot');
+    await mkdir(out, { recursive: true });
+    await writeFile(join(out, 'stale.md'), 'old');
+    const files = new Map([['agents/a.agent.md', { content: Buffer.from('new'), mode: 0o644 }]]);
+    await writeTreeAtomically(files, out);
+    expect(await walk(out)).toEqual(['agents/a.agent.md']);
+    expect((await readdir(fx)).sort()).toEqual(['copilot']);
+  });
+
+  it('a failure mid-write leaves the previous tree byte-identical and cleans the temp dir', async () => {
+    const out = join(fx, 'copilot');
+    await mkdir(join(out, 'agents'), { recursive: true });
+    await writeFile(join(out, 'agents', 'keep.agent.md'), 'previous');
+    // A file and a directory at the same path: the second write must fail.
+    const files = new Map([
+      ['x', { content: Buffer.from('file'), mode: 0o644 }],
+      ['x/y', { content: Buffer.from('child'), mode: 0o644 }],
+    ]);
+    await expect(writeTreeAtomically(files, out)).rejects.toThrow();
+    expect(await readFile(join(out, 'agents', 'keep.agent.md'), 'utf-8')).toBe('previous');
+    expect((await readdir(fx)).sort()).toEqual(['copilot']);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Whole-tree freshness against the COMMITTED output
+// ---------------------------------------------------------------------------
+
+describe('committed assets/copilot/ is fresh — the WHOLE tree, not just SKILL.md', () => {
+  let built;
+  beforeEach(async () => { built = built || await buildCopilotTree(); });
+
+  it('the canonical sources build with zero errors', () => {
+    expect(built.errors).toEqual([]);
+  });
+
+  it('the on-disk file set equals the built file set exactly (nothing missing, nothing without a source)', async () => {
+    const onDisk = await walk(COPILOT_DIR);
+    expect(onDisk).toEqual([...built.files.keys()].sort());
+  });
+
+  it('every on-disk file (references, scripts, instructions, prompts) is byte-identical and mode-identical', async () => {
+    const drift = [];
+    for (const [rel, { content, mode }] of built.files) {
+      const abs = join(COPILOT_DIR, ...rel.split('/'));
+      const disk = await readFile(abs);
+      if (!disk.equals(content)) drift.push(`${rel}: content`);
+      if (((await stat(abs)).mode & 0o111) !== (mode & 0o111)) drift.push(`${rel}: exec bit`);
+    }
+    expect(drift, 'run: node scripts/generate-claude-assets.js').toEqual([]);
+  });
+
+  it('assets/manifest.json equals the built manifest and carries no timestamp', async () => {
+    const disk = await readFile(join(ASSETS_DIR, 'manifest.json'), 'utf-8');
+    expect(disk).toBe(serializeManifest(built.manifest));
+    expect(JSON.parse(disk).generated).toBeUndefined();
+  });
+
+  it('no per-skill manifest.json is emitted (its only reader, remote.js, is dead code)', () => {
+    expect([...built.files.keys()].filter((k) => k.endsWith('/manifest.json'))).toEqual([]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Residue over the entire generated tree, with an explicit allow-list
+// ---------------------------------------------------------------------------
+
+describe('Claude-host residue — the entire generated tree', () => {
+  it('no generated markdown file carries residue outside the allow-list (third-party skills are upstream\'s text, copied verbatim — exempt)', async () => {
+    const findings = [];
+    const files = await walk(COPILOT_DIR);
+    // Epic J: a mirrored skill folder holding `UPSTREAM.json` is third-party — its markdown is
+    // upstream's, not Vesper's, and the generator neither renders nor scans it.
+    const thirdPartyDirs = files.filter((f) => f.endsWith('/UPSTREAM.json')).map((f) => f.slice(0, -'UPSTREAM.json'.length));
+    expect(thirdPartyDirs.length).toBeGreaterThan(0);
+    for (const rel of files) {
+      if (!rel.endsWith('.md') || thirdPartyDirs.some((d) => rel.startsWith(d))) continue;
+      findings.push(...findCopilotResidue(rel, await readFile(join(COPILOT_DIR, rel), 'utf-8')));
+    }
+    expect(findings).toEqual([]);
+  });
+
+  it('every allow-list entry is still needed (no stale exemptions)', async () => {
+    for (const [rel, tokens] of Object.entries(COPILOT_RESIDUE_ALLOW_LIST)) {
+      const text = await readFile(join(COPILOT_DIR, rel), 'utf-8');
+      for (const token of tokens) {
+        const spec = COPILOT_RESIDUE_TOKENS.find((t) => t.token === token);
+        expect(spec, `unknown token ${token}`).toBeTruthy();
+        expect(spec.pattern.test(text), `${rel} no longer contains ${token} — drop the exemption`).toBe(true);
+      }
+    }
+  });
+
+  it('the four maintenance prompts and every orchestration reference delegate by plain name', async () => {
+    const targets = [
+      ...(await readdir(join(COPILOT_DIR, 'prompts'))).map((f) => `prompts/${f}`),
+      ...(await walk(join(COPILOT_DIR, 'skills', 'vesper-orchestration'))).map((f) => `skills/vesper-orchestration/${f}`),
+    ];
+    expect(targets.length).toBeGreaterThan(8);
+    for (const rel of targets) {
+      const text = await readFile(join(COPILOT_DIR, rel), 'utf-8');
+      expect(text, rel).not.toMatch(/`vesper:[a-z]/);
+      expect(text, rel).not.toMatch(/\.claude\/(rules|vesper)\//);
+    }
+  });
+
+  it('the Team Roster has ONE Copilot path: the one the rendered playbook reads', async () => {
+    const seen = new Set();
+    for (const rel of await walk(COPILOT_DIR)) {
+      if (!rel.endsWith('.md')) continue;
+      for (const m of (await readFile(join(COPILOT_DIR, rel), 'utf-8')).matchAll(/[.\w/-]*vesper-orchestration(?:\.[a-z]+)*\.md/g)) seen.add(m[0]);
+    }
+    expect([...seen].sort()).toEqual(['.vesper/instructions/vesper-orchestration.instructions.md']);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// The generator never runs on import
+// ---------------------------------------------------------------------------
+
+describe('import safety', () => {
+  it('importing the generator does not touch the output tree', async () => {
+    const before = (await stat(COPILOT_DIR)).mtimeMs;
+    await import('./generate-claude-assets.js');
+    expect((await stat(COPILOT_DIR)).mtimeMs).toBe(before);
+    expect(existsSync(COPILOT_DIR)).toBe(true);
+  });
+
+  // Explicit timeout: this copies the whole canonical asset tree and builds it twice, so its
+  // runtime scales with the tree and with machine load (~4.7s under the full suite, past the
+  // 5s default intermittently). 60s is headroom for a loaded machine, not a performance budget.
+  it('a fixture copy of the real sources builds identically to the real sources (assetsDir seam is faithful)', async () => {
+    for (const root of ['claude', 'claude-committed']) await cp(join(ASSETS_DIR, root), join(fx, root), { recursive: true });
+    const a = await buildCopilotTree();
+    const b = await buildCopilotTree({ assetsDir: fx });
+    expect([...b.files.keys()]).toEqual([...a.files.keys()]);
+  }, 60000);
+});