@unrdf/diataxis-kit 26.4.2

package/test/determinism.test.mjs

@@ -0,0 +1,321 @@
+/**
+ * @file Determinism tests for Diátaxis Kit
+ * @description Validates that DETERMINISTIC=1 produces identical output across runs
+ */
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { readFile, readdir, rm } from 'node:fs/promises';
+import { join, resolve, dirname } from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const execFileAsync = promisify(execFile);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageRoot = resolve(__dirname, '..');
+
+let passCount = 0;
+let failCount = 0;
+
+/**
+ * Test runner with timeout support
+ * @param {string} name - Test name
+ * @param {Function} fn - Async test function
+ * @param {number} [timeout=30000] - Timeout in milliseconds
+ */
+async function test(name, fn, timeout = 30000) {
+  const timeoutPromise = new Promise((_, reject) => {
+    setTimeout(() => reject(new Error(`Test timeout after ${timeout}ms`)), timeout);
+  });
+
+  try {
+    await Promise.race([fn(), timeoutPromise]);
+    console.log(`✅ ${name}`);
+    passCount++;
+  } catch (err) {
+    console.log(`❌ ${name}: ${err.message}`);
+    if (err.stack && process.env.VERBOSE) {
+      console.log(err.stack);
+    }
+    failCount++;
+  }
+}
+
+/**
+ * Hash a file's contents
+ * @param {string} filePath - Path to file
+ * @returns {Promise<string>} SHA256 hash
+ */
+async function hashFile(filePath) {
+  const content = await readFile(filePath, 'utf8');
+  return createHash('sha256').update(content, 'utf8').digest('hex');
+}
+
+/**
+ * Recursively find all files in a directory
+ * @param {string} dir - Directory to search
+ * @param {string[]} results - Accumulator for results
+ * @returns {Promise<string[]>} Array of file paths
+ */
+async function findAllFiles(dir, results = []) {
+  if (!existsSync(dir)) {
+    return results;
+  }
+
+  const entries = await readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await findAllFiles(fullPath, results);
+    } else if (entry.isFile()) {
+      results.push(fullPath);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Clean output directories
+ * @returns {Promise<void>}
+ */
+async function cleanOutputs() {
+  const artifactsDir = join(packageRoot, 'ARTIFACTS');
+  const outDir = join(packageRoot, 'OUT');
+
+  if (existsSync(artifactsDir)) {
+    await rm(artifactsDir, { recursive: true, force: true });
+  }
+  if (existsSync(outDir)) {
+    await rm(outDir, { recursive: true, force: true });
+  }
+}
+
+/**
+ * Run bin/run.mjs with DETERMINISTIC=1
+ * @returns {Promise<void>}
+ */
+async function runPipeline() {
+  const binPath = join(packageRoot, 'bin', 'run.mjs');
+
+  await execFileAsync('node', [binPath], {
+    cwd: packageRoot,
+    env: {
+      ...process.env,
+      DETERMINISTIC: '1'
+    }
+  });
+}
+
+/**
+ * Test: Determinism - running twice produces identical hashes
+ */
+async function testDeterminism() {
+  // Run 1
+  await cleanOutputs();
+  await runPipeline();
+
+  const artifactsDir = join(packageRoot, 'ARTIFACTS');
+  const outDir = join(packageRoot, 'OUT');
+
+  const files1 = await findAllFiles(artifactsDir);
+  const hashes1 = new Map();
+
+  for (const file of files1) {
+    const hash = await hashFile(file);
+    const relativePath = file.replace(artifactsDir + '/', '');
+    hashes1.set(relativePath, hash);
+  }
+
+  // Sleep 100ms
+  await new Promise(resolve => setTimeout(resolve, 100));
+
+  // Run 2
+  await cleanOutputs();
+  await runPipeline();
+
+  const files2 = await findAllFiles(artifactsDir);
+  const hashes2 = new Map();
+
+  for (const file of files2) {
+    const hash = await hashFile(file);
+    const relativePath = file.replace(artifactsDir + '/', '');
+    hashes2.set(relativePath, hash);
+  }
+
+  // Compare file counts
+  if (files1.length !== files2.length) {
+    throw new Error(`File count mismatch: ${files1.length} vs ${files2.length}`);
+  }
+
+  // Compare hashes
+  const mismatches = [];
+  for (const [path, hash1] of hashes1) {
+    const hash2 = hashes2.get(path);
+    if (!hash2) {
+      mismatches.push(`Missing in run 2: ${path}`);
+    } else if (hash1 !== hash2) {
+      mismatches.push(`Hash mismatch: ${path}\n  Run 1: ${hash1}\n  Run 2: ${hash2}`);
+    }
+  }
+
+  if (mismatches.length > 0) {
+    throw new Error(`Determinism failed:\n${mismatches.join('\n')}`);
+  }
+
+  console.log(`  Verified ${hashes1.size} files identical across runs`);
+}
+
+/**
+ * Test: Consistent timestamps when DETERMINISTIC=1
+ */
+async function testConsistentTimestamps() {
+  await cleanOutputs();
+  await runPipeline();
+
+  const artifactsDir = join(packageRoot, 'ARTIFACTS', 'diataxis');
+
+  // Check inventory.json
+  const inventoryPath = join(artifactsDir, 'inventory.json');
+  const inventory = JSON.parse(await readFile(inventoryPath, 'utf8'));
+
+  if (inventory.generatedAt !== '2000-01-01T00:00:00.000Z') {
+    throw new Error(`Inventory timestamp not fixed: ${inventory.generatedAt}`);
+  }
+
+  // Check 5 random diataxis.json files
+  const packageDirs = await readdir(artifactsDir, { withFileTypes: true });
+  const validDirs = packageDirs.filter(d => d.isDirectory()).slice(0, 5);
+
+  for (const dir of validDirs) {
+    const diataxisPath = join(artifactsDir, dir.name, 'diataxis.json');
+    if (existsSync(diataxisPath)) {
+      const diataxis = JSON.parse(await readFile(diataxisPath, 'utf8'));
+      if (diataxis.generatedAt !== '2000-01-01T00:00:00.000Z') {
+        throw new Error(`${dir.name} timestamp not fixed: ${diataxis.generatedAt}`);
+      }
+    }
+  }
+
+  console.log(`  Verified timestamps fixed at 2000-01-01T00:00:00.000Z`);
+}
+
+/**
+ * Test: Stable ordering in output files
+ */
+async function testStableOrdering() {
+  await cleanOutputs();
+  await runPipeline();
+
+  const artifactsDir = join(packageRoot, 'ARTIFACTS', 'diataxis');
+  const inventoryPath = join(artifactsDir, 'inventory.json');
+  const inventory = JSON.parse(await readFile(inventoryPath, 'utf8'));
+
+  // Verify packages are sorted by name
+  const packageNames = inventory.packages.map(p => p.name);
+  const sortedNames = [...packageNames].sort((a, b) => a.localeCompare(b));
+
+  for (let i = 0; i < packageNames.length; i++) {
+    if (packageNames[i] !== sortedNames[i]) {
+      throw new Error(`Packages not sorted: ${packageNames[i]} at index ${i}, expected ${sortedNames[i]}`);
+    }
+  }
+
+  // Check one diataxis.json for sorted tutorials/howtos
+  const firstPackageDir = inventory.packages[0].name;
+  const diataxisPath = join(artifactsDir, firstPackageDir, 'diataxis.json');
+
+  if (existsSync(diataxisPath)) {
+    const diataxis = JSON.parse(await readFile(diataxisPath, 'utf8'));
+
+    // Verify tutorials sorted by id
+    if (diataxis.tutorials && diataxis.tutorials.length > 1) {
+      const tutorialIds = diataxis.tutorials.map(t => t.id);
+      const sortedIds = [...tutorialIds].sort((a, b) => a.localeCompare(b));
+      for (let i = 0; i < tutorialIds.length; i++) {
+        if (tutorialIds[i] !== sortedIds[i]) {
+          throw new Error(`Tutorials not sorted in ${firstPackageDir}`);
+        }
+      }
+    }
+
+    // Verify howtos sorted by id
+    if (diataxis.howtos && diataxis.howtos.length > 1) {
+      const howtoIds = diataxis.howtos.map(h => h.id);
+      const sortedIds = [...howtoIds].sort((a, b) => a.localeCompare(b));
+      for (let i = 0; i < howtoIds.length; i++) {
+        if (howtoIds[i] !== sortedIds[i]) {
+          throw new Error(`How-tos not sorted in ${firstPackageDir}: ${howtoIds[i]} at index ${i}, expected ${sortedIds[i]}`);
+        }
+      }
+    }
+  }
+
+  console.log(`  Verified stable ordering for ${packageNames.length} packages`);
+}
+
+/**
+ * Test: Proof hashes in generated markdown files
+ */
+async function testProofHashes() {
+  await cleanOutputs();
+  await runPipeline();
+
+  const outDir = join(packageRoot, 'OUT');
+  const allMarkdownFiles = await findAllFiles(outDir);
+  const markdownFiles = allMarkdownFiles.filter(f => f.endsWith('.md'));
+
+  if (markdownFiles.length === 0) {
+    throw new Error('No markdown files generated');
+  }
+
+  let proofCount = 0;
+  const invalidProofs = [];
+
+  for (const mdFile of markdownFiles) {
+    const content = await readFile(mdFile, 'utf8');
+
+    // Parse frontmatter for proof field
+    const frontmatterMatch = content.match(/^---\n([\s\S]+?)\n---/);
+    if (!frontmatterMatch) {
+      continue; // Skip files without frontmatter
+    }
+
+    const frontmatter = frontmatterMatch[1];
+    const proofMatch = frontmatter.match(/^proof:\s*([a-f0-9]{64})$/m);
+
+    if (proofMatch) {
+      const proof = proofMatch[1];
+      proofCount++;
+
+      // Verify it's a valid 64-character hex string (SHA256)
+      if (!/^[a-f0-9]{64}$/.test(proof)) {
+        invalidProofs.push(`${mdFile}: invalid proof format "${proof}"`);
+      }
+    }
+  }
+
+  if (invalidProofs.length > 0) {
+    throw new Error(`Invalid proofs:\n${invalidProofs.join('\n')}`);
+  }
+
+  console.log(`  Verified ${proofCount} proof hashes in ${markdownFiles.length} markdown files`);
+}
+
+// Run all tests
+async function runTests() {
+  console.log('Running determinism tests...\n');
+
+  await test('determinism: inventory and artifacts hashes match', testDeterminism, 60000);
+  await test('determinism: timestamps fixed at 2000-01-01', testConsistentTimestamps, 30000);
+  await test('determinism: stable ordering', testStableOrdering, 30000);
+  await test('determinism: proof hashes', testProofHashes, 30000);
+
+  console.log(`\n${passCount}/${passCount + failCount} tests passed`);
+  process.exit(failCount > 0 ? 1 : 0);
+}
+
+runTests();

package/test/evidence.test.mjs

@@ -0,0 +1,145 @@
+/**
+ * @file evidence.test.mjs
+ * @description Tests for evidence collection module
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { collectEvidence, hashEvidence } from '../src/evidence.mjs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const fixturesDir = join(__dirname, 'fixtures', 'test-package');
+
+describe('evidence.mjs', () => {
+  describe('collectEvidence', () => {
+    it('should return valid EvidenceSnapshot structure', async () => {
+      const packageJson = {
+        name: 'test-package',
+        version: '1.0.0',
+        keywords: ['test', 'example'],
+        bin: { 'test-cli': './bin/cli.js' },
+        exports: { '.': './src/index.mjs' }
+      };
+
+      const evidence = await collectEvidence(fixturesDir, packageJson);
+
+      // Verify structure
+      assert.ok(evidence);
+      assert.ok(typeof evidence.readmeContent === 'string' || evidence.readmeContent === null);
+      assert.ok(Array.isArray(evidence.readmeHeadings));
+      assert.ok(Array.isArray(evidence.examplesFiles));
+      assert.ok(typeof evidence.examplesSnippets === 'object');
+      assert.ok(Array.isArray(evidence.docsFiles));
+      assert.ok(typeof evidence.docsSnippets === 'object');
+      assert.ok(Array.isArray(evidence.srcFiles));
+      assert.ok(typeof evidence.testFileCount === 'number');
+      assert.ok(typeof evidence.binEntries === 'object');
+      assert.ok(typeof evidence.exportSurface === 'object');
+      assert.ok(Array.isArray(evidence.keywords));
+      assert.ok(typeof evidence.hasLicense === 'boolean');
+      assert.ok(typeof evidence.hasTsConfig === 'boolean');
+      assert.ok(typeof evidence.fingerprint === 'string');
+      assert.equal(evidence.fingerprint.length, 64); // SHA256 hex = 64 chars
+    });
+
+    it('should extract package.json fields correctly', async () => {
+      const packageJson = {
+        keywords: ['rdf', 'semantic'],
+        bin: { 'my-cli': './bin/cli.js' },
+        exports: { '.': './index.js' }
+      };
+
+      const evidence = await collectEvidence(fixturesDir, packageJson);
+
+      assert.deepEqual(evidence.keywords, ['rdf', 'semantic']);
+      assert.deepEqual(evidence.binEntries, { 'my-cli': './bin/cli.js' });
+      assert.deepEqual(evidence.exportSurface, { '.': './index.js' });
+    });
+
+    it('should handle missing directories gracefully', async () => {
+      const packageJson = { name: 'test' };
+      const evidence = await collectEvidence('/tmp/nonexistent-package-dir', packageJson);
+
+      assert.equal(evidence.readmeContent, null);
+      assert.deepEqual(evidence.readmeHeadings, []);
+      assert.deepEqual(evidence.examplesFiles, []);
+      assert.deepEqual(evidence.docsFiles, []);
+      assert.deepEqual(evidence.srcFiles, []);
+      assert.equal(evidence.testFileCount, 0);
+    });
+
+    it('should throw on invalid package.json', async () => {
+      await assert.rejects(
+        async () => await collectEvidence('/tmp', null),
+        { message: /Invalid package.json/ }
+      );
+    });
+
+    it('should compute deterministic fingerprint', async () => {
+      const packageJson = { name: 'test', keywords: ['a', 'b'] };
+      const evidence1 = await collectEvidence(fixturesDir, packageJson);
+      const evidence2 = await collectEvidence(fixturesDir, packageJson);
+
+      assert.equal(evidence1.fingerprint, evidence2.fingerprint);
+    });
+  });
+
+  describe('hashEvidence', () => {
+    it('should produce deterministic hash', () => {
+      const evidence = {
+        readmeContent: 'test',
+        readmeHeadings: ['Heading 1'],
+        examplesFiles: ['example.js'],
+        examplesSnippets: { 'example.js': 'code' },
+        docsFiles: ['doc.md'],
+        docsSnippets: { 'doc.md': 'docs' },
+        srcFiles: ['index.js'],
+        testFileCount: 5,
+        binEntries: {},
+        exportSurface: {},
+        keywords: ['test'],
+        hasLicense: true,
+        hasTsConfig: false,
+        fingerprint: 'abc123'
+      };
+
+      const hash1 = hashEvidence(evidence);
+      const hash2 = hashEvidence(evidence);
+
+      assert.equal(hash1, hash2);
+      assert.equal(hash1.length, 64); // SHA256 hex
+    });
+
+    it('should produce different hashes for different evidence', () => {
+      const evidence1 = {
+        readmeContent: 'test1',
+        readmeHeadings: [],
+        examplesFiles: [],
+        examplesSnippets: {},
+        docsFiles: [],
+        docsSnippets: {},
+        srcFiles: [],
+        testFileCount: 0,
+        binEntries: {},
+        exportSurface: {},
+        keywords: [],
+        hasLicense: false,
+        hasTsConfig: false,
+        fingerprint: 'abc'
+      };
+
+      const evidence2 = {
+        ...evidence1,
+        readmeContent: 'test2'
+      };
+
+      const hash1 = hashEvidence(evidence1);
+      const hash2 = hashEvidence(evidence2);
+
+      assert.notEqual(hash1, hash2);
+    });
+  });
+});

package/test/fixtures/scaffold-det1/explanation/explanation.md

@@ -0,0 +1,35 @@
+---
+title: "test-deterministic Explanation"
+type: "explanation"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+confidenceScore: 0
+proof: "3f8072cee79349fae3971f82f5c1d412e5a32fbe09ed9860a00bdc84d583b953"
+---
+
+## Concepts
+
+- (No concepts documented)
+
+## Architecture
+
+(No architecture documented)
+
+## Tradeoffs
+
+- (No tradeoffs documented)
+
+## Proof
+
+This file was generated from the following evidence sources:
+
+- (No sources recorded)
+
+```json
+{
+  "fingerprintInput": "|test-deterministic Explanation|0",
+  "hash": "3f8072cee79349fae3971f82f5c1d412e5a32fbe09ed9860a00bdc84d583b953",
+  "sources": []
+}
+```

package/test/fixtures/scaffold-det1/index.md

@@ -0,0 +1,29 @@
+---
+title: "test-deterministic Documentation"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+---
+
+# test-deterministic Documentation
+
+Generated documentation using the Diátaxis framework.
+
+## Quick Stats
+
+- **Tutorials**: 1
+- **How-To Guides**: 0
+- **Reference**: Not available
+- **Explanation**: Not available
+
+## Documentation Sections
+
+### Tutorials
+
+- [Test Tutorial](tutorials/tutorial-test-tutorial.md)
+
+
+
+---
+
+*Generated at 2000-01-01T00:00:00.000Z*

package/test/fixtures/scaffold-det1/reference/reference.md

@@ -0,0 +1,34 @@
+---
+title: "test-deterministic Reference"
+type: "reference"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+confidenceScore: 0
+proof: "1db83d70e898a847f11d662317d65d4a5326b670fe1cc181774ba086078fd6a7"
+---
+
+## Overview
+
+This reference documentation provides technical details for test-deterministic.
+
+## API Reference
+
+| Name | Type | Description |
+|------|------|-------------|
+| - | - | (No items documented) |
+
+
+## Proof
+
+This file was generated from the following evidence sources:
+
+- (No sources recorded)
+
+```json
+{
+  "fingerprintInput": "|test-deterministic Reference|0",
+  "hash": "1db83d70e898a847f11d662317d65d4a5326b670fe1cc181774ba086078fd6a7",
+  "sources": []
+}
+```

package/test/fixtures/scaffold-det1/tutorials/tutorial-test-tutorial.md

@@ -0,0 +1,37 @@
+---
+title: "Test Tutorial"
+type: "tutorial"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+confidenceScore: 0.9
+proof: "fa5ec3edd8fd24ada2fbe509ea3369703fea30ef1c5d0707430893d876099862"
+---
+
+## Overview
+
+Test
+
+## Prerequisites
+
+- None
+
+## Steps
+
+1. Step 1
+
+## Proof
+
+This file was generated from the following evidence sources:
+
+- test.md
+
+```json
+{
+  "fingerprintInput": "test.md|Test Tutorial|0.9",
+  "hash": "fa5ec3edd8fd24ada2fbe509ea3369703fea30ef1c5d0707430893d876099862",
+  "sources": [
+    "test.md"
+  ]
+}
+```

package/test/fixtures/scaffold-det2/explanation/explanation.md

@@ -0,0 +1,35 @@
+---
+title: "test-deterministic Explanation"
+type: "explanation"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+confidenceScore: 0
+proof: "3f8072cee79349fae3971f82f5c1d412e5a32fbe09ed9860a00bdc84d583b953"
+---
+
+## Concepts
+
+- (No concepts documented)
+
+## Architecture
+
+(No architecture documented)
+
+## Tradeoffs
+
+- (No tradeoffs documented)
+
+## Proof
+
+This file was generated from the following evidence sources:
+
+- (No sources recorded)
+
+```json
+{
+  "fingerprintInput": "|test-deterministic Explanation|0",
+  "hash": "3f8072cee79349fae3971f82f5c1d412e5a32fbe09ed9860a00bdc84d583b953",
+  "sources": []
+}
+```

package/test/fixtures/scaffold-det2/index.md

@@ -0,0 +1,29 @@
+---
+title: "test-deterministic Documentation"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+---
+
+# test-deterministic Documentation
+
+Generated documentation using the Diátaxis framework.
+
+## Quick Stats
+
+- **Tutorials**: 1
+- **How-To Guides**: 0
+- **Reference**: Not available
+- **Explanation**: Not available
+
+## Documentation Sections
+
+### Tutorials
+
+- [Test Tutorial](tutorials/tutorial-test-tutorial.md)
+
+
+
+---
+
+*Generated at 2000-01-01T00:00:00.000Z*

package/test/fixtures/scaffold-det2/reference/reference.md

@@ -0,0 +1,34 @@
+---
+title: "test-deterministic Reference"
+type: "reference"
+packageName: "test-deterministic"
+version: "1.0.0"
+generatedAt: "2000-01-01T00:00:00.000Z"
+confidenceScore: 0
+proof: "1db83d70e898a847f11d662317d65d4a5326b670fe1cc181774ba086078fd6a7"
+---
+
+## Overview
+
+This reference documentation provides technical details for test-deterministic.
+
+## API Reference
+
+| Name | Type | Description |
+|------|------|-------------|
+| - | - | (No items documented) |
+
+
+## Proof
+
+This file was generated from the following evidence sources:
+
+- (No sources recorded)
+
+```json
+{
+  "fingerprintInput": "|test-deterministic Reference|0",
+  "hash": "1db83d70e898a847f11d662317d65d4a5326b670fe1cc181774ba086078fd6a7",
+  "sources": []
+}
+```