ai-design-context 0.4.0

package/tools/init.mjs

@@ -0,0 +1,99 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+const startMarker = '<!-- ai-design-context:start -->';
+const endMarker = '<!-- ai-design-context:end -->';
+const templates = [
+  'docs/PRD.md',
+  'docs/PERSONAS.md',
+  'docs/USER_FLOWS.md',
+  'docs/INFORMATION_ARCHITECTURE.md',
+  'docs/DESIGN_DECISIONS.md',
+  'templates/FEATURE_TEMPLATE.md',
+  'templates/TASK_TEMPLATE.md',
+  'reviews/DESIGN_REVIEW.md',
+  'benchmarks/BENCHMARK_CHECKLIST.md',
+];
+
+const instructions = `${startMarker}
+## AI Design Context
+
+Use the installed, lockfile-pinned AI Design Context knowledge graph for user-facing product, UX, UI, accessibility, motion, and design-prompt tasks. Existing project instructions, business logic, and the project's design system remain authoritative. Skip graph retrieval for backend-only, infrastructure-only, or unrelated maintenance tasks.
+
+Before implementation:
+
+\`\`\`bash
+npx --no-install ai-design-context context --task "<task phrase>" --intent implement
+npx --no-install ai-design-context context --task "<task phrase>" --platform mobile --intent implement
+\`\`\`
+
+Before review, resolve a known graph object or matching task phrase:
+
+\`\`\`bash
+npx --no-install ai-design-context context --review "<object ID, graph path, or matching phrase>" --intent qa
+\`\`\`
+
+Read the returned research and rules before making product decisions. Markdown lists absolute reading paths; JSON (\`--format json\`) retains graph-relative \`path\` and adds \`absolutePath\`. Review queries resolve graph context; they do not inspect arbitrary application files. Inspect the implementation separately.
+
+Treat \`draft\` and \`seed\` guidance as bounded evidence. Matching is lexical, not translation or semantic search: prefer known slugs or IDs when a phrase does not match. If retrieval fails, record an observation or research need instead of inventing design advice. Report applied rules, patterns, state coverage, mobile assumptions, accessibility checks, and validation limits.
+
+Use existing product documentation first. Fill any newly created starter-kit placeholders with actual product context; use \`reviews/DESIGN_REVIEW.md\` for UI review.
+${endMarker}`;
+
+function inspectTarget(root, relativePath) {
+  const parts = relativePath.split('/');
+  let current = root;
+  let result;
+  for (const [index, part] of parts.entries()) {
+    current = path.join(current, part);
+    try {
+      result = fs.lstatSync(current);
+    } catch (error) {
+      if (error.code === 'ENOENT') return undefined;
+      throw error;
+    }
+    if (result.isSymbolicLink()) throw new Error(`Refusing to write through symlink: ${relativePath}`);
+    if (index < parts.length - 1 && !result.isDirectory()) {
+      throw new Error(`Expected a directory in template path: ${relativePath}`);
+    }
+  }
+  if (!result.isFile()) throw new Error(`Expected a regular file: ${relativePath}`);
+  return result;
+}
+
+export function initProject(projectRoot, packageRoot) {
+  const root = fs.realpathSync(projectRoot);
+  const agentsPath = path.join(root, 'AGENTS.md');
+  const agentsExists = inspectTarget(root, 'AGENTS.md');
+  const original = agentsExists ? fs.readFileSync(agentsPath, 'utf8') : '';
+  const starts = original.split(startMarker).length - 1;
+  const ends = original.split(endMarker).length - 1;
+  if (starts !== ends || starts > 1 || (starts === 1 && original.indexOf(startMarker) > original.indexOf(endMarker))) {
+    throw new Error('Invalid AI Design Context markers in AGENTS.md; repair the marked block before running init.');
+  }
+
+  // Preflight all destinations and sources before writing any project files.
+  const missing = templates.filter((file) => !inspectTarget(root, file));
+  const contents = missing.map((file) => [file, fs.readFileSync(path.join(packageRoot, 'starter-kit', file))]);
+  const created = [];
+  for (const [file, content] of contents) {
+    const target = path.join(root, file);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.writeFileSync(target, content, { flag: 'wx' });
+    created.push(file);
+  }
+
+  if (starts === 0) {
+    const newline = original.includes('\r\n') ? '\r\n' : '\n';
+    const block = instructions.replaceAll('\n', newline);
+    if (agentsExists) {
+      const separator = original.endsWith(newline) ? newline : `${newline}${newline}`;
+      fs.appendFileSync(agentsPath, `${separator}${block}${newline}`);
+    } else {
+      fs.writeFileSync(agentsPath, `# Project Agent Instructions${newline}${newline}${block}${newline}`, { flag: 'wx' });
+      created.unshift('AGENTS.md');
+    }
+  }
+
+  return { created, instructionsAdded: starts === 0 };
+}

package/tools/validate-benchmark-evidence.mjs

@@ -0,0 +1,248 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+const root = process.cwd();
+const evidenceRoot = path.join(root, 'evidence');
+const requiredMetadataFields = [
+  'scenario',
+  'run_type',
+  'date',
+  'model',
+  'provider',
+  'generation_surface',
+  'temperature',
+  'tool_access',
+  'implementation_target',
+  'evaluator',
+  'rubric_version',
+  'evidence_level',
+  'screenshots',
+  'limitation',
+];
+const requiredArtifacts = ['metadata.json', 'prompt.md', 'generated-output.md', 'notes.md', 'scores.md'];
+const rubricCategories = [
+  'Product Thinking',
+  'UX',
+  'Information Architecture',
+  'Navigation',
+  'Accessibility',
+  'Mobile-first',
+  'Visual Hierarchy',
+  'State Design',
+  'Consistency',
+  'Performance Awareness',
+  'Simplicity',
+  'Overall Product Quality',
+];
+const errors = [];
+const warnings = [];
+const runs = [];
+const screenshotExtensions = new Set(['.avif', '.gif', '.jpeg', '.jpg', '.png', '.webp']);
+
+function hasSupportedImageSignature(filePath, extension) {
+  const bytes = fs.readFileSync(filePath).subarray(0, 16);
+  const ascii = bytes.toString('ascii');
+  const hex = bytes.toString('hex');
+
+  switch (extension) {
+    case '.png':
+      return hex.startsWith('89504e470d0a1a0a');
+    case '.jpeg':
+    case '.jpg':
+      return hex.startsWith('ffd8ff');
+    case '.gif':
+      return ascii.startsWith('GIF87a') || ascii.startsWith('GIF89a');
+    case '.webp':
+      return ascii.startsWith('RIFF') && ascii.slice(8, 12) === 'WEBP';
+    case '.avif':
+      return ascii.slice(4, 8) === 'ftyp' && ['avif', 'avis'].includes(ascii.slice(8, 12));
+    default:
+      return false;
+  }
+}
+
+function listDirectories(directory) {
+  if (!fs.existsSync(directory)) {
+    return [];
+  }
+
+  return fs
+    .readdirSync(directory, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.'))
+    .map((entry) => entry.name)
+    .sort();
+}
+
+function readMetadata(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return null;
+  }
+}
+
+function validateArtifacts(runPath, runType) {
+  for (const artifact of requiredArtifacts) {
+    const artifactPath = path.join(runPath, runType, artifact);
+    if (!fs.existsSync(artifactPath)) {
+      errors.push(`${path.relative(root, runPath)}: missing ${runType}/${artifact}`);
+    }
+  }
+}
+
+function validateMetadata(runPath, runType) {
+  const metadataPath = path.join(runPath, runType, 'metadata.json');
+  if (!fs.existsSync(metadataPath)) {
+    return null;
+  }
+
+  const metadata = readMetadata(metadataPath);
+  if (!metadata) {
+    return null;
+  }
+
+  for (const field of requiredMetadataFields) {
+    if (!(field in metadata)) {
+      errors.push(`${path.relative(root, metadataPath)}: missing required field ${field}`);
+    }
+  }
+
+  if (metadata.run_type !== runType) {
+    errors.push(`${path.relative(root, metadataPath)}: run_type must be ${runType}`);
+  }
+
+  if (!Array.isArray(metadata.tool_access)) {
+    errors.push(`${path.relative(root, metadataPath)}: tool_access must be an array`);
+  }
+
+  if (!['directional', 'rendered'].includes(metadata.evidence_level)) {
+    errors.push(`${path.relative(root, metadataPath)}: evidence_level must be directional or rendered`);
+  }
+
+  if (metadata.evidence_level === 'rendered') {
+    if (!Array.isArray(metadata.screenshots) || metadata.screenshots.length === 0) {
+      errors.push(`${path.relative(root, metadataPath)}: rendered evidence requires non-empty screenshots`);
+    } else {
+      for (const screenshot of metadata.screenshots) {
+        if (typeof screenshot !== 'string' || path.isAbsolute(screenshot)) {
+          errors.push(`${path.relative(root, metadataPath)}: screenshot paths must be relative strings`);
+          continue;
+        }
+
+        const screenshotRoot = path.resolve(runPath, runType);
+        const screenshotPath = path.resolve(runPath, screenshot);
+        const relativeScreenshotPath = path.relative(screenshotRoot, screenshotPath);
+        if (
+          relativeScreenshotPath === '' ||
+          relativeScreenshotPath.startsWith(`..${path.sep}`) ||
+          path.isAbsolute(relativeScreenshotPath)
+        ) {
+          errors.push(
+            `${path.relative(root, metadataPath)}: declared screenshot ${screenshot} must stay inside ${runType}/`,
+          );
+          continue;
+        }
+
+        const extension = path.extname(screenshotPath).toLowerCase();
+        if (!screenshotExtensions.has(extension)) {
+          errors.push(`${path.relative(root, metadataPath)}: declared screenshot ${screenshot} must be an image file`);
+          continue;
+        }
+
+        if (!fs.existsSync(screenshotPath)) {
+          errors.push(`${path.relative(root, metadataPath)}: declared screenshot ${screenshot} does not exist`);
+          continue;
+        }
+
+        if (!fs.lstatSync(screenshotPath).isFile()) {
+          errors.push(`${path.relative(root, metadataPath)}: declared screenshot ${screenshot} must be a regular file`);
+          continue;
+        }
+
+        if (!hasSupportedImageSignature(screenshotPath, extension)) {
+          errors.push(
+            `${path.relative(root, metadataPath)}: declared screenshot ${screenshot} has invalid image data`,
+          );
+        }
+      }
+    }
+  }
+
+  return metadata;
+}
+
+function validateRun(runPath) {
+  const relativeRunPath = path.relative(root, runPath);
+  const evaluationPath = path.join(runPath, 'EVALUATION.md');
+  if (!fs.existsSync(evaluationPath)) {
+    errors.push(`${relativeRunPath}: missing EVALUATION.md`);
+  } else {
+    const evaluation = fs.readFileSync(evaluationPath, 'utf8');
+    for (const category of rubricCategories) {
+      if (!evaluation.includes(`| ${category} |`)) {
+        errors.push(`${relativeRunPath}/EVALUATION.md: missing rubric category ${category}`);
+      }
+    }
+  }
+
+  validateArtifacts(runPath, 'baseline');
+  validateArtifacts(runPath, 'ai-design-rules');
+  const baseline = validateMetadata(runPath, 'baseline');
+  const rules = validateMetadata(runPath, 'ai-design-rules');
+
+  if (!baseline || !rules) {
+    return;
+  }
+
+  for (const field of [
+    'scenario',
+    'date',
+    'model',
+    'provider',
+    'generation_surface',
+    'temperature',
+    'tool_access',
+    'implementation_target',
+    'evaluator',
+    'rubric_version',
+    'evidence_level',
+  ]) {
+    if (JSON.stringify(baseline[field]) !== JSON.stringify(rules[field])) {
+      errors.push(`${relativeRunPath}: baseline and ai-design-rules must use the same ${field}`);
+    }
+  }
+
+  if (baseline.evidence_level === 'directional') {
+    warnings.push(`${relativeRunPath}: directional evidence only; no rendered visual proof is asserted`);
+  }
+
+  runs.push({ evidenceLevel: baseline.evidence_level, runPath: relativeRunPath });
+}
+
+for (const scenario of listDirectories(evidenceRoot)) {
+  for (const run of listDirectories(path.join(evidenceRoot, scenario))) {
+    validateRun(path.join(evidenceRoot, scenario, run));
+  }
+}
+
+if (runs.length === 0) {
+  errors.push('evidence/: no benchmark run directories found');
+}
+
+if (errors.length > 0) {
+  console.error(`Benchmark evidence validation failed with ${errors.length} error(s):`);
+  for (const error of errors) {
+    console.error(`- ${error}`);
+  }
+  process.exit(1);
+}
+
+const renderedCount = runs.filter((run) => run.evidenceLevel === 'rendered').length;
+const directionalCount = runs.length - renderedCount;
+console.log(
+  `Benchmark evidence validation passed: ${runs.length} run(s), ${renderedCount} rendered, ${directionalCount} directional.`,
+);
+for (const warning of warnings) {
+  console.warn(`Warning: ${warning}`);
+}