@open-agent-toolkit/cli 0.2.26 → 0.2.28

package/assets/skills/explainer-kit/scripts/lib/qa.mjs

@@ -1,5 +1,12 @@
+import { readFile, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { assertBrowserProbeSession } from './browser-runtime.mjs';
+import { writeJsonAtomic } from './fs-safe.mjs';
 import { findUnpinnedResourceRefs } from './html-safety.mjs';
+import { decodeBrowserPng } from './png.mjs';
 import { recipeFloor, recipeRequiredNarrative } from './recipes.mjs';
+import { cohesionEvidenceFromLedger } from './visual-review.mjs';
 
 const VOID_ELEMENTS = new Set([
   'area',
@@ -21,6 +28,7 @@ const INLINE_ASSET_VIOLATION_PATTERN =
   /<link\b|@import\b|url\(\s*["']?(?!data:|#)/i;
 const TOKEN_PATTERN = /{{\s*[A-Z][A-Z0-9_]*\s*}}/g;
 const ARROW_KEYS = ['ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown'];
+const MAX_SCREENSHOT_BYTES = 20 * 1024 * 1024;
 const INLINE_DIAGRAM_PATTERN =
   /<svg\b(?=[^>]*(?:\bclass\s*=\s*["'][^"']*\bdiagram\b|\baria-label\s*=\s*["'][^"']*(?:architecture|diagram)))[^>]*>/i;
 const STRUCTURED_BLOCK_PATTERNS = [
@@ -420,12 +428,33 @@ export function checkHtmlStructure({
   return { valid: issues.length === 0, issues };
 }
 
-export function checkArtifactCohesion(artifacts) {
+export function checkArtifactCohesion(artifacts, { ledger = null } = {}) {
   if (!Array.isArray(artifacts)) {
     throw new TypeError('Artifact cohesion input must be an array.');
   }
   const issues = [];
   const groups = ['terminology', 'numericClaims', 'statuses'];
+  const expected = ledger
+    ? {
+        terminology: new Map(
+          (ledger.terminology ?? []).map(({ term }) => [term, term]),
+        ),
+        numericClaims: new Map(
+          (ledger.numbers ?? []).map(({ subject, value }) => [subject, value]),
+        ),
+        statuses: new Map(
+          (ledger.statuses ?? []).map(({ subject, value }) => [subject, value]),
+        ),
+      }
+    : null;
+
+  if (expected && groups.some((group) => expected[group].size === 0)) {
+    issues.push({
+      code: 'cohesion-ledger-empty',
+      message:
+        'Adaptive recap cohesion requires non-empty terminology, numeric, and status ledger entries.',
+    });
+  }
 
   for (const group of groups) {
     const claims = new Map();
@@ -458,6 +487,38 @@ export function checkArtifactCohesion(artifacts) {
         }
       }
     }
+    if (expected) {
+      for (const [claim, expectedValue] of expected[group]) {
+        const observed = claims.get(claim);
+        if (
+          !observed ||
+          observed.normalized !== normalizeClaim(expectedValue)
+        ) {
+          issues.push({
+            code: 'cohesion-claim-unobserved',
+            message: `Rendered artifacts do not observably support ${group}.${claim}.`,
+            claim,
+          });
+        }
+      }
+    }
+  }
+
+  if (expected) {
+    for (const artifact of artifacts) {
+      const count = groups.reduce(
+        (total, group) =>
+          total + Object.keys(artifact?.cohesion?.[group] ?? {}).length,
+        0,
+      );
+      if (count === 0) {
+        issues.push({
+          code: 'cohesion-observations-empty',
+          message: `Artifact ${String(artifact?.id)} has no observed shared-ledger evidence.`,
+          artifactId: artifact?.id,
+        });
+      }
+    }
   }
 
   return { valid: issues.length === 0, issues };
@@ -466,14 +527,36 @@ export function checkArtifactCohesion(artifacts) {
 export async function runBrowserProbes({
   artifacts,
   probe,
+  browserSession,
   widths = REPRESENTATIVE_WIDTHS,
+  evidenceRoot,
+  requireEvidence = false,
+  onProbeResult,
 }) {
   if (!Array.isArray(artifacts) || artifacts.length === 0) {
     throw new TypeError('Browser QA requires at least one artifact.');
   }
-  if (typeof probe !== 'function') {
+  if (browserSession !== undefined && probe !== undefined) {
+    throw new TypeError(
+      'Browser QA accepts either a trusted browser session or a bare non-retaining probe, not both.',
+    );
+  }
+  const session =
+    browserSession === undefined
+      ? null
+      : assertBrowserProbeSession(browserSession, { allowFixture: true });
+  const resolvedProbe = session?.probe ?? probe;
+  if (typeof resolvedProbe !== 'function') {
     throw new TypeError('Browser QA requires a probe callback.');
   }
+  if (evidenceRoot && !session) {
+    throw new TypeError(
+      'A trusted browser session is required for retained evidence.',
+    );
+  }
+  if (onProbeResult !== undefined && typeof onProbeResult !== 'function') {
+    throw new TypeError('Browser QA probe observer must be a callback.');
+  }
   if (
     !Array.isArray(widths) ||
     widths.length === 0 ||
@@ -483,10 +566,21 @@ export async function runBrowserProbes({
   }
 
   const issues = [];
+  const evidence = [];
   let probes = 0;
   for (const artifact of artifacts) {
+    const evidenceId = browserEvidenceId(artifact.id);
     for (const width of widths) {
       for (const scenario of browserScenarios(artifact)) {
+        const viewport = viewportName(width);
+        const screenshotPath =
+          evidenceRoot && scenario === 'default'
+            ? `qa/browser/${evidenceId}/${viewport}.png`
+            : undefined;
+        const metricsPath =
+          evidenceRoot && scenario === 'default'
+            ? `qa/browser/${evidenceId}/${viewport}.json`
+            : undefined;
         const request = {
           artifact,
           scenario,
@@ -500,6 +594,9 @@ export async function runBrowserProbes({
             artifact.type === 'deck' && scenario === 'default'
               ? { tab: true, arrows: [...ARROW_KEYS] }
               : { tab: true },
+          ...(screenshotPath && {
+            screenshotPath: join(evidenceRoot, screenshotPath),
+          }),
           ...(scenario === 'no-js' && { javascriptEnabled: false }),
           ...(scenario === 'print' && { media: 'print' }),
           ...(artifact.type === 'deck' &&
@@ -520,11 +617,62 @@ export async function runBrowserProbes({
               },
             }),
         };
-        const result = await probe(request);
+        let result;
+        try {
+          result = await resolvedProbe(request);
+        } catch (cause) {
+          const error = new Error(
+            `Browser evidence callback failed: ${cause?.message ?? String(cause)}`,
+            { cause },
+          );
+          error.code = 'E_VISUAL_REVIEW';
+          throw error;
+        }
         probes += 1;
         validateProbeResult(result, artifact.id, width, request);
+        if (onProbeResult) {
+          try {
+            await onProbeResult(
+              structuredClone({
+                artifactId: artifact.id,
+                artifactType: artifact.type,
+                scenario,
+                viewport: request.viewport,
+                result,
+              }),
+            );
+          } catch (cause) {
+            const error = new Error(
+              `Browser evidence observer failed: ${cause?.message ?? String(cause)}`,
+              { cause },
+            );
+            error.code = 'E_VISUAL_REVIEW';
+            throw error;
+          }
+        }
 
         const context = { artifactId: artifact.id, width, scenario };
+        if (screenshotPath && metricsPath) {
+          const retained = await retainBrowserEvidence({
+            evidenceRoot,
+            artifactId: artifact.id,
+            viewport,
+            viewportSize: request.viewport,
+            screenshotPath,
+            metricsPath,
+            result,
+            browserSession: session,
+          });
+          if (retained.valid) {
+            evidence.push(retained.evidence);
+          } else if (requireEvidence) {
+            issues.push({
+              ...context,
+              code: 'browser-evidence-missing',
+              message: retained.message,
+            });
+          }
+        }
         if (result.pageOverflowX) {
           issues.push({
             ...context,
@@ -625,29 +773,51 @@ export async function runBrowserProbes({
     }
   }
 
-  return { valid: issues.length === 0, issues, probes };
+  return {
+    valid: issues.length === 0,
+    issues,
+    probes,
+    ...(evidenceRoot && { evidence }),
+  };
 }
 
 export async function auditArtifactSet({
   artifacts,
   denylist = [],
   browserProbe,
+  browserSession,
   widths,
+  evidenceRoot,
+  requireBrowserEvidence = false,
+  onProbeResult,
+  setPlan,
 }) {
   if (!Array.isArray(artifacts) || artifacts.length === 0) {
     throw new TypeError('Render QA requires at least one artifact.');
   }
 
-  const structural = artifacts.map((artifact) => ({
+  const artifactsWithCohesion = cohesionEvidenceFromLedger(artifacts, setPlan);
+  const structural = artifactsWithCohesion.map((artifact) => ({
     id: artifact.id,
     ...checkHtmlStructure({ ...artifact, denylist }),
   }));
-  const cohesion = checkArtifactCohesion(artifacts);
-  const browser = browserProbe
+  const cohesion = checkArtifactCohesion(artifactsWithCohesion, {
+    ...(setPlan?.recipe?.id === 'project-recap' &&
+      Object.values(setPlan.ledger ?? {}).some(
+        (entries) => Array.isArray(entries) && entries.length > 0,
+      ) && {
+        ledger: setPlan.ledger,
+      }),
+  });
+  const browserProvider = browserSession ?? browserProbe;
+  const browser = browserProvider
     ? await runBrowserProbes({
         artifacts,
-        probe: browserProbe,
+        ...(browserSession ? { browserSession } : { probe: browserProbe }),
         ...(widths && { widths }),
+        ...(evidenceRoot && { evidenceRoot }),
+        ...(requireBrowserEvidence && { requireEvidence: true }),
+        ...(onProbeResult !== undefined && { onProbeResult }),
       })
     : null;
   const issues = [
@@ -665,6 +835,108 @@ export async function auditArtifactSet({
   };
 }
 
+async function retainBrowserEvidence({
+  evidenceRoot,
+  artifactId: artifactIdentifier,
+  viewport,
+  viewportSize,
+  screenshotPath,
+  metricsPath,
+  result,
+  browserSession,
+}) {
+  let screenshot;
+  try {
+    screenshot = await stat(join(evidenceRoot, screenshotPath));
+  } catch {
+    return {
+      valid: false,
+      message: `Browser screenshot evidence is missing for ${artifactIdentifier} at ${viewportSize.width}px.`,
+    };
+  }
+  if (
+    !screenshot.isFile() ||
+    screenshot.size === 0 ||
+    screenshot.size > MAX_SCREENSHOT_BYTES
+  ) {
+    return {
+      valid: false,
+      message: `Browser screenshot evidence for ${artifactIdentifier} at ${viewportSize.width}px is empty or exceeds ${MAX_SCREENSHOT_BYTES} bytes.`,
+    };
+  }
+  const screenshotBytes = await readFile(join(evidenceRoot, screenshotPath));
+  const decoded = decodedPng(screenshotBytes);
+  if (
+    !decoded ||
+    decoded.width !== viewportSize.width ||
+    decoded.height !== viewportSize.height
+  ) {
+    return {
+      valid: false,
+      message: `Browser screenshot evidence for ${artifactIdentifier} at ${viewportSize.width}px must be a viewport-matched PNG.`,
+    };
+  }
+  await writeJsonAtomic(evidenceRoot, metricsPath, {
+    schemaVersion: 'explainer-kit.browser-evidence/v2',
+    artifactId: artifactIdentifier,
+    viewport,
+    viewportSize,
+    scenario: 'default',
+    runtime: structuredClone(browserSession.runtime),
+    capture: structuredClone(browserSession.capture),
+    captureIdentity: browserSession.captureIdentity,
+    screenshotPath,
+    metrics: structuredClone(result),
+  });
+  return {
+    valid: true,
+    evidence: {
+      artifactId: artifactIdentifier,
+      viewport,
+      width: viewportSize.width,
+      height: viewportSize.height,
+      screenshotPath,
+      decodedScreenshotHash: decoded.decodedHash,
+      metricsPath,
+      runtime: structuredClone(browserSession.runtime),
+      captureIdentity: browserSession.captureIdentity,
+    },
+  };
+}
+
+export function pngDimensions(bytes) {
+  const decoded = decodedPng(bytes);
+  return decoded ? { width: decoded.width, height: decoded.height } : null;
+}
+
+function decodedPng(bytes) {
+  try {
+    return decodeBrowserPng(bytes);
+  } catch {
+    return null;
+  }
+}
+
+function browserEvidenceId(value) {
+  if (
+    typeof value !== 'string' ||
+    !/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(value)
+  ) {
+    throw new TypeError('Browser evidence requires a safe artifact id.');
+  }
+  return value;
+}
+
+function viewportName(width) {
+  return (
+    {
+      320: 'mobile',
+      768: 'tablet',
+      1440: 'desktop',
+    }[width] ?? `viewport-${width}`
+  );
+}
+
 function checkHeadings(html, add) {
   const headings = [
     ...html.matchAll(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi),

package/assets/skills/explainer-kit/scripts/lib/recipes.mjs

@@ -1,7 +1,7 @@
 import { readFileSync } from 'node:fs';
 
 const RECIPE_SCHEMA_V2 = 'explainer-kit.recipe/v2';
-const RECIPE_V2_ROOT_KEYS = [
+const RECIPE_V2_REQUIRED_ROOT_KEYS = [
   'discoveryLimits',
   'expansion',
   'floor',
@@ -10,6 +10,7 @@ const RECIPE_V2_ROOT_KEYS = [
   'sourceRoles',
   'version',
 ];
+const RECIPE_V2_ROOT_KEYS = [...RECIPE_V2_REQUIRED_ROOT_KEYS, 'fallback'];
 const SOURCE_ROLE_KEYS = [
   'accepts',
   'maxBindings',
@@ -34,9 +35,14 @@ const PROFILE_REQUIRED_KEYS = [
   'profileId',
   'type',
 ];
-const PROFILE_KEYS = [...PROFILE_REQUIRED_KEYS, 'shell'];
+const PROFILE_KEYS = [
+  ...PROFILE_REQUIRED_KEYS,
+  'allowedJustificationKinds',
+  'shell',
+];
 const EXPANSION_LIMIT_KEYS = ['maxArtifacts', 'maxPerType'];
 const DISCOVERY_LIMIT_KEYS = ['consecutiveNoNewFindingsRounds', 'maxRounds'];
+const FALLBACK_KEYS = ['authoring', 'mode', 'scope', 'selection'];
 const SOURCE_KINDS = new Set([
   'file',
   'directory',
@@ -81,6 +87,54 @@ export function loadRecipe(id, version) {
   return structuredClone(recipe);
 }
 
+export function selectRecipeAuthoring(recipe, mode = 'artistic') {
+  const selected = structuredClone(recipe);
+  if (mode === 'artistic') {
+    return selected;
+  }
+  if (selected.fallback?.mode !== mode) {
+    const error = new Error(
+      `Recipe ${selected.id}@${selected.version} does not support recap mode ${mode}`,
+    );
+    error.code = 'E_RECIPE_UNSUPPORTED';
+    throw error;
+  }
+
+  selected.floor = selected.floor.map((artifact) => ({
+    ...artifact,
+    authoring: selected.fallback.authoring,
+  }));
+  selected.expansion.profiles = selected.expansion.profiles.map(
+    ({ shell: _shell, ...profile }) => ({
+      ...profile,
+      authoring: selected.fallback.authoring,
+    }),
+  );
+  return selected;
+}
+
+export function resolveDiagramRenderingRoute(recipe, artifact, diagrams) {
+  if (
+    !recipe ||
+    typeof recipe !== 'object' ||
+    !artifact ||
+    typeof artifact !== 'object' ||
+    !Array.isArray(diagrams)
+  ) {
+    throw new TypeError(
+      'Diagram routing requires a recipe, artifact, and diagram analyses.',
+    );
+  }
+  const supported = diagrams.filter(({ valid }) => valid === true);
+  if (
+    supported.length === 0 ||
+    supported.every(({ inlineSupported }) => inlineSupported === true)
+  ) {
+    return 'inline';
+  }
+  return artifact.authoring === 'html' ? 'artistic' : 'reject';
+}
+
 export function recipeFloor(recipe) {
   if (recipe?.schemaVersion === RECIPE_SCHEMA_V2) {
     return structuredClone(recipe.floor);
@@ -304,6 +358,54 @@ export function validateContentModel(recipe, contentModel) {
   return { valid: errors.length === 0, errors };
 }
 
+export function validatePlannedPortfolio(recipe, portfolio) {
+  const errors = [];
+  if (!Array.isArray(portfolio)) {
+    return { valid: false, errors: ['Planned portfolio must be an array'] };
+  }
+  const floorIds = new Set(recipeFloor(recipe).map(({ id }) => id));
+  const expansion = recipeExpansion(recipe);
+  const profiles = new Map(
+    expansion.profiles.map((profile) => [profile.profileId, profile]),
+  );
+  const counts = new Map();
+  let optionalCount = 0;
+
+  for (const artifact of portfolio) {
+    if (floorIds.has(artifact?.artifactId)) continue;
+    optionalCount += 1;
+    const profile = profiles.get(artifact?.profileId);
+    if (!profile) {
+      errors.push(`Unknown planned profile: ${artifact?.profileId}`);
+      continue;
+    }
+    counts.set(profile.profileId, (counts.get(profile.profileId) ?? 0) + 1);
+    if (!isObject(artifact.justification)) {
+      errors.push(
+        `Optional artifact ${artifact.artifactId} requires a source-backed justification`,
+      );
+      continue;
+    }
+    if (
+      profile.allowedJustificationKinds !== undefined &&
+      !profile.allowedJustificationKinds.includes(artifact.justification.kind)
+    ) {
+      errors.push(
+        `Optional artifact ${artifact.artifactId} justification ${artifact.justification.kind} is not allowed for ${profile.profileId}`,
+      );
+    }
+  }
+  if (optionalCount > expansion.limits.maxArtifacts) {
+    errors.push('Planned portfolio exceeds the recipe optional-artifact limit');
+  }
+  for (const [profileId, count] of counts) {
+    if (count > profiles.get(profileId).maxCount) {
+      errors.push(`Planned portfolio exceeds the ${profileId} profile limit`);
+    }
+  }
+  return { valid: errors.length === 0, errors };
+}
+
 export function shouldStopDiscovery(recipe, findingsByRound) {
   if (
     !Array.isArray(findingsByRound) ||
@@ -329,7 +431,12 @@ export function shouldStopDiscovery(recipe, findingsByRound) {
 export function validateRecipe(recipe, file = 'recipe') {
   assertObject(recipe, `${file} recipe`);
   if (recipe.schemaVersion === RECIPE_SCHEMA_V2) {
-    assertExactKeys(recipe, RECIPE_V2_ROOT_KEYS, `${file} recipe`);
+    assertAllowedKeys(
+      recipe,
+      RECIPE_V2_ROOT_KEYS,
+      RECIPE_V2_REQUIRED_ROOT_KEYS,
+      `${file} recipe`,
+    );
     validateV2Shape(recipe, file);
   } else {
     assert(false, `${file} has unsupported schemaVersion`);
@@ -368,6 +475,22 @@ export function validateRecipe(recipe, file = 'recipe') {
   }
   assertUnique(roleNames, `${file} source role names`);
 
+  if ('fallback' in recipe) {
+    assert(
+      recipe.id === 'project-recap',
+      `${file} fallback is allowed only for project-recap`,
+    );
+    assertObject(recipe.fallback, `${file} fallback`);
+    assertExactKeys(recipe.fallback, FALLBACK_KEYS, `${file} fallback`);
+    assert(
+      recipe.fallback.mode === 'deterministic-markdown' &&
+        recipe.fallback.selection === 'explicit' &&
+        recipe.fallback.authoring === 'markdown' &&
+        recipe.fallback.scope === 'portfolio',
+      `${file} has an unsupported fallback policy`,
+    );
+  }
+
   validateDiscoveryLimits(recipe.discoveryLimits, file);
   return recipe;
 }
@@ -435,6 +558,12 @@ function validateV2Shape(recipe, file) {
       `${file} expansion profile briefRef`,
     );
     assertFiniteCount(profile.maxCount, `${file} expansion profile maxCount`);
+    if ('allowedJustificationKinds' in profile) {
+      assertUniqueNonEmptyStrings(
+        profile.allowedJustificationKinds,
+        `${file} expansion profile allowedJustificationKinds`,
+      );
+    }
     if (profile.authoring === 'html' || 'shell' in profile) {
       assertNonEmptyString(
         profile.shell,