@design-ai/cli 4.56.0 → 4.57.0

package/cli/lib/signals-readiness.mjs

@@ -0,0 +1,247 @@
+// Learning signal readiness checks and readiness report rendering helpers.
+
+import { readinessCountByStatus, statusRank, yesNo } from "./signals-shared.mjs";
+
+export function optionalGapDetail(check = {}) {
+  if (check.id === "check-capture") {
+    return {
+      id: check.id,
+      label: check.label || "Check learning capture",
+      status: check.status || "info",
+      summary: check.summary || "",
+      reason: "No real warn/fail check result has been intentionally captured into the local learning profile yet.",
+      nextCondition: "Run `design-ai check <artifact.md> --learn --yes` only after reviewing an actual warning or failure that should improve future outputs.",
+      automationPolicy: "Do not emit placeholder mutation commands for this advisory gap; wait for real check evidence.",
+    };
+  }
+
+  return {
+    id: check.id || "unknown",
+    label: check.label || "Optional readiness check",
+    status: check.status || "info",
+    summary: check.summary || "",
+    reason: "Optional evidence is incomplete.",
+    nextCondition: "Collect real local evidence before treating this optional signal as complete.",
+    automationPolicy: "Keep this advisory unless a future required gate explicitly depends on it.",
+  };
+}
+
+export function renderOptionalGapDetails(lines, readiness = {}) {
+  const details = Array.isArray(readiness.optionalGapDetails) ? readiness.optionalGapDetails : [];
+  if (details.length === 0) return;
+  lines.push("", "Optional gap details:");
+  for (const detail of details) {
+    lines.push(`- ${detail.id || "unknown"}: ${detail.reason || detail.summary || ""}`);
+    if (detail.nextCondition) lines.push(`  Next condition: ${detail.nextCondition}`);
+    if (detail.automationPolicy) lines.push(`  Automation policy: ${detail.automationPolicy}`);
+  }
+}
+
+export function renderReadinessCheckIndex(lines, readiness = {}) {
+  const requiredIds = Array.isArray(readiness.requiredCheckIds) ? readiness.requiredCheckIds : [];
+  const optionalIds = Array.isArray(readiness.optionalCheckIds) ? readiness.optionalCheckIds : [];
+  const statusById = readiness.checkStatusById && typeof readiness.checkStatusById === "object"
+    ? readiness.checkStatusById
+    : {};
+  const requiredById = readiness.checkRequiredById && typeof readiness.checkRequiredById === "object"
+    ? readiness.checkRequiredById
+    : {};
+  const checks = Array.isArray(readiness.checks) ? readiness.checks : [];
+  const indexIds = [...new Set([
+    ...checks.map((item) => item?.id).filter(Boolean),
+    ...requiredIds,
+    ...optionalIds,
+    ...Object.keys(statusById),
+    ...Object.keys(requiredById),
+  ])];
+  if (indexIds.length === 0) return;
+
+  lines.push("", "Readiness check index:");
+  lines.push(`- Required ids: ${requiredIds.length > 0 ? requiredIds.join(", ") : "none"}`);
+  lines.push(`- Optional ids: ${optionalIds.length > 0 ? optionalIds.join(", ") : "none"}`);
+  lines.push(`- Status index: ${indexIds.map((id) => `${id}=${statusById[id] || "unknown"}`).join(", ")}`);
+  lines.push(`- Required index: ${indexIds.map((id) => `${id}=${yesNo(Boolean(requiredById[id]))}`).join(", ")}`);
+  if (readiness.checkCountByStatus && typeof readiness.checkCountByStatus === "object") {
+    const formatCounts = (counts = {}) => ["pass", "info", "warn", "fail", "missing", "template", "unknown"]
+      .map((key) => `${key}=${counts[key] ?? 0}`)
+      .join(", ");
+    lines.push(`- Status counts: ${formatCounts(readiness.checkCountByStatus)}`);
+    lines.push(`- Required status counts: ${formatCounts(readiness.requiredCheckCountByStatus)}`);
+    lines.push(`- Optional status counts: ${formatCounts(readiness.optionalCheckCountByStatus)}`);
+  }
+}
+export function signalCheck({
+  id,
+  label,
+  status,
+  required,
+  summary,
+  evidence = {},
+}) {
+  return {
+    id,
+    label,
+    status,
+    required: Boolean(required),
+    summary,
+    evidence,
+  };
+}
+
+export function buildLearningSignalReadiness({
+  audit,
+  usage,
+  evals,
+  checkCapture,
+  workspace,
+  agentDevelopment,
+  status,
+}) {
+  const profileStatus = !audit.exists
+    ? "missing"
+    : audit.summary.failures > 0
+      ? "fail"
+      : audit.summary.warnings > 0
+        ? "warn"
+        : "pass";
+  const usageStatus = usage.staleSelectedEntryCount > 0
+    ? "warn"
+    : usage.exists && usage.eventCount > 0
+      ? "pass"
+      : "info";
+  const evalStatus = evals.failed > 0
+    ? "fail"
+    : evals.warned > 0 || evals.templates > 0 || evals.count === 0
+      ? "warn"
+      : "pass";
+  const checkCaptureStatus = checkCapture.count > 0 ? "pass" : "info";
+  const workspaceStatus = workspace.nextActionCounts.fail > 0
+    ? "fail"
+    : workspace.nextActionCounts.warn > 0
+      ? "warn"
+      : "pass";
+  const agentStatus = agentDevelopment.status || "unknown";
+  const checks = [
+    signalCheck({
+      id: "learning-profile",
+      label: "Learning profile",
+      status: profileStatus,
+      required: true,
+      summary: audit.exists
+        ? `Profile has ${audit.count} entries with ${audit.summary.failures} audit failure(s) and ${audit.summary.warnings} warning(s).`
+        : "Learning profile is missing.",
+      evidence: {
+        exists: Boolean(audit.exists),
+        entries: audit.count,
+        failures: audit.summary.failures,
+        warnings: audit.summary.warnings,
+      },
+    }),
+    signalCheck({
+      id: "usage-sidecar",
+      label: "Usage sidecar",
+      status: usageStatus,
+      required: false,
+      summary: usage.exists && usage.eventCount > 0
+        ? `Usage sidecar has ${usage.eventCount} event(s) and ${usage.staleSelectedEntryCount} stale selected id(s).`
+        : "Usage sidecar has no prompt/pack usage events yet.",
+      evidence: {
+        exists: Boolean(usage.exists),
+        events: usage.eventCount,
+        staleSelectedEntryCount: usage.staleSelectedEntryCount,
+      },
+    }),
+    signalCheck({
+      id: "eval-signals",
+      label: "Eval signals",
+      status: evalStatus,
+      required: true,
+      summary: evals.count > 0
+        ? `Eval signals include ${evals.reports} report(s), ${evals.templates} unresolved template(s), ${evals.failed} failed report(s), and ${evals.warned} warned report(s).`
+        : "No route, prompt, pack, or learning eval signal files were found.",
+      evidence: {
+        files: evals.count,
+        reports: evals.reports,
+        templates: evals.templates,
+        failed: evals.failed,
+        warned: evals.warned,
+      },
+    }),
+    signalCheck({
+      id: "check-capture",
+      label: "Check learning capture",
+      status: checkCaptureStatus,
+      required: false,
+      summary: checkCapture.count > 0
+        ? `Profile includes ${checkCapture.count} check-capture learning entr${checkCapture.count === 1 ? "y" : "ies"}.`
+        : "No check-capture entries are present; this is advisory until real warn/fail checks are captured.",
+      evidence: {
+        entries: checkCapture.count,
+        categoryCounts: checkCapture.categoryCounts,
+      },
+    }),
+    signalCheck({
+      id: "workspace-readiness",
+      label: "Workspace readiness",
+      status: workspaceStatus,
+      required: true,
+      summary: `Workspace has ${workspace.nextActionCounts.fail || 0} fail action(s), ${workspace.nextActionCounts.warn || 0} warn action(s), and ${workspace.nextActionCount || 0} total next action(s).`,
+      evidence: {
+        fail: workspace.nextActionCounts.fail || 0,
+        warn: workspace.nextActionCounts.warn || 0,
+        nextActionCount: workspace.nextActionCount || 0,
+      },
+    }),
+    signalCheck({
+      id: "agent-development",
+      label: "Agent development backlog",
+      status: agentStatus,
+      required: true,
+      summary: `Agent backlog has ${agentDevelopment.actionCount || 0} action(s): ${agentDevelopment.p0Count || 0} P0, ${agentDevelopment.p1Count || 0} P1, ${agentDevelopment.p2Count || 0} P2, ${agentDevelopment.p3Count || 0} P3.`,
+      evidence: {
+        actions: agentDevelopment.actionCount || 0,
+        p0: agentDevelopment.p0Count || 0,
+        p1: agentDevelopment.p1Count || 0,
+        p2: agentDevelopment.p2Count || 0,
+        p3: agentDevelopment.p3Count || 0,
+      },
+    }),
+  ];
+  const requiredChecks = checks.filter((item) => item.required);
+  const optionalChecks = checks.filter((item) => !item.required);
+  const blockingChecks = requiredChecks.filter((item) => statusRank(item.status) >= statusRank("warn"));
+  const optionalGaps = optionalChecks.filter((item) => item.status !== "pass");
+  const requiredPassCount = requiredChecks.filter((item) => item.status === "pass").length;
+  const checkStatusById = Object.fromEntries(checks.map((item) => [item.id, item.status]));
+  const checkRequiredById = Object.fromEntries(checks.map((item) => [item.id, Boolean(item.required)]));
+  const checkCountByStatus = readinessCountByStatus(checks);
+  const requiredCheckCountByStatus = readinessCountByStatus(requiredChecks);
+  const optionalCheckCountByStatus = readinessCountByStatus(optionalChecks);
+  const summary = blockingChecks.length === 0
+    ? optionalGaps.length === 0
+      ? "Required and optional local learning signal surfaces are complete."
+      : "Required local learning signal surfaces are ready; optional evidence gaps remain."
+    : "Required local learning signal surfaces need review before this can be used as a gate.";
+
+  return {
+    version: 1,
+    status,
+    summary,
+    requiredPassCount,
+    requiredCount: requiredChecks.length,
+    requiredReady: blockingChecks.length === 0,
+    blockingCount: blockingChecks.length,
+    optionalGapCount: optionalGaps.length,
+    blockingChecks: blockingChecks.map((item) => item.id),
+    optionalGaps: optionalGaps.map((item) => item.id),
+    optionalGapDetails: optionalGaps.map(optionalGapDetail),
+    requiredCheckIds: requiredChecks.map((item) => item.id),
+    optionalCheckIds: optionalChecks.map((item) => item.id),
+    checkStatusById,
+    checkRequiredById,
+    checkCountByStatus,
+    requiredCheckCountByStatus,
+    optionalCheckCountByStatus,
+    checks,
+  };
+}

package/cli/lib/signals-registry.mjs

@@ -0,0 +1,377 @@
+// Read-only learning signal registry and agent backlog report payload builders.
+
+import path from "node:path";
+
+import {
+  auditLearningProfile,
+  defaultLearningFile,
+  defaultLearningUsageFile,
+  learningStats,
+  learningUsageStats,
+  loadLearningProfile,
+} from "./learn.mjs";
+import {
+  buildAgentBacklogActionPlan,
+  buildAgentDevelopmentBacklog,
+} from "./signals-backlog.mjs";
+import {
+  defaultLearningEvalReportPath,
+  resolveSignalFiles,
+  summarizeEvalSignals,
+  summarizeSignalEvalFile,
+} from "./signals-eval.mjs";
+import { buildLearningSignalReadiness } from "./signals-readiness.mjs";
+import {
+  commandFromArgs,
+  countBy,
+  previewText,
+  worstStatus,
+} from "./signals-shared.mjs";
+import { collectWorkspaceReport, defaultLearningEvalPath } from "./workspace.mjs";
+
+export function summarizeCheckCapture(profile) {
+  const entries = (profile.entries || []).filter((entry) => String(entry.source || "").startsWith("check:"));
+  const sorted = [...entries].sort((a, b) => String(b.createdAt || "").localeCompare(String(a.createdAt || "")));
+  return {
+    count: entries.length,
+    categoryCounts: countBy(entries, (entry) => entry.category || "other"),
+    sourceCounts: countBy(entries, (entry) => entry.source || "check:artifact"),
+    latestEntries: sorted.slice(0, 5).map((entry) => ({
+      id: entry.id,
+      category: entry.category,
+      source: entry.source,
+      createdAt: entry.createdAt || "",
+      textPreview: previewText(entry.text),
+    })),
+  };
+}
+
+export function summarizeWorkspace(report) {
+  const nextActions = Array.isArray(report?.nextActions) ? report.nextActions : [];
+  const actionCounts = countBy(nextActions, (item) => item.level || "info");
+  return {
+    root: report?.context?.root || "",
+    version: report?.context?.version || "",
+    git: {
+      isRepo: Boolean(report?.git?.isRepo),
+      branch: report?.git?.branch || "",
+      clean: Boolean(report?.git?.clean),
+      ahead: Number.isInteger(report?.git?.ahead) ? report.git.ahead : 0,
+      behind: Number.isInteger(report?.git?.behind) ? report.git.behind : 0,
+    },
+    repository: {
+      status: report?.repository?.status || "",
+      canonical: Boolean(report?.repository?.canonical),
+    },
+    learning: {
+      status: report?.learning?.readiness?.status || report?.learning?.auditSummary?.status || "",
+      reason: report?.learning?.readiness?.reason || "",
+    },
+    learningUsage: report?.learningUsage?.readiness || null,
+    learningEval: report?.learningEval?.freshness || report?.learningEval?.readiness || null,
+    nextActionCounts: actionCounts,
+    nextActionCount: nextActions.length,
+  };
+}
+
+export function buildRecommendations({ audit, usage, evals, checkCapture, workspace }) {
+  const recommendations = [];
+  if (!audit.exists) {
+    recommendations.push({
+      level: "info",
+      text: "Initialize or import a local learning profile before treating signal registry output as complete.",
+    });
+  } else if (audit.summary.failures > 0) {
+    recommendations.push({
+      level: "fail",
+      text: "Fix learning profile audit failures before using local learning signals as release evidence.",
+    });
+  } else if (audit.summary.warnings > 0) {
+    recommendations.push({
+      level: "warn",
+      text: "Review learning profile audit warnings before promoting learned preferences.",
+    });
+  }
+
+  if (!usage.exists || usage.eventCount === 0) {
+    recommendations.push({
+      level: "info",
+      text: "Run prompt or pack with --with-learning to create usage signals for selected learning entries.",
+    });
+  }
+  if (usage.staleSelectedEntryCount > 0) {
+    recommendations.push({
+      level: "warn",
+      text: "Usage sidecar references learning entry ids that are no longer active.",
+    });
+  }
+  if (evals.files.length === 0) {
+    recommendations.push({
+      level: "warn",
+      text: "No route/prompt/pack/learning eval signal files were found; generate eval reports before relying on this registry as a gate.",
+    });
+  }
+  if (evals.failed > 0) {
+    recommendations.push({
+      level: "fail",
+      text: "At least one eval signal report failed; inspect the failing checkpoint before continuing agent development.",
+    });
+  }
+  if (evals.templates > 0) {
+    recommendations.push({
+      level: "warn",
+      text: "Some eval signals are templates rather than executed reports; replay them with --eval --strict for stronger evidence.",
+    });
+  }
+  if (checkCapture.count === 0) {
+    recommendations.push({
+      level: "info",
+      text: "No check learning capture entries are present yet; run check --learn --yes on real warnings/failures when appropriate.",
+    });
+  }
+  if (workspace.nextActionCounts.fail > 0 || workspace.nextActionCounts.warn > 0) {
+    recommendations.push({
+      level: workspace.nextActionCounts.fail > 0 ? "fail" : "warn",
+      text: "Workspace readiness has unresolved next actions; review workspace output before using signals as release evidence.",
+    });
+  }
+  return recommendations;
+}
+
+export function learningSignalRegistry({
+  filePath = defaultLearningFile(),
+  usageFile = "",
+  signalSource = "",
+  root = process.cwd(),
+  now = new Date(),
+  workspaceReportProvider = collectWorkspaceReport,
+  learningStatsProvider = learningStats,
+  learningUsageStatsProvider = learningUsageStats,
+} = {}) {
+  const resolvedFile = path.resolve(filePath);
+  const resolvedUsageFile = path.resolve(usageFile || defaultLearningUsageFile(resolvedFile));
+  const profile = loadLearningProfile(resolvedFile);
+  const audit = auditLearningProfile({ filePath: resolvedFile });
+  const stats = learningStatsProvider({ filePath: resolvedFile });
+  const usage = learningUsageStatsProvider({
+    filePath: resolvedFile,
+    usageFile: resolvedUsageFile,
+  });
+  const signalFiles = resolveSignalFiles({
+    signalSource,
+    root,
+    extraFiles: [defaultLearningEvalPath(resolvedFile), defaultLearningEvalReportPath(resolvedFile)],
+  });
+  const evalFiles = signalFiles.map((file) => summarizeSignalEvalFile(file));
+  const evalSignalSummary = summarizeEvalSignals(evalFiles);
+  const evalSummary = {
+    source: signalSource ? path.resolve(signalSource) : path.resolve(root),
+    count: evalFiles.length,
+    reports: evalSignalSummary.reports,
+    templates: evalSignalSummary.templates,
+    rawTemplates: evalSignalSummary.rawTemplates,
+    templateFiles: evalSignalSummary.templateFiles,
+    failed: evalSignalSummary.failed,
+    warned: evalSignalSummary.warned,
+    passed: evalSignalSummary.passed,
+    files: evalFiles,
+  };
+  const workspaceReport = workspaceReportProvider({
+    root,
+    learningFilePath: resolvedFile,
+    learningUsagePath: resolvedUsageFile,
+  });
+  const workspace = summarizeWorkspace(workspaceReport);
+  const checkCapture = summarizeCheckCapture(profile);
+  const recommendations = buildRecommendations({
+    audit,
+    usage,
+    evals: evalSummary,
+    checkCapture,
+    workspace,
+  });
+  const agentDevelopment = buildAgentDevelopmentBacklog({
+    audit,
+    usage,
+    evals: evalSummary,
+    checkCapture,
+    workspace,
+    filePath: resolvedFile,
+    usageFile: resolvedUsageFile,
+    signalSource: evalSummary.source,
+  });
+  const status = worstStatus([
+    audit.summary.failures > 0 ? "fail" : "",
+    audit.summary.warnings > 0 ? "warn" : "",
+    usage.staleSelectedEntryCount > 0 ? "warn" : "",
+    evalSummary.failed > 0 ? "fail" : "",
+    evalSummary.warned > 0 || evalSummary.templates > 0 || evalSummary.count === 0 ? "warn" : "",
+    workspace.nextActionCounts.fail > 0 ? "fail" : "",
+    workspace.nextActionCounts.warn > 0 ? "warn" : "",
+    "pass",
+  ]);
+  const readiness = buildLearningSignalReadiness({
+    audit,
+    usage,
+    evals: evalSummary,
+    checkCapture,
+    workspace,
+    agentDevelopment,
+    status,
+  });
+
+  return {
+    version: 1,
+    generatedAt: (now instanceof Date ? now : new Date(now)).toISOString(),
+    status,
+    file: resolvedFile,
+    signalSource: evalSummary.source,
+    learning: {
+      exists: audit.exists,
+      version: audit.version,
+      updatedAt: audit.updatedAt,
+      count: audit.count,
+      categoryCounts: audit.categoryCounts,
+      sourceCounts: stats.sourceCounts || {},
+      auditSummary: audit.summary,
+    },
+    usage: {
+      usageFile: usage.usageFile,
+      exists: usage.exists,
+      eventCount: usage.eventCount,
+      usedEntryCount: usage.usedEntryCount,
+      unusedEntryCount: usage.unusedEntryCount,
+      staleSelectedEntryCount: usage.staleSelectedEntryCount,
+      commandCounts: usage.commandCounts,
+      routeCounts: usage.routeCounts,
+      latestEvent: usage.latestEvent || null,
+      privacy: usage.privacy,
+    },
+    evals: evalSummary,
+    checkCapture,
+    workspace,
+    agentDevelopment,
+    readiness,
+    recommendations,
+    privacy: {
+      mutatesProfile: false,
+      storesRawBriefText: false,
+      exposesEntryTextPreview: true,
+      readsSignalFilesOnly: true,
+    },
+  };
+}
+
+export function agentBacklogReport({
+  filePath = defaultLearningFile(),
+  usageFile = "",
+  signalSource = "",
+  root = process.cwd(),
+  now = new Date(),
+  signalRegistryProvider = learningSignalRegistry,
+} = {}) {
+  const registry = signalRegistryProvider({
+    filePath,
+    usageFile,
+    signalSource,
+    root,
+    now,
+  });
+  const agentDevelopment = registry.agentDevelopment || {
+    status: "unknown",
+    actionCount: 0,
+    p0Count: 0,
+    p1Count: 0,
+    p2Count: 0,
+    p3Count: 0,
+    actions: [],
+    privacy: {
+      mutatesProfile: false,
+      mutatesSkillFiles: false,
+      callsExternalAiApis: false,
+      storesRawBriefText: false,
+    },
+  };
+  const actions = Array.isArray(agentDevelopment.actions) ? agentDevelopment.actions : [];
+  const signalsJsonArgs = [
+    "design-ai",
+    "learn",
+    "--signals",
+    "--from-file",
+    registry.signalSource || ".",
+    "--file",
+    registry.file || filePath,
+    "--usage-file",
+    registry.usage?.usageFile || usageFile || defaultLearningUsageFile(path.resolve(filePath)),
+    "--json",
+  ];
+  const signalsReportArgs = [
+    "design-ai",
+    "learn",
+    "--signals",
+    "--from-file",
+    registry.signalSource || ".",
+    "--file",
+    registry.file || filePath,
+    "--usage-file",
+    registry.usage?.usageFile || usageFile || defaultLearningUsageFile(path.resolve(filePath)),
+    "--report",
+    "--out",
+    "learning-signals.md",
+  ];
+  const agentBacklogJsonArgs = [
+    "design-ai",
+    "learn",
+    "--agent-backlog",
+    "--from-file",
+    registry.signalSource || ".",
+    "--file",
+    registry.file || filePath,
+    "--usage-file",
+    registry.usage?.usageFile || usageFile || defaultLearningUsageFile(path.resolve(filePath)),
+    "--strict",
+    "--json",
+  ];
+  const commands = {
+    signalsJson: commandFromArgs(signalsJsonArgs),
+    signalsJsonArgs,
+    signalsReport: commandFromArgs(signalsReportArgs),
+    signalsReportArgs,
+    agentBacklogJson: commandFromArgs(agentBacklogJsonArgs),
+    agentBacklogJsonArgs,
+  };
+  const privacy = {
+    mutatesProfile: false,
+    mutatesSkillFiles: false,
+    callsExternalAiApis: false,
+    storesRawBriefText: false,
+    readsSignalFilesOnly: true,
+  };
+  return {
+    version: 1,
+    generatedAt: registry.generatedAt || (now instanceof Date ? now : new Date(now)).toISOString(),
+    status: agentDevelopment.status || "unknown",
+    signalStatus: registry.status || "unknown",
+    file: registry.file || path.resolve(filePath),
+    usageFile: registry.usage?.usageFile || path.resolve(usageFile || defaultLearningUsageFile(path.resolve(filePath))),
+    signalSource: registry.signalSource || (signalSource ? path.resolve(signalSource) : path.resolve(root)),
+    counts: {
+      actions: agentDevelopment.actionCount ?? actions.length,
+      p0: agentDevelopment.p0Count ?? 0,
+      p1: agentDevelopment.p1Count ?? 0,
+      p2: agentDevelopment.p2Count ?? 0,
+      p3: agentDevelopment.p3Count ?? 0,
+      learningEntries: registry.learning?.count ?? 0,
+      usageEvents: registry.usage?.eventCount ?? 0,
+      evalSignals: registry.evals?.count ?? 0,
+      checkCaptures: registry.checkCapture?.count ?? 0,
+      workspaceNextActions: registry.workspace?.nextActionCount ?? 0,
+    },
+    actions,
+    actionPlan: buildAgentBacklogActionPlan({ actions, commands, privacy }),
+    readiness: registry.readiness || null,
+    commands,
+    recommendations: registry.recommendations || [],
+    privacy,
+  };
+}