@warpgogol/forge 4.0.0 → 4.1.2

package/os/session/handlers/metrics-rfc.ts

@@ -0,0 +1,339 @@
+/*
+<MODULE_CONTRACT>
+<purpose>
+RFC-1053: Generate per-RFC skill effectiveness metrics as structured YAML.
+Reconstructs the RFC pipeline (audit, enhance, plan, implement, review, fix)
+from git commit history, review reports, and verification evidence.
+Writes to docs/metrics/rfcs/<rfc-id>.metrics.yaml via writeFileAtomic.
+</purpose>
+<non-goals>
+  <item>Does not execute kernel commands — reads existing artifacts only.</item>
+  <item>Does not extract quantitative results from acceptance criteria.</item>
+  <item>Does not block the stamp on metrics failure — caller wraps in try/catch.</item>
+</non-goals>
+</MODULE_CONTRACT>
+<CHANGE_SUMMARY>
+  <item>RFC-1053: initial generateRfcMetrics pure function.</item>
+</CHANGE_SUMMARY>
+*/
+
+import { execFile } from "node:child_process";
+import { readFile, readdir, mkdir } from "node:fs/promises";
+import { join, dirname } from "node:path";
+
+import { writeFileAtomic } from "../../../src/utils/fs-atomic.ts";
+import { parse as yamlParse, stringify as yamlStringify } from "yaml";
+
+import {
+  METRICS_DIR,
+  type RfcMetrics,
+  type RfcPipelineStep,
+  type ReviewMetrics,
+  type FixMetrics,
+  type VerificationMetrics,
+  type ResultMetrics,
+  type RfcTimings,
+} from "../types.ts";
+
+// ─── Git helper ──────────────────────────────────────────────────────────────
+
+function execGit(workspaceRoot: string, args: string[]): Promise<string> {
+  return new Promise((resolve) => {
+    execFile("git", args, { cwd: workspaceRoot, timeout: 10000 }, (err, stdout) => {
+      if (err) {
+        resolve("");
+        return;
+      }
+      resolve(stdout.trim());
+    });
+  });
+}
+
+// ─── Pipeline step classification ────────────────────────────────────────────
+
+const STEP_PREFIXES: Record<string, { step: string; skill: string | null }> = {
+  "audit:": { step: "audit", skill: "fo-idea-audit" },
+  "enhance:": { step: "enhance", skill: "fo-idea-enhance" },
+  "plan:": { step: "plan", skill: "fo-idea-plan" },
+  "implement:": { step: "implement", skill: "fo-idea-implement" },
+  "review:": { step: "review", skill: "fo-review" },
+  "fix:": { step: "fix", skill: "fo-fix" },
+  "evidence:": { step: "evidence", skill: null },
+  "test:": { step: "test", skill: null },
+  "docs:": { step: "docs", skill: null },
+  "trace:": { step: "trace", skill: null },
+};
+
+interface GitLogEntry {
+  sha: string;
+  message: string;
+  timestamp: string;
+}
+
+async function getRfcCommits(workspaceRoot: string, rfcId: string): Promise<GitLogEntry[]> {
+  const format = "%H%x1f%s%x1f%cI";
+  const raw = await execGit(workspaceRoot, [
+    "log",
+    "--no-merges",
+    `--grep=${rfcId}`,
+    `--format=${format}`,
+  ]);
+  if (!raw) return [];
+
+  const entries: GitLogEntry[] = [];
+  const lines = raw.split("\n");
+  for (const line of lines) {
+    const parts = line.split("\x1f");
+    if (parts.length >= 3) {
+      entries.push({
+        sha: parts[0]!.trim(),
+        message: parts[1]!.trim(),
+        timestamp: parts[2]!.trim(),
+      });
+    }
+  }
+  return entries;
+}
+
+function classifyCommit(message: string): { step: string; skill: string | null } | null {
+  const firstLine = message.split("\n")[0] ?? "";
+  for (const [prefix, mapping] of Object.entries(STEP_PREFIXES)) {
+    if (firstLine.startsWith(prefix)) {
+      return mapping;
+    }
+  }
+  return null;
+}
+
+// ─── Review report parsing ───────────────────────────────────────────────────
+
+async function findReviewReport(workspaceRoot: string, rfcId: string): Promise<string | null> {
+  const reviewDir = join(workspaceRoot, "docs", "reviews", "code");
+  let files: string[];
+  try {
+    files = await readdir(reviewDir, { recursive: true });
+  } catch {
+    return null;
+  }
+
+  const rfcLower = rfcId.toLowerCase();
+  for (const file of files) {
+    if (!file.endsWith(".md")) continue;
+    if (file.toLowerCase().includes(rfcLower)) {
+      try {
+        const content = await readFile(join(reviewDir, file), "utf-8");
+        return content;
+      } catch {
+        continue;
+      }
+    }
+  }
+  return null;
+}
+
+function parseReviewReport(content: string): ReviewMetrics {
+  const findingsByAxis: Record<string, number> = {};
+  let findingsCount = 0;
+  let verdict: string | null = null;
+
+  const verdictMatch = content.match(/##\s*Verdict[:\s]+(.+)/i);
+  if (verdictMatch) {
+    verdict = verdictMatch[1]!.trim().toLowerCase();
+  }
+
+  const axisRegex = /##\s*Axis\s+[A-Z]\s*—\s*(.+)/gi;
+  let match;
+  while ((match = axisRegex.exec(content)) !== null) {
+    const axisName = match[1]!.trim();
+    const sectionStart = match.index! + match[0].length;
+    const nextAxisMatch = content.slice(sectionStart).match(/##\s*Axis\s+[A-Z]\s*—/i);
+    const sectionEnd = nextAxisMatch ? sectionStart + nextAxisMatch.index! : content.length;
+    const section = content.slice(sectionStart, sectionEnd);
+
+    const failMatches = section.match(/\*\*FAIL\*\*/g);
+    const failCount = failMatches ? failMatches.length : 0;
+    if (failCount > 0) {
+      findingsByAxis[axisName] = failCount;
+      findingsCount += failCount;
+    }
+  }
+
+  return { findingsCount, findingsByAxis, verdict };
+}
+
+// ─── Verification evidence parsing ───────────────────────────────────────────
+
+async function parseVerificationEvidence(
+  workspaceRoot: string,
+  rfcId: string,
+): Promise<VerificationMetrics | null> {
+  const rfcNum = rfcId.replace(/^RFC-/, "").toLowerCase();
+  const evidencePath = join(
+    workspaceRoot,
+    "docs",
+    "rfcs",
+    "verification",
+    `rfc-${rfcNum}.generated.yaml`,
+  );
+  let content: string;
+  try {
+    content = await readFile(evidencePath, "utf-8");
+  } catch {
+    return null;
+  }
+
+  try {
+    const data = yamlParse(content) as Record<string, unknown>;
+    const probes = Array.isArray(data["probes"])
+      ? (data["probes"] as Array<Record<string, unknown>>)
+      : [];
+    const probesTotal = probes.length;
+    const probesPassed = probes.filter((p) => p["ok"] === true).length;
+    return {
+      probesTotal,
+      probesPassed,
+      evidencePath: `docs/rfcs/verification/rfc-${rfcNum}.generated.yaml`,
+    };
+  } catch {
+    return null;
+  }
+}
+
+// ─── Acceptance criteria parsing ─────────────────────────────────────────────
+
+async function parseAcceptanceCriteria(
+  workspaceRoot: string,
+  rfcId: string,
+): Promise<ResultMetrics> {
+  const rfcDir = join(workspaceRoot, "docs", "rfcs");
+  let files: string[];
+  try {
+    files = await readdir(rfcDir);
+  } catch {
+    return { acceptanceCriteriaTotal: 0, acceptanceCriteriaMet: 0 };
+  }
+
+  const rfcLower = rfcId.toLowerCase();
+  const rfcFile = files.find(
+    (f) => f.toLowerCase().startsWith(rfcLower + "-") || f.toLowerCase().startsWith(rfcLower + "_"),
+  );
+  if (!rfcFile) {
+    return { acceptanceCriteriaTotal: 0, acceptanceCriteriaMet: 0 };
+  }
+
+  let content: string;
+  try {
+    content = await readFile(join(rfcDir, rfcFile), "utf-8");
+  } catch {
+    return { acceptanceCriteriaTotal: 0, acceptanceCriteriaMet: 0 };
+  }
+
+  const acSectionMatch = content.match(/##\s*Acceptance\s+criteria[\s\S]*?(?=\n##\s|$)/i);
+  if (!acSectionMatch) {
+    return { acceptanceCriteriaTotal: 0, acceptanceCriteriaMet: 0 };
+  }
+
+  const acSection = acSectionMatch[0]!;
+  const totalMatches = acSection.match(/-\s*\[x\]/gi);
+  const uncheckedMatches = acSection.match(/-\s*\[\s\]/g);
+  const acceptanceCriteriaMet = totalMatches ? totalMatches.length : 0;
+  const acceptanceCriteriaUnmet = uncheckedMatches ? uncheckedMatches.length : 0;
+  return {
+    acceptanceCriteriaTotal: acceptanceCriteriaMet + acceptanceCriteriaUnmet,
+    acceptanceCriteriaMet,
+  };
+}
+
+// ─── Fix metrics parsing ─────────────────────────────────────────────────────
+
+function extractFixMetrics(commits: GitLogEntry[]): FixMetrics | null {
+  const fixCommits = commits.filter((c) => c.message.split("\n")[0]!.startsWith("fix:"));
+  if (fixCommits.length === 0) return null;
+  return {
+    fixesApplied: fixCommits.length,
+    commitSha: fixCommits[fixCommits.length - 1]!.sha,
+    iterations: fixCommits.length,
+  };
+}
+
+// ─── Timings ─────────────────────────────────────────────────────────────────
+
+function computeTimings(commits: GitLogEntry[]): RfcTimings {
+  if (commits.length === 0) {
+    return { firstCommitAt: null, lastCommitAt: null, totalDurationMs: null };
+  }
+  const sorted = [...commits].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+  const first = sorted[0]!;
+  const last = sorted[sorted.length - 1]!;
+  const durationMs = new Date(last.timestamp).getTime() - new Date(first.timestamp).getTime();
+  return {
+    firstCommitAt: first.timestamp,
+    lastCommitAt: last.timestamp,
+    totalDurationMs: isNaN(durationMs) ? null : durationMs,
+  };
+}
+
+// ─── Main function ───────────────────────────────────────────────────────────
+
+export async function generateRfcMetrics(
+  workspaceRoot: string,
+  rfcId: string,
+  logger: { warn: (msg: string) => void },
+): Promise<string> {
+  const commits = await getRfcCommits(workspaceRoot, rfcId);
+
+  const pipeline: RfcPipelineStep[] = [];
+  for (const commit of commits) {
+    const classified = classifyCommit(commit.message);
+    if (classified) {
+      pipeline.push({
+        step: classified.step,
+        skill: classified.skill,
+        commitSha: commit.sha,
+        timestamp: commit.timestamp,
+      });
+    }
+  }
+
+  let review: ReviewMetrics | null = null;
+  try {
+    const reviewContent = await findReviewReport(workspaceRoot, rfcId);
+    if (reviewContent) {
+      review = parseReviewReport(reviewContent);
+    }
+  } catch {
+    logger.warn(`[metrics] Could not parse review report for ${rfcId}`);
+  }
+
+  if (!review) {
+    review = { findingsCount: 0, findingsByAxis: {}, verdict: null };
+  }
+
+  const fix = extractFixMetrics(commits);
+  const verification = await parseVerificationEvidence(workspaceRoot, rfcId);
+  const result = await parseAcceptanceCriteria(workspaceRoot, rfcId);
+  const timings = computeTimings(commits);
+
+  const metrics: RfcMetrics = {
+    rfcId,
+    generatedAt: new Date().toISOString(),
+    pipeline,
+    review,
+    fix,
+    verification,
+    result,
+    timings,
+  };
+
+  const yamlContent = yamlStringify(metrics, { lineWidth: 120 });
+  const metricsFilePath = join(
+    workspaceRoot,
+    METRICS_DIR,
+    "rfcs",
+    `${rfcId.toLowerCase()}.metrics.yaml`,
+  );
+  await mkdir(dirname(metricsFilePath), { recursive: true });
+  await writeFileAtomic(metricsFilePath, yamlContent);
+
+  return `${METRICS_DIR}/rfcs/${rfcId.toLowerCase()}.metrics.yaml`;
+}

package/os/session/handlers/metrics-session.ts

@@ -0,0 +1,209 @@
+/*
+<MODULE_CONTRACT>
+<purpose>
+RFC-1053: Generate per-session skill effectiveness metrics as structured YAML.
+Extracts skill invocations from parsed ATIF messages (not rendered markdown)
+with approximate durations from message timestamps. Writes to
+docs/metrics/sessions/<session-id>.metrics.yaml via writeFileAtomic.
+</purpose>
+<non-goals>
+  <item>Does not parse the rendered markdown transcript — uses AtifMessage[] directly.</item>
+  <item>Does not block session.save on metrics failure — caller wraps in try/catch.</item>
+  <item>Does not extract skill durations with precision — all durations are approximate.</item>
+</non-goals>
+</MODULE_CONTRACT>
+<CHANGE_SUMMARY>
+  <item>RFC-1053: initial generateSessionMetrics pure function using ATIF metadata.</item>
+</CHANGE_SUMMARY>
+*/
+
+import { readFile, readdir, mkdir } from "node:fs/promises";
+import { join, dirname } from "node:path";
+
+import { writeFileAtomic } from "../../../src/utils/fs-atomic.ts";
+import { stringify as yamlStringify } from "yaml";
+
+import { METRICS_DIR, type SessionMetrics, type SessionSkillInvocation, type SessionDocumentRef, type InsightSummary } from "../types.ts";
+import type { AtifMessage } from "../atif-parser.ts";
+
+// ─── Skill name extraction ───────────────────────────────────────────────────
+
+const SKILL_PATTERN = /\b(fo-[a-z]+(?:-[a-z]+)*)\b/g;
+
+const KNOWN_SKILLS = new Set([
+  "fo-idea",
+  "fo-idea-audit",
+  "fo-idea-enhance",
+  "fo-idea-plan",
+  "fo-idea-implement",
+  "fo-idea-create-rfc",
+  "fo-idea-create-adr",
+  "fo-idea-i-just-want-to-see-the-plan",
+  "fo-idea-i-just-want-to-see-the-result",
+  "fo-idea-status",
+  "fo-idea-audit",
+  "fo-review",
+  "fo-fix",
+  "fo-session-retro",
+  "fo-session-save",
+  "fo-doc-audit",
+  "fo-architecture",
+  "fo-compass-annotate",
+  "fo-explore",
+  "fo-extract-dna",
+  "fo-handoff",
+  "fo-harvest",
+  "fo-knowledge-distill",
+  "fo-memory-sync",
+  "fo-qa",
+  "fo-step-commit",
+  "fo-triage",
+  "fo-add-tests",
+]);
+
+function extractSkillInvocations(messages: AtifMessage[]): SessionSkillInvocation[] {
+  const invocations: SessionSkillInvocation[] = [];
+  const seenSkills = new Map<string, { firstMsgIndex: number; lastMsgIndex: number }>();
+
+  for (let i = 0; i < messages.length; i++) {
+    const msg = messages[i]!;
+    const matches = [...msg.content.matchAll(SKILL_PATTERN)];
+    const skillNames = new Set(matches.map((m) => m[1]!));
+    for (const skill of skillNames) {
+      if (!KNOWN_SKILLS.has(skill)) continue;
+      const existing = seenSkills.get(skill);
+      if (existing) {
+        existing.lastMsgIndex = i;
+      } else {
+        seenSkills.set(skill, { firstMsgIndex: i, lastMsgIndex: i });
+      }
+    }
+  }
+
+  for (const [skill, indices] of seenSkills) {
+    const firstMsg = messages[indices.firstMsgIndex]!;
+    const lastMsg = messages[indices.lastMsgIndex]!;
+    let approximateDurationMs: number | null = null;
+
+    if (firstMsg.timestamp && lastMsg.timestamp) {
+      const firstTime = new Date(firstMsg.timestamp).getTime();
+      const lastTime = new Date(lastMsg.timestamp).getTime();
+      if (!isNaN(firstTime) && !isNaN(lastTime)) {
+        approximateDurationMs = Math.max(0, lastTime - firstTime);
+      }
+    }
+
+    invocations.push({ skill, approximateDurationMs });
+  }
+
+  return invocations;
+}
+
+// ─── Document references ─────────────────────────────────────────────────────
+
+async function extractDocumentRefs(
+  workspaceRoot: string,
+  relatedRfcs: string[],
+): Promise<SessionDocumentRef[]> {
+  const refs: SessionDocumentRef[] = [];
+  const metricsRfcDir = join(workspaceRoot, METRICS_DIR, "rfcs");
+
+  let existingMetricsFiles: string[] = [];
+  try {
+    existingMetricsFiles = await readdir(metricsRfcDir);
+  } catch {
+    // No metrics directory yet
+  }
+
+  for (const rfcId of relatedRfcs) {
+    const rfcLower = rfcId.toLowerCase();
+    const metricsFileName = `${rfcLower}.metrics.yaml`;
+    const hasMetrics = existingMetricsFiles.includes(metricsFileName);
+    refs.push({
+      rfcId,
+      status: "unknown",
+      metricsFile: hasMetrics ? `${METRICS_DIR}/rfcs/${metricsFileName}` : null,
+    });
+  }
+
+  return refs;
+}
+
+// ─── Insight extraction ──────────────────────────────────────────────────────
+
+function extractInsights(messages: AtifMessage[]): InsightSummary | null {
+  let total = 0;
+  const byCategory: Record<string, number> = {};
+
+  for (const msg of messages) {
+    const lowerContent = msg.content.toLowerCase();
+    if (!lowerContent.includes("insight")) continue;
+
+    const categoryPattern = /(?:category|type|route):\s*([a-z-]+)/gi;
+    let match;
+    while ((match = categoryPattern.exec(msg.content)) !== null) {
+      const category = match[1]!.trim();
+      total++;
+      byCategory[category] = (byCategory[category] ?? 0) + 1;
+    }
+  }
+
+  if (total === 0) return null;
+  return { total, byCategory };
+}
+
+// ─── Duration calculation ────────────────────────────────────────────────────
+
+function computeSessionDurationMs(messages: AtifMessage[]): number {
+  const timestamps = messages
+    .map((m) => (m.timestamp ? new Date(m.timestamp).getTime() : null))
+    .filter((t): t is number => t !== null && !isNaN(t));
+
+  if (timestamps.length < 2) return 0;
+  const min = Math.min(...timestamps);
+  const max = Math.max(...timestamps);
+  return Math.max(0, max - min);
+}
+
+// ─── Main function ───────────────────────────────────────────────────────────
+
+export async function generateSessionMetrics(
+  workspaceRoot: string,
+  sessionId: string,
+  messages: AtifMessage[],
+  sessionFrontmatter: {
+    date?: string;
+    relatedRfcs?: string[];
+    commits?: string[];
+  },
+  logger: { warn: (msg: string) => void },
+): Promise<string> {
+  const skills = extractSkillInvocations(messages);
+  const documents = await extractDocumentRefs(workspaceRoot, sessionFrontmatter.relatedRfcs ?? []);
+  const insights = extractInsights(messages);
+  const durationMs = computeSessionDurationMs(messages);
+
+  const metrics: SessionMetrics = {
+    sessionId,
+    generatedAt: new Date().toISOString(),
+    date: sessionFrontmatter.date ?? new Date().toISOString().split("T")[0]!,
+    durationMs,
+    approximateDurations: true,
+    documents,
+    skills,
+    insights,
+    commits: sessionFrontmatter.commits ?? [],
+  };
+
+  const yamlContent = yamlStringify(metrics, { lineWidth: 120 });
+  const metricsFilePath = join(
+    workspaceRoot,
+    METRICS_DIR,
+    "sessions",
+    `${sessionId}.metrics.yaml`,
+  );
+  await mkdir(dirname(metricsFilePath), { recursive: true });
+  await writeFileAtomic(metricsFilePath, yamlContent);
+
+  return `${METRICS_DIR}/sessions/${sessionId}.metrics.yaml`;
+}

package/os/session/handlers/save.ts

@@ -13,6 +13,7 @@ Deterministic only — no LLM/intelligent annotation (that is fo-session-save sk
 <CHANGE_SUMMARY>
   <item>RFC-0537: implement session.save command handler.</item>
   <item>Replace fs.unlink with trashPath for raw file deletion (trash bin for LLM-initiated deletions).</item>
+  <item>RFC-1053: integrate generateSessionMetrics after markdown write (non-fatal, guarded by !dryRun).</item>
 </CHANGE_SUMMARY>
 */
 
@@ -27,6 +28,7 @@ import type {
 } from "../../../src/types.ts";
 import { parseAtif, messagesToTranscriptMarkdown } from "../atif-parser.ts";
 import { trashPath } from "../../../src/utils/fs-trash.ts";
+import { generateSessionMetrics } from "./metrics-session.ts";
 import {
   SESSION_DIR,
   SESSION_RAW_SUBDIR,
@@ -289,10 +291,26 @@ export async function runSessionSave(
       transcriptText,
     );
 
+    let metricsPath: string | undefined;
     if (!dryRun) {
       await fs.mkdir(sessionDirPath, { recursive: true });
       await fs.writeFile(outputPath, markdown, "utf-8");
 
+      // ── RFC-1053: Generate session metrics from ATIF messages (non-fatal) ──
+      try {
+        metricsPath = await generateSessionMetrics(
+          workspaceRoot,
+          id,
+          atifResult.messages,
+          { date: timestamp.toISOString(), relatedRfcs, commits },
+          logger,
+        );
+      } catch (metricsErr) {
+        logger.warn(
+          `[metrics] Failed to generate session metrics for ${id}: ${(metricsErr as Error).message}`,
+        );
+      }
+
       // Delete raw file unless --keep-raw
       if (!keepRaw) {
         try {
@@ -320,6 +338,7 @@ export async function runSessionSave(
         files,
         commands,
       },
+      ...(metricsPath ? { metricsPath } : {}),
       dryRun,
     };
     saved.push(result);

package/os/session/index.ts

@@ -7,10 +7,11 @@
 </MODULE_CONTRACT>
 <CHANGE_SUMMARY>
   <item>RFC-0537: initial session module barrel.</item>
+  <item>RFC-1053: export metrics types and METRICS_DIR constant.</item>
 </CHANGE_SUMMARY>
 */
 
-export { forgeSessionModule } from "./session.module.ts";
+export { createForgeSessionModule } from "./session.module.ts";
 export { runSessionSave } from "./handlers/save.ts";
 export { runSessionArchive } from "./handlers/archive.ts";
 export { runSessionValidate } from "./handlers/validate.ts";
@@ -34,6 +35,7 @@ export {
   SESSION_DIR,
   SESSION_RAW_SUBDIR,
   SESSION_ARCHIVE_SUBDIR,
+  METRICS_DIR,
   SESSION_TYPES,
   SES_RULES,
   type SessionType,
@@ -47,4 +49,17 @@ export {
   type SessionListResult,
   type SessionListEntry,
   type SesRule,
+  type RfcMetrics,
+  type RfcPipelineStep,
+  type ReviewMetrics,
+  type FixMetrics,
+  type VerificationMetrics,
+  type ResultMetrics,
+  type RfcTimings,
+  type SessionMetrics,
+  type SessionDocumentRef,
+  type SessionSkillInvocation,
+  type InsightSummary,
+  type SkillAggregate,
+  type MetricsAggregateResult,
 } from "./types.ts";