forgedock 1.0.16 → 1.0.18

package/docs/design/fixtures/runs/full-abc/runs.json

@@ -0,0 +1,190 @@
+{
+  "corpus_version": "seed-v1",
+  "generation_model": "claude-sonnet-4-6",
+  "judge_model": "claude-sonnet-4-6",
+  "benchmark_date": "2026-06-22",
+  "note": "Full A-vs-B-vs-C benchmark. Arm A = ForgeDock harness (archetype sampling + anti-slop + critique). Arm B = raw one-shot. Slipstream arm C viewport-only (vercel.com rendering issue).",
+  "products": [
+    {
+      "product": "cadence",
+      "reference": "resend.com",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "B"},
+          "rubric": {
+            "A": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 2, "originality": 2, "mobile": 3},
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 3, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 5}
+          },
+          "slop": {"A": 4, "B": 3, "C": 0}
+        },
+        {
+          "run": 2,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "tie"},
+          "rubric": {
+            "A": {"hierarchy": 3, "typography": 4, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 5}
+          },
+          "slop": {"A": 3, "B": 4, "C": 0}
+        },
+        {
+          "run": 3,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "B"},
+          "rubric": {
+            "A": {"hierarchy": 3, "typography": 3, "color": 2, "whitespace": 2, "originality": 2, "mobile": 3},
+            "B": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 3, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 5}
+          },
+          "slop": {"A": 5, "B": 3, "C": 0}
+        }
+      ]
+    },
+    {
+      "product": "tender",
+      "reference": "stripe.com",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 3, "mobile": 3},
+            "B": {"hierarchy": 3, "typography": 3, "color": 2, "whitespace": 3, "originality": 2, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 3, "C": 1}
+        },
+        {
+          "run": 2,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "B"},
+          "rubric": {
+            "A": {"hierarchy": 3, "typography": 4, "color": 3, "whitespace": 2, "originality": 3, "mobile": 2},
+            "B": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 4}
+          },
+          "slop": {"A": 2, "B": 2, "C": 1}
+        },
+        {
+          "run": 3,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 4, "mobile": 3},
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 2, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 3, "C": 1}
+        }
+      ]
+    },
+    {
+      "product": "slipstream",
+      "reference": "vercel.com",
+      "note": "Arm C viewport-only (vercel.com rendering timeout). A_vs_C and B_vs_C comparisons may be miscalibrated.",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"A_vs_C": "A", "B_vs_C": "B", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 3, "originality": 5, "mobile": 3},
+            "B": {"hierarchy": 4, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 4, "whitespace": 5, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 2, "C": 0}
+        },
+        {
+          "run": 2,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 4, "originality": 3, "mobile": 3},
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 4, "whitespace": 5, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 2, "B": 1, "C": 0}
+        },
+        {
+          "run": 3,
+          "pairwise": {"A_vs_C": "A", "B_vs_C": "B", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 5, "typography": 4, "color": 4, "whitespace": 4, "originality": 4, "mobile": 4},
+            "B": {"hierarchy": 4, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 4, "whitespace": 5, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 3, "C": 0}
+        }
+      ]
+    },
+    {
+      "product": "voltage",
+      "reference": "modal.com",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 3, "mobile": 3},
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 2, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 4, "color": 5, "whitespace": 4, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 3, "C": 1}
+        },
+        {
+          "run": 2,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 3},
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 4, "color": 5, "whitespace": 4, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 0, "B": 2, "C": 1}
+        },
+        {
+          "run": 3,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 5, "color": 4, "whitespace": 5, "originality": 4, "mobile": 4},
+            "B": {"hierarchy": 3, "typography": 2, "color": 2, "whitespace": 2, "originality": 2, "mobile": 2},
+            "C": {"hierarchy": 5, "typography": 4, "color": 5, "whitespace": 4, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 4, "C": 1}
+        }
+      ]
+    },
+    {
+      "product": "plume",
+      "reference": "linear.app",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "B"},
+          "rubric": {
+            "A": {"hierarchy": 2, "typography": 3, "color": 2, "whitespace": 3, "originality": 3, "mobile": 2},
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 3, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 3, "C": 2}
+        },
+        {
+          "run": 2,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "A"},
+          "rubric": {
+            "A": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 3, "originality": 5, "mobile": 3},
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 4, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 2, "B": 3, "C": 2}
+        },
+        {
+          "run": 3,
+          "pairwise": {"A_vs_C": "C", "B_vs_C": "C", "A_vs_B": "B"},
+          "rubric": {
+            "A": {"hierarchy": 3, "typography": 4, "color": 3, "whitespace": 4, "originality": 4, "mobile": 2},
+            "B": {"hierarchy": 5, "typography": 4, "color": 4, "whitespace": 4, "originality": 4, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 4, "mobile": 4}
+          },
+          "slop": {"A": 1, "B": 2, "C": 2}
+        }
+      ]
+    }
+  ]
+}

package/docs/design/fixtures/runs/full-abc/scorecard.json

@@ -0,0 +1,144 @@
+{
+  "corpus_version": "seed-v1",
+  "generation_model": "claude-sonnet-4-6",
+  "judge_model": "claude-sonnet-4-6",
+  "n_products": 5,
+  "n_runs_per_product": 3,
+  "win_rate_vs_C": {
+    "A": {
+      "mean": 0.133,
+      "by_product": {
+        "cadence": 0,
+        "tender": 0,
+        "slipstream": 0.667,
+        "voltage": 0,
+        "plume": 0
+      }
+    },
+    "B": {
+      "mean": 0.133,
+      "by_product": {
+        "cadence": 0,
+        "tender": 0,
+        "slipstream": 0.667,
+        "voltage": 0,
+        "plume": 0
+      }
+    }
+  },
+  "a_vs_b_win_rate": 0.633,
+  "rubric": {
+    "A": {
+      "hierarchy": {
+        "mean": 3.667,
+        "stdev": 0.816
+      },
+      "typography": {
+        "mean": 3.933,
+        "stdev": 0.594
+      },
+      "color": {
+        "mean": 3.467,
+        "stdev": 0.834
+      },
+      "whitespace": {
+        "mean": 3.4,
+        "stdev": 0.91
+      },
+      "originality": {
+        "mean": 3.533,
+        "stdev": 0.99
+      },
+      "mobile": {
+        "mean": 2.933,
+        "stdev": 0.594
+      }
+    },
+    "B": {
+      "hierarchy": {
+        "mean": 3.733,
+        "stdev": 0.594
+      },
+      "typography": {
+        "mean": 3.467,
+        "stdev": 0.64
+      },
+      "color": {
+        "mean": 3.267,
+        "stdev": 0.704
+      },
+      "whitespace": {
+        "mean": 3.2,
+        "stdev": 0.561
+      },
+      "originality": {
+        "mean": 2.867,
+        "stdev": 0.64
+      },
+      "mobile": {
+        "mean": 3.4,
+        "stdev": 0.632
+      }
+    },
+    "C": {
+      "hierarchy": {
+        "mean": 5,
+        "stdev": 0
+      },
+      "typography": {
+        "mean": 4.8,
+        "stdev": 0.414
+      },
+      "color": {
+        "mean": 4.8,
+        "stdev": 0.414
+      },
+      "whitespace": {
+        "mean": 4.4,
+        "stdev": 0.507
+      },
+      "originality": {
+        "mean": 4.4,
+        "stdev": 0.507
+      },
+      "mobile": {
+        "mean": 4.2,
+        "stdev": 0.414
+      }
+    }
+  },
+  "slop": {
+    "A": 1.733,
+    "B": 2.733,
+    "C": 0.8
+  },
+  "judge_calibration": {
+    "miscalibrated_runs": [
+      {
+        "product": "slipstream",
+        "run": 1,
+        "comparison": "A_vs_C",
+        "winner": "A"
+      },
+      {
+        "product": "slipstream",
+        "run": 1,
+        "comparison": "B_vs_C",
+        "winner": "B"
+      },
+      {
+        "product": "slipstream",
+        "run": 3,
+        "comparison": "A_vs_C",
+        "winner": "A"
+      },
+      {
+        "product": "slipstream",
+        "run": 3,
+        "comparison": "B_vs_C",
+        "winner": "B"
+      }
+    ],
+    "ok": false
+  }
+}