forgedock 1.0.17 → 1.0.19

package/docs/design/fixtures/runs/b-vs-c-baseline/runs.json

@@ -0,0 +1,175 @@
+{
+  "corpus_version": "seed-v1",
+  "generation_model": "claude-sonnet-4-6",
+  "judge_model": "claude-sonnet-4-6",
+  "benchmark_date": "2026-06-22",
+  "note": "B-vs-C baseline only — no arm A. Slipstream arm C was viewport-only (vercel.com rendering timeout), flagged as calibration artifact.",
+  "products": [
+    {
+      "product": "cadence",
+      "reference": "resend.com",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 5, "C": 0}
+        },
+        {
+          "run": 2,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 4, "C": 0}
+        },
+        {
+          "run": 3,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 5, "C": 0}
+        }
+      ]
+    },
+    {
+      "product": "tender",
+      "reference": "stripe.com",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 4, "C": 1}
+        },
+        {
+          "run": 2,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 6, "C": 1}
+        },
+        {
+          "run": 3,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 5, "C": 1}
+        }
+      ]
+    },
+    {
+      "product": "slipstream",
+      "reference": "vercel.com",
+      "note": "CALIBRATION ARTIFACT: Arm C screenshot was viewport-only (vercel.com rendering timeout produced 54KB vs 1-3MB for other references). Judge rated B > C due to incomplete C capture, not genuine design superiority. Exclude from aggregate metrics or treat as suspect.",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"B_vs_C": "B"},
+          "rubric": {
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 3, "originality": 4, "mobile": 3}
+          },
+          "slop": {"B": 4, "C": 2}
+        },
+        {
+          "run": 2,
+          "pairwise": {"B_vs_C": "B"},
+          "rubric": {
+            "B": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 4, "originality": 3, "mobile": 4},
+            "C": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 3, "originality": 4, "mobile": 3}
+          },
+          "slop": {"B": 5, "C": 2}
+        },
+        {
+          "run": 3,
+          "pairwise": {"B_vs_C": "B"},
+          "rubric": {
+            "B": {"hierarchy": 5, "typography": 5, "color": 4, "whitespace": 4, "originality": 4, "mobile": 4},
+            "C": {"hierarchy": 4, "typography": 4, "color": 3, "whitespace": 3, "originality": 4, "mobile": 3}
+          },
+          "slop": {"B": 3, "C": 2}
+        }
+      ]
+    },
+    {
+      "product": "voltage",
+      "reference": "modal.com",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 2, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 5, "C": 1}
+        },
+        {
+          "run": 2,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 6, "C": 1}
+        },
+        {
+          "run": 3,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 2, "typography": 2, "color": 2, "whitespace": 3, "originality": 2, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 4, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 7, "C": 1}
+        }
+      ]
+    },
+    {
+      "product": "plume",
+      "reference": "linear.app",
+      "runs": [
+        {
+          "run": 1,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 3, "color": 3, "whitespace": 3, "originality": 2, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 6, "C": 1}
+        },
+        {
+          "run": 2,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 4, "typography": 4, "color": 4, "whitespace": 4, "originality": 4, "mobile": 4},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 2, "C": 1}
+        },
+        {
+          "run": 3,
+          "pairwise": {"B_vs_C": "C"},
+          "rubric": {
+            "B": {"hierarchy": 3, "typography": 4, "color": 3, "whitespace": 3, "originality": 3, "mobile": 3},
+            "C": {"hierarchy": 5, "typography": 5, "color": 5, "whitespace": 5, "originality": 5, "mobile": 4}
+          },
+          "slop": {"B": 5, "C": 1}
+        }
+      ]
+    }
+  ]
+}

package/docs/design/fixtures/runs/b-vs-c-baseline/scorecard.json

@@ -0,0 +1,138 @@
+{
+  "corpus_version": "seed-v1",
+  "generation_model": "claude-sonnet-4-6",
+  "judge_model": "claude-sonnet-4-6",
+  "n_products": 5,
+  "n_runs_per_product": 3,
+  "win_rate_vs_C": {
+    "A": {
+      "mean": null,
+      "by_product": {
+        "cadence": null,
+        "tender": null,
+        "slipstream": null,
+        "voltage": null,
+        "plume": null
+      }
+    },
+    "B": {
+      "mean": 0.2,
+      "by_product": {
+        "cadence": 0,
+        "tender": 0,
+        "slipstream": 1,
+        "voltage": 0,
+        "plume": 0
+      }
+    }
+  },
+  "a_vs_b_win_rate": null,
+  "rubric": {
+    "A": {
+      "hierarchy": {
+        "mean": null,
+        "stdev": null
+      },
+      "typography": {
+        "mean": null,
+        "stdev": null
+      },
+      "color": {
+        "mean": null,
+        "stdev": null
+      },
+      "whitespace": {
+        "mean": null,
+        "stdev": null
+      },
+      "originality": {
+        "mean": null,
+        "stdev": null
+      },
+      "mobile": {
+        "mean": null,
+        "stdev": null
+      }
+    },
+    "B": {
+      "hierarchy": {
+        "mean": 3.4,
+        "stdev": 0.737
+      },
+      "typography": {
+        "mean": 3.467,
+        "stdev": 0.743
+      },
+      "color": {
+        "mean": 3.2,
+        "stdev": 0.561
+      },
+      "whitespace": {
+        "mean": 3.333,
+        "stdev": 0.488
+      },
+      "originality": {
+        "mean": 2.933,
+        "stdev": 0.594
+      },
+      "mobile": {
+        "mean": 3.267,
+        "stdev": 0.458
+      }
+    },
+    "C": {
+      "hierarchy": {
+        "mean": 4.8,
+        "stdev": 0.414
+      },
+      "typography": {
+        "mean": 4.8,
+        "stdev": 0.414
+      },
+      "color": {
+        "mean": 4.6,
+        "stdev": 0.828
+      },
+      "whitespace": {
+        "mean": 4.4,
+        "stdev": 0.828
+      },
+      "originality": {
+        "mean": 4.8,
+        "stdev": 0.414
+      },
+      "mobile": {
+        "mean": 3.8,
+        "stdev": 0.414
+      }
+    }
+  },
+  "slop": {
+    "A": null,
+    "B": 4.8,
+    "C": 1
+  },
+  "judge_calibration": {
+    "miscalibrated_runs": [
+      {
+        "product": "slipstream",
+        "run": 1,
+        "comparison": "B_vs_C",
+        "winner": "B"
+      },
+      {
+        "product": "slipstream",
+        "run": 2,
+        "comparison": "B_vs_C",
+        "winner": "B"
+      },
+      {
+        "product": "slipstream",
+        "run": 3,
+        "comparison": "B_vs_C",
+        "winner": "B"
+      }
+    ],
+    "ok": false
+  }
+}