@motion-proto/live-tokens 0.87.1 → 0.88.0

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.88.0 — Eval results on the testing page
+
+### Changed
+
+- **The testing page reports the eval results.** A new Measured value
+  chapter tabulates the pick-component eval across four arms: with and
+  without the skills, and with and without the CLI. It gives the requirements
+  each arm got right, with its turns, time, and cost. The Skill Atlas loses
+  its Measured value tab, and a `#measured-value` link to the atlas now opens
+  the chapter.
+
 ## 0.87.1 — Floating parts stay above the page in sketch mode
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@motion-proto/live-tokens",
-  "version": "0.87.1",
+  "version": "0.88.0",
   "type": "module",
   "description": "Design token editor with live CSS variable editing. Svelte 5 + Vite 8.",
   "keywords": [

package/src/demo/TestingLoops.svelte

@@ -1,10 +1,12 @@
 <script lang="ts">
+  import { onMount } from 'svelte';
   import CollapsibleSection from '../system/components/CollapsibleSection.svelte';
   import Panel from '../system/components/Panel.svelte';
   import Table from '../system/components/Table.svelte';
   import Card from '../system/components/Card.svelte';
   import { portal } from '../system/internal/portal';
   import { navigate } from '../editor/core/routing/router';
+  import { pickComponentEval as result } from './evalResults';
 
   let { homeHref = '/demo' }: { homeHref?: string } = $props();
 
@@ -14,6 +16,7 @@
     { id: 'test-runs', title: 'Testing pages and components' },
     { id: 'walkthrough', title: 'Build and check a page' },
     { id: 'reference', title: 'Rule reference' },
+    { id: 'measured-value', title: 'Measured value' },
   ];
 
   // Closed by default: open, the fixed panel covers the full-width figures.
@@ -21,6 +24,12 @@
 
   let openRules = $state<Record<string, boolean>>({});
 
+  // The router loads this page lazily, after the browser's own jump to a hash has passed.
+  onMount(() => {
+    const id = window.location.hash.slice(1);
+    if (id) document.getElementById(id)?.scrollIntoView({ block: 'start' });
+  });
+
   function jump(event: MouseEvent, id: string) {
     event.preventDefault();
     contentsOpen = false;
@@ -344,6 +353,63 @@
     </div>
   </section>
 
+  <section class="chapter" id="measured-value" aria-labelledby="measured-value-title">
+    <div class="chapter-body">
+      <h2 id="measured-value-title">Measured value</h2>
+      <p>An eval asked an agent to choose a component for {result.requirements} requirements. It ran in four arms of {result.runsPerArm} runs each: with and without the skills, and with and without the CLI. The arms chose the same components, with one miss among 144 answers, and differ in turns, time, and cost.</p>
+      <div class="eval-results">
+        <Table>
+          <table aria-label="Eval arms">
+            <thead>
+              <tr>
+                <th scope="col">Skills</th>
+                <th scope="col">CLI</th>
+                <th scope="col">Requirements right</th>
+                <th scope="col">Turns</th>
+                <th scope="col">Seconds</th>
+                <th scope="col">Cost</th>
+              </tr>
+            </thead>
+            <tbody>
+              {#each result.arms as arm (`${arm.skills}-${arm.cli}`)}
+                <tr>
+                  <td>{arm.skills ? 'Yes' : 'No'}</td>
+                  <td>{arm.cli ? 'Yes' : 'No'}</td>
+                  <td>{arm.rowsRight}</td>
+                  <td>{arm.turns}</td>
+                  <td>{arm.seconds}</td>
+                  <td>{arm.cost}</td>
+                </tr>
+              {/each}
+            </tbody>
+          </table>
+        </Table>
+      </div>
+      <ul class="eval-summary">
+        <li><strong>Turns:</strong> the skill saves turns only through the CLI command. Without the CLI, the skill arm takes as many turns as the agent with neither.</li>
+        <li><strong>Time and cost:</strong> the skills and the CLI each save time on their own, and together the run costs less than half as much as with neither.</li>
+        <li><strong>Accuracy:</strong> every arm scores about the same. The choices come from the entries in the component files.</li>
+        <li><strong>Change since 2026-09-20:</strong> in that run, no agent without the skills found the CLI; this time 2 of 3 did, so that arm varies from run to run.</li>
+      </ul>
+      <div class="text-columns">
+        <div>
+          <h3>Method</h3>
+          <ul>
+            <li>Cases <code>{result.cases[0]}</code> and <code>{result.cases[1]}</code>, run on {result.date} at package {result.packageVersion}.</li>
+            <li>Ten requirements with one right component, and two that nothing shipped fits.</li>
+            <li>The second case seeds the same project without the package’s <code>bin</code> folder, so the command the skill names fails. The skills are unchanged.</li>
+            <li>The runner adds the arm without skills to each case.</li>
+            <li>Each requirement has its own pattern grader, so a failure names its row.</li>
+          </ul>
+        </div>
+        <div>
+          <h3>Limits</h3>
+          <p>Three runs per arm is a small sample. The twelve requirements each have a clear answer in the entries. A harder case, where two components both fit, has not been measured.</p>
+        </div>
+      </div>
+    </div>
+  </section>
+
   <!-- NOTES, open questions for this page:
   - "Review by eye" (walkthrough step 7) and Figure 2's "Read the page yourself".
     Leaning toward removing both. The create-page skill says "Open the page at
@@ -438,6 +504,11 @@
     grid-column: 2 / -2;
   }
 
+  .chapter-body > .eval-results {
+    grid-column: 2 / -2;
+    margin-block: var(--space-12) var(--space-40);
+  }
+
   .chapter-body > .reference-groups {
     grid-column: 2 / 11;
   }
@@ -475,6 +546,35 @@
     margin: var(--space-0) var(--space-0) var(--space-24);
   }
 
+  .text-columns h3 {
+    font-family: var(--heading-sm-font-family);
+    font-size: var(--heading-sm-font-size);
+    font-weight: var(--heading-sm-font-weight);
+    line-height: var(--heading-sm-line-height);
+    letter-spacing: var(--heading-sm-letter-spacing);
+    margin: var(--space-0) var(--space-0) var(--space-12);
+  }
+
+  .eval-summary {
+    margin: var(--space-0) var(--space-0) var(--space-40);
+    padding-left: var(--space-24);
+    color: var(--text-secondary);
+  }
+
+  .eval-summary li + li {
+    margin-top: var(--space-12);
+  }
+
+  .text-columns ul {
+    margin: var(--space-0);
+    padding-left: var(--space-24);
+    color: var(--text-secondary);
+  }
+
+  .text-columns li + li {
+    margin-top: var(--space-8);
+  }
+
   p,
   li,
   figcaption {

package/src/demo/evalResults.ts

@@ -0,0 +1,62 @@
+/** One arm of an eval: the same case run with or without the skills, and with or without the CLI. */
+export interface EvalArm {
+  skills: boolean;
+  cli: boolean;
+  rowsRight: string;
+  turns: string;
+  seconds: string;
+  cost: string;
+}
+
+export interface EvalResult {
+  cases: string[];
+  date: string;
+  packageVersion: string;
+  runsPerArm: number;
+  requirements: number;
+  arms: EvalArm[];
+}
+
+// Recorded by hand from `npm run eval` in the live-tokens repository. The
+// evals do not ship, so the page cannot read a result file at runtime.
+export const pickComponentEval: EvalResult = {
+  cases: ['outcome-pick-component', 'outcome-pick-component-no-cli'],
+  date: '2026-09-22',
+  packageVersion: '0.87.1',
+  runsPerArm: 3,
+  requirements: 12,
+  arms: [
+    {
+      skills: true,
+      cli: true,
+      rowsRight: '12 of 12 in all 3 runs',
+      turns: '9 to 10',
+      seconds: '46 to 76',
+      cost: '$0.41 to $0.46',
+    },
+    {
+      skills: false,
+      cli: true,
+      rowsRight: '12 of 12 in all 3 runs; 2 of 3 found the CLI on their own',
+      turns: '15 to 17',
+      seconds: '65 to 102',
+      cost: '$0.63 to $0.74',
+    },
+    {
+      skills: true,
+      cli: false,
+      rowsRight: '12 of 12 in all 3 runs',
+      turns: '14 to 20',
+      seconds: '66 to 84',
+      cost: '$0.73 to $0.91',
+    },
+    {
+      skills: false,
+      cli: false,
+      rowsRight: '11 of 12 in one run, which missed requirement 12',
+      turns: '17 to 20',
+      seconds: '113 to 169',
+      cost: '$1.02 to $1.14',
+    },
+  ],
+};

package/src/editor/skill-atlas/SkillAtlas.svelte

@@ -1,9 +1,8 @@
 <script lang="ts">
   import { onMount, tick } from 'svelte';
-  import { navigate } from '../core/routing/router';
+  import { navigate, route } from '../core/routing/router';
   import Button from '../../system/components/Button.svelte';
   import TabBar from '../../system/components/TabBar.svelte';
-  import SkillValue from './SkillValue.svelte';
   import SourcePane from './SourcePane.svelte';
   import TreeCanvas from './TreeCanvas.svelte';
   import { linkHash, linkTargets, resolveLink } from './atlasLink';
@@ -13,35 +12,23 @@
 
   // `#set-type` opens that skill and `#set-type/write-the-font-pairing/voice`
   // also selects that block, so a link can hand someone one step of one tree.
-  const VALUE_TAB = 'measured-value';
   const firstSkill = Object.keys(skillTrees)[0];
 
-  function isValueHash(hash: string) {
-    return hash === `#${VALUE_TAB}`;
-  }
-
   const linked = resolveLink(window.location.hash, skillTrees);
-  let active = $state(
-    isValueHash(window.location.hash) ? VALUE_TAB : (linked?.skill ?? firstSkill),
-  );
+  let active = $state(linked?.skill ?? firstSkill);
   let selection: Selection | null = $state(linked?.target ?? null);
   /** Which of the skill's documents the source pane shows. */
   let doc: string = $state(SKILL_DOC);
 
-  // The value tab has no tree; the panes it hides keep reading a real skill.
-  let skill = $derived(active === VALUE_TAB ? firstSkill : active);
-  let tree = $derived(skillTrees[skill]);
-  let docs = $derived(skillDocs[skill]);
+  let tree = $derived(skillTrees[active]);
+  let docs = $derived(skillDocs[active]);
   let lines = $derived(docs[doc] ?? docs[SKILL_DOC]);
   let siblings = $derived(Object.keys(docs).filter((name) => name !== SKILL_DOC));
 
-  let tabs = $derived([
-    ...Object.entries(skillTrees).map(([id, t]) => ({
-      id,
-      label: `${t.title}\n${skillDocs[id][SKILL_DOC].length} lines`,
-    })),
-    { id: VALUE_TAB, label: 'Measured value\n1 eval' },
-  ]);
+  const tabs = Object.entries(skillTrees).map(([id, t]) => ({
+    id,
+    label: `${t.title}\n${skillDocs[id][SKILL_DOC].length} lines`,
+  }));
 
   let docTabs = $derived(
     Object.keys(docs).map((name) => ({
@@ -65,10 +52,6 @@
   }
 
   function shareSelection() {
-    if (active === VALUE_TAB) {
-      history.replaceState(null, '', `${window.location.pathname}#${VALUE_TAB}`);
-      return;
-    }
     const target = linkTargets(tree).find((t) => t.key === selection?.key) ?? null;
     history.replaceState(null, '', `${window.location.pathname}${linkHash(active, target)}`);
   }
@@ -111,10 +94,10 @@
   }
 
   function openLink(hash: string) {
-    if (isValueHash(hash)) {
-      active = VALUE_TAB;
-      selection = null;
-      doc = SKILL_DOC;
+    // The eval report moved to the testing page; replace, so Back skips the old link.
+    if (hash === '#measured-value') {
+      history.replaceState(null, '', '/testing-loops#measured-value');
+      route.set('/testing-loops');
       return;
     }
     const link = resolveLink(hash, skillTrees);
@@ -157,42 +140,38 @@
     <TabBar {tabs} value={active} onchange={changeTab} />
   </div>
 
-  {#if active === VALUE_TAB}
-    <SkillValue />
-  {:else}
-    <div class="split">
-      <section class="pane" aria-label="{tree.id} decision tree">
-        <div class="pane-head">
-          <span class="pane-title">{tree.id}</span>
-          <span class="pane-note">{tree.nodes.length} steps</span>
-        </div>
-        <div class="pane-body" bind:this={treePane}>
-          <h2 class="tagline">{tree.tagline}</h2>
-
-          <TreeCanvas {tree} selected={selection?.key ?? null} onselect={selectTarget} onopen={openDoc} />
-        </div>
-      </section>
-
-      <section class="pane pane-source" aria-label="{tree.id} source">
-        <div class="pane-head">
-          <span class="pane-title">{tree.id}/{doc}</span>
-          <span class="pane-note">{lines.length} lines</span>
-        </div>
-        <div class="doc-tabs">
-          <TabBar tabs={docTabs} value={doc} onchange={openDoc} />
-        </div>
-        <div class="pane-body" bind:this={sourcePane}>
-          <SourcePane
-            {lines}
-            {siblings}
-            highlight={doc === SKILL_DOC ? (selection?.lines ?? null) : null}
-            onpick={selectFromLine}
-            onopen={openDoc}
-          />
-        </div>
-      </section>
-    </div>
-  {/if}
+  <div class="split">
+    <section class="pane" aria-label="{tree.id} decision tree">
+      <div class="pane-head">
+        <span class="pane-title">{tree.id}</span>
+        <span class="pane-note">{tree.nodes.length} steps</span>
+      </div>
+      <div class="pane-body" bind:this={treePane}>
+        <h2 class="tagline">{tree.tagline}</h2>
+
+        <TreeCanvas {tree} selected={selection?.key ?? null} onselect={selectTarget} onopen={openDoc} />
+      </div>
+    </section>
+
+    <section class="pane pane-source" aria-label="{tree.id} source">
+      <div class="pane-head">
+        <span class="pane-title">{tree.id}/{doc}</span>
+        <span class="pane-note">{lines.length} lines</span>
+      </div>
+      <div class="doc-tabs">
+        <TabBar tabs={docTabs} value={doc} onchange={openDoc} />
+      </div>
+      <div class="pane-body" bind:this={sourcePane}>
+        <SourcePane
+          {lines}
+          {siblings}
+          highlight={doc === SKILL_DOC ? (selection?.lines ?? null) : null}
+          onpick={selectFromLine}
+          onopen={openDoc}
+        />
+      </div>
+    </section>
+  </div>
 </div>
 
 <style>

package/src/editor/skill-atlas/SkillValue.svelte

@@ -1,202 +0,0 @@
-<script lang="ts">
-  import Table from '../../system/components/Table.svelte';
-  import { pickComponentEval } from './evalResults';
-
-  const result = pickComponentEval;
-  const [withSkills, without] = result.arms;
-  const difference = (withSkills.score - without.score).toFixed(2);
-</script>
-
-<article class="skill-value" aria-labelledby="skill-value-title">
-  <div class="measure">
-    <p class="eyebrow">Measured value</p>
-    <h2 id="skill-value-title">The skills make an agent faster. The component entries make it right.</h2>
-    <p class="lead">
-      An eval asked an agent to choose a component for twelve requirements, three times with the
-      skills loaded and three times with none. Every run that finished chose all twelve correctly,
-      in both arms. The skills changed the cost of getting there.
-    </p>
-  </div>
-
-  <div class="results">
-    <Table>
-      <table>
-        <thead>
-          <tr>
-            <th scope="col"><span class="visually-hidden">Arm</span></th>
-            <th scope="col">Score</th>
-            <th scope="col">Requirements right</th>
-            <th scope="col">Ran the catalogue command</th>
-            <th scope="col">Turns</th>
-            <th scope="col">Seconds</th>
-          </tr>
-        </thead>
-        <tbody>
-          {#each result.arms as arm (arm.label)}
-            <tr>
-              <td class="arm">{arm.label}</td>
-              <td>{arm.score.toFixed(2)}</td>
-              <td>{arm.rowsRight}</td>
-              <td>{arm.ranCatalogue}</td>
-              <td>{arm.turns}</td>
-              <td>{arm.seconds}</td>
-            </tr>
-          {/each}
-        </tbody>
-      </table>
-    </Table>
-  </div>
-
-  <div class="measure">
-    <h3>Accuracy comes from the component</h3>
-    <p>
-      Each component file carries its own entry: <code>whenToUse</code> states the condition that
-      makes it right, and <code>whenNotToUse</code> lists the conditions that rule it out, each with
-      the component to use in its place. An agent with no skills opened the component files, found
-      those entries, and made the same twelve choices, including the two requirements nothing
-      shipped fits and the destructive action that belongs in a Dialog.
-    </p>
-
-    <h3>Efficiency comes from the skill</h3>
-    <p>
-      The pick-component skill sends the agent to one command,
-      <code>npx live-tokens components --json</code>, which returns every entry at once. With the
-      skills the agent answered in {withSkills.turns} turns and under a minute. Without them it
-      never found the command, read files one at a time for {without.turns} turns, and one of its
-      three runs reached the five minute limit before it answered.
-    </p>
-
-    <h3>Method</h3>
-    <ul>
-      <li>Case <code>{result.id}</code>, run on {result.date}.</li>
-      <li>Twelve requirements: ten with one right component, two that nothing shipped fits.</li>
-      <li>Three runs per arm. The runner adds the arm without skills on its own.</li>
-      <li>
-        Each requirement has its own pattern grader, so a failure names its row. One more grader
-        checks that the agent ran the catalogue command.
-      </li>
-      <li>
-        The score is the share of graders that pass. The difference of {difference} comes from one
-        timeout and from the catalogue-command grader. No finished run chose a wrong component.
-      </li>
-    </ul>
-
-    <h3>Limits</h3>
-    <p>
-      Three runs per arm is a small sample, and one timeout moves the score a long way. The twelve
-      requirements each have a clear answer in the entries. A harder case, where two components
-      both fit, has not been measured.
-    </p>
-  </div>
-</article>
-
-<style>
-  /* The atlas locks to the viewport on desktop, so the report scrolls itself. */
-  .skill-value {
-    flex: 1 1 auto;
-    min-height: 0;
-    overflow-y: auto;
-    padding-bottom: var(--space-48);
-    color: var(--text-secondary);
-  }
-
-  .measure {
-    max-width: 70ch;
-  }
-
-  .results {
-    max-width: 110ch;
-    margin: var(--space-32) 0 var(--space-40);
-  }
-
-  .eyebrow {
-    margin: 0 0 var(--space-8);
-    font-family: var(--eyebrow-font-family);
-    font-size: var(--eyebrow-font-size);
-    font-weight: var(--eyebrow-font-weight);
-    line-height: var(--eyebrow-line-height);
-    letter-spacing: var(--eyebrow-letter-spacing);
-    text-transform: var(--eyebrow-text-transform);
-    color: var(--text-tertiary);
-  }
-
-  h2 {
-    margin: 0 0 var(--space-24);
-    font-family: var(--heading-lg-font-family);
-    font-size: var(--heading-lg-font-size);
-    font-weight: var(--heading-lg-font-weight);
-    line-height: var(--heading-lg-line-height);
-    letter-spacing: var(--heading-lg-letter-spacing);
-    color: var(--text-primary);
-  }
-
-  h3 {
-    margin: var(--space-40) 0 var(--space-12);
-    font-family: var(--heading-sm-font-family);
-    font-size: var(--heading-sm-font-size);
-    font-weight: var(--heading-sm-font-weight);
-    line-height: var(--heading-sm-line-height);
-    letter-spacing: var(--heading-sm-letter-spacing);
-    color: var(--text-primary);
-  }
-
-  .measure > h3:first-child {
-    margin-top: 0;
-  }
-
-  p,
-  li {
-    font-family: var(--body-md-font-family);
-    font-size: var(--body-md-font-size);
-    font-weight: var(--body-md-font-weight);
-    line-height: var(--body-md-line-height);
-    letter-spacing: var(--body-md-letter-spacing);
-  }
-
-  p {
-    margin: 0 0 var(--space-16);
-  }
-
-  .lead {
-    margin-bottom: 0;
-    color: var(--text-primary);
-  }
-
-  ul {
-    margin: 0;
-    padding-left: var(--space-24);
-  }
-
-  li + li {
-    margin-top: var(--space-8);
-  }
-
-  code {
-    font-family: var(--code-font-family);
-    font-size: var(--code-font-size);
-    font-weight: var(--code-font-weight);
-    line-height: var(--code-line-height);
-    letter-spacing: var(--code-letter-spacing);
-    background: var(--tint-low);
-    padding-inline: var(--space-4);
-    border-radius: var(--radius-sm);
-    /* The code face joins `--` into one dash, and a line may break between the
-       two hyphens. Either misprints a CLI flag. */
-    font-variant-ligatures: none;
-    white-space: nowrap;
-  }
-
-  .results td.arm {
-    color: var(--text-primary);
-    white-space: nowrap;
-  }
-
-  .visually-hidden {
-    position: absolute;
-    width: 1px;
-    height: 1px;
-    overflow: hidden;
-    clip-path: inset(50%);
-    white-space: nowrap;
-  }
-</style>

package/src/editor/skill-atlas/evalResults.ts

@@ -1,46 +0,0 @@
-/** One arm of an eval: the same case run with the skills loaded, or without them. */
-export interface EvalArm {
-  label: string;
-  score: number;
-  rowsRight: string;
-  ranCatalogue: string;
-  turns: string;
-  seconds: string;
-}
-
-export interface EvalResult {
-  id: string;
-  date: string;
-  packageVersion: string;
-  runsPerArm: number;
-  requirements: number;
-  arms: [withSkills: EvalArm, without: EvalArm];
-}
-
-// Recorded by hand from `npm run eval` in the live-tokens repository. The
-// evals do not ship, so the page cannot read a result file at runtime.
-export const pickComponentEval: EvalResult = {
-  id: 'outcome-pick-component',
-  date: '2026-09-20',
-  packageVersion: '0.82.0, plus the unreleased catalogue entry',
-  runsPerArm: 3,
-  requirements: 12,
-  arms: [
-    {
-      label: 'With the skills',
-      score: 1,
-      rowsRight: '12 of 12 in all 3 runs',
-      ranCatalogue: '3 of 3 runs',
-      turns: '9 to 10',
-      seconds: '42 to 58',
-    },
-    {
-      label: 'Without',
-      score: 0.62,
-      rowsRight: '12 of 12 in the 2 runs that finished',
-      ranCatalogue: '0 of 3 runs',
-      turns: '15 to 16',
-      seconds: '70 to 72, and one run timed out at 300',
-    },
-  ],
-};