onbuzz 4.8.3 → 4.9.1

package/scripts/memory-bench/live-v2-baseline-2026-05-23T22-43-20-757Z.json

@@ -0,0 +1,628 @@
+{
+  "timestamp": "2026-05-23T22:43:20.758Z",
+  "args": {
+    "models": [
+      "Kimi-K2.6",
+      "gpt-5.4",
+      "gpt-4.1-mini"
+    ],
+    "trials": 3,
+    "scenarios": null,
+    "label": "baseline",
+    "out": null
+  },
+  "results": [
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "Kimi-K2.6",
+      "trial": 0,
+      "hit": true,
+      "reason": "recall=Y save=Y",
+      "sub": {
+        "recall": {
+          "hit": true,
+          "reason": "called memory.read"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 33,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779573165324-1779573165387"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "Kimi-K2.6",
+      "trial": 1,
+      "hit": true,
+      "reason": "recall=Y save=Y",
+      "sub": {
+        "recall": {
+          "hit": true,
+          "reason": "called memory.list"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 25,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779573527656-1779573527658"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "Kimi-K2.6",
+      "trial": 2,
+      "hit": true,
+      "reason": "recall=Y save=Y",
+      "sub": {
+        "recall": {
+          "hit": true,
+          "reason": "called memory.read"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 44,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779573770623-1779573770629"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "gpt-5.4",
+      "trial": 0,
+      "hit": true,
+      "reason": "recall=Y save=Y",
+      "sub": {
+        "recall": {
+          "hit": true,
+          "reason": "called memory.read"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 19,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779574007750-1779574007755"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "gpt-5.4",
+      "trial": 1,
+      "hit": false,
+      "reason": "recall=N save=Y",
+      "sub": {
+        "recall": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 11,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779574040977-1779574040980"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "gpt-5.4",
+      "trial": 2,
+      "hit": true,
+      "reason": "recall=Y save=Y",
+      "sub": {
+        "recall": {
+          "hit": true,
+          "reason": "called memory.read"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 14,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779574068164-1779574068166"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "gpt-4.1-mini",
+      "trial": 0,
+      "hit": false,
+      "reason": "recall=N save=Y",
+      "sub": {
+        "recall": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 19,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779574102888-1779574102891"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "gpt-4.1-mini",
+      "trial": 1,
+      "hit": false,
+      "reason": "recall=N save=Y",
+      "sub": {
+        "recall": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 14,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779574154184-1779574154187"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S1-midtask-rule",
+      "mechanism": "catalog-driven-recall + save-on-rule-observation",
+      "model": "gpt-4.1-mini",
+      "trial": 2,
+      "hit": true,
+      "reason": "recall=Y save=Y",
+      "sub": {
+        "recall": {
+          "hit": true,
+          "reason": "called memory.search"
+        },
+        "save": {
+          "hit": true,
+          "reason": "called memory.add"
+        }
+      },
+      "totalCallCount": 13,
+      "error": null,
+      "agentId": "agent-v2-v2-s1-midtask-rule-1779574194939-1779574194942"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "Kimi-K2.6",
+      "trial": 0,
+      "hit": false,
+      "reason": "1/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"30 seconds\""
+        },
+        "task3": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        }
+      },
+      "partialHits": 1,
+      "totalPhases": 3,
+      "totalCallCount": 75,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779574241762-1779574241765"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "Kimi-K2.6",
+      "trial": 1,
+      "hit": false,
+      "reason": "2/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": true,
+          "reason": "fingerprint \"#F59E0B\""
+        }
+      },
+      "partialHits": 2,
+      "totalPhases": 3,
+      "totalCallCount": 42,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779574784866-1779574784869"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "Kimi-K2.6",
+      "trial": 2,
+      "hit": true,
+      "reason": "3/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": true,
+          "reason": "fingerprint \"#F59E0B\""
+        }
+      },
+      "partialHits": 3,
+      "totalPhases": 3,
+      "totalCallCount": 26,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575169491-1779575169494"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "gpt-5.4",
+      "trial": 0,
+      "hit": true,
+      "reason": "3/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": true,
+          "reason": "fingerprint \"#F59E0B\""
+        }
+      },
+      "partialHits": 3,
+      "totalPhases": 3,
+      "totalCallCount": 22,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575478671-1779575478674"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "gpt-5.4",
+      "trial": 1,
+      "hit": true,
+      "reason": "3/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": true,
+          "reason": "fingerprint \"#F59E0B\""
+        }
+      },
+      "partialHits": 3,
+      "totalPhases": 3,
+      "totalCallCount": 22,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575534560-1779575534563"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "gpt-5.4",
+      "trial": 2,
+      "hit": true,
+      "reason": "3/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": true,
+          "reason": "fingerprint \"#F59E0B\""
+        }
+      },
+      "partialHits": 3,
+      "totalPhases": 3,
+      "totalCallCount": 22,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575590423-1779575590426"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "gpt-4.1-mini",
+      "trial": 0,
+      "hit": false,
+      "reason": "1/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        },
+        "task3": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        }
+      },
+      "partialHits": 1,
+      "totalPhases": 3,
+      "totalCallCount": 55,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575641736-1779575641739"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "gpt-4.1-mini",
+      "trial": 1,
+      "hit": false,
+      "reason": "2/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": false,
+          "reason": "no action/fingerprint match"
+        }
+      },
+      "partialHits": 2,
+      "totalPhases": 3,
+      "totalCallCount": 30,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575818259-1779575818262"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S2-multi-topic",
+      "mechanism": "catalog-injection across topic shifts",
+      "model": "gpt-4.1-mini",
+      "trial": 2,
+      "hit": true,
+      "reason": "3/3 tasks-with-memory",
+      "sub": {
+        "task1": {
+          "hit": true,
+          "reason": "fingerprint \"tok-<\""
+        },
+        "task2": {
+          "hit": true,
+          "reason": "fingerprint \"exponential backoff\""
+        },
+        "task3": {
+          "hit": true,
+          "reason": "fingerprint \"#F59E0B\""
+        }
+      },
+      "partialHits": 3,
+      "totalPhases": 3,
+      "totalCallCount": 27,
+      "error": null,
+      "agentId": "agent-v2-v2-s2-multi-topic-1779575961620-1779575961623"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "Kimi-K2.6",
+      "trial": 0,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 3,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576049132-1779576049135"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "Kimi-K2.6",
+      "trial": 1,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 3,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576061234-1779576061236"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "Kimi-K2.6",
+      "trial": 2,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 1,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576071819-1779576071821"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "gpt-5.4",
+      "trial": 0,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 4,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576080933-1779576080935"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "gpt-5.4",
+      "trial": 1,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 4,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576091559-1779576091561"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "gpt-5.4",
+      "trial": 2,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 3,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576102169-1779576102175"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "gpt-4.1-mini",
+      "trial": 0,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 6,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576109836-1779576109841"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "gpt-4.1-mini",
+      "trial": 1,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 17,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576125084-1779576125090"
+    },
+    {
+      "label": "baseline",
+      "scenario": "V2-S3-negative-control",
+      "mechanism": "no-trigger-happy",
+      "model": "gpt-4.1-mini",
+      "trial": 2,
+      "hit": true,
+      "reason": "clean",
+      "sub": {
+        "negative": {
+          "hit": true,
+          "reason": "clean"
+        }
+      },
+      "totalCallCount": 17,
+      "error": null,
+      "agentId": "agent-v2-v2-s3-negative-control-1779576162926-1779576162930"
+    }
+  ]
+}