onbuzz 4.8.3 → 4.9.1

package/scripts/memory-bench/scenarios-v2.mjs

@@ -0,0 +1,123 @@
+/**
+ * v2 scenarios — production-shape multi-turn sessions designed to
+ * meaningfully exercise the memory feedback loop, NOT just the
+ * single-turn nudge response.
+ *
+ * Self-adversarial-pass rationale (vs v1 in scenarios.mjs):
+ *   - Longer sessions (10-20 turns) to expose compounding behavior
+ *   - Vague user trigger messages: the SEED MEMORY contains the
+ *     specific answer; the user's wording does NOT echo it. This
+ *     prevents fingerprint gaming (agent reading user's words back).
+ *   - Multi-task structure in S2 mimics real agent sessions
+ *   - Pre-committed scoring: per-turn binary metric, summed/aggregated
+ *
+ * Each scenario:
+ *   id                  short tag
+ *   mechanism           which feature it probes
+ *   seedMemories        pre-populated memory entries (server assigns ids)
+ *   userTurns           scripted user messages, sent in order
+ *   expect              scoring rubric:
+ *     anyOfActions      list of tool.action strings — any one counts as a hit
+ *     contentFingerprint optional: substrings to grep in agent's text answers
+ *                       (MUST be distinctive content NOT present in any userTurn)
+ *     negativeSpace     optional: tool.actions that, if called, count as a MISS
+ *                       (e.g. catalog spam — too many memory.list calls)
+ */
+
+export const SCENARIOS_V2 = [
+  // ── S1: Mid-task rule introduction ─────────────────────────────
+  // Vague debugging question whose answer is in memory.
+  // Mid-session, user adds a NEW rule the agent should save.
+  // Total turns: ~10-12.
+  {
+    id: 'V2-S1-midtask-rule',
+    mechanism: 'catalog-driven-recall + save-on-rule-observation',
+    description: 'Debugging question whose answer is in memory + mid-session rule introduction',
+    seedMemories: [
+      { id: 'mem-quota',
+        title: 'finding/upstream-quota',
+        description: 'Notes on the upstream provider quota we observed',
+        // The content has distinctive tokens (numbers/phrases) NOT
+        // present in the user turn. If the agent's answer contains
+        // these, it pulled them from memory.
+        content: 'Upstream provider enforces 60 req/min on our API key. Failed-auth retries DO count against the bucket. We hit this on the Talisman integration — auth-test retry-on-401 burned 20 requests in seconds.' },
+    ],
+    userTurns: [
+      'Our integration tests are flaky again — sometimes pass, sometimes a few fail. Can you investigate and tell me what might be going on?',
+      'Quick rule for going forward: never use bare `console.log` in production code — always use the structured logger from `lib/log.js`. Apply this from now on.',
+    ],
+    expect: {
+      // Recall phase (after turn 1): consult catalog or memory tool
+      recall: {
+        anyOfActions: ['memory.read', 'memory.search', 'memory.list'],
+        contentFingerprint: ['60 req/min', '60 requests per minute', 'failed-auth retries', 'failed auth retries'],
+      },
+      // Save phase (after turn 2): persist the new rule
+      save: {
+        anyOfActions: ['memory.add'],
+        contentFingerprint: ['console.log', 'structured logger'],
+      },
+    },
+  },
+
+  // ── S2: Sequential multi-topic session ─────────────────────────
+  // Three different tasks in one session. Each has a topically-
+  // relevant pre-seeded memory the agent should consult.
+  // Total turns: ~15-20.
+  {
+    id: 'V2-S2-multi-topic',
+    mechanism: 'catalog-injection across topic shifts',
+    description: 'Three back-to-back tasks; each has its own relevant memory.',
+    seedMemories: [
+      { id: 'mem-auth',
+        title: 'finding/auth-token-format',
+        description: 'How we mint and verify bearer tokens',
+        content: 'Bearer tokens use `tok-<userId>-<timestampMs>` format. Verification is a Map lookup keyed by user, NOT a JWT decode. The store is in-memory; tokens are invalidated on logout, NOT by TTL — there is no TTL enforcement today.' },
+      { id: 'mem-retries',
+        title: 'finding/retry-policy',
+        description: 'Decided retry policy',
+        content: 'Exponential backoff base 2, capped at 30 seconds, max 5 retries. ANY 5xx triggers; 4xx do not. Idempotency keys are required for retried writes.' },
+      { id: 'mem-dashboard',
+        title: 'rule/dashboard-error-display',
+        description: 'How error UI components should render error states',
+        content: 'Dashboard error displays must NOT use red. Use amber `#F59E0B` for transient errors, slate `#475569` for permanent. Icons go on the right, never left.' },
+    ],
+    userTurns: [
+      'I need to add token expiry to our auth. What\'s the right approach given how we currently mint and verify tokens?',
+      'OK now switch gears — review the retry behavior in api.js. Are we doing exponential backoff correctly?',
+      'One more — the error display in the dashboard looks off in production. Tell me which colors and icons we should be using.',
+    ],
+    expect: {
+      // For each task, did the agent consult the topically-relevant memory?
+      // (memory.search/list/read on the relevant entry OR content fingerprint)
+      perTask: [
+        { contentFingerprint: ['tok-<', 'Map lookup', 'no TTL'] },
+        { contentFingerprint: ['exponential backoff', '30 seconds', '5 retries', 'idempotency'] },
+        { contentFingerprint: ['#F59E0B', 'amber', 'slate', '#475569'] },
+      ],
+    },
+  },
+
+  // ── S3: Negative control ───────────────────────────────────────
+  // Routine acknowledgement; should not trigger memory writes or
+  // excessive memory.list spam.
+  {
+    id: 'V2-S3-negative-control',
+    mechanism: 'no-trigger-happy',
+    description: 'Routine ack — should not trigger memory writes or excessive list calls',
+    seedMemories: [
+      { id: 'mem-x', title: 'finding/something', description: 'noise', content: 'noise' },
+    ],
+    userTurns: [
+      'Thanks, I think I have what I need. I will take it from here.',
+    ],
+    expect: {
+      negative: {
+        // ANY memory.add/append is a fail.
+        forbiddenActions: ['memory.add', 'memory.append'],
+        // More than 2 memory.list calls = catalog spam → fail.
+        maxCallsPerAction: { 'memory.list': 2 },
+      },
+    },
+  },
+];

package/scripts/memory-bench/scenarios.mjs

@@ -0,0 +1,178 @@
+/**
+ * Multi-turn memory-feedback scenarios.
+ *
+ * Each scenario exercises ONE specific mechanism from the memory work
+ * shipped in 5bca65d (catalog injection / transition auto-load /
+ * conditional nudge / post-compaction nudge / append-vs-add).
+ *
+ * Each scenario specifies:
+ *   - mechanism: which feature it tests (for grouped scoring)
+ *   - seedMemories: pre-populated memories the agent starts with
+ *   - seedTasks: pre-populated tasks
+ *   - postCompactionFlag: whether to inject the post-compaction nudge
+ *     before the first turn (simulates running right after compaction)
+ *   - userTurns: scripted user messages (one per turn, in order)
+ *   - expect: scoring rubric
+ */
+
+export const SCENARIOS = [
+  // ─── Catalog injection ─────────────────────────────────────────
+  {
+    id: 'M1-catalog-recall',
+    mechanism: 'catalog-injection',
+    description: 'Pre-existing memory addresses the user question. Agent should consult catalog → read the relevant memory.',
+    seedMemories: [
+      { id: 'mem-rate-limits', title: 'finding/upstream-rate-limits',
+        description: 'API limits 60 req/min, counts failed auth retries',
+        content: 'The upstream provider enforces 60 req/min on our key. Failed auth retries DO count against the bucket. We hit this on the Talisman integration last sprint — the auth tests retry on 401 and burned through 20 requests in a few seconds.' },
+      { id: 'mem-unrelated', title: 'plan/dashboard-redesign',
+        description: 'Dashboard redesign plan, 3 phases',
+        content: 'Phase 1: tokens. Phase 2: navigation. Phase 3: analytics.' },
+    ],
+    seedTasks: [],
+    postCompactionFlag: false,
+    userTurns: [
+      'Our integration tests are flaky again — sometimes they pass, sometimes 5 of them fail with 429. Any idea why?',
+    ],
+    expect: {
+      // Mechanism: "catalog injection drives the agent to USE the
+      // memory's content". Three success paths, any one counts:
+      //   1. memory.read on the seeded entry (most explicit)
+      //   2. memory.list or memory.search (consulting the tool)
+      //   3. The agent's text answer references content from the
+      //      seed memory verbatim (the catalog description already
+      //      contains the answer; this is "answered from catalog")
+      // The only failure: agent improvises without ANY signal that
+      // it saw the memory.
+      action: 'memory.read',
+      orActions: ['memory.search', 'memory.list'],
+      memoryId: 'mem-rate-limits',
+      // Content fingerprints (any-of) to grep in the agent's final
+      // assistant text / jobdone summary.
+      answerContentAny: ['60 req/min', '60 requests per minute', 'failed auth retries'],
+      reason: 'rate-limit memory directly explains the 429 flakiness; agent should consult memory OR pull its content into the answer',
+    },
+  },
+
+  // ─── Task transition auto-load (memory attached) ───────────────
+  {
+    id: 'M2-transition-attached',
+    mechanism: 'transition-handler-attached',
+    description: 'Task references a memory. When task → in_progress, the memory is auto-loaded in the result. Agent should USE that content.',
+    seedMemories: [
+      { id: 'mem-auth-plan', title: 'plan/auth-migration',
+        description: 'Auth migration: 3-step rollout',
+        content: 'Step 1: dual-issue tokens for 7 days. Step 2: switch reads to new path. Step 3: deprecate v1. Hard requirement: NO downtime; we MUST keep v1 verifying tokens until step 3.' },
+    ],
+    seedTasks: [
+      { id: 'task-auth', title: 'Start auth migration',
+        description: '[memory:mem-auth-plan] Begin rollout per the linked plan',
+        status: 'pending', priority: 'high' },
+    ],
+    postCompactionFlag: false,
+    userTurns: [
+      'Go ahead and start working on the auth migration task. Tell me which step you\'re tackling first and why.',
+    ],
+    expect: {
+      // Either path counts: (a) taskmanager.update transitions task →
+      // auto-load fires → content lands in tool result; (b) the agent
+      // reads the memory directly off the catalog. Both deliver the
+      // plan content to the agent; both are successful outcomes for
+      // the user's question.
+      action: 'taskmanager.update',
+      orAction: 'memory.read',
+      memoryId: 'mem-auth-plan',
+      reason: 'transition handler auto-loads OR catalog drives direct read; either delivers plan content',
+    },
+  },
+
+  // ─── Conditional nudge (no memory attached) ────────────────────
+  {
+    id: 'M3-transition-nudge',
+    mechanism: 'transition-handler-nudge',
+    description: 'Task with NO memory marker, transitioned to in_progress. Nudge fires pointing at catalog. Catalog contains a relevant memory.',
+    seedMemories: [
+      { id: 'mem-style', title: 'rule/never-emojis',
+        description: 'Team rule: no emojis in code or commits',
+        content: 'Never use emojis in code, comments, or commit messages. Hard rule from review.' },
+    ],
+    seedTasks: [],  // live-probe creates the task inline via the user message; the simulated bench seeds it via the in-memory store
+    postCompactionFlag: false,
+    // Combined trigger: creates the task + asks the agent to begin
+    // work in one message. Ensures the transition (and therefore the
+    // nudge) fires within the measured window. The pre-restart
+    // scenario used a 2-phase setup which leaked the measured work
+    // into the setup turn.
+    userTurns: [
+      'Please create a task titled "Polish the README" with description "Rewrite the intro paragraph to feel less robotic", then BEGIN WORKING on it (transition it to in_progress) and tell me which step you\'re doing first.',
+    ],
+    expect: {
+      action: 'memory.list',
+      orAction: 'memory.read',
+      reason: 'no attached memory → nudge → agent should at least scan catalog before improvising',
+    },
+  },
+
+  // ─── Post-compaction nudge ─────────────────────────────────────
+  {
+    id: 'M4-post-compaction',
+    mechanism: 'post-compaction-nudge',
+    description: 'Compaction just happened. The relevant detail now lives only as a memory. Nudge should drive the agent to consult it.',
+    seedMemories: [
+      { id: 'mem-decision', title: 'finding/retry-policy',
+        description: 'Decided: exponential backoff, cap at 30s, max 5 retries',
+        content: 'After discussing trade-offs we landed on exponential backoff (base 2), capped at 30s, max 5 retries. Decided last Thursday.' },
+    ],
+    seedTasks: [],
+    postCompactionFlag: true,
+    userTurns: [
+      'Quick check: what did we decide about retries? I forget the cap.',
+    ],
+    expect: {
+      action: 'memory.read',
+      memoryId: 'mem-decision',
+      reason: 'post-compaction nudge + catalog should drive read, not improvise',
+    },
+  },
+
+  // ─── Append-vs-add (memory evolution) ──────────────────────────
+  {
+    id: 'M5-append-evolution',
+    mechanism: 'append-action',
+    description: 'Plan memory exists; user shares a new constraint that augments it. Agent should `memory append`, not `add` a new entry.',
+    seedMemories: [
+      { id: 'mem-plan', title: 'plan/onboarding',
+        description: 'New-hire onboarding flow plan',
+        content: 'Day 1: accounts. Day 2-3: shadow. Week 2: first task.' },
+    ],
+    seedTasks: [],
+    postCompactionFlag: false,
+    userTurns: [
+      'Add this to our onboarding plan: each new hire also gets paired with a senior eng for code reviews for the first month.',
+    ],
+    expect: {
+      action: 'memory.append',
+      memoryId: 'mem-plan',
+      reason: 'augmenting an existing plan should append; creating new would duplicate the topic',
+    },
+  },
+
+  // ─── Negative control: small talk ──────────────────────────────
+  {
+    id: 'M6-negative-control',
+    mechanism: 'negative-control',
+    description: 'Routine ack. Catalog injection should NOT make the agent trigger-happy on memory writes.',
+    seedMemories: [
+      { id: 'mem-x', title: 'finding/something', description: '...', content: 'noise' },
+    ],
+    seedTasks: [],
+    postCompactionFlag: false,
+    userTurns: [
+      'Thanks, sounds good.',
+    ],
+    expect: {
+      action: 'none',
+      reason: 'agent should NOT call memory.add on a simple ack',
+    },
+  },
+];

package/scripts/office-bench/generate-samples.mjs

@@ -0,0 +1,180 @@
+/**
+ * Office tool fidelity bench — generate one rich sample per family
+ * (.docx, .xlsx, .pptx) into ./bench-out so the team can open them
+ * in real Word / Excel / PowerPoint and verify rendering fidelity.
+ *
+ * Run:
+ *   node scripts/office-bench/generate-samples.mjs
+ *
+ * Output:
+ *   bench-out/office/sample.docx
+ *   bench-out/office/sample.xlsx
+ *   bench-out/office/sample.pptx
+ */
+
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { mkdir } from 'fs/promises';
+
+import OfficeTool from '../../src/tools/officeTool.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const projectRoot = path.resolve(__dirname, '..', '..');
+const outDir = path.join(projectRoot, 'bench-out', 'office');
+
+await mkdir(outDir, { recursive: true });
+
+const tool = new OfficeTool({}, { info() {}, warn() {}, error() {}, debug() {} });
+const ctx = { projectDir: projectRoot, agentId: 'bench' };
+
+const docSpec = {
+  title: 'Office Bench — Word Fidelity Sample',
+  author: 'Loxia Bench',
+  pageSize: 'letter',
+  sections: [{
+    children: [
+      { type: 'heading', level: 1, text: 'Word — Rich Features Showcase' },
+      { type: 'paragraph', text: 'This document exercises every feature of the office.doc.create surface so the team can verify rendering fidelity in Microsoft Word, Google Docs, and LibreOffice.' },
+
+      { type: 'heading', level: 2, text: 'Lists' },
+      { type: 'list', items: ['Bullet one', 'Bullet two', { text: 'Nested', level: 1 }, { text: 'Deeper', level: 2 }] },
+      { type: 'list', ordered: true, items: ['First', 'Second', 'Third'] },
+
+      { type: 'heading', level: 2, text: 'Tables' },
+      { type: 'table',
+        headers: ['Region', 'Q1', 'Q2', 'Q3'],
+        rows: [
+          ['East',  '1,200', '1,310', '1,420'],
+          ['West',  '980',   '1,120', '1,150'],
+          ['North', '1,430', '1,500', '1,600'],
+        ],
+        headerFill: '#1F3864',
+        alternateFill: '#F2F6FC',
+      },
+
+      { type: 'heading', level: 2, text: 'Callouts' },
+      { type: 'callout', label: 'Rule of thumb', content: 'Every batch operation uses the same plural-canonical envelope.', color: '#2E75B6' },
+      { type: 'callout', label: 'Warning', content: 'Do not bypass the catalog.', color: '#C00000' },
+
+      { type: 'heading', level: 2, text: 'Hyperlinks' },
+      { type: 'hyperlink', text: 'Anthropic', url: 'https://www.anthropic.com' },
+
+      { type: 'pageBreak' },
+      { type: 'heading', level: 1, text: 'Page Two' },
+      { type: 'paragraph', text: 'Page break worked.' },
+      { type: 'horizontalRule' },
+      { type: 'richParagraph', runs: [
+        { text: 'Mixed runs: ', bold: true },
+        { text: 'italic ', italic: true },
+        { text: 'colored ', color: '#2E75B6' },
+        { text: 'Consolas ', font: 'Consolas' },
+        { text: 'size 14pt.', size: 14 },
+      ]},
+    ],
+  }],
+};
+
+const sheetSpec = {
+  creator: 'Loxia Bench',
+  properties: { title: 'Office Bench — Excel Fidelity Sample', company: 'Loxia' },
+  sheets: [
+    {
+      name: 'Sales',
+      tabColor: '#1F3864',
+      columns: [
+        { header: 'Region', key: 'region',  width: 16 },
+        { header: 'Q1',     key: 'q1',      width: 12, type: 'currency' },
+        { header: 'Q2',     key: 'q2',      width: 12, type: 'currency' },
+        { header: 'Q3',     key: 'q3',      width: 12, type: 'currency' },
+        { header: 'Total',  key: 'total',   width: 14, type: 'currency' },
+      ],
+      headerStyle: { bold: true, fontColor: '#FFFFFF', fill: '#1F3864', height: 22 },
+      alternateRowFill: '#F2F6FC',
+      rows: [
+        { region: 'East',  q1: 1200, q2: 1310, q3: 1420, total: '=SUM(B2:D2)' },
+        { region: 'West',  q1: 980,  q2: 1120, q3: 1150, total: '=SUM(B3:D3)' },
+        { region: 'North', q1: 1430, q2: 1500, q3: 1600, total: '=SUM(B4:D4)' },
+      ],
+      freezeRow: 1,
+      autoFilter: true,
+      cellStyles: [
+        { cell: 'E2', bold: true },
+        { cell: 'E3', bold: true },
+        { cell: 'E4', bold: true },
+      ],
+      mergedCells: [],
+      conditionalFormatting: [
+        { range: 'B2:D4', type: 'colorScale', min: '#F8696B', mid: '#FFEB84', max: '#63BE7B' },
+      ],
+      namedRanges: [{ name: 'Q1Totals', range: 'B2:B4' }],
+    },
+    {
+      name: 'Pivot Notes',
+      columns: [{ header: 'Note', key: 'note', width: 80 }],
+      rows: [
+        { note: 'Color scale shows hot/cold quarters across regions.' },
+        { note: 'Sales!E2:E4 are SUM formulas — touch a value and watch them recompute.' },
+      ],
+      headerStyle: { bold: true, fontColor: '#FFFFFF', fill: '#2E75B6' },
+    },
+  ],
+};
+
+const presSpec = {
+  title: 'Office Bench — PowerPoint Fidelity Sample',
+  author: 'Loxia Bench',
+  company: 'Loxia',
+  layout: '16:9',
+  theme: { primaryColor: '#1F3864', accentColor: '#2E75B6', fontFamily: 'Calibri' },
+  slides: [
+    { layout: 'title', title: 'Office Bench', subtitle: 'PowerPoint fidelity sample' },
+    { layout: 'section-header', title: 'Section: Features' },
+    { layout: 'content', title: 'Bullets', content: [
+      { type: 'bullets', items: ['First point', 'Second point', { text: 'Nested', level: 1, ordered: false }, 'Third'], x: 0.5, y: 1.5, w: 9, h: 4 },
+    ], notes: 'Speaker notes — this is the bullets slide.' },
+    { layout: 'content', title: 'Table', content: [
+      { type: 'table',
+        headers: ['Region', 'Q1', 'Q2'],
+        rows: [['East', '1200', '1310'], ['West', '980', '1120'], ['North', '1430', '1500']],
+        x: 0.5, y: 1.5, w: 9, h: 2.5,
+        headerStyle: { fill: '#1F3864', color: '#FFFFFF' },
+      },
+    ]},
+    { layout: 'content', title: 'Shapes', content: [
+      { type: 'shape', shape: 'rect',     x: 0.5, y: 1.5, w: 2, h: 1.5, fill: '#1F3864' },
+      { type: 'shape', shape: 'rounded',  x: 2.8, y: 1.5, w: 2, h: 1.5, fill: '#2E75B6' },
+      { type: 'shape', shape: 'ellipse',  x: 5.1, y: 1.5, w: 2, h: 1.5, fill: '#5B9BD5' },
+      { type: 'text', text: 'Shapes — fills + borders work.', x: 0.5, y: 3.5, w: 9, h: 0.5, fontSize: 14, color: '#595959' },
+    ]},
+    { layout: 'content', title: 'Chart', content: [
+      { type: 'chart', chartType: 'bar', title: 'Quarterly Sales', x: 0.5, y: 1.5, w: 9, h: 4.5,
+        data: [
+          { name: 'Q1', labels: ['East', 'West', 'North'], values: [1200, 980, 1430] },
+          { name: 'Q2', labels: ['East', 'West', 'North'], values: [1310, 1120, 1500] },
+        ],
+      },
+    ]},
+  ],
+};
+
+async function emit(action, spec, name) {
+  const out = path.join(outDir, name);
+  const r = await tool.execute({
+    actions: [{ type: action, outputPath: out, content: spec }],
+  }, ctx);
+  if (!r.success) {
+    console.error(`✗ ${action} failed: ${r.error || r.output}`);
+    process.exit(1);
+  }
+  console.log(`✓ ${action}  →  ${r.outputPath}  (${(r.fileSize / 1024).toFixed(1)} KB)`);
+  return r;
+}
+
+await emit('doc.create',   docSpec,   'sample.docx');
+await emit('sheet.create', sheetSpec, 'sample.xlsx');
+await emit('pres.create',  presSpec,  'sample.pptx');
+
+console.log(`\nOpen the three files in real Office apps and verify:`);
+console.log(`  - sample.docx: heading colors, list nesting, table zebra, callout left bar, page break, hyperlink`);
+console.log(`  - sample.xlsx: header bar, currency formatting, SUM formulas, color-scale conditional formatting, frozen row 1, autofilter`);
+console.log(`  - sample.pptx: title slide, bullets, table, shapes (rect/rounded/ellipse with fills), bar chart with two series`);