@1mbrain/benchmarks 0.1.1

package/fixtures/graph-stress-hard/generate_graph_stress_hard.js

@@ -0,0 +1,526 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const scenarios = [
+  {
+    id: 'gsh-c01',
+    agent: 'agent-launch-ops',
+    domain: 'software',
+    person: 'Rina',
+    codename: 'Aurora',
+    project: 'customer-retention project',
+    bridge: 'renewal-risk initiative',
+    answer: 'Postgres event pipeline',
+    ritual: 'Copper Gate',
+    team: 'release assurance team',
+    artifact: 'signed rollback matrix',
+    stale: 'email-only weekly digest',
+    middle: 'Slack digest with email fallback',
+    final: 'in-app digest with Slack fallback',
+    distractorName: 'Aurelia',
+    distractorAnswer: 'Redis queue',
+    entityAnswer: 'Sofia Marin',
+    missingTopic: 'budget cap',
+  },
+  {
+    id: 'gsh-c02',
+    agent: 'agent-clinical-research',
+    domain: 'research',
+    person: 'Devon',
+    codename: 'Blue Lantern',
+    project: 'rural asthma cohort',
+    bridge: 'inhaler-adherence study',
+    answer: 'FHIR observation export',
+    ritual: 'Silver File',
+    team: 'ethics submission team',
+    artifact: 'redacted consent ledger',
+    stale: 'manual spreadsheet tracking',
+    middle: 'Airtable tracking',
+    final: 'REDCap tracking with nightly export',
+    distractorName: 'Devan',
+    distractorAnswer: 'HL7 batch import',
+    entityAnswer: 'Priya Shah',
+    missingTopic: 'participant compensation amount',
+  },
+  {
+    id: 'gsh-c03',
+    agent: 'agent-finance-ops',
+    domain: 'finance',
+    person: 'Mika',
+    codename: 'Northstar',
+    project: 'invoice anomaly monitor',
+    bridge: 'vendor-risk workflow',
+    answer: 'BigQuery audit table',
+    ritual: 'Pine Review',
+    team: 'controls validation team',
+    artifact: 'variance exception register',
+    stale: 'threshold at USD 500',
+    middle: 'threshold at USD 750',
+    final: 'threshold at USD 1,200 with CFO override',
+    distractorName: 'Mica',
+    distractorAnswer: 'Snowflake staging table',
+    entityAnswer: 'Elena Costa',
+    missingTopic: 'insurance carrier',
+  },
+  {
+    id: 'gsh-c04',
+    agent: 'agent-legal-casework',
+    domain: 'legal',
+    person: 'Amina',
+    codename: 'Harbor',
+    project: 'lease-renewal dispute',
+    bridge: 'tenant-notice timeline',
+    answer: 'Clause 14 cure-period memo',
+    ritual: 'Green Tab',
+    team: 'paralegal review team',
+    artifact: 'service affidavit packet',
+    stale: 'hearing expected in March',
+    middle: 'hearing moved to April',
+    final: 'hearing stayed pending mediation',
+    distractorName: 'Ameena',
+    distractorAnswer: 'Clause 9 insurance memo',
+    entityAnswer: 'Marco Bell',
+    missingTopic: 'settlement floor',
+  },
+  {
+    id: 'gsh-c05',
+    agent: 'agent-education-design',
+    domain: 'education',
+    person: 'Leo',
+    codename: 'Riverglass',
+    project: 'ninth-grade climate unit',
+    bridge: 'field-data practicum',
+    answer: 'watershed sensor dataset',
+    ritual: 'Lantern Check',
+    team: 'curriculum alignment team',
+    artifact: 'rubric calibration sheet',
+    stale: 'final project as a poster',
+    middle: 'final project as a slide deck',
+    final: 'final project as a data story notebook',
+    distractorName: 'Lio',
+    distractorAnswer: 'weather-station photo set',
+    entityAnswer: 'Nora Iqbal',
+    missingTopic: 'bus pickup location',
+  },
+  {
+    id: 'gsh-c06',
+    agent: 'agent-health-coach',
+    domain: 'healthcare',
+    person: 'Samir',
+    codename: 'Quiet Mile',
+    project: 'post-surgery walking plan',
+    bridge: 'mobility milestone tracker',
+    answer: '10-minute evening walk',
+    ritual: 'White Card',
+    team: 'care-transition team',
+    artifact: 'pain-escalation checklist',
+    stale: 'stairs avoided completely',
+    middle: 'stairs allowed with supervision',
+    final: 'stairs allowed twice daily with handrail',
+    distractorName: 'Samira',
+    distractorAnswer: '15-minute morning bike',
+    entityAnswer: 'Dr. Lin Patel',
+    missingTopic: 'pharmacy copay',
+  },
+  {
+    id: 'gsh-c07',
+    agent: 'agent-creative-studio',
+    domain: 'creative',
+    person: 'Iskandar',
+    codename: 'Glass Orchard',
+    project: 'ambient album launch',
+    bridge: 'vinyl preorder campaign',
+    answer: 'Bandcamp private presale page',
+    ritual: 'Blue Room',
+    team: 'mastering coordination team',
+    artifact: 'final lacquer approval note',
+    stale: 'release set for July',
+    middle: 'release moved to August',
+    final: 'release locked for September 6',
+    distractorName: 'Iskander',
+    distractorAnswer: 'Spotify canvas package',
+    entityAnswer: 'Marta Velasquez',
+    missingTopic: 'tour opener',
+  },
+  {
+    id: 'gsh-c08',
+    agent: 'agent-travel-planner',
+    domain: 'personal_assistant',
+    person: 'Raf',
+    codename: 'Lighthouse',
+    project: 'Lisbon family itinerary',
+    bridge: 'museum-access plan',
+    answer: 'Tile Museum quiet-entry slot',
+    ritual: 'Map Fold',
+    team: 'reservation support team',
+    artifact: 'accessibility confirmation email',
+    stale: 'hostel in Baixa',
+    middle: 'apartment in Alfama',
+    final: 'boutique hotel in Chiado',
+    distractorName: 'Rafi',
+    distractorAnswer: 'tram pass kiosk',
+    entityAnswer: 'Tomas Freire',
+    missingTopic: 'airport transfer vendor',
+  },
+  {
+    id: 'gsh-c09',
+    agent: 'agent-security-review',
+    domain: 'software',
+    person: 'Nadia',
+    codename: 'Iron Finch',
+    project: 'API abuse detection',
+    bridge: 'credential-stuffing response',
+    answer: 'edge rate-limit rule set',
+    ritual: 'Black Seal',
+    team: 'incident readiness team',
+    artifact: 'pager escalation map',
+    stale: 'block after 20 failed attempts',
+    middle: 'challenge after 15 failed attempts',
+    final: 'challenge after 8 failed attempts plus IP reputation',
+    distractorName: 'Nadya',
+    distractorAnswer: 'WAF geo-block list',
+    entityAnswer: 'Owen Hart',
+    missingTopic: 'annual security budget',
+  },
+  {
+    id: 'gsh-c10',
+    agent: 'agent-supply-chain',
+    domain: 'operations',
+    person: 'Jonas',
+    codename: 'Cedar Path',
+    project: 'warehouse slotting refresh',
+    bridge: 'cold-chain pick route',
+    answer: 'zone C thermal scanner',
+    ritual: 'Orange Tag',
+    team: 'inventory audit team',
+    artifact: 'lot-expiry reconciliation sheet',
+    stale: 'refresh every quarter',
+    middle: 'refresh every two months',
+    final: 'refresh every six weeks during peak season',
+    distractorName: 'Jona',
+    distractorAnswer: 'zone A RFID gate',
+    entityAnswer: 'Mina Park',
+    missingTopic: 'forklift maintenance vendor',
+  },
+  {
+    id: 'gsh-c11',
+    agent: 'agent-civic-data',
+    domain: 'research',
+    person: 'Yara',
+    codename: 'Civic Loom',
+    project: 'public-comment clustering',
+    bridge: 'transportation hearing archive',
+    answer: 'speaker-topic matrix',
+    ritual: 'Grey Index',
+    team: 'open-records team',
+    artifact: 'redaction exception log',
+    stale: 'archive limited to 2024 hearings',
+    middle: 'archive expanded to 2025 hearings',
+    final: 'archive covers 2023 through 2026 hearings',
+    distractorName: 'Yarra',
+    distractorAnswer: 'council-vote matrix',
+    entityAnswer: 'Hugo Stern',
+    missingTopic: 'grant renewal date',
+  },
+  {
+    id: 'gsh-c12',
+    agent: 'agent-product-analytics',
+    domain: 'software',
+    person: 'Talia',
+    codename: 'Prism Desk',
+    project: 'support-ticket insight panel',
+    bridge: 'agent-escalation taxonomy',
+    answer: 'DuckDB local analytics store',
+    ritual: 'Crimson Pass',
+    team: 'quality review team',
+    artifact: 'escalation-label audit file',
+    stale: 'charts refreshed daily',
+    middle: 'charts refreshed hourly',
+    final: 'charts refreshed every fifteen minutes',
+    distractorName: 'Thalia',
+    distractorAnswer: 'ClickHouse metrics cluster',
+    entityAnswer: 'Irene Moss',
+    missingTopic: 'customer advisory board schedule',
+  },
+];
+
+function assoc(targetId, relation, strength = 0.9) {
+  return { target_id: targetId, strength, relation };
+}
+
+function mem(id, type, timestamp, content, tags, importance, associations = [], metadata = {}) {
+  return {
+    id,
+    type,
+    timestamp,
+    content,
+    tags,
+    importance,
+    metadata,
+    associations,
+  };
+}
+
+function question(id, category, text, expected, required, forbidden = [], difficulty = 'hard', note = '') {
+  return {
+    question_id: id,
+    category,
+    question: text,
+    expected_answer: expected,
+    required_memory_ids: required,
+    forbidden_memory_ids: forbidden,
+    difficulty,
+    architecture_bias_risk: category === 'atomic_fact_recall' ? 'low' : 'intentional_graph_stress',
+    fairness_note: note,
+  };
+}
+
+function buildConversation(s, index) {
+  const p = s.id;
+  const m = (n) => `${p}-m${String(n).padStart(2, '0')}`;
+  const q = (n) => `${p}-q${String(n).padStart(2, '0')}`;
+  const baseDay = String(index + 1).padStart(2, '0');
+  const ts = (hour) => `2026-05-${baseDay}T${String(hour).padStart(2, '0')}:00:00Z`;
+
+  const memories = [
+    mem(
+      m(1),
+      'semantic',
+      ts(9),
+      `${s.person} uses codename ${s.codename} for the active workstream, but the codename itself does not state the operational dependency.`,
+      [s.codename.toLowerCase().replace(/\s+/g, '-'), 'codename'],
+      0.7,
+      [assoc(m(2), 'alias_of')],
+      { role: 'anchor', source_turn: 't01' },
+    ),
+    mem(
+      m(2),
+      'semantic',
+      ts(10),
+      `Codename ${s.codename} refers to the ${s.project}, which is governed through the ${s.bridge}.`,
+      [s.project.split(' ')[0].toLowerCase(), 'project-link'],
+      0.8,
+      [assoc(m(1), 'alias_of'), assoc(m(3), 'depends_on')],
+      { role: 'bridge', source_turn: 't02' },
+    ),
+    mem(
+      m(3),
+      'semantic',
+      ts(11),
+      `The ${s.bridge} depends on the ${s.answer}; this dependency is not named in the codename discussion.`,
+      [s.bridge.split(' ')[0].toLowerCase(), 'dependency'],
+      0.9,
+      [assoc(m(2), 'supports')],
+      { role: 'answer', source_turn: 't03' },
+    ),
+    mem(
+      m(4),
+      'episodic',
+      ts(12),
+      `${s.person} said the approval ritual is internally called ${s.ritual}; outside notes describe it as the release approval meeting.`,
+      ['approval', 'ritual'],
+      0.75,
+      [assoc(m(5), 'owned_by')],
+      { role: 'weak-overlap-anchor', source_turn: 't04' },
+    ),
+    mem(
+      m(5),
+      'semantic',
+      ts(13),
+      `${s.ritual} is owned by the ${s.team}, not by the team whose name resembles the project codename.`,
+      ['approval', 'team'],
+      0.82,
+      [assoc(m(4), 'owns'), assoc(m(6), 'requires')],
+      { role: 'weak-overlap-bridge', source_turn: 't05' },
+    ),
+    mem(
+      m(6),
+      'procedural',
+      ts(14),
+      `Before ${s.team} signs off, they require the ${s.artifact}.`,
+      ['approval', 'artifact'],
+      0.88,
+      [assoc(m(5), 'required_by')],
+      { role: 'weak-overlap-answer', source_turn: 't06' },
+    ),
+    mem(
+      m(7),
+      'semantic',
+      ts(15),
+      `Initial state for ${s.project}: ${s.stale}.`,
+      ['current-state', 'stale'],
+      0.55,
+      [assoc(m(8), 'replaced_by', 0.95)],
+      { role: 'stale', source_turn: 't07' },
+    ),
+    mem(
+      m(8),
+      'episodic',
+      ts(16),
+      `Interim update for ${s.project}: ${s.middle}, replacing the initial state.`,
+      ['current-state', 'interim'],
+      0.65,
+      [assoc(m(7), 'replaces', 0.95), assoc(m(9), 'replaced_by', 0.98)],
+      { role: 'interim', source_turn: 't08' },
+    ),
+    mem(
+      m(9),
+      'semantic',
+      ts(17),
+      `Final resolved state for ${s.project}: ${s.final}. This supersedes all earlier versions.`,
+      ['current-state', 'final'],
+      0.95,
+      [assoc(m(8), 'replaces', 0.98), assoc(m(7), 'replaces', 0.98)],
+      { role: 'final', source_turn: 't09' },
+    ),
+    mem(
+      m(10),
+      'semantic',
+      ts(18),
+      `Distractor: ${s.distractorName} runs a similar-sounding effort with ${s.distractorAnswer}, but it is not ${s.person}'s ${s.project}.`,
+      ['near-entity', 'distractor'],
+      0.5,
+      [],
+      { role: 'distractor', source_turn: 't10' },
+    ),
+    mem(
+      m(11),
+      'semantic',
+      ts(19),
+      `For ${s.person}'s ${s.project}, the accountable owner is ${s.entityAnswer}.`,
+      ['near-entity', 'owner'],
+      0.86,
+      [assoc(m(2), 'owned_by')],
+      { role: 'entity-answer', source_turn: 't11' },
+    ),
+    mem(
+      m(12),
+      'semantic',
+      ts(20),
+      `Tempting gap: ${s.distractorName}'s similar effort mentions ${s.missingTopic}, but ${s.person}'s records do not state that value.`,
+      ['abstention', 'tempting-distractor'],
+      0.45,
+      [],
+      { role: 'abstention-distractor', source_turn: 't12' },
+    ),
+  ];
+
+  const questions = [
+    question(
+      q(1),
+      'multi_hop_association',
+      `What operational dependency is ultimately used by ${s.person}'s codename ${s.codename}?`,
+      s.answer,
+      [m(1), m(2), m(3)],
+      [m(10)],
+      'hard',
+      'Requires codename to project to bridge to dependency; no single memory contains both codename and answer.',
+    ),
+    question(
+      q(2),
+      'multi_hop_association',
+      `Which artifact is needed before the release approval meeting can be signed off for ${s.person}'s work?`,
+      s.artifact,
+      [m(4), m(5), m(6)],
+      [m(10)],
+      'hard',
+      'Uses weak lexical overlap between release approval meeting and the internal ritual name.',
+    ),
+    question(
+      q(3),
+      'contradiction_resolution',
+      `What is the current resolved state for ${s.project}?`,
+      s.final,
+      [m(9)],
+      [m(7), m(8)],
+      'hard',
+      'Final memory must beat stale and interim states connected by replaces/replaced_by edges.',
+    ),
+    question(
+      q(4),
+      'graph_traversal',
+      `For the approval chain in ${s.person}'s project, which team owns the internal ritual?`,
+      s.team,
+      [m(4), m(5)],
+      [m(10)],
+      'medium',
+      'Retrieval should cross from the approval phrasing to the internal ritual and owner team.',
+    ),
+    index < 7
+      ? question(
+          q(5),
+          'entity_disambiguation',
+          `Who is the accountable owner for ${s.person}'s project, not ${s.distractorName}'s similar effort?`,
+          s.entityAnswer,
+          [m(11)],
+          [m(10), m(12)],
+          'medium',
+          'Near-name distractors are intentionally tempting but wrong.',
+        )
+      : question(
+          q(5),
+          'abstention',
+          `What is the ${s.missingTopic} for ${s.person}'s project?`,
+          `Not stated in the available memories for ${s.person}'s project.`,
+          [],
+          [m(12), m(10)],
+          'hard',
+          'The dataset contains a tempting value for a similar entity, but no answer for the target entity.',
+        ),
+  ];
+
+  return {
+    conversation_id: s.id,
+    agent_id: s.agent,
+    domain: s.domain,
+    memory_records: memories,
+    questions,
+  };
+}
+
+const conversations = scenarios.map(buildConversation);
+const questionCategories = {};
+let memoryCount = 0;
+let questionCount = 0;
+for (const conversation of conversations) {
+  memoryCount += conversation.memory_records.length;
+  questionCount += conversation.questions.length;
+  for (const q of conversation.questions) {
+    questionCategories[q.category] = (questionCategories[q.category] ?? 0) + 1;
+  }
+}
+
+const dataset = {
+  name: 'graph-stress-hard',
+  description:
+    'Diagnostic benchmark for multi-hop association, conflict resolution, graph traversal with weak lexical overlap, near-entity distractors, and abstention under tempting distractors.',
+  generated_at: '2026-06-19',
+  generated_by: 'deterministic-template',
+  version: '1.0.0',
+  fairness_notes: [
+    'This is intentionally not a provider-neutral public leaderboard fixture; it is a graph-stress diagnostic for tuning association traversal and conflict semantics.',
+    'Many questions require two or three required memories, and no single memory contains both the query anchor and final answer.',
+    'Conflict questions include stale, interim, and final states linked by replaces/replaced_by associations.',
+    'Weak-overlap graph traversal questions use different surface wording for the query anchor and the target memory.',
+    'Abstention questions contain tempting distractor facts for similar entities but no answer for the target entity.',
+  ],
+  stats: {
+    conversations: conversations.length,
+    memory_records: memoryCount,
+    questions: questionCount,
+    question_categories: questionCategories,
+  },
+  conversations,
+};
+
+fs.writeFileSync(
+  path.join(__dirname, 'dataset_graph_stress_hard.json'),
+  `${JSON.stringify(dataset, null, 2)}\n`,
+);
+
+console.log(
+  `Generated graph-stress-hard: ${conversations.length} conversations, ${memoryCount} memories, ${questionCount} questions.`,
+);