@runchr/gstack-antigravity 0.1.0

package/gstack-origin/test/helpers/observability.test.ts

@@ -0,0 +1,283 @@
+/**
+ * Unit tests for E2E observability infrastructure.
+ *
+ * Tests heartbeat, progress.log, NDJSON persistence, savePartial(),
+ * finalize() cleanup, failure transcript paths, watcher rendering,
+ * and non-fatal I/O guarantees.
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { sanitizeTestName } from './session-runner';
+import { EvalCollector } from './eval-store';
+import { renderDashboard } from '../../scripts/eval-watch';
+import type { HeartbeatData, PartialData } from '../../scripts/eval-watch';
+
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'obs-test-'));
+});
+
+afterEach(() => {
+  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+});
+
+// --- Test 1: runDir created when runId set ---
+
+describe('session-runner observability', () => {
+  test('1: sanitizeTestName strips slashes and leading dashes', () => {
+    expect(sanitizeTestName('/plan-ceo-review')).toBe('plan-ceo-review');
+    expect(sanitizeTestName('browse-basic')).toBe('browse-basic');
+    expect(sanitizeTestName('/qa/deep/test')).toBe('qa-deep-test');
+    expect(sanitizeTestName('///leading')).toBe('leading');
+  });
+
+  test('2: heartbeat file path uses ~/.gstack-dev/e2e-live.json', () => {
+    // Just verify the constant is correct — actual write is tested by E2E
+    const expected = path.join(os.homedir(), '.gstack-dev', 'e2e-live.json');
+    // Import the module and check HEARTBEAT_PATH exists in the file
+    const sessionRunnerSrc = fs.readFileSync(
+      path.resolve(__dirname, 'session-runner.ts'), 'utf-8'
+    );
+    expect(sessionRunnerSrc).toContain("'e2e-live.json'");
+    expect(sessionRunnerSrc).toContain('atomicWriteSync');
+  });
+
+  test('3: heartbeat JSON schema has expected fields', () => {
+    // Verify the heartbeat write code includes all required fields
+    const src = fs.readFileSync(
+      path.resolve(__dirname, 'session-runner.ts'), 'utf-8'
+    );
+    for (const field of ['runId', 'startedAt', 'currentTest', 'status', 'turn', 'toolCount', 'lastTool', 'lastToolAt', 'elapsedSec']) {
+      expect(src).toContain(field);
+    }
+    // Should NOT contain completedTests (removed per plan)
+    expect(src).not.toContain('completedTests');
+  });
+
+  test('4: progress.log format matches expected pattern', () => {
+    // The progress line format is: "  [Ns] turn T tool #C: Name(...)"
+    const src = fs.readFileSync(
+      path.resolve(__dirname, 'session-runner.ts'), 'utf-8'
+    );
+    // Both stderr and progress.log use the same progressLine variable
+    expect(src).toContain('progressLine');
+    expect(src).toContain("'progress.log'");
+    expect(src).toContain('appendFileSync');
+  });
+
+  test('5: NDJSON file uses sanitized test name', () => {
+    const src = fs.readFileSync(
+      path.resolve(__dirname, 'session-runner.ts'), 'utf-8'
+    );
+    expect(src).toContain('safeName');
+    expect(src).toContain('.ndjson');
+  });
+
+  test('8: failure transcript goes to runDir when available', () => {
+    const src = fs.readFileSync(
+      path.resolve(__dirname, 'session-runner.ts'), 'utf-8'
+    );
+    // Should use runDir as primary, workingDirectory as fallback
+    expect(src).toContain('runDir || path.join(workingDirectory');
+    expect(src).toContain('-failure.json');
+  });
+
+  test('11: all new I/O is wrapped in try/catch (non-fatal)', () => {
+    const src = fs.readFileSync(
+      path.resolve(__dirname, 'session-runner.ts'), 'utf-8'
+    );
+    // Count non-fatal comments — should be present for each new I/O path
+    const nonFatalCount = (src.match(/\/\* non-fatal \*\//g) || []).length;
+    // Original had 2 (promptFile unlink + failure transcript), we added 4 more
+    // (runDir creation, progress.log, heartbeat, NDJSON append)
+    expect(nonFatalCount).toBeGreaterThanOrEqual(6);
+  });
+});
+
+// --- Tests 6, 7: eval-store savePartial() and finalize() ---
+
+describe('eval-store observability', () => {
+  test('6: savePartial() writes valid JSON with _partial: true', () => {
+    const evalDir = path.join(tmpDir, 'evals');
+    const collector = new EvalCollector('e2e', evalDir);
+
+    collector.addTest({
+      name: 'test-one',
+      suite: 'test',
+      tier: 'e2e',
+      passed: true,
+      duration_ms: 1000,
+      cost_usd: 0.05,
+      exit_reason: 'success',
+    });
+
+    const partialPath = path.join(evalDir, '_partial-e2e.json');
+    expect(fs.existsSync(partialPath)).toBe(true);
+
+    const partial = JSON.parse(fs.readFileSync(partialPath, 'utf-8'));
+    expect(partial._partial).toBe(true);
+    expect(partial.tests).toHaveLength(1);
+    expect(partial.tests[0].name).toBe('test-one');
+    expect(partial.tests[0].exit_reason).toBe('success');
+    expect(partial.schema_version).toBe(1);
+    expect(partial.total_tests).toBe(1);
+    expect(partial.passed).toBe(1);
+  });
+
+  test('6b: savePartial() accumulates multiple tests', () => {
+    const evalDir = path.join(tmpDir, 'evals');
+    const collector = new EvalCollector('e2e', evalDir);
+
+    collector.addTest({
+      name: 'test-one', suite: 'test', tier: 'e2e',
+      passed: true, duration_ms: 1000, cost_usd: 0.05,
+    });
+    collector.addTest({
+      name: 'test-two', suite: 'test', tier: 'e2e',
+      passed: false, duration_ms: 2000, cost_usd: 0.10,
+      exit_reason: 'timeout', timeout_at_turn: 5, last_tool_call: 'Bash(ls)',
+    });
+
+    const partialPath = path.join(evalDir, '_partial-e2e.json');
+    const partial = JSON.parse(fs.readFileSync(partialPath, 'utf-8'));
+    expect(partial.tests).toHaveLength(2);
+    expect(partial.total_tests).toBe(2);
+    expect(partial.passed).toBe(1);
+    expect(partial.failed).toBe(1);
+    expect(partial.tests[1].exit_reason).toBe('timeout');
+    expect(partial.tests[1].timeout_at_turn).toBe(5);
+    expect(partial.tests[1].last_tool_call).toBe('Bash(ls)');
+  });
+
+  test('7: finalize() preserves partial file alongside final', async () => {
+    const evalDir = path.join(tmpDir, 'evals');
+    const collector = new EvalCollector('e2e', evalDir);
+
+    collector.addTest({
+      name: 'test-one', suite: 'test', tier: 'e2e',
+      passed: true, duration_ms: 1000, cost_usd: 0.05,
+    });
+
+    const partialPath = path.join(evalDir, '_partial-e2e.json');
+    expect(fs.existsSync(partialPath)).toBe(true);
+
+    await collector.finalize();
+
+    // Partial file preserved for observability — never cleaned up
+    expect(fs.existsSync(partialPath)).toBe(true);
+
+    // Final eval file should also exist
+    const files = fs.readdirSync(evalDir).filter(f => f.endsWith('.json') && !f.startsWith('_'));
+    expect(files.length).toBeGreaterThanOrEqual(1);
+  });
+
+  test('EvalTestEntry includes diagnostic fields', () => {
+    const evalDir = path.join(tmpDir, 'evals');
+    const collector = new EvalCollector('e2e', evalDir);
+
+    collector.addTest({
+      name: 'diagnostic-test', suite: 'test', tier: 'e2e',
+      passed: false, duration_ms: 5000, cost_usd: 0.20,
+      exit_reason: 'error_max_turns',
+      timeout_at_turn: undefined,
+      last_tool_call: 'Write(review-output.md)',
+    });
+
+    const partialPath = path.join(evalDir, '_partial-e2e.json');
+    const partial = JSON.parse(fs.readFileSync(partialPath, 'utf-8'));
+    const t = partial.tests[0];
+    expect(t.exit_reason).toBe('error_max_turns');
+    expect(t.last_tool_call).toBe('Write(review-output.md)');
+  });
+});
+
+// --- Tests 9, 10: watcher dashboard rendering ---
+
+describe('eval-watch dashboard', () => {
+  test('9: renderDashboard shows completed tests and current test', () => {
+    const heartbeat: HeartbeatData = {
+      runId: '20260314-143022',
+      startedAt: '2026-03-14T14:30:22Z',
+      currentTest: 'plan-ceo-review',
+      status: 'running',
+      turn: 4,
+      toolCount: 3,
+      lastTool: 'Write(review-output.md)',
+      lastToolAt: new Date().toISOString(), // recent — not stale
+      elapsedSec: 285,
+    };
+
+    const partial: PartialData = {
+      tests: [
+        { name: 'browse basic', passed: true, cost_usd: 0.07, duration_ms: 24000, turns_used: 6 },
+        { name: '/review', passed: true, cost_usd: 0.17, duration_ms: 63000, turns_used: 13 },
+      ],
+      total_cost_usd: 0.24,
+      _partial: true,
+    };
+
+    const output = renderDashboard(heartbeat, partial);
+
+    // Should contain run ID
+    expect(output).toContain('20260314-143022');
+
+    // Should show completed tests
+    expect(output).toContain('browse basic');
+    expect(output).toContain('/review');
+    expect(output).toContain('$0.07');
+    expect(output).toContain('$0.17');
+
+    // Should show current test
+    expect(output).toContain('plan-ceo-review');
+    expect(output).toContain('turn 4');
+    expect(output).toContain('Write(review-output.md)');
+
+    // Should NOT show stale warning (lastToolAt is recent)
+    expect(output).not.toContain('STALE');
+  });
+
+  test('10: renderDashboard warns on stale heartbeat', () => {
+    const staleTime = new Date(Date.now() - 15 * 60 * 1000).toISOString(); // 15 min ago
+
+    const heartbeat: HeartbeatData = {
+      runId: '20260314-143022',
+      startedAt: '2026-03-14T14:30:22Z',
+      currentTest: 'plan-ceo-review',
+      status: 'running',
+      turn: 4,
+      toolCount: 3,
+      lastTool: 'Write(review-output.md)',
+      lastToolAt: staleTime,
+      elapsedSec: 900,
+    };
+
+    const output = renderDashboard(heartbeat, null);
+
+    expect(output).toContain('STALE');
+    expect(output).toContain('may have crashed');
+  });
+
+  test('renderDashboard handles no active run', () => {
+    const output = renderDashboard(null, null);
+    expect(output).toContain('No active run');
+    expect(output).toContain('bun test');
+  });
+
+  test('renderDashboard handles partial-only (heartbeat gone)', () => {
+    const partial: PartialData = {
+      tests: [
+        { name: 'browse basic', passed: true, cost_usd: 0.07, duration_ms: 24000 },
+      ],
+      total_cost_usd: 0.07,
+      _partial: true,
+    };
+
+    const output = renderDashboard(null, partial);
+    expect(output).toContain('browse basic');
+    expect(output).toContain('$0.07');
+  });
+});

package/gstack-origin/test/helpers/session-runner.test.ts

@@ -0,0 +1,96 @@
+import { describe, test, expect } from 'bun:test';
+import { parseNDJSON } from './session-runner';
+
+// Fixture: minimal NDJSON session (system init, assistant with tool_use, tool result, assistant text, result)
+const FIXTURE_LINES = [
+  '{"type":"system","subtype":"init","session_id":"test-123"}',
+  '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tu1","name":"Bash","input":{"command":"echo hello"}}]}}',
+  '{"type":"user","tool_use_result":{"tool_use_id":"tu1","stdout":"hello\\n","stderr":""}}',
+  '{"type":"assistant","message":{"content":[{"type":"text","text":"The command printed hello."}]}}',
+  '{"type":"assistant","message":{"content":[{"type":"text","text":"Let me also read a file."},{"type":"tool_use","id":"tu2","name":"Read","input":{"file_path":"/tmp/test"}}]}}',
+  '{"type":"result","subtype":"success","total_cost_usd":0.05,"num_turns":3,"usage":{"input_tokens":100,"output_tokens":50},"result":"Done."}',
+];
+
+describe('parseNDJSON', () => {
+  test('parses valid NDJSON with system + assistant + result events', () => {
+    const parsed = parseNDJSON(FIXTURE_LINES);
+    expect(parsed.transcript).toHaveLength(6);
+    expect(parsed.transcript[0].type).toBe('system');
+    expect(parsed.transcript[5].type).toBe('result');
+  });
+
+  test('extracts tool calls from assistant.message.content[].type === tool_use', () => {
+    const parsed = parseNDJSON(FIXTURE_LINES);
+    expect(parsed.toolCalls).toHaveLength(2);
+    expect(parsed.toolCalls[0]).toEqual({
+      tool: 'Bash',
+      input: { command: 'echo hello' },
+      output: '',
+    });
+    expect(parsed.toolCalls[1]).toEqual({
+      tool: 'Read',
+      input: { file_path: '/tmp/test' },
+      output: '',
+    });
+    expect(parsed.toolCallCount).toBe(2);
+  });
+
+  test('skips malformed lines without throwing', () => {
+    const lines = [
+      '{"type":"system"}',
+      'this is not json',
+      '{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}}',
+      '{incomplete json',
+      '{"type":"result","subtype":"success","result":"done"}',
+    ];
+    const parsed = parseNDJSON(lines);
+    expect(parsed.transcript).toHaveLength(3); // system, assistant, result
+    expect(parsed.resultLine?.subtype).toBe('success');
+  });
+
+  test('skips empty and whitespace-only lines', () => {
+    const lines = [
+      '',
+      '  ',
+      '{"type":"system"}',
+      '\t',
+      '{"type":"result","subtype":"success","result":"ok"}',
+    ];
+    const parsed = parseNDJSON(lines);
+    expect(parsed.transcript).toHaveLength(2);
+  });
+
+  test('extracts resultLine from type: "result" event', () => {
+    const parsed = parseNDJSON(FIXTURE_LINES);
+    expect(parsed.resultLine).not.toBeNull();
+    expect(parsed.resultLine.subtype).toBe('success');
+    expect(parsed.resultLine.total_cost_usd).toBe(0.05);
+    expect(parsed.resultLine.num_turns).toBe(3);
+    expect(parsed.resultLine.result).toBe('Done.');
+  });
+
+  test('counts turns correctly — one per assistant event, not per text block', () => {
+    const parsed = parseNDJSON(FIXTURE_LINES);
+    // 3 assistant events in fixture (tool_use, text, text+tool_use)
+    expect(parsed.turnCount).toBe(3);
+  });
+
+  test('handles empty input', () => {
+    const parsed = parseNDJSON([]);
+    expect(parsed.transcript).toHaveLength(0);
+    expect(parsed.resultLine).toBeNull();
+    expect(parsed.turnCount).toBe(0);
+    expect(parsed.toolCallCount).toBe(0);
+    expect(parsed.toolCalls).toHaveLength(0);
+  });
+
+  test('handles assistant event with no content array', () => {
+    const lines = [
+      '{"type":"assistant","message":{}}',
+      '{"type":"assistant"}',
+    ];
+    const parsed = parseNDJSON(lines);
+    expect(parsed.turnCount).toBe(2);
+    expect(parsed.toolCalls).toHaveLength(0);
+  });
+});