dsh-llm-verifier 0.1.13 → 0.1.16

package/src/team-gate.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, it } from 'vitest'
+import { Session } from '@deepseek-ai/dsh-session'
+import { inspectTeamTasks, buildTeamTaskVerificationPrompt } from './team-gate.ts'
+
+describe('team-gate', () => {
+  it('inspects team tasks and detects recently completed tasks', () => {
+    const session = Session.create('session-team-1' as never)
+    session.append('team/task' as never, { task: { id: 'task-1', revision: 1, subject: 'Write unit tests', status: 'in_progress' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-2', revision: 1, subject: 'Update docs', status: 'pending' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-1', revision: 2, subject: 'Write unit tests', description: 'Ensure 100% coverage', status: 'completed' } } as never)
+
+    const inspection = inspectTeamTasks(session.events, 0)
+    expect(inspection.hasRecentCompletedTask).toBe(true)
+    expect(inspection.latestCompletedTask?.id).toBe('task-1')
+    expect(inspection.latestCompletedTask?.status).toBe('completed')
+    expect(inspection.latestCompletedTask?.description).toBe('Ensure 100% coverage')
+    expect(inspection.activeTasks).toHaveLength(2)
+    expect(inspection.completedTasks).toHaveLength(1)
+    expect(inspection.completedTasks[0]?.seq).toBe(2)
+  })
+
+  it('lists every completed task once, oldest first', () => {
+    const session = Session.create('session-team-order' as never)
+    session.append('team/task' as never, { task: { id: 'task-a', revision: 1, subject: 'A', status: 'completed' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-b', revision: 1, subject: 'B', status: 'in_progress' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-b', revision: 2, subject: 'B', status: 'completed' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-a', revision: 2, subject: 'A', status: 'in_progress' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-a', revision: 3, subject: 'A', status: 'completed' } } as never)
+
+    const inspection = inspectTeamTasks(session.events, 0)
+    expect(inspection.completedTasks.map(entry => entry.task.id)).toEqual(['task-b', 'task-a'])
+    expect(inspection.completedTasks.map(entry => entry.seq)).toEqual([2, 4])
+    // A reopened task is verified once, at its newest completion.
+    expect(inspection.latestCompletedTask?.id).toBe('task-a')
+    expect(inspection.latestCompletedTask?.revision).toBe(3)
+  })
+
+  it('drops a completion that was reopened or deleted afterwards', () => {
+    const session = Session.create('session-team-ghost' as never)
+    session.append('team/task' as never, { task: { id: 'task-x', revision: 1, subject: 'X', status: 'completed' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-x', revision: 2, subject: 'X', status: 'in_progress' } } as never)
+    const reopened = inspectTeamTasks(session.events, 0)
+    expect(reopened.completedTasks).toEqual([])
+    expect(reopened.latestCompletedTask).toBeUndefined()
+    expect(reopened.hasRecentCompletedTask).toBe(false)
+
+    session.append('team/task' as never, { task: { id: 'task-x', revision: 3, subject: 'X', status: 'deleted' } } as never)
+    expect(inspectTeamTasks(session.events, 0).completedTasks).toEqual([])
+
+    session.append('team/task' as never, { task: { id: 'task-x', revision: 4, subject: 'X', status: 'completed' } } as never)
+    expect(inspectTeamTasks(session.events, 0).completedTasks.map(entry => entry.seq)).toEqual([3])
+  })
+
+  it('filters deleted tasks from active tasks', () => {
+    const session = Session.create('session-team-2' as never)
+    session.append('team/task' as never, { task: { id: 'task-1', revision: 1, subject: 'Task A', status: 'in_progress' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-2', revision: 1, subject: 'Task B', status: 'pending' } } as never)
+    session.append('team/task' as never, { task: { id: 'task-2', revision: 2, subject: 'Task B', status: 'deleted' } } as never)
+
+    const inspection = inspectTeamTasks(session.events, 0)
+    expect(inspection.activeTasks).toHaveLength(1)
+    expect(inspection.activeTasks[0]?.id).toBe('task-1')
+    expect(inspection.hasRecentCompletedTask).toBe(false)
+    expect(inspection.completedTasks).toEqual([])
+  })
+
+  it('respects fromSeq boundary for recent completed task', () => {
+    const session = Session.create('session-team-3' as never)
+    session.append('team/task' as never, { task: { id: 'task-1', revision: 1, subject: 'Initial setup', status: 'completed' } } as never)
+    // seq 0 is task-1 completed
+    session.append('team/task' as never, { task: { id: 'task-2', revision: 1, subject: 'Next step', status: 'in_progress' } } as never)
+
+    const inspection = inspectTeamTasks(session.events, 1)
+    expect(inspection.hasRecentCompletedTask).toBe(false)
+    expect(inspection.latestCompletedTask).toBeUndefined()
+    expect(inspection.completedTasks).toEqual([])
+  })
+
+  it('builds verification prompt containing task metadata and trace evidence', () => {
+    const prompt = buildTeamTaskVerificationPrompt(
+      { id: 'task-100', revision: 2, subject: 'Build Artifact Bundle', description: 'Produce dist/index.js and dist/client.js', status: 'completed' },
+      'Executed pnpm build: completed in 1.2s with zero errors.',
+      10000
+    )
+    expect(prompt).toContain('You are an expert independent technical verifier reviewing a completed Agent Teams task')
+    expect(prompt).toContain('ID: task-100')
+    expect(prompt).toContain('Subject: Build Artifact Bundle')
+    expect(prompt).toContain('Description: Produce dist/index.js')
+    expect(prompt).toContain('Executed pnpm build: completed in 1.2s')
+    expect(prompt).toContain('Verdict: <single uppercase letter A-T>')
+  })
+})

package/src/team-gate.ts

@@ -0,0 +1,80 @@
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+import type { TeamTaskItem } from './router.ts'
+import { sanitizeVerifierText } from './session.ts'
+
+/** One completed task and the sequence number of its completion. */
+export interface CompletedTeamTask {
+  task: TeamTaskItem
+  seq: number
+}
+
+export interface TeamTaskInspection {
+  /** Latest completion in the window; kept for callers that only need one. */
+  latestCompletedTask?: TeamTaskItem
+  completedSeq?: number
+  /** Every task completed in the window, one entry per task id, oldest first. */
+  completedTasks: CompletedTeamTask[]
+  /** Known non-deleted tasks, including completed ones. */
+  activeTasks: TeamTaskItem[]
+  hasRecentCompletedTask: boolean
+}
+
+export function inspectTeamTasks(events: readonly SessionEvent[], fromSeq = 0): TeamTaskInspection {
+  const taskMap = new Map<string, TeamTaskItem>()
+  // Sequence of the newest completion per task id. Reopening or deleting a task
+  // clears its entry, so a stale completion can never outlive the task status it
+  // came from; completing the same task again records the new sequence.
+  const completionSeq = new Map<string, number>()
+
+  for (const rawEvent of events) {
+    const event = rawEvent as unknown as { type: string; seq: number; data: any }
+    if (event.type === 'team/task') {
+      const data = event.data as { task?: TeamTaskItem }
+      if (data?.task) {
+        taskMap.set(data.task.id, { ...data.task })
+        if (data.task.status === 'completed') completionSeq.set(data.task.id, event.seq)
+        else completionSeq.delete(data.task.id)
+      }
+    }
+  }
+
+  const completedTasks: CompletedTeamTask[] = [...completionSeq.entries()]
+    .filter(([id, seq]) => seq >= fromSeq && taskMap.get(id)?.status === 'completed')
+    .map(([id, seq]) => ({ task: { ...(taskMap.get(id) as TeamTaskItem) }, seq }))
+    .sort((left, right) => left.seq - right.seq)
+  const latest = completedTasks.at(-1)
+
+  return {
+    latestCompletedTask: latest?.task,
+    completedSeq: latest?.seq,
+    completedTasks,
+    activeTasks: [...taskMap.values()].filter(t => t.status !== 'deleted'),
+    hasRecentCompletedTask: latest !== undefined,
+  }
+}
+
+export function buildTeamTaskVerificationPrompt(task: TeamTaskItem, executionTrace: string, maxChars = 20000): string {
+  return [
+    'You are an expert independent technical verifier reviewing a completed Agent Teams task.',
+    'Verify if the task goal has been satisfied by concrete evidence (code edits, test runs, successful commands).',
+    'Task Details:',
+    'ID: ' + task.id,
+    'Subject: ' + task.subject,
+    task.description ? 'Description: ' + task.description : '',
+    '',
+    'Execution Evidence & Trace:',
+    sanitizeVerifierText(executionTrace, maxChars),
+    '',
+    'Evaluate whether the task is genuinely completed with verified proof.',
+    'Scoring criteria:',
+    'A: Completely fulfilled with solid execution/test evidence.',
+    'B-J: Mostly complete, minor gaps or edge cases not fully asserted.',
+    'K-S: Incomplete, missing key requirements or untested.',
+    'T: Unfulfilled, failed, or contradictory evidence.',
+    '',
+    'Output format:',
+    'The first line must be exactly "Verdict: <single uppercase letter A-T>" — one letter, nothing else on the line.',
+    'Line 2: Summary: <One sentence evaluation of task fulfillment>',
+    'Line 3+: Remaining gaps or next steps for the team.',
+  ].filter(Boolean).join('\n\n')
+}

package/src/top-logprobs.test.ts

@@ -1,84 +1,84 @@
-import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
-import { readFileSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { describe, expect, it } from 'vitest'
-import { CAPABILITY_TTL_MS, TopLogprobCapabilityCache, resolveCapabilityFile } from './top-logprobs.ts'
-
-describe('TopLogprobCapabilityCache persistence', () => {
-  it('round-trips marks across instances through the capability file', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
-    try {
-      const file = resolveCapabilityFile(dir)
-      const first = new TopLogprobCapabilityCache(file)
-      expect(first.isUnsupported('openai', 'gpt-5')).toBe(false)
-      first.markUnsupported('openai', 'gpt-5')
-      await first.flush()
-      expect(existsSync(file)).toBe(true)
-
-      const second = new TopLogprobCapabilityCache(file)
-      await second.ensureLoaded()
-      expect(second.isUnsupported('openai', 'gpt-5')).toBe(true)
-      expect(second.isUnsupported('openai', 'other')).toBe(false)
-
-      // A mark from a later instance must merge with hydrated entries, not clobber them.
-      second.markUnsupported('anthropic', 'claude')
-      await second.flush()
-      const raw = JSON.parse(readFileSync(file, 'utf8')) as { version: number; entries: Record<string, number> }
-      expect(raw.version).toBe(1)
-      expect(Object.keys(raw.entries)).toHaveLength(2)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-
-  it('drops marks older than the TTL on hydration', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
-    try {
-      const file = join(dir, 'capabilities-v1.json')
-      writeFileSync(file, JSON.stringify({ version: 1, entries: { 'openai\0gpt-5': Date.now() - CAPABILITY_TTL_MS - 1, 'openai\0fresh': Date.now() } }))
-      const cache = new TopLogprobCapabilityCache(file)
-      await cache.ensureLoaded()
-      expect(cache.isUnsupported('openai', 'gpt-5')).toBe(false)
-      expect(cache.isUnsupported('openai', 'fresh')).toBe(true)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-
-  it('ignores missing and corrupt capability files', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
-    try {
-      const corrupt = join(dir, 'corrupt.json')
-      writeFileSync(corrupt, 'not json at all')
-      const corrupted = new TopLogprobCapabilityCache(corrupt)
-      await expect(corrupted.ensureLoaded()).resolves.toBeUndefined()
-      expect(corrupted.isUnsupported('openai', 'gpt-5')).toBe(false)
-
-      const absent = new TopLogprobCapabilityCache(join(dir, 'absent.json'))
-      await expect(absent.ensureLoaded()).resolves.toBeUndefined()
-      absent.markUnsupported('openai', 'gpt-5')
-      await absent.flush()
-      expect(absent.isUnsupported('openai', 'gpt-5')).toBe(true)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-
-  it('keeps the freshest mark when hydration races an in-process probe', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
-    try {
-      const file = join(dir, 'capabilities-v1.json')
-      writeFileSync(file, JSON.stringify({ version: 1, entries: { 'openai\0gpt-5': Date.now() - 5_000 } }))
-      const cache = new TopLogprobCapabilityCache(file)
-      const floor = Date.now()
-      cache.markUnsupported('openai', 'gpt-5')
-      await cache.ensureLoaded()
-      await cache.flush()
-      const raw = JSON.parse(readFileSync(file, 'utf8')) as { entries: Record<string, number> }
-      expect(raw.entries['openai\0gpt-5']).toBeGreaterThanOrEqual(floor)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-
-  it('keeps the in-memory-only behavior when no file is configured', async () => {
-    const cache = new TopLogprobCapabilityCache()
-    await expect(cache.ensureLoaded()).resolves.toBeUndefined()
-    cache.markUnsupported('p', 'm')
-    expect(cache.isUnsupported('p', 'm')).toBe(true)
-    expect(cache.isUnsupported('p', 'other')).toBe(false)
-  })
-})
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { CAPABILITY_TTL_MS, TopLogprobCapabilityCache, resolveCapabilityFile } from './top-logprobs.ts'
+
+describe('TopLogprobCapabilityCache persistence', () => {
+  it('round-trips marks across instances through the capability file', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
+    try {
+      const file = resolveCapabilityFile(dir)
+      const first = new TopLogprobCapabilityCache(file)
+      expect(first.isUnsupported('openai', 'gpt-5')).toBe(false)
+      first.markUnsupported('openai', 'gpt-5')
+      await first.flush()
+      expect(existsSync(file)).toBe(true)
+
+      const second = new TopLogprobCapabilityCache(file)
+      await second.ensureLoaded()
+      expect(second.isUnsupported('openai', 'gpt-5')).toBe(true)
+      expect(second.isUnsupported('openai', 'other')).toBe(false)
+
+      // A mark from a later instance must merge with hydrated entries, not clobber them.
+      second.markUnsupported('anthropic', 'claude')
+      await second.flush()
+      const raw = JSON.parse(readFileSync(file, 'utf8')) as { version: number; entries: Record<string, number> }
+      expect(raw.version).toBe(1)
+      expect(Object.keys(raw.entries)).toHaveLength(2)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+
+  it('drops marks older than the TTL on hydration', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
+    try {
+      const file = join(dir, 'capabilities-v1.json')
+      writeFileSync(file, JSON.stringify({ version: 1, entries: { 'openai\0gpt-5': Date.now() - CAPABILITY_TTL_MS - 1, 'openai\0fresh': Date.now() } }))
+      const cache = new TopLogprobCapabilityCache(file)
+      await cache.ensureLoaded()
+      expect(cache.isUnsupported('openai', 'gpt-5')).toBe(false)
+      expect(cache.isUnsupported('openai', 'fresh')).toBe(true)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+
+  it('ignores missing and corrupt capability files', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
+    try {
+      const corrupt = join(dir, 'corrupt.json')
+      writeFileSync(corrupt, 'not json at all')
+      const corrupted = new TopLogprobCapabilityCache(corrupt)
+      await expect(corrupted.ensureLoaded()).resolves.toBeUndefined()
+      expect(corrupted.isUnsupported('openai', 'gpt-5')).toBe(false)
+
+      const absent = new TopLogprobCapabilityCache(join(dir, 'absent.json'))
+      await expect(absent.ensureLoaded()).resolves.toBeUndefined()
+      absent.markUnsupported('openai', 'gpt-5')
+      await absent.flush()
+      expect(absent.isUnsupported('openai', 'gpt-5')).toBe(true)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+
+  it('keeps the freshest mark when hydration races an in-process probe', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-caps-'))
+    try {
+      const file = join(dir, 'capabilities-v1.json')
+      writeFileSync(file, JSON.stringify({ version: 1, entries: { 'openai\0gpt-5': Date.now() - 5_000 } }))
+      const cache = new TopLogprobCapabilityCache(file)
+      const floor = Date.now()
+      cache.markUnsupported('openai', 'gpt-5')
+      await cache.ensureLoaded()
+      await cache.flush()
+      const raw = JSON.parse(readFileSync(file, 'utf8')) as { entries: Record<string, number> }
+      expect(raw.entries['openai\0gpt-5']).toBeGreaterThanOrEqual(floor)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+
+  it('keeps the in-memory-only behavior when no file is configured', async () => {
+    const cache = new TopLogprobCapabilityCache()
+    await expect(cache.ensureLoaded()).resolves.toBeUndefined()
+    cache.markUnsupported('p', 'm')
+    expect(cache.isUnsupported('p', 'm')).toBe(true)
+    expect(cache.isUnsupported('p', 'other')).toBe(false)
+  })
+})