@pikku/inspector 0.12.75 → 0.12.82

package/src/add/add-agent-scorer.test.ts

@@ -1,197 +0,0 @@
-import { strict as assert } from 'assert'
-import { describe, test } from 'node:test'
-import { mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { inspect } from '../inspector.js'
-import { ErrorCode } from '../error-codes.js'
-import type { InspectorLogger } from '../types.js'
-
-const makeLogger = (criticals: Array<{ code: ErrorCode; message: string }>) =>
-  ({
-    debug: () => {},
-    info: () => {},
-    warn: () => {},
-    error: () => {},
-    diagnostic: ({ code, message }) => {
-      criticals.push({ code, message })
-    },
-    critical: (code: ErrorCode, message: string) => {
-      criticals.push({ code, message })
-    },
-    hasCriticalErrors: () => criticals.length > 0,
-  }) satisfies InspectorLogger
-
-const inspectSources = async (sources: Record<string, string>) => {
-  const rootDir = await mkdtemp(join(tmpdir(), 'pikku-add-agent-scorer-'))
-  const files: Record<string, string> = {}
-  for (const [name, source] of Object.entries(sources)) {
-    files[name] = join(rootDir, name)
-    await writeFile(files[name]!, source)
-  }
-  const criticals: Array<{ code: ErrorCode; message: string }> = []
-  try {
-    const state = await inspect(makeLogger(criticals), Object.values(files), {
-      rootDir,
-    })
-    return { state, criticals }
-  } finally {
-    await rm(rootDir, { recursive: true, force: true })
-  }
-}
-
-const inspectSource = (source: string) =>
-  inspectSources({ 'grading.scorer.ts': source })
-
-describe('addAgentScorer', () => {
-  test('reads the lane off which constructor was called, not off a field', async () => {
-    const { state, criticals } = await inspectSource(
-      [
-        `export const brevity = pikkuAgentScorer({`,
-        `  name: 'brevity',`,
-        `  description: 'Shorter answers score higher',`,
-        `  score: () => ({ score: 1 }),`,
-        `})`,
-        `export const helpfulness = pikkuAgentJudge({`,
-        `  name: 'helpfulness',`,
-        `  description: 'Does the answer help',`,
-        `  model: 'claude-opus-5',`,
-        `  goal: 'Grade helpfulness.',`,
-        `})`,
-      ].join('\n')
-    )
-
-    assert.deepEqual(criticals, [])
-    assert.equal(state.scorers.scorersMeta['brevity']?.lane, 'fast')
-    assert.equal(state.scorers.scorersMeta['helpfulness']?.lane, 'slow')
-  })
-
-  test('defaults an unstated sample rate to grading every run', async () => {
-    const { state } = await inspectSource(
-      [
-        `export const brevity = pikkuAgentScorer({`,
-        `  name: 'brevity',`,
-        `  description: 'Shorter answers score higher',`,
-        `  score: () => ({ score: 1 }),`,
-        `})`,
-      ].join('\n')
-    )
-
-    assert.equal(state.scorers.scorersMeta['brevity']?.sampleRate, 1)
-    assert.equal(state.scorers.scorersMeta['brevity']?.requiresReference, false)
-  })
-
-  test('carries the sample rate and the reference requirement into the meta', async () => {
-    const { state } = await inspectSource(
-      [
-        `export const correctness = pikkuAgentJudge({`,
-        `  name: 'correctness',`,
-        `  description: 'Is the answer right',`,
-        `  model: 'claude-opus-5',`,
-        `  goal: 'Grade correctness.',`,
-        `  sampleRate: 0.1,`,
-        `  requiresReference: true,`,
-        `})`,
-      ].join('\n')
-    )
-
-    assert.equal(state.scorers.scorersMeta['correctness']?.sampleRate, 0.1)
-    assert.equal(
-      state.scorers.scorersMeta['correctness']?.requiresReference,
-      true
-    )
-  })
-
-  test('rejects a sample rate that is not a fraction', async () => {
-    const { state, criticals } = await inspectSource(
-      [
-        `export const brevity = pikkuAgentScorer({`,
-        `  name: 'brevity',`,
-        `  description: 'Shorter answers score higher',`,
-        `  sampleRate: 50,`,
-        `  score: () => ({ score: 1 }),`,
-        `})`,
-      ].join('\n')
-    )
-
-    assert.equal(criticals[0]?.code, ErrorCode.INVALID_VALUE)
-    assert.match(criticals[0]!.message, /between 0 and 1/)
-    assert.deepEqual(state.scorers.scorersMeta, {})
-  })
-
-  test('a scorer with no description is refused, since the meta is what names it', async () => {
-    const { criticals } = await inspectSource(
-      [
-        `export const brevity = pikkuAgentScorer({`,
-        `  name: 'brevity',`,
-        `  score: () => ({ score: 1 }),`,
-        `})`,
-      ].join('\n')
-    )
-
-    assert.equal(criticals[0]?.code, ErrorCode.MISSING_DESCRIPTION)
-  })
-
-  test('records where a scorer was declared, so codegen can import it back', async () => {
-    const { state } = await inspectSource(
-      [
-        `export const brevity = pikkuAgentScorer({`,
-        `  name: 'brevity',`,
-        `  description: 'Shorter answers score higher',`,
-        `  score: () => ({ score: 1 }),`,
-        `})`,
-      ].join('\n')
-    )
-
-    assert.equal(state.scorers.files.get('brevity')?.exportedName, 'brevity')
-    assert.match(
-      state.scorers.files.get('brevity')?.path ?? '',
-      /grading\.scorer\.ts$/
-    )
-  })
-})
-
-describe('an agent asking to be graded', () => {
-  const scorerFile = [
-    `export const brevity = pikkuAgentScorer({`,
-    `  name: 'brevity',`,
-    `  description: 'Shorter answers score higher',`,
-    `  score: () => ({ score: 1 }),`,
-    `})`,
-  ].join('\n')
-
-  const agent = (scorers: string) =>
-    [
-      `export const assistant = pikkuAgent({`,
-      `  name: 'assistant',`,
-      `  description: 'Answers questions',`,
-      `  goal: 'Answer the question.',`,
-      `  model: 'anthropic/claude-opus-5',`,
-      `  scorers: [${scorers}],`,
-      `})`,
-    ].join('\n')
-
-  test('carries the named scorers onto the agent meta', async () => {
-    const { state, criticals } = await inspectSources({
-      'grading.scorer.ts': scorerFile,
-      'assistant.agent.ts': agent(`'brevity'`),
-    })
-
-    assert.deepEqual(criticals, [])
-    assert.deepEqual(state.agents.agentsMeta['assistant']?.scorers, ['brevity'])
-  })
-
-  test('naming a scorer that does not exist fails the build rather than grading nothing', async () => {
-    const { criticals } = await inspectSources({
-      'grading.scorer.ts': scorerFile,
-      'assistant.agent.ts': agent(`'brevty'`),
-    })
-
-    const finding = criticals.find(
-      (critical) => critical.code === ErrorCode.AGENT_SCORER_NOT_FOUND
-    )
-    assert.ok(finding, `expected a scorer-not-found critical`)
-    assert.match(finding!.message, /brevty/)
-    assert.match(finding!.message, /Declared scorers: brevity/)
-  })
-})

package/src/add/add-agent-scorer.ts

@@ -1,83 +0,0 @@
-import * as ts from 'typescript'
-import { getPropertyValue } from '../utils/get-property-value.js'
-import type { AddWiring } from '../types.js'
-import { extractFunctionName } from '../utils/extract-function-name.js'
-import { ErrorCode } from '../error-codes.js'
-
-/**
- * `pikkuAgentScorer` grades in code and `pikkuAgentJudge` grades with a model, which
- * is the whole difference between the two lanes — so the lane is read off which
- * constructor was called rather than from a field an author could contradict.
- */
-const LANES: Record<string, 'fast' | 'slow'> = {
-  pikkuAgentScorer: 'fast',
-  pikkuAgentJudge: 'slow',
-}
-
-export const addAgentScorer: AddWiring = (logger, node, checker, state) => {
-  if (!ts.isCallExpression(node)) return
-
-  const expression = node.expression
-  if (!ts.isIdentifier(expression)) return
-
-  const lane = LANES[expression.text]
-  if (!lane) return
-
-  const firstArg = node.arguments[0]
-  if (!firstArg || !ts.isObjectLiteralExpression(firstArg)) return
-
-  const obj = firstArg
-  const { exportedName } = extractFunctionName(node, checker, state.rootDir)
-
-  const nameValue = getPropertyValue(obj, 'name') as string | null
-  if (!nameValue) {
-    logger.critical(
-      ErrorCode.MISSING_NAME,
-      "AI scorer is missing the required 'name' property."
-    )
-    return
-  }
-
-  const description = getPropertyValue(obj, 'description') as string | null
-  if (!description) {
-    logger.critical(
-      ErrorCode.MISSING_DESCRIPTION,
-      `AI scorer '${nameValue}' is missing the required 'description' property.`
-    )
-    return
-  }
-
-  const sampleRateValue = getPropertyValue(obj, 'sampleRate') as number | null
-  if (
-    sampleRateValue !== null &&
-    (sampleRateValue < 0 || sampleRateValue > 1)
-  ) {
-    logger.critical(
-      ErrorCode.INVALID_VALUE,
-      `AI scorer '${nameValue}' has a sampleRate of ${sampleRateValue} — it is the fraction of live runs to grade, so it must be between 0 and 1.`
-    )
-    return
-  }
-
-  const requiresReference =
-    (getPropertyValue(obj, 'requiresReference') as boolean | null) ?? false
-
-  const scorerKey = exportedName || nameValue
-
-  if (exportedName) {
-    state.scorers.files.set(scorerKey, {
-      path: node.getSourceFile().fileName,
-      exportedName,
-    })
-  }
-
-  state.scorers.scorersMeta[scorerKey] = {
-    name: nameValue,
-    description,
-    lane,
-    sampleRate: sampleRateValue ?? 1,
-    requiresReference,
-    sourceFile: node.getSourceFile().fileName,
-    exportedName: exportedName || undefined,
-  }
-}

package/src/add/add-agent.test.ts

@@ -1,87 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert'
-import * as ts from 'typescript'
-
-import { resolveWorkflowReferences } from './add-agent.js'
-import type { InspectorLogger } from '../types.js'
-
-function createSourceFile(code: string): ts.SourceFile {
-  return ts.createSourceFile('test.ts', code, ts.ScriptTarget.ESNext, true)
-}
-
-function getObjectLiteral(
-  sourceFile: ts.SourceFile
-): ts.ObjectLiteralExpression {
-  let result: ts.ObjectLiteralExpression | undefined
-  ts.forEachChild(sourceFile, function visit(node) {
-    if (ts.isObjectLiteralExpression(node) && !result) {
-      result = node
-    }
-    ts.forEachChild(node, visit)
-  })
-  if (!result) throw new Error('No object literal found')
-  return result
-}
-
-function createLogger(): {
-  logger: InspectorLogger
-  criticals: Array<{ code: unknown; message: string }>
-} {
-  const criticals: Array<{ code: unknown; message: string }> = []
-  const logger = {
-    info: () => {},
-    error: () => {},
-    warn: () => {},
-    debug: () => {},
-    diagnostic: () => {},
-    critical: (code: unknown, message: string) =>
-      criticals.push({ code, message }),
-    hasCriticalErrors: () => criticals.length > 0,
-  } as unknown as InspectorLogger
-  return { logger, criticals }
-}
-
-describe('resolveWorkflowReferences', () => {
-  test('returns null when no workflows property exists', () => {
-    const obj = getObjectLiteral(createSourceFile('const x = { name: "a" }'))
-    const { logger } = createLogger()
-    assert.strictEqual(
-      resolveWorkflowReferences(obj, {} as any, 'a', logger),
-      null
-    )
-  })
-
-  test('returns null for an empty workflows array', () => {
-    const obj = getObjectLiteral(
-      createSourceFile('const x = { workflows: [] }')
-    )
-    const { logger } = createLogger()
-    assert.strictEqual(
-      resolveWorkflowReferences(obj, {} as any, 'a', logger),
-      null
-    )
-  })
-
-  test('resolves ref() and workflow() string-literal references', () => {
-    const obj = getObjectLiteral(
-      createSourceFile(
-        `const x = { workflows: [ref('buildReport'), workflow('country_capitals')] }`
-      )
-    )
-    const { logger, criticals } = createLogger()
-    const result = resolveWorkflowReferences(obj, {} as any, 'planner', logger)
-    assert.deepStrictEqual(result, ['buildReport', 'country_capitals'])
-    assert.strictEqual(criticals.length, 0)
-  })
-
-  test('flags a raw string literal as a critical error', () => {
-    const obj = getObjectLiteral(
-      createSourceFile(`const x = { workflows: ['buildReport'] }`)
-    )
-    const { logger, criticals } = createLogger()
-    const result = resolveWorkflowReferences(obj, {} as any, 'planner', logger)
-    assert.strictEqual(result, null)
-    assert.strictEqual(criticals.length, 1)
-    assert.match(criticals[0].message, /string literal/)
-  })
-})