@pikku/inspector 0.12.56 → 0.12.59

package/dist/utils/post-process.js

@@ -2,7 +2,6 @@ import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { extractTypeKeys } from './type-utils.js';
 import { ErrorCode } from '../error-codes.js';
-import { isSecretBrokerFunction } from './secret-brokers.js';
 import { findSecretAliasServices } from './secret-alias-services.js';
 import { relative } from 'node:path';
 import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js';
@@ -318,14 +317,21 @@ export function validateSecretOverrides(logger, state) {
     // older meta without a secretId field.
     const secretIds = new Set(state.secrets.definitions.map((d) => d.secretId ?? d.name));
     for (const [namespace, addonDecl] of wireAddonDeclarations.entries()) {
-        if (!addonDecl.secretOverrides)
-            continue;
-        for (const [logicalName, resolvedName] of Object.entries(addonDecl.secretOverrides)) {
+        for (const [logicalName, resolvedName] of Object.entries(addonDecl.secretOverrides ?? {})) {
             if (!secretIds.has(resolvedName)) {
                 const availableSecrets = Array.from(secretIds);
                 logger.critical(ErrorCode.INVALID_VALUE, `Secret override '${logicalName}' -> '${resolvedName}' in addon '${namespace}' (${addonDecl.package}) targets a secret that does not exist. Available secrets: ${availableSecrets.join(', ') || 'none'}`);
             }
         }
+        // A grant names the secret as the addon reads it, so it resolves through
+        // the override map before it can be looked up in the project.
+        for (const logicalName of addonDecl.secretGrants ?? []) {
+            const resolvedName = addonDecl.secretOverrides?.[logicalName] ?? logicalName;
+            if (!secretIds.has(resolvedName)) {
+                const availableSecrets = Array.from(secretIds);
+                logger.critical(ErrorCode.INVALID_VALUE, `Secret grant '${logicalName}' in addon '${namespace}' (${addonDecl.package}) targets a secret that does not exist. Available secrets: ${availableSecrets.join(', ') || 'none'}`);
+            }
+        }
     }
 }
 export function validateCredentialOverrides(logger, state) {
@@ -334,14 +340,19 @@ export function validateCredentialOverrides(logger, state) {
         return;
     const credentialNames = new Set(state.credentials?.definitions.map((d) => d.name) ?? []);
     for (const [namespace, addonDecl] of wireAddonDeclarations.entries()) {
-        if (!addonDecl.credentialOverrides)
-            continue;
-        for (const [logicalName, resolvedName] of Object.entries(addonDecl.credentialOverrides)) {
+        for (const [logicalName, resolvedName] of Object.entries(addonDecl.credentialOverrides ?? {})) {
             if (!credentialNames.has(resolvedName)) {
                 const availableCredentials = Array.from(credentialNames);
                 logger.critical(ErrorCode.INVALID_VALUE, `Credential override '${logicalName}' -> '${resolvedName}' in addon '${namespace}' (${addonDecl.package}) targets a credential that does not exist. Available credentials: ${availableCredentials.join(', ') || 'none'}`);
             }
         }
+        for (const logicalName of addonDecl.credentialGrants ?? []) {
+            const resolvedName = addonDecl.credentialOverrides?.[logicalName] ?? logicalName;
+            if (!credentialNames.has(resolvedName)) {
+                const availableCredentials = Array.from(credentialNames);
+                logger.critical(ErrorCode.INVALID_VALUE, `Credential grant '${logicalName}' in addon '${namespace}' (${addonDecl.package}) targets a credential that does not exist. Available credentials: ${availableCredentials.join(', ') || 'none'}`);
+            }
+        }
     }
 }
 export function validateVariableOverrides(logger, state) {
@@ -606,15 +617,26 @@ export function validateSchemaReferences(logger, state) {
             `is declared but not exported — export it, or inline the type at the call site.`,
     });
 }
-export function validateAgentModels(logger, state) {
+/**
+ * A model is either concrete (`provider/model`) or an alias, which must appear
+ * in the `models` table from pikku.config.json.
+ */
+export function validateAgentModels(logger, state, modelAliases = {}) {
     for (const [, meta] of Object.entries(state.agents.agentsMeta)) {
         const model = meta.model;
         if (!model) {
             logger.critical(ErrorCode.MISSING_MODEL, `AI agent '${meta.name}' is missing the 'model' property.`);
             continue;
         }
-        if (!model.includes('/')) {
-            logger.critical(ErrorCode.INVALID_MODEL, `AI agent '${meta.name}' uses model '${model}', which must be provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4').`);
+        // `hasOwn`, not a truthy lookup: every object literal inherits `toString`,
+        // `constructor` and `__proto__`, so a bare model named after one of them
+        // would read as a configured alias and pass a check nothing else repeats.
+        if (!model.includes('/') && !Object.hasOwn(modelAliases, model)) {
+            const known = Object.keys(modelAliases).sort();
+            logger.critical(ErrorCode.INVALID_MODEL, `AI agent '${meta.name}' uses model '${model}', which is neither provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4') nor an alias in the "models" table of pikku.config.json. ` +
+                (known.length
+                    ? `Known aliases: ${known.join(', ')}.`
+                    : `No "models" aliases are configured.`));
         }
     }
 }
@@ -793,14 +815,6 @@ export function validateSchemaWiringSeparation(logger, state) {
         }
     }
 }
-/** Marks the functions that keep the full `SecretService`. */
-export function computeSecretBrokers(state) {
-    for (const [id, meta] of Object.entries(state.functions.meta)) {
-        if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
-            meta.secretBroker = true;
-        }
-    }
-}
 /**
  * Refuses a singleton service that is a `SecretService` under another name.
  *
@@ -830,9 +844,6 @@ export function validateNoSecretAliasServices(logger, checker, state) {
             `SecretService is confined to pikkuServices, pikkuWireServices, addon service factories and middleware — give a service the secret value when you construct it and expose only what the function needs.`);
     };
     for (const [id, meta] of Object.entries(state.functions.meta)) {
-        if (meta.secretBroker) {
-            continue;
-        }
         report('Function', id, meta.services?.services ?? []);
     }
     for (const [id, def] of Object.entries(state.permissions.definitions)) {
@@ -1038,6 +1049,25 @@ export function validateAgentToolReferences(logger, state, options = {}) {
         }
     }
 }
+/**
+ * Validates that every scorer an agent asks to be graded by exists.
+ *
+ * The runtime warns and skips an unresolvable name rather than failing a run
+ * that has already answered the user — which means a typo would otherwise cost
+ * nothing visible and simply grade nothing, forever.
+ */
+export function validateAgentScorerReferences(logger, state) {
+    const known = Object.keys(state.scorers.scorersMeta);
+    for (const [agentKey, agent] of Object.entries(state.agents.agentsMeta)) {
+        const where = agent.sourceFile ? ` (${agent.sourceFile})` : '';
+        for (const scorer of agent.scorers ?? []) {
+            if (known.includes(scorer))
+                continue;
+            logger.critical(ErrorCode.AGENT_SCORER_NOT_FOUND, `AI agent '${agentKey}'${where} asks to be graded by scorer '${scorer}', which is not declared in this project. ` +
+                `Declared scorers: ${known.join(', ') || 'none'}.`);
+        }
+    }
+}
 /**
  * Validates that every scope granted by a system role is declared via
  * `defineScope`. Runs after all visitors, so declaration order does not matter.

package/dist/utils/serialize-inspector-state.d.ts

@@ -195,6 +195,10 @@ export interface SerializableInspectorState {
                 secretOverrides?: Record<string, string>;
                 variableOverrides?: Record<string, string>;
                 credentialOverrides?: Record<string, string>;
+                secretGrants?: string[];
+                credentialGrants?: string[];
+                globalSecrets?: string;
+                globalCredentials?: string;
             }
         ]>;
         wireAddonFiles: string[];
@@ -212,6 +216,13 @@ export interface SerializableInspectorState {
             exportedName: string;
         }][];
     };
+    scorers: {
+        scorersMeta: InspectorState['scorers']['scorersMeta'];
+        files: [string, {
+            path: string;
+            exportedName: string;
+        }][];
+    };
     cli: {
         meta: InspectorState['cli']['meta'];
         files: string[];

package/dist/utils/serialize-inspector-state.js

@@ -95,6 +95,10 @@ export function serializeInspectorState(state) {
             agentsMeta: state.agents?.agentsMeta ?? {},
             files: Array.from(state.agents?.files?.entries() ?? []),
         },
+        scorers: {
+            scorersMeta: state.scorers?.scorersMeta ?? {},
+            files: Array.from(state.scorers?.files?.entries() ?? []),
+        },
         cli: {
             meta: state.cli.meta,
             files: Array.from(state.cli.files),
@@ -268,6 +272,10 @@ export function deserializeInspectorState(data) {
             agentsMeta: data.agents?.agentsMeta || {},
             files: new Map(data.agents?.files || []),
         },
+        scorers: {
+            scorersMeta: data.scorers?.scorersMeta || {},
+            files: new Map(data.scorers?.files || []),
+        },
         cli: {
             meta: data.cli.meta,
             files: new Set(data.cli.files),

package/dist/utils/workflow/dsl/extract-dsl-workflow.js

@@ -803,19 +803,28 @@ function extractApprovalStep(call, context, outputVar) {
             step.outputVar = outputVar;
         }
         // The `schema` option is a runtime value validated inside the workflow body,
-        // so it is deliberately not serialized here — only `expiry`, which the graph
-        // and planned-step ladder need in order to describe the gate.
+        // so it is deliberately not serialized here — only the literal options the
+        // graph and planned-step ladder need in order to describe the gate.
         const options = args[1];
         if (options && ts.isObjectLiteralExpression(options)) {
             for (const prop of options.properties) {
-                if (ts.isPropertyAssignment(prop) &&
-                    prop.name.getText() === 'expiry' &&
+                if (!ts.isPropertyAssignment(prop))
+                    continue;
+                const name = prop.name.getText();
+                if (name === 'expiry' &&
                     (ts.isStringLiteral(prop.initializer) ||
                         ts.isNumericLiteral(prop.initializer))) {
                     step.expiry = ts.isNumericLiteral(prop.initializer)
                         ? Number(prop.initializer.text)
                         : prop.initializer.text;
                 }
+                if (name === 'approvers' && ts.isStringLiteral(prop.initializer)) {
+                    step.approvers = prop.initializer
+                        .text;
+                }
+                if (name === 'approverScope' && ts.isStringLiteral(prop.initializer)) {
+                    step.approverScope = prop.initializer.text;
+                }
             }
         }
         return step;

package/dist/utils/workflow/dsl/patterns.d.ts

@@ -35,7 +35,8 @@ export declare function isScenarioStepCall(node: ts.CallExpression): boolean;
 export declare function getScenarioStepPhase(node: ts.CallExpression): ScenarioStepPhaseName | undefined;
 /**
  * Check if a call expression is one of the scenario expectation helpers —
- * `scenario.expectService()`, `.expectError()` or `.expectEventually()`.
+ * `scenario.expectService()`, `.expectError()`, `.expectEventually()` or
+ * `.expectScore()`.
  */
 export declare function isScenarioExpectationCall(node: ts.CallExpression): boolean;
 /**

package/dist/utils/workflow/dsl/patterns.js

@@ -57,19 +57,22 @@ export function getScenarioStepPhase(node) {
 /**
  * The expectation helpers that assert without being a `then`.
  *
- * `expectService` and `expectError` are inline steps rather than scenario
- * steps, so they carry no phase and never reach the step meta — but each one is
- * a witness: a recorded service call, or a refusal. PKU680 has to see them, or
- * a scenario whose only assertion is one of these reads as asserting nothing.
+ * `expectService`, `expectError` and `expectScore` are inline steps rather than
+ * scenario steps, so they carry no phase and never reach the step meta — but
+ * each one is a witness: a recorded service call, a refusal, or a graded run.
+ * PKU680 has to see them, or a scenario whose only assertion is one of these
+ * reads as asserting nothing.
  */
 const SCENARIO_EXPECTATION_CALLS = [
     'expectService',
     'expectError',
     'expectEventually',
+    'expectScore',
 ];
 /**
  * Check if a call expression is one of the scenario expectation helpers —
- * `scenario.expectService()`, `.expectError()` or `.expectEventually()`.
+ * `scenario.expectService()`, `.expectError()`, `.expectEventually()` or
+ * `.expectScore()`.
  */
 export function isScenarioExpectationCall(node) {
     return (ts.isPropertyAccessExpression(node.expression) &&

package/dist/utils/workflow/graph/workflow-graph.types.d.ts

@@ -67,7 +67,8 @@ export interface NodeOptions {
  * Flow node types for control flow (no RPC call)
  */
 export type FlowType = 'sleep' | 'suspend' | 'approval' | 'branch' | 'parallel' | 'fanout' | 'inline' | 'switch' | 'filter' | 'arrayPredicate' | 'return' | 'cancel' | 'set';
-import type { ContextVariable, ScenarioStepPhase, WorkflowContext, WorkflowPlannedStep } from '@pikku/core/workflow';
+import type { ContextVariable, WorkflowContext, WorkflowPlannedStep } from '@pikku/core/workflow';
+import type { ScenarioStepPhase } from '@pikku/core/scenario';
 export type { ContextVariable, WorkflowContext };
 /**
  * Base node properties shared by all node types

package/dist/visit.js

@@ -29,6 +29,7 @@ import { addVariable } from './add/add-variable.js';
 import { addWorkflowGraph } from './add/add-workflow-graph.js';
 import { addFeature } from './add/add-feature.js';
 import { addAIAgent } from './add/add-ai-agent.js';
+import { addAIScorer } from './add/add-ai-scorer.js';
 import { addApprovalDescription } from './add/add-approval-description.js';
 export const visitSetup = (logger, checker, node, state, options) => {
     addFileExtendsCoreType(node, checker, state.singletonServicesTypeImportMap, 'CoreSingletonServices', state);
@@ -88,5 +89,6 @@ export const visitRoutes = (logger, checker, node, state, options) => {
     addWorkflowGraph(logger, node, checker, state, nextOptions);
     addFeature(logger, node, checker, state, nextOptions);
     addAIAgent(logger, node, checker, state, nextOptions);
+    addAIScorer(logger, node, checker, state, nextOptions);
     ts.forEachChild(node, (child) => visitRoutes(logger, checker, child, state, nextOptions));
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/inspector",
-  "version": "0.12.56",
+  "version": "0.12.59",
   "description": "TypeScript inspector for Pikku — discovers functions and wirings to drive code generation",
   "author": "yasser.fadl@gmail.com",
   "license": "BUSL-1.1",
@@ -45,10 +45,10 @@
     "zod-to-ts": "^2.1.0"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.80"
+    "@pikku/core": "^0.12.83"
   },
   "devDependencies": {
-    "@pikku/core": "^0.12.80",
+    "@pikku/core": "^0.12.83",
     "@types/node": "^24.11.0"
   },
   "engines": {

package/src/add/add-ai-agent.ts

@@ -330,6 +330,39 @@ function resolveIdentifierToAgentName(
   return null
 }
 
+/**
+ * The scorers an agent asks to be graded by, which are plain names rather than
+ * references: a scorer is resolved out of the registry at grading time, not
+ * imported by the agent, so nothing here has a declaration to chase.
+ */
+function resolveScorerNames(
+  obj: ts.ObjectLiteralExpression,
+  agentName: string,
+  logger: InspectorLogger
+): string[] | null {
+  const property = obj.properties.find(
+    (p) =>
+      ts.isPropertyAssignment(p) &&
+      ts.isIdentifier(p.name) &&
+      p.name.text === 'scorers'
+  )
+  if (!property || !ts.isPropertyAssignment(property)) return null
+  if (!ts.isArrayLiteralExpression(property.initializer)) return null
+
+  const names: string[] = []
+  for (const element of property.initializer.elements) {
+    if (!ts.isStringLiteral(element)) {
+      logger.critical(
+        ErrorCode.INVALID_VALUE,
+        `AI agent '${agentName}' lists a scorer that is not an inline string: \`${element.getText()}\`. A scorer is named, not imported, so the name has to be readable from source.`
+      )
+      continue
+    }
+    names.push(element.text)
+  }
+  return names.length > 0 ? names : null
+}
+
 export const addAIAgent: AddWiring = (
   logger,
   node,
@@ -377,6 +410,7 @@ export const addAIAgent: AddWiring = (
       | number
       | null
     const toolChoiceValue = getPropertyValue(obj, 'toolChoice') as string | null
+    const scorersValue = resolveScorerNames(obj, nameValue || '', logger)
     const toolsValue = resolveToolReferences(
       obj,
       checker,
@@ -582,6 +616,7 @@ export const addAIAgent: AddWiring = (
       ...(toolsValue !== null && { tools: toolsValue }),
       ...(agentsValue !== null && { agents: agentsValue }),
       ...(workflowsValue !== null && { workflows: workflowsValue }),
+      ...(scorersValue !== null && { scorers: scorersValue }),
       tags,
       inputSchema,
       outputSchema,

package/src/add/add-ai-scorer.test.ts

@@ -0,0 +1,197 @@
+import { strict as assert } from 'assert'
+import { describe, test } from 'node:test'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { inspect } from '../inspector.js'
+import { ErrorCode } from '../error-codes.js'
+import type { InspectorLogger } from '../types.js'
+
+const makeLogger = (criticals: Array<{ code: ErrorCode; message: string }>) =>
+  ({
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    diagnostic: ({ code, message }) => {
+      criticals.push({ code, message })
+    },
+    critical: (code: ErrorCode, message: string) => {
+      criticals.push({ code, message })
+    },
+    hasCriticalErrors: () => criticals.length > 0,
+  }) satisfies InspectorLogger
+
+const inspectSources = async (sources: Record<string, string>) => {
+  const rootDir = await mkdtemp(join(tmpdir(), 'pikku-add-ai-scorer-'))
+  const files: Record<string, string> = {}
+  for (const [name, source] of Object.entries(sources)) {
+    files[name] = join(rootDir, name)
+    await writeFile(files[name]!, source)
+  }
+  const criticals: Array<{ code: ErrorCode; message: string }> = []
+  try {
+    const state = await inspect(makeLogger(criticals), Object.values(files), {
+      rootDir,
+    })
+    return { state, criticals }
+  } finally {
+    await rm(rootDir, { recursive: true, force: true })
+  }
+}
+
+const inspectSource = (source: string) =>
+  inspectSources({ 'grading.scorer.ts': source })
+
+describe('addAIScorer', () => {
+  test('reads the lane off which constructor was called, not off a field', async () => {
+    const { state, criticals } = await inspectSource(
+      [
+        `export const brevity = pikkuAIScorer({`,
+        `  name: 'brevity',`,
+        `  description: 'Shorter answers score higher',`,
+        `  score: () => ({ score: 1 }),`,
+        `})`,
+        `export const helpfulness = pikkuAIJudge({`,
+        `  name: 'helpfulness',`,
+        `  description: 'Does the answer help',`,
+        `  model: 'claude-opus-5',`,
+        `  goal: 'Grade helpfulness.',`,
+        `})`,
+      ].join('\n')
+    )
+
+    assert.deepEqual(criticals, [])
+    assert.equal(state.scorers.scorersMeta['brevity']?.lane, 'fast')
+    assert.equal(state.scorers.scorersMeta['helpfulness']?.lane, 'slow')
+  })
+
+  test('defaults an unstated sample rate to grading every run', async () => {
+    const { state } = await inspectSource(
+      [
+        `export const brevity = pikkuAIScorer({`,
+        `  name: 'brevity',`,
+        `  description: 'Shorter answers score higher',`,
+        `  score: () => ({ score: 1 }),`,
+        `})`,
+      ].join('\n')
+    )
+
+    assert.equal(state.scorers.scorersMeta['brevity']?.sampleRate, 1)
+    assert.equal(state.scorers.scorersMeta['brevity']?.requiresReference, false)
+  })
+
+  test('carries the sample rate and the reference requirement into the meta', async () => {
+    const { state } = await inspectSource(
+      [
+        `export const correctness = pikkuAIJudge({`,
+        `  name: 'correctness',`,
+        `  description: 'Is the answer right',`,
+        `  model: 'claude-opus-5',`,
+        `  goal: 'Grade correctness.',`,
+        `  sampleRate: 0.1,`,
+        `  requiresReference: true,`,
+        `})`,
+      ].join('\n')
+    )
+
+    assert.equal(state.scorers.scorersMeta['correctness']?.sampleRate, 0.1)
+    assert.equal(
+      state.scorers.scorersMeta['correctness']?.requiresReference,
+      true
+    )
+  })
+
+  test('rejects a sample rate that is not a fraction', async () => {
+    const { state, criticals } = await inspectSource(
+      [
+        `export const brevity = pikkuAIScorer({`,
+        `  name: 'brevity',`,
+        `  description: 'Shorter answers score higher',`,
+        `  sampleRate: 50,`,
+        `  score: () => ({ score: 1 }),`,
+        `})`,
+      ].join('\n')
+    )
+
+    assert.equal(criticals[0]?.code, ErrorCode.INVALID_VALUE)
+    assert.match(criticals[0]!.message, /between 0 and 1/)
+    assert.deepEqual(state.scorers.scorersMeta, {})
+  })
+
+  test('a scorer with no description is refused, since the meta is what names it', async () => {
+    const { criticals } = await inspectSource(
+      [
+        `export const brevity = pikkuAIScorer({`,
+        `  name: 'brevity',`,
+        `  score: () => ({ score: 1 }),`,
+        `})`,
+      ].join('\n')
+    )
+
+    assert.equal(criticals[0]?.code, ErrorCode.MISSING_DESCRIPTION)
+  })
+
+  test('records where a scorer was declared, so codegen can import it back', async () => {
+    const { state } = await inspectSource(
+      [
+        `export const brevity = pikkuAIScorer({`,
+        `  name: 'brevity',`,
+        `  description: 'Shorter answers score higher',`,
+        `  score: () => ({ score: 1 }),`,
+        `})`,
+      ].join('\n')
+    )
+
+    assert.equal(state.scorers.files.get('brevity')?.exportedName, 'brevity')
+    assert.match(
+      state.scorers.files.get('brevity')?.path ?? '',
+      /grading\.scorer\.ts$/
+    )
+  })
+})
+
+describe('an agent asking to be graded', () => {
+  const scorerFile = [
+    `export const brevity = pikkuAIScorer({`,
+    `  name: 'brevity',`,
+    `  description: 'Shorter answers score higher',`,
+    `  score: () => ({ score: 1 }),`,
+    `})`,
+  ].join('\n')
+
+  const agent = (scorers: string) =>
+    [
+      `export const assistant = pikkuAIAgent({`,
+      `  name: 'assistant',`,
+      `  description: 'Answers questions',`,
+      `  goal: 'Answer the question.',`,
+      `  model: 'anthropic/claude-opus-5',`,
+      `  scorers: [${scorers}],`,
+      `})`,
+    ].join('\n')
+
+  test('carries the named scorers onto the agent meta', async () => {
+    const { state, criticals } = await inspectSources({
+      'grading.scorer.ts': scorerFile,
+      'assistant.agent.ts': agent(`'brevity'`),
+    })
+
+    assert.deepEqual(criticals, [])
+    assert.deepEqual(state.agents.agentsMeta['assistant']?.scorers, ['brevity'])
+  })
+
+  test('naming a scorer that does not exist fails the build rather than grading nothing', async () => {
+    const { criticals } = await inspectSources({
+      'grading.scorer.ts': scorerFile,
+      'assistant.agent.ts': agent(`'brevty'`),
+    })
+
+    const finding = criticals.find(
+      (critical) => critical.code === ErrorCode.AGENT_SCORER_NOT_FOUND
+    )
+    assert.ok(finding, `expected a scorer-not-found critical`)
+    assert.match(finding!.message, /brevty/)
+    assert.match(finding!.message, /Declared scorers: brevity/)
+  })
+})

package/src/add/add-ai-scorer.ts

@@ -0,0 +1,83 @@
+import * as ts from 'typescript'
+import { getPropertyValue } from '../utils/get-property-value.js'
+import type { AddWiring } from '../types.js'
+import { extractFunctionName } from '../utils/extract-function-name.js'
+import { ErrorCode } from '../error-codes.js'
+
+/**
+ * `pikkuAIScorer` grades in code and `pikkuAIJudge` grades with a model, which
+ * is the whole difference between the two lanes — so the lane is read off which
+ * constructor was called rather than from a field an author could contradict.
+ */
+const LANES: Record<string, 'fast' | 'slow'> = {
+  pikkuAIScorer: 'fast',
+  pikkuAIJudge: 'slow',
+}
+
+export const addAIScorer: AddWiring = (logger, node, checker, state) => {
+  if (!ts.isCallExpression(node)) return
+
+  const expression = node.expression
+  if (!ts.isIdentifier(expression)) return
+
+  const lane = LANES[expression.text]
+  if (!lane) return
+
+  const firstArg = node.arguments[0]
+  if (!firstArg || !ts.isObjectLiteralExpression(firstArg)) return
+
+  const obj = firstArg
+  const { exportedName } = extractFunctionName(node, checker, state.rootDir)
+
+  const nameValue = getPropertyValue(obj, 'name') as string | null
+  if (!nameValue) {
+    logger.critical(
+      ErrorCode.MISSING_NAME,
+      "AI scorer is missing the required 'name' property."
+    )
+    return
+  }
+
+  const description = getPropertyValue(obj, 'description') as string | null
+  if (!description) {
+    logger.critical(
+      ErrorCode.MISSING_DESCRIPTION,
+      `AI scorer '${nameValue}' is missing the required 'description' property.`
+    )
+    return
+  }
+
+  const sampleRateValue = getPropertyValue(obj, 'sampleRate') as number | null
+  if (
+    sampleRateValue !== null &&
+    (sampleRateValue < 0 || sampleRateValue > 1)
+  ) {
+    logger.critical(
+      ErrorCode.INVALID_VALUE,
+      `AI scorer '${nameValue}' has a sampleRate of ${sampleRateValue} — it is the fraction of live runs to grade, so it must be between 0 and 1.`
+    )
+    return
+  }
+
+  const requiresReference =
+    (getPropertyValue(obj, 'requiresReference') as boolean | null) ?? false
+
+  const scorerKey = exportedName || nameValue
+
+  if (exportedName) {
+    state.scorers.files.set(scorerKey, {
+      path: node.getSourceFile().fileName,
+      exportedName,
+    })
+  }
+
+  state.scorers.scorersMeta[scorerKey] = {
+    name: nameValue,
+    description,
+    lane,
+    sampleRate: sampleRateValue ?? 1,
+    requiresReference,
+    sourceFile: node.getSourceFile().fileName,
+    exportedName: exportedName || undefined,
+  }
+}

package/src/add/add-functions.ts

@@ -9,7 +9,7 @@ import {
 import { extractFunctionNode } from '../utils/extract-function-node.js'
 import { extractUsedWires } from '../utils/extract-services.js'
 import type { AuditDurability, FunctionServicesMeta } from '@pikku/core'
-import type { ScenarioStepKind, ScenarioSurface } from '@pikku/core/workflow'
+import type { ScenarioStepKind, ScenarioSurface } from '@pikku/core/scenario'
 import { formatVersionedId, parseVersionedId } from '@pikku/core'
 
 /** Binding keys a `pikkuScenarioStep` may declare, in report order. */