@pikku/core 0.12.83 → 0.12.86

package/tsconfig.type-tests.json

@@ -7,6 +7,7 @@
     "declarationMap": false
   },
   "files": [
-    "src/secret-value.test.ts"
+    "src/classification/secret-value.test.ts",
+    "src/types/typed-credentials.test.ts"
   ]
-}
+}

package/dist/testing/service-tests/ai-storage-service-tests.d.ts

@@ -1,3 +0,0 @@
-import type { ServiceTestConfig } from '../service-tests.js';
-/** Conformance suite for `aiStorageService`. Runs only when a backend supplies one. */
-export declare const defineAiStorageServiceTests: (name: string, aiStorageService: NonNullable<ServiceTestConfig["services"]["aiStorageService"]>) => void;

package/dist/wirings/ai-agent/ai-agent-registry.d.ts

@@ -1,16 +0,0 @@
-import type { CoreAIAgent } from './ai-agent.types.js';
-import type { AIRunStateService } from '../../services/ai-run-state-service.js';
-import { type RunAIAgentParams } from './ai-agent-prepare.js';
-export declare const addAIAgent: (agentName: string, agent: CoreAIAgent<any, any>, packageName?: string | null) => void;
-export declare function approveAIAgent(aiRunState: AIRunStateService, runId: string, approvals: {
-    toolCallId: string;
-    approved: boolean;
-}[], expectedAgentName?: string, params?: RunAIAgentParams): Promise<{
-    status: 'resumed' | 'suspended';
-    runId: string;
-    approved: string[];
-    rejected: string[];
-    remainingApprovals: number;
-}>;
-export declare const getAIAgents: () => Map<string, CoreAIAgent>;
-export declare const getAIAgentsMeta: () => import("./ai-agent.types.js").AIAgentMeta;

package/dist/wirings/ai-agent/ai-agent-runner.d.ts

@@ -1,7 +0,0 @@
-import type { AIAgentInput, AIAgentOutput } from './ai-agent.types.js';
-import { type RunAIAgentParams } from './ai-agent-prepare.js';
-export declare function runAIAgent(agentName: string, input: AIAgentInput, params: RunAIAgentParams, agentSessionMap?: Map<string, string>): Promise<AIAgentOutput>;
-export declare function resumeAIAgentSync(runId: string, approvals: {
-    toolCallId: string;
-    approved: boolean;
-}[], params: RunAIAgentParams, expectedAgentName?: string): Promise<AIAgentOutput>;

package/dist/wirings/ai-agent/ai-agent-utils.d.ts

@@ -1 +0,0 @@
-export declare const randomUUID: () => string;

package/dist/wirings/ai-agent/ai-agent-utils.js

@@ -1 +0,0 @@
-export const randomUUID = () => globalThis.crypto.randomUUID();

package/dist/wirings/ai-agent/index.d.ts

@@ -1,12 +0,0 @@
-export { agent, agentStream, agentResume, agentApprove, agentInterrupt, } from './ai-agent-helpers.js';
-export { wrapChannelWithAGUI } from './ai-agent-agui.js';
-export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js';
-export { resolveModelAlias } from './ai-agent-model-config.js';
-export { streamAIAgent, resumeAIAgent, interruptAIAgent, } from './ai-agent-stream.js';
-export { voiceInput, NoSpeechDetectedError, SPOKEN_TURN, SPOKEN_TRANSCRIPT, } from './voice-input.js';
-export { voiceOutput, unspeakableScripts, voiceForText, type SpeakableScripts, } from './voice-output.js';
-export { AgentInterruptedError, signalRunInterrupt, } from './ai-agent-interrupt.js';
-export type { AgentInterruption, AgentInterruptResult, InterruptibleRunHandle, } from './ai-agent-interrupt.js';
-export { type RunAIAgentParams, type StreamAIAgentOptions, ToolApprovalRequired, ToolCredentialRequired, canAccessThread, isOwnedByPrincipal, threadOwnerConstraint, } from './ai-agent-prepare.js';
-export { addAIAgent, } from './ai-agent-registry.js';
-export type { AIAgentInput, AIAgentMeta, AIAgentMemoryConfig, AIAgentStep, AIContentPart, AgentRunRow, AgentRunService, AgentRunState, AIMessage, AIStreamChannel, AIStreamEvent, AIThread, CoreAIAgent, PendingApproval, PikkuAIMiddlewareHooks, } from './ai-agent.types.js';

package/dist/wirings/ai-agent/index.js

@@ -1,10 +0,0 @@
-export { agent, agentStream, agentResume, agentApprove, agentInterrupt, } from './ai-agent-helpers.js';
-export { wrapChannelWithAGUI } from './ai-agent-agui.js';
-export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js';
-export { resolveModelAlias } from './ai-agent-model-config.js';
-export { streamAIAgent, resumeAIAgent, interruptAIAgent, } from './ai-agent-stream.js';
-export { voiceInput, NoSpeechDetectedError, SPOKEN_TURN, SPOKEN_TRANSCRIPT, } from './voice-input.js';
-export { voiceOutput, unspeakableScripts, voiceForText, } from './voice-output.js';
-export { AgentInterruptedError, signalRunInterrupt, } from './ai-agent-interrupt.js';
-export { ToolApprovalRequired, ToolCredentialRequired, canAccessThread, isOwnedByPrincipal, threadOwnerConstraint, } from './ai-agent-prepare.js';
-export { addAIAgent, } from './ai-agent-registry.js';

package/dist/wirings/ai-scorer/ai-scorer-judge.d.ts

@@ -1,17 +0,0 @@
-import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js';
-import type { PikkuAIScorer, ScorerInput, ScorerJudgeConfig, ScorerOutput } from './ai-scorer.types.js';
-/**
- * The message a judge grades when its scorer supplies no `prompt`.
- *
- * A reference-based judge is shown the answer key; a reference-free one is not,
- * and grades the answer on its own merits.
- */
-export declare const buildJudgePrompt: (judge: ScorerJudgeConfig, input: ScorerInput) => string;
-/**
- * Run an LLM judge and force the `{ score, reason }` contract, so a scorer
- * author writes a rubric rather than a parser.
- *
- * A judge has no tools, which is the one case where an output schema is
- * reliably honoured.
- */
-export declare const runJudge: (scorer: PikkuAIScorer<any>, input: ScorerInput, aiAgentRunner: AIAgentRunnerService | undefined) => Promise<ScorerOutput>;

package/dist/wirings/ai-scorer/ai-scorer.types.js

@@ -1,4 +0,0 @@
-export const SCORER_LANE_QUEUES = {
-    fast: 'ai-score-fast',
-    slow: 'ai-score-slow',
-};

package/dist/wirings/ai-scorer/index.d.ts

@@ -1,6 +0,0 @@
-export { pikkuAIScorer, pikkuAIJudge } from './ai-scorer.js';
-export { addAIScorer, getAIScorers, getAIScorersMeta, } from './ai-scorer-registry.js';
-export { gradeRun } from './ai-scorer-grade.js';
-export { enableScoreSnapshots, getScoreSnapshot, } from './ai-scorer-snapshots.js';
-export { wireAIScorerQueueWorkers, } from './ai-scorer-worker.js';
-export type { AIRunScore, PikkuAIScorer, ScoreJob, ScorerInput, ScorerJudgeConfig, ScorerLane, ScorerMeta, ScorerOutput, } from './ai-scorer.types.js';

package/dist/wirings/ai-scorer/index.js

@@ -1,5 +0,0 @@
-export { pikkuAIScorer, pikkuAIJudge } from './ai-scorer.js';
-export { addAIScorer, getAIScorers, getAIScorersMeta, } from './ai-scorer-registry.js';
-export { gradeRun } from './ai-scorer-grade.js';
-export { enableScoreSnapshots, getScoreSnapshot, } from './ai-scorer-snapshots.js';
-export { wireAIScorerQueueWorkers, } from './ai-scorer-worker.js';

package/src/ecosystem.ts

@@ -1,40 +0,0 @@
-/**
- * The surface a package in the Pikku ecosystem reaches for — a runtime adapter,
- * a service, the code generator's output, the CLI — and application code does
- * not.
- *
- * Split out from the package root so 1.0's compatibility promise covers what
- * applications actually import. These signatures move when the runtime needs
- * them to — `runPikkuFunc` was reshaped over the last year while every field on
- * `PikkuWire` survived — and promising stability on both would mean promising
- * the weaker of the two.
- *
- * Not `/internal`: generated bootstrap files import from here, so the specifier
- * lands in the user's own `.pikku` directory, and telling someone they are
- * touching internals when the code generator put it there is both wrong and
- * self-defeating — it could never be broken anyway. Not `/runtime` either: that
- * reads as runtime-versus-compile-time, i.e. the real API, and `packages/runtimes/*`
- * already claims the word while the CLI is the largest consumer here.
- *
- * `/internal` stays as an alias because the pinned bootstrap CLI still emits it.
- *
- * knowledge: decisions/internals/the-ecosystem-entry-point-carries-the-adapter-surface.md
- */
-export {
-  pikkuState,
-  resetPikkuState,
-  getAllPackageStates,
-  getSingletonServices,
-  getCreateWireServices,
-  setSingletonServices,
-} from './pikku-state.js'
-export { runPikkuFunc, addFunction } from './function/function-runner.js'
-export {
-  addGlobalMiddleware,
-  addTagMiddleware as addMiddleware,
-} from './middleware-runner.js'
-export { httpRouter } from './wirings/http/routers/http-router.js'
-export type {
-  CreateSingletonServices,
-  CreateWireServices,
-} from './types/core.types.js'

package/src/index.ts

@@ -1,255 +0,0 @@
-/**
- * @module @pikku/core
- */
-export type {
-  AuthInstance,
-  CommonWireMeta,
-  CoreConfig,
-  CorePikkuMiddleware,
-  CorePikkuMiddlewareConfig,
-  CorePikkuMiddlewareFactory,
-  CorePikkuMiddlewareGroup,
-  CoreServices,
-  CoreSecretlessSingletonServices,
-  CoreSingletonServices,
-  CoreUserSession,
-  SecretlessServices,
-  CreateConfig,
-  ServerLifecycle,
-  FunctionMeta,
-  FunctionServicesMeta,
-  FunctionWiresMeta,
-  FunctionsMeta,
-  FunctionsRuntimeMeta,
-  JSONValue,
-  MakeRequired,
-  MiddlewareMetadata,
-  MiddlewarePriority,
-  PermissionMetadata,
-  PickOptional,
-  PickRequired,
-  PikkuAIMiddlewareHooks,
-  PikkuWire,
-  PikkuRawWire,
-  PikkuWiringTypes,
-  PostgresConfig,
-  RequireAtLeastOne,
-  SecurityAuditIssue,
-  SecurityAuditReport,
-  SecurityAuditSummary,
-  SecurityAuditUpdate,
-  SecuritySeverity,
-  SecurityUpdateLevel,
-  SerializedError,
-} from './types/core.types.js'
-export {
-  pikkuAIMiddleware,
-  pikkuChannelMiddleware,
-  pikkuChannelMiddlewareFactory,
-  pikkuMiddleware,
-  pikkuMiddlewareFactory,
-} from './types/core.types.js'
-export type {
-  CorePikkuAuth,
-  CorePikkuAuthConfig,
-  CorePikkuFunction,
-  CorePikkuFunctionConfig,
-  CorePikkuPermission,
-  CorePikkuPermissionConfig,
-  CorePikkuPermissionFactory,
-  CorePikkuApprovalDescription,
-  CorePermissionGroup,
-} from './function/functions.types.js'
-export {
-  pikkuAuth,
-  pikkuPermission,
-  pikkuPermissionFactory,
-  pikkuApprovalDescription,
-} from './function/functions.types.js'
-export { getAllFunctionNames } from './function/index.js'
-export type {
-  ListInput,
-  ListOutput,
-  Filter,
-} from './function/list.types.js'
-export { pikkuCLIRender } from './wirings/cli/cli-runner.js'
-export { PikkuRequest } from './pikku-request.js'
-export {
-  getRelativeTimeOffsetFromNow,
-  parseDurationString,
-} from './time-utils.js'
-export type { RelativeTimeInput } from './time-utils.js'
-export {
-  formatVersionedId,
-  isVersionedId,
-  parseVersionedId,
-} from './version.js'
-export {
-  AbandonedError,
-  type AbortScope,
-} from './function/abort-scope.js'
-export { fetch } from './wirings/http/http-runner.js'
-export type {
-  MCPToolResponse,
-  MCPResourceResponse,
-  MCPPromptResponse,
-} from './wirings/mcp/mcp.types.js'
-export {
-  AIProviderAuthError,
-  AIProviderNotConfiguredError,
-  BadGatewayError,
-  BadRequestError,
-  ConflictError,
-  ExpectationFailedError,
-  ForbiddenError,
-  GatewayTimeoutError,
-  GoneError,
-  HTTPVersionNotSupportedError,
-  InternalServerError,
-  InvalidMiddlewareWireError,
-  InvalidOriginError,
-  InvalidSessionError,
-  LengthRequiredError,
-  LocalEnvironmentOnlyError,
-  LockedError,
-  MaxComputeTimeReachedError,
-  MethodNotAllowedError,
-  MissingCredentialError,
-  MissingSchemaError,
-  MissingScopeError,
-  MissingServiceError,
-  MissingSessionError,
-  NotAcceptableError,
-  NotFoundError,
-  NotImplementedError,
-  PayloadTooLargeError,
-  PaymentRequiredError,
-  PikkuMissingMetaError,
-  PreconditionFailedError,
-  ProxyAuthenticationRequiredError,
-  RangeNotSatisfiableError,
-  ReadonlySessionError,
-  RequestTimeoutError,
-  ServiceUnavailableError,
-  TooManyRequestsError,
-  URITooLongError,
-  UnauthorizedError,
-  UnprocessableContentError,
-  UnsupportedMediaTypeError,
-} from './errors/errors.js'
-export { PikkuError, isExpectedError } from './errors/error-handler.js'
-export type { EventHubService } from './wirings/channel/eventhub-service.js'
-export type { QueueService } from './wirings/queue/queue.types.js'
-export type { JWTService } from './services/jwt-service.js'
-export type {
-  EmailService,
-  SendEmailInput,
-  SendEmailResult,
-  SendHTMLEmailInput,
-  SendTemplateEmailInput,
-  SendTextEmailInput,
-} from './services/email-service.js'
-export type { SecretService } from './services/secret-service.js'
-export {
-  SecretAccessDeniedError,
-  withoutSecrets,
-} from './services/secretless.js'
-export {
-  SecretHostNotAllowedError,
-  assertSecretAllowedForHost,
-} from './services/secret-host-binding.js'
-export type { VariablesService } from './services/variables-service.js'
-export type {
-  ContentService,
-  SignContentKeyArgs,
-  SignURLArgs,
-  GetUploadURLArgs,
-  UploadURLResult,
-  BucketKeyArgs,
-  WriteFileArgs,
-  CopyFileArgs,
-} from './services/content-service.js'
-export type { DeploymentService } from './services/deployment-service.js'
-export type { WorkflowService } from './services/workflow-service.js'
-export type { GatewayService } from './services/gateway-service.js'
-export type { TriggerService } from './services/trigger-service.js'
-export type { SchemaService } from './services/schema-service.js'
-export type { SessionService } from './services/user-session-service.js'
-export {
-  NoopAuditService,
-  createInvocationAudit,
-} from './services/audit-service.js'
-export type {
-  AuditConfig,
-  AuditDurability,
-  AuditEvent,
-  AuditEventBatch,
-  AuditFacets,
-  AuditLog,
-  AuditQuery,
-  AuditQueryResult,
-  AuditService,
-  AuditUserIdentity,
-  ResolvedAuditConfig,
-} from './services/audit-service.js'
-export type {
-  AIAgentRunnerService,
-} from './services/ai-agent-runner-service.js'
-export type { AIEmbeddingService } from './services/ai-embedding-service.js'
-export type { AIRunStateService } from './services/ai-run-state-service.js'
-export type { AIStorageService } from './services/ai-storage-service.js'
-export type {
-  EmailsMeta,
-  EmailTemplateMeta,
-  MetaService,
-} from './services/meta-service.js'
-export type { HTTPMethod } from './wirings/http/http.types.js'
-export type { GraphNodeConfig } from './wirings/workflow/graph/workflow-graph.types.js'
-export { createGraph } from './wirings/workflow/graph/graph-node.js'
-export { wireAddon } from './wirings/rpc/wire-addon.js'
-export type { WireAddonConfig } from './wirings/rpc/wire-addon.js'
-export { wireRemoteAddon } from './wirings/rpc/wire-remote-addon.js'
-export type {
-  WireRemoteAddonConfig,
-  RemoteAddonAuth,
-} from './wirings/rpc/wire-remote-addon.js'
-export type { PikkuPackageState } from './types/state.types.js'
-export { runMiddleware, addTagMiddleware } from './middleware-runner.js'
-export { addGlobalPermission, checkAuthPermissions } from './permissions.js'
-export { hasScopes, verifyScopes } from './scopes.js'
-export {
-  isSerializable,
-  stopSingletonServices,
-  pikkuServerLifecycle,
-} from './utils.js'
-export { clearPikkuRuntimeState } from './test-utils.js'
-export {
-  type ScheduledTaskInfo,
-  type ScheduledTaskSummary,
-} from './services/scheduler-service.js'
-export { SchedulerService } from './services/scheduler-service.js'
-
-export type {
-  Private,
-  Pii,
-  Secret,
-  Classification,
-  AnonymizeStrategy,
-  ClassificationManifest,
-  ColumnForm,
-  WrappedValue,
-  SealedValue,
-  HashedValue,
-} from './data-classification.js'
-
-export {
-  hashToken,
-} from './column-form.js'
-
-export type { SecretValue, Safe } from './secret-value.js'
-export {
-  createSecretValue,
-  isSecretValue,
-  SecretCoercionError,
-  REDACTED,
-} from './secret-value.js'

package/src/wirings/ai-agent/ai-agent-utils.ts

@@ -1 +0,0 @@
-export const randomUUID = (): string => globalThis.crypto.randomUUID()

package/src/wirings/ai-agent/index.ts

@@ -1,65 +0,0 @@
-export {
-  agent,
-  agentStream,
-  agentResume,
-  agentApprove,
-  agentInterrupt,
-} from './ai-agent-helpers.js'
-export { wrapChannelWithAGUI } from './ai-agent-agui.js'
-export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
-export { resolveModelAlias } from './ai-agent-model-config.js'
-export {
-  streamAIAgent,
-  resumeAIAgent,
-  interruptAIAgent,
-} from './ai-agent-stream.js'
-export {
-  voiceInput,
-  NoSpeechDetectedError,
-  SPOKEN_TURN,
-  SPOKEN_TRANSCRIPT,
-} from './voice-input.js'
-export {
-  voiceOutput,
-  unspeakableScripts,
-  voiceForText,
-  type SpeakableScripts,
-} from './voice-output.js'
-export {
-  AgentInterruptedError,
-  signalRunInterrupt,
-} from './ai-agent-interrupt.js'
-export type {
-  AgentInterruption,
-  AgentInterruptResult,
-  InterruptibleRunHandle,
-} from './ai-agent-interrupt.js'
-export {
-  type RunAIAgentParams,
-  type StreamAIAgentOptions,
-  ToolApprovalRequired,
-  ToolCredentialRequired,
-  canAccessThread,
-  isOwnedByPrincipal,
-  threadOwnerConstraint,
-} from './ai-agent-prepare.js'
-export {
-  addAIAgent,
-} from './ai-agent-registry.js'
-export type {
-  AIAgentInput,
-  AIAgentMeta,
-  AIAgentMemoryConfig,
-  AIAgentStep,
-  AIContentPart,
-  AgentRunRow,
-  AgentRunService,
-  AgentRunState,
-  AIMessage,
-  AIStreamChannel,
-  AIStreamEvent,
-  AIThread,
-  CoreAIAgent,
-  PendingApproval,
-  PikkuAIMiddlewareHooks,
-} from './ai-agent.types.js'

package/src/wirings/ai-scorer/ai-scorer-judge.test.ts

@@ -1,143 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-import { buildJudgePrompt, runJudge } from './ai-scorer-judge.js'
-import { pikkuAIJudge } from './ai-scorer.js'
-import type { ScorerInput } from './ai-scorer.types.js'
-import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
-
-const input = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
-  runId: 'run-1',
-  agentName: 'assistant',
-  input: 'what is the capital of France?',
-  output: 'Paris',
-  toolCalls: [],
-  usage: { inputTokens: 10, outputTokens: 5 },
-  ...overrides,
-})
-
-const runner = (
-  object: unknown,
-  seen?: { params: any }
-): AIAgentRunnerService =>
-  ({
-    run: async (params: any) => {
-      if (seen) seen.params = params
-      return { object, usage: { inputTokens: 100, outputTokens: 20 } }
-    },
-  }) as unknown as AIAgentRunnerService
-
-describe('buildJudgePrompt', () => {
-  test('shows the answer key to a reference-based judge', () => {
-    const scorer = pikkuAIJudge({
-      name: 'correctness',
-      description: 'Is the answer right',
-      model: 'claude-opus-5',
-      goal: 'Grade correctness.',
-      requiresReference: true,
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({ reference: 'Paris' })
-    )
-
-    assert.match(prompt, /Reference answer:\nParis/)
-  })
-
-  test('withholds a reference section when there is no answer key', () => {
-    const scorer = pikkuAIJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-
-    const prompt = buildJudgePrompt(scorer.judge!, input())
-
-    assert.doesNotMatch(prompt, /Reference answer/)
-    assert.match(prompt, /Grade helpfulness\./)
-  })
-
-  test('a scorer that supplies its own prompt replaces the framing entirely', () => {
-    const scorer = pikkuAIJudge({
-      name: 'custom',
-      description: 'Custom framing',
-      model: 'claude-opus-5',
-      goal: 'ignored',
-      prompt: (run) => `Only this: ${run.output}`,
-    })
-
-    assert.equal(buildJudgePrompt(scorer.judge!, input()), 'Only this: Paris')
-  })
-})
-
-describe('runJudge', () => {
-  const helpfulness = pikkuAIJudge({
-    name: 'helpfulness',
-    description: 'Is the answer useful',
-    model: 'claude-opus-5',
-    goal: 'Grade helpfulness.',
-  })
-
-  test('records the score, the reason and what the judgement cost', async () => {
-    const result = await runJudge(
-      helpfulness,
-      input(),
-      runner({ score: 0.8, reason: 'Direct and correct.' })
-    )
-
-    assert.equal(result.score, 0.8)
-    assert.equal(result.reason, 'Direct and correct.')
-    assert.deepEqual(result.metadata, {
-      judgeModel: 'claude-opus-5',
-      judgeTokens: 120,
-    })
-  })
-
-  test('clamps an out-of-range score rather than losing the judgement', async () => {
-    const high = await runJudge(helpfulness, input(), runner({ score: 1.4 }))
-    const low = await runJudge(helpfulness, input(), runner({ score: -2 }))
-
-    assert.equal(high.score, 1)
-    assert.equal(low.score, 0)
-  })
-
-  test('rejects a non-numeric score, which is not a judgement at all', async () => {
-    await assert.rejects(
-      () => runJudge(helpfulness, input(), runner({ score: 'great' })),
-      /non-numeric score/
-    )
-  })
-
-  test('gives the judge no tools, so its structured output is actually honoured', async () => {
-    const seen = { params: undefined as any }
-    await runJudge(helpfulness, input(), runner({ score: 1 }, seen))
-
-    assert.deepEqual(seen.params.tools, [])
-    assert.equal(seen.params.toolChoice, 'none')
-    assert.equal(seen.params.maxSteps, 1)
-    assert.equal(seen.params.model, 'claude-opus-5')
-  })
-
-  test('names the missing provider rather than failing as an undefined call', async () => {
-    await assert.rejects(
-      () => runJudge(helpfulness, input(), undefined),
-      /needs an AI provider/
-    )
-  })
-
-  test('refuses to grade a reference-based judge with no answer key', async () => {
-    const correctness = pikkuAIJudge({
-      name: 'correctness',
-      description: 'Is the answer right',
-      model: 'claude-opus-5',
-      goal: 'Grade correctness.',
-      requiresReference: true,
-    })
-
-    await assert.rejects(
-      () => runJudge(correctness, input(), runner({ score: 1 })),
-      /grades against a reference answer/
-    )
-  })
-})