@exercode/problem-utils 2.2.2 → 2.3.1

package/dist/presets/command.js.map

@@ -1 +1 @@
-{"version":3,"file":"command.js","names":["readFileTestCases"],"sources":["../../src/presets/command.ts"],"sourcesContent":["import path from 'node:path';\n\nimport { z } from 'zod';\n\nimport { checkProblemDirIsolation } from '../helpers/checkProblemDirIsolation.js';\nimport { cleanWorkingDirectory, snapshotWorkingDirectory } from '../helpers/cleanWorkingDirectory.js';\nimport { copyTestCaseFileInput } from '../helpers/copyTestCaseFileInput.js';\nimport { findEntryPointFile } from '../helpers/findEntryPointFile.js';\nimport { findLanguageDefinitionByPath } from '../helpers/findLanguageDefinitionByPath.js';\nimport { judgeByStaticAnalysis } from '../helpers/judgeByStaticAnalysis.js';\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { printDebugBanner } from '../helpers/printDebugBanner.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { readOutputFiles } from '../helpers/readOutputFiles.js';\nimport { runCustomRunner } from '../helpers/runCustomRunner.js';\nimport {\n  getSandboxUserEnvOverrides,\n  makeAccessibleToSandboxUser,\n  wrapCommandWithSandboxUser,\n} from '../helpers/sandboxUser.js';\nimport { readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { readTestCases as readFileTestCases } from '../helpers/readTestCases.js';\nimport {\n  printDebugCwdBanner,\n  printDebugExpectationFailureBanner,\n  resolveCwds,\n  type ResolvedCwd,\n} from '../helpers/resolveCwds.js';\nimport { spawnSyncWithTimeout } from '../helpers/spawnSyncWithTimeout.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { ProblemMarkdownFrontMatter } from '../types/problem.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst BUILD_TIMEOUT_SECONDS = 10;\nconst JUDGE_DEFAULT_TIMEOUT_SECONDS = 2;\nconst MAX_STDOUT_LENGTH = 50_000;\n\nconst judgeParamsSchema = z.object({\n  language: z.union([z.string(), z.array(z.string())]).optional(),\n});\n\ntype JudgeParams = z.infer<typeof judgeParamsSchema>;\n\ninterface BaseCommandTestCase {\n  id: string;\n  input?: string;\n  fileInputPath?: string;\n}\n\ntype CommandJudgeCaseResult = Pick<TestCaseResult, 'decisionCode' | 'feedbackMarkdown' | 'stderr'>;\n\nexport interface CommandRunResult {\n  stdin: string;\n  stdout: string;\n  stderr: string;\n  status: number | undefined;\n  timeSeconds: number;\n  memoryBytes: number;\n  outputLimitExceeded?: boolean;\n}\n\ninterface CommandJudgeContext {\n  timeLimitSeconds: number;\n  outputLimitLength: number;\n  problemMarkdownFrontMatter: Pick<ProblemMarkdownFrontMatter, 'memoryLimitByte' | 'requiredOutputFilePaths'>;\n}\n\nexport interface CommandJudgeLimits {\n  buildTimeoutSeconds: number;\n  maxOutputLength: number;\n}\n\nexport interface CommandJudgePresetOptions<\n  TTestCase extends BaseCommandTestCase = BaseCommandTestCase,\n  TRunResult extends CommandRunResult = CommandRunResult,\n> {\n  /**\n   * Set to false when the custom runner builds the submitted source itself.\n   * Defaults to true.\n   */\n  buildSubmission?: boolean;\n  limits?: CommandJudgeLimits;\n  runTimeoutSeconds?: number;\n  readTestCases?: (problemDir: string) => Promise<readonly TTestCase[]>;\n  /**\n   * Runs as the trusted harness user with the submission's `cwd`. Fixture files it creates there\n   * must be written with `createDirectoryWithoutFollowingSymlinks`/`writeFileWithoutFollowingSymlinks`\n   * (both exported): a submission can plant a symlink at a fixture path, and a plain `fs.writeFile`\n   * would follow it into a file only the harness can write.\n   */\n  resolveInput?: (context: { testCase: TTestCase; cwd: string; env: NodeJS.ProcessEnv }) => Promise<string> | string;\n  runCommand?: (context: {\n    testCase: TTestCase;\n    /**\n     * The command to run. Under `EXERCODE_SANDBOX_USER` delegation it is already wrapped so it\n     * executes as the sandbox user; spawn it as given (with the supplied `env`) instead of\n     * reconstructing it, or the submission runs as the trusted harness user.\n     *\n     * Its direct child is then a root-owned `sudo` whose descendants belong to the sandbox user, so\n     * the handler cannot signal them: enforce `timeLimitSeconds` with `startSandboxTimeoutWatchdog`\n     * and `killSandboxUserProcesses` (both exported) rather than `child.kill()` or an outer\n     * `timeout`. The preset terminates leftover sandbox processes after the handler returns.\n     */\n    command: readonly [string, ...string[]];\n    stdin: string;\n    cwd: string;\n    env: NodeJS.ProcessEnv;\n    timeLimitSeconds: number;\n  }) => Promise<TRunResult> | TRunResult;\n  test?: (context: {\n    testCase: TTestCase;\n    runResult: TRunResult;\n    outputFiles: NonNullable<TestCaseResult['outputFiles']>;\n    context: CommandJudgeContext;\n  }) => Promise<Partial<CommandJudgeCaseResult>> | Partial<CommandJudgeCaseResult> | undefined;\n}\n\n/**\n * A preset function for judging by executable command.\n *\n * Keep problem-specific logic in `resolveInput` and `test`.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { commandJudgePreset } from '@exercode/problem-utils/presets/command';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await commandJudgePreset(import.meta.dirname, {\n *   readTestCases: async () => [\n *     { id: '01', input: '1 2' },\n *   ],\n *   test: ({ runResult }) => {\n *     return runResult.stdout.trim() === '3'\n *       ? { decisionCode: DecisionCode.ACCEPTED }\n *       : { decisionCode: DecisionCode.WRONG_ANSWER };\n *   },\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/python '{ \"language\": \"python\" }'\n * ```\n *\n * Run without a cwd argument to judge each `<problemDir>/model_answers/*` directory\n * for debugging. A prominent banner is printed for each run.\n * ```bash\n * bun judge.ts\n * ```\n */\nexport async function commandJudgePreset<\n  TTestCase extends BaseCommandTestCase = BaseCommandTestCase,\n  TRunResult extends CommandRunResult = CommandRunResult,\n>(problemDir: string, options: CommandJudgePresetOptions<TTestCase, TRunResult>): Promise<void> {\n  const args = parseArgs(process.argv);\n  const params = judgeParamsSchema.parse(args.params);\n\n  const { cwds, isDebugMode } = await resolveCwds(problemDir, args.cwd);\n\n  if (isDebugMode) {\n    const acceptedCwd = cwds.find((cwd) => cwd.expectedResult === 'accepted');\n    if (acceptedCwd) {\n      const isolationCheckResult = await checkProblemDirIsolation(problemDir, acceptedCwd, params);\n      if (!isolationCheckResult.passed) {\n        process.exitCode = 1;\n        return;\n      }\n    } else {\n      printDebugBanner([\n        '[DEBUG MODE] isolated problem directory check skipped',\n        '',\n        'No accepted model answer is available for checking that the copied judge still accepts a valid submission.',\n      ]);\n    }\n  }\n\n  for (const resolvedCwd of cwds) {\n    if (isDebugMode) printDebugCwdBanner(problemDir, resolvedCwd);\n    const result = await runCommandJudgeForCwd<TTestCase, TRunResult>(problemDir, resolvedCwd.cwd, params, options);\n    if (isDebugMode && !matchesExpectedResult(resolvedCwd, result)) {\n      process.exitCode = 1;\n      printDebugExpectationFailureBanner(problemDir, resolvedCwd);\n    }\n  }\n}\n\nasync function runCommandJudgeForCwd<\n  TTestCase extends BaseCommandTestCase,\n  TRunResult extends CommandRunResult = CommandRunResult,\n>(\n  problemDir: string,\n  cwd: string,\n  params: JudgeParams,\n  options: CommandJudgePresetOptions<TTestCase, TRunResult>\n): Promise<{ allAccepted: boolean }> {\n  // The sandboxed submission must read its sources and write build/run outputs in its directory.\n  makeAccessibleToSandboxUser(cwd);\n\n  const problemMarkdownFrontMatter = await readProblemMarkdownFrontMatter(problemDir);\n  const testCases = await (options.readTestCases ?? readCommandTestCases)(problemDir);\n  const prebuildTestCaseId = testCases[0]?.id ?? 'prebuild';\n  const limits = {\n    buildTimeoutSeconds: options.limits?.buildTimeoutSeconds ?? BUILD_TIMEOUT_SECONDS,\n    maxOutputLength: options.limits?.maxOutputLength ?? MAX_STDOUT_LENGTH,\n  };\n  const runTimeoutSeconds = options.runTimeoutSeconds ?? JUDGE_DEFAULT_TIMEOUT_SECONDS;\n\n  const staticAnalysisResult = await judgeByStaticAnalysis(cwd, problemMarkdownFrontMatter);\n  if (staticAnalysisResult) {\n    printTestCaseResult({ testCaseId: prebuildTestCaseId, ...staticAnalysisResult });\n    return { allAccepted: false };\n  }\n\n  const originalMainFilePath = await findEntryPointFile(cwd, params.language);\n  if (!originalMainFilePath) {\n    printTestCaseResult({\n      testCaseId: prebuildTestCaseId,\n      decisionCode: DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,\n      stderr: `main file not found${params.language ? `: language: ${params.language}` : ''}`,\n    });\n    return { allAccepted: false };\n  }\n\n  const languageDefinition = findLanguageDefinitionByPath(originalMainFilePath);\n  if (!languageDefinition) {\n    printTestCaseResult({\n      testCaseId: prebuildTestCaseId,\n      decisionCode: DecisionCode.WRONG_ANSWER,\n      stderr: 'unsupported language',\n    });\n    return { allAccepted: false };\n  }\n\n  // `CI` changes affects Chainlit. `FORCE_COLOR` affects Bun.\n  const env = { ...process.env, CI: '', FORCE_COLOR: '0' };\n\n  let mainFilePath = originalMainFilePath;\n  if (options.buildSubmission !== false) {\n    if (languageDefinition.prebuild) {\n      try {\n        await languageDefinition.prebuild(cwd);\n        const prebuiltMainFilePath = await findEntryPointFile(cwd, params.language);\n        if (prebuiltMainFilePath) mainFilePath = prebuiltMainFilePath;\n      } catch (error) {\n        printTestCaseResult({\n          testCaseId: prebuildTestCaseId,\n          decisionCode: DecisionCode.BUILD_ERROR,\n          stderr: error instanceof Error ? error.message : String(error),\n        });\n        return { allAccepted: false };\n      }\n    }\n\n    const buildCommand = languageDefinition.buildCommand?.(mainFilePath);\n    if (buildCommand) {\n      const buildResult = runBuild(buildCommand, {\n        cwd,\n        env,\n        testCaseId: prebuildTestCaseId,\n        limits,\n      });\n      if (buildResult) {\n        printTestCaseResult(buildResult);\n        return { allAccepted: false };\n      }\n    }\n  }\n\n  const cwdSnapshot = await snapshotWorkingDirectory(cwd);\n\n  if (testCases.length === 0) {\n    printTestCaseResult({ testCaseId: 'default', decisionCode: DecisionCode.ACCEPTED });\n    return { allAccepted: true };\n  }\n\n  const sharedFileInputPath = (testCases as { shared?: { fileInputPath?: string } }).shared?.fileInputPath;\n\n  for (const testCase of testCases) {\n    if (sharedFileInputPath) await copyTestCaseFileInput(sharedFileInputPath, cwd);\n    if (testCase.fileInputPath) await copyTestCaseFileInput(testCase.fileInputPath, cwd);\n\n    const timeLimitSeconds =\n      typeof problemMarkdownFrontMatter.timeLimitMs === 'number'\n        ? problemMarkdownFrontMatter.timeLimitMs / 1000\n        : runTimeoutSeconds;\n\n    const command = languageDefinition.command(mainFilePath);\n    let stdin = testCase.input ?? '';\n    let runResult: TRunResult;\n    try {\n      if (options.resolveInput) {\n        stdin = await options.resolveInput({ testCase, cwd, env });\n      }\n\n      runResult = options.runCommand\n        ? await runCustomRunner(\n            () =>\n              // Hand custom runners a command that is already sandbox-wrapped, and the matching\n              // environment: they spawn it themselves, so an unwrapped command would run the\n              // submission as the trusted harness user and defeat the delegation boundary.\n              options.runCommand?.({\n                testCase,\n                command: wrapCommandWithSandboxUser(command),\n                stdin,\n                cwd,\n                env: { ...env, ...getSandboxUserEnvOverrides(env) },\n                timeLimitSeconds,\n              }) as Promise<TRunResult>\n          )\n        : (runCommand(command, {\n            stdin,\n            cwd,\n            env,\n            timeLimitSeconds,\n          }) as TRunResult);\n    } catch (error) {\n      printTestCaseResult({\n        testCaseId: testCase.id,\n        decisionCode: DecisionCode.RUNTIME_ERROR,\n        stdin,\n        stderr: errorToMessage(error),\n      });\n      await cleanWorkingDirectory(cwd, cwdSnapshot);\n      return { allAccepted: false };\n    }\n\n    const outputFiles = await readOutputFiles(cwd, problemMarkdownFrontMatter.requiredOutputFilePaths ?? []);\n    const judgeContext: CommandJudgeContext = {\n      timeLimitSeconds,\n      outputLimitLength: limits.maxOutputLength,\n      problemMarkdownFrontMatter: {\n        memoryLimitByte: problemMarkdownFrontMatter.memoryLimitByte,\n        requiredOutputFilePaths: problemMarkdownFrontMatter.requiredOutputFilePaths,\n      },\n    };\n    const baseJudgeResult = evaluateByLimits({ runResult, outputFiles, context: judgeContext });\n    let judgeResult = baseJudgeResult;\n    if (baseJudgeResult.decisionCode === DecisionCode.ACCEPTED) {\n      try {\n        const extendedJudgeResult = await options.test?.({ testCase, runResult, outputFiles, context: judgeContext });\n        if (extendedJudgeResult) {\n          judgeResult = {\n            decisionCode: extendedJudgeResult.decisionCode ?? baseJudgeResult.decisionCode,\n            feedbackMarkdown: extendedJudgeResult.feedbackMarkdown,\n            stderr: extendedJudgeResult.stderr,\n          };\n        }\n      } catch (error) {\n        judgeResult = {\n          decisionCode: DecisionCode.RUNTIME_ERROR,\n          stderr: errorToMessage(error),\n        };\n      }\n    }\n\n    printTestCaseResult({\n      testCaseId: testCase.id,\n      ...judgeResult,\n      exitStatus: runResult.status,\n      stdin: runResult.stdin,\n      stdout: runResult.stdout.slice(0, limits.maxOutputLength) || undefined,\n      stderr: (judgeResult.stderr ?? runResult.stderr).slice(0, limits.maxOutputLength) || undefined,\n      timeSeconds: runResult.timeSeconds,\n      memoryBytes: runResult.memoryBytes,\n      outputFiles: outputFiles.length > 0 ? outputFiles : undefined,\n    });\n\n    await cleanWorkingDirectory(cwd, cwdSnapshot);\n    if (judgeResult.decisionCode !== DecisionCode.ACCEPTED) return { allAccepted: false };\n  }\n\n  return { allAccepted: true };\n}\n\nfunction matchesExpectedResult(resolvedCwd: ResolvedCwd, result: { allAccepted: boolean }): boolean {\n  return result.allAccepted === (resolvedCwd.expectedResult === 'accepted');\n}\n\nfunction runBuild(\n  buildCommand: readonly [string, ...string[]],\n  context: { cwd: string; env: NodeJS.ProcessEnv; testCaseId: string; limits: CommandJudgeLimits }\n): (Omit<TestCaseResult, 'testCaseId'> & { testCaseId: string }) | undefined {\n  const spawnResult = spawnSyncWithTimeout(\n    buildCommand[0],\n    buildCommand.slice(1),\n    { cwd: context.cwd, encoding: 'utf8', env: context.env },\n    context.limits.buildTimeoutSeconds\n  );\n  const exitStatus = spawnResult.status ?? undefined;\n\n  if (spawnResult.status !== 0) {\n    return {\n      testCaseId: context.testCaseId,\n      decisionCode: DecisionCode.BUILD_ERROR,\n      exitStatus,\n      stdout: spawnResult.stdout.slice(0, context.limits.maxOutputLength),\n      stderr: spawnResult.stderr.slice(0, context.limits.maxOutputLength),\n      timeSeconds: spawnResult.timeSeconds,\n      memoryBytes: spawnResult.memoryBytes,\n    };\n  }\n\n  if (spawnResult.timeSeconds > context.limits.buildTimeoutSeconds) {\n    return {\n      testCaseId: context.testCaseId,\n      decisionCode: DecisionCode.BUILD_TIME_LIMIT_EXCEEDED,\n      exitStatus,\n      stdout: spawnResult.stdout.slice(0, context.limits.maxOutputLength),\n      stderr: spawnResult.stderr.slice(0, context.limits.maxOutputLength),\n      timeSeconds: spawnResult.timeSeconds,\n      memoryBytes: spawnResult.memoryBytes,\n    };\n  }\n\n  if (\n    spawnResult.stdout.length > context.limits.maxOutputLength ||\n    spawnResult.stderr.length > context.limits.maxOutputLength\n  ) {\n    return {\n      testCaseId: context.testCaseId,\n      decisionCode: DecisionCode.BUILD_OUTPUT_SIZE_LIMIT_EXCEEDED,\n      exitStatus,\n      stdout: spawnResult.stdout.slice(0, context.limits.maxOutputLength),\n      stderr: spawnResult.stderr.slice(0, context.limits.maxOutputLength),\n      timeSeconds: spawnResult.timeSeconds,\n      memoryBytes: spawnResult.memoryBytes,\n    };\n  }\n\n  return;\n}\n\nfunction errorToMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nfunction toCommandTestCase(value: {\n  id: string;\n  input?: string;\n  fileInputPath?: string;\n  fileOutputPath?: string;\n  output?: string;\n}): BaseCommandTestCase {\n  return { id: value.id, input: value.input, fileInputPath: value.fileInputPath };\n}\n\nasync function readCommandTestCases<TTestCase extends BaseCommandTestCase = BaseCommandTestCase>(\n  problemDir: string\n): Promise<readonly TTestCase[]> {\n  const fileTestCases = await readFileTestCases(path.join(problemDir, 'test_cases'));\n  const commandTestCases = fileTestCases.map((testCase) => toCommandTestCase(testCase) as TTestCase);\n  if (fileTestCases.shared?.fileInputPath) {\n    return Object.assign(commandTestCases, { shared: { fileInputPath: fileTestCases.shared.fileInputPath } });\n  }\n  return commandTestCases;\n}\n\nfunction runCommand(\n  command: readonly [string, ...string[]],\n  context: {\n    stdin: string;\n    cwd: string;\n    env: NodeJS.ProcessEnv;\n    timeLimitSeconds: number;\n  }\n): CommandRunResult {\n  const spawnResult = spawnSyncWithTimeout(\n    command[0],\n    command.slice(1),\n    { cwd: context.cwd, encoding: 'utf8', input: context.stdin, env: context.env },\n    context.timeLimitSeconds\n  );\n\n  return {\n    stdin: context.stdin,\n    stdout: spawnResult.stdout,\n    stderr: spawnResult.stderr,\n    status: spawnResult.status ?? undefined,\n    timeSeconds: spawnResult.timeSeconds,\n    memoryBytes: spawnResult.memoryBytes,\n  };\n}\n\nfunction evaluateByLimits(context: {\n  runResult: CommandRunResult;\n  outputFiles: NonNullable<TestCaseResult['outputFiles']>;\n  context: CommandJudgeContext;\n}): CommandJudgeCaseResult {\n  if (context.runResult.status !== 0) {\n    return { decisionCode: DecisionCode.RUNTIME_ERROR, stderr: context.runResult.stderr };\n  }\n\n  if (context.runResult.timeSeconds > context.context.timeLimitSeconds) {\n    return { decisionCode: DecisionCode.TIME_LIMIT_EXCEEDED, stderr: context.runResult.stderr };\n  }\n\n  if (\n    context.runResult.memoryBytes >\n    (context.context.problemMarkdownFrontMatter.memoryLimitByte ?? Number.POSITIVE_INFINITY)\n  ) {\n    return { decisionCode: DecisionCode.MEMORY_LIMIT_EXCEEDED, stderr: context.runResult.stderr };\n  }\n\n  if (\n    context.runResult.outputLimitExceeded ||\n    context.runResult.stdout.length > context.context.outputLimitLength ||\n    context.runResult.stderr.length > context.context.outputLimitLength\n  ) {\n    return { decisionCode: DecisionCode.OUTPUT_SIZE_LIMIT_EXCEEDED, stderr: context.runResult.stderr };\n  }\n\n  const requiredOutputFileCount = context.context.problemMarkdownFrontMatter.requiredOutputFilePaths?.length ?? 0;\n  if (context.outputFiles.length < requiredOutputFileCount) {\n    return { decisionCode: DecisionCode.MISSING_REQUIRED_OUTPUT_FILE_ERROR };\n  }\n\n  return { decisionCode: DecisionCode.ACCEPTED };\n}\n"],"mappings":"g4CAiCA,MAIM,EAAoB,EAAE,OAAO,CACjC,SAAU,EAAE,MAAM,CAAC,EAAE,OAAO,EAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAChE,CAAC,EAgHD,eAAsB,EAGpB,EAAoB,EAA0E,CAC9F,IAAM,EAAO,EAAU,QAAQ,IAAI,EAC7B,EAAS,EAAkB,MAAM,EAAK,MAAM,EAE5C,CAAE,OAAM,eAAgB,MAAM,EAAY,EAAY,EAAK,GAAG,EAEpE,GAAI,EAAa,CACf,IAAM,EAAc,EAAK,KAAM,GAAQ,EAAI,iBAAmB,UAAU,EACxE,GAAI,EAEE,IAAA,EAAC,MAD8B,EAAyB,EAAY,EAAa,CAAM,EAAA,CACjE,OAAQ,CAChC,QAAQ,SAAW,EACnB,MACF,OAEA,EAAiB,CACf,wDACA,GACA,4GACF,CAAC,CAEL,CAEA,IAAK,IAAM,KAAe,EAAM,CAC1B,GAAa,EAAoB,EAAY,CAAW,EAC5D,IAAM,EAAS,MAAM,EAA6C,EAAY,EAAY,IAAK,EAAQ,CAAO,EAC1G,GAAe,CAAC,EAAsB,EAAa,CAAM,IAC3D,QAAQ,SAAW,EACnB,EAAmC,EAAY,CAAW,EAE9D,CACF,CAEA,eAAe,EAIb,EACA,EACA,EACA,EACmC,CAEnC,EAA4B,CAAG,EAE/B,IAAM,EAA6B,MAAM,EAA+B,CAAU,EAC5E,EAAY,MAAO,EAAQ,eAAiB,EAAA,CAAsB,CAAU,EAC5E,EAAqB,EAAU,EAAE,EAAE,IAAM,WACzC,EAAS,CACb,oBAAqB,EAAQ,QAAQ,qBAAuB,GAC5D,gBAAiB,EAAQ,QAAQ,iBAAmB,GACtD,EACM,EAAoB,EAAQ,mBAAqB,EAEjD,EAAuB,MAAM,EAAsB,EAAK,CAA0B,EACxF,GAAI,EAEF,OADA,EAAoB,CAAE,WAAY,EAAoB,GAAG,CAAqB,CAAC,EACxE,CAAE,YAAa,EAAM,EAG9B,IAAM,EAAuB,MAAM,EAAmB,EAAK,EAAO,QAAQ,EAC1E,GAAI,CAAC,EAMH,OALA,EAAoB,CAClB,WAAY,EACZ,aAAc,EAAa,uCAC3B,OAAQ,sBAAsB,EAAO,SAAW,eAAe,EAAO,WAAa,IACrF,CAAC,EACM,CAAE,YAAa,EAAM,EAG9B,IAAM,EAAqB,EAA6B,CAAoB,EAC5E,GAAI,CAAC,EAMH,OALA,EAAoB,CAClB,WAAY,EACZ,aAAc,EAAa,aAC3B,OAAQ,sBACV,CAAC,EACM,CAAE,YAAa,EAAM,EAI9B,IAAM,EAAM,CAAE,GAAG,QAAQ,IAAK,GAAI,GAAI,YAAa,GAAI,EAEnD,EAAe,EACnB,GAAI,EAAQ,kBAAoB,GAAO,CACrC,GAAI,EAAmB,SACrB,GAAI,CACF,MAAM,EAAmB,SAAS,CAAG,EACrC,IAAM,EAAuB,MAAM,EAAmB,EAAK,EAAO,QAAQ,EACtE,IAAsB,EAAe,EAC3C,OAAS,EAAO,CAMd,OALA,EAAoB,CAClB,WAAY,EACZ,aAAc,EAAa,YAC3B,OAAQ,aAAiB,MAAQ,EAAM,QAAU,OAAO,CAAK,CAC/D,CAAC,EACM,CAAE,YAAa,EAAM,CAC9B,CAGF,IAAM,EAAe,EAAmB,eAAe,CAAY,EACnE,GAAI,EAAc,CAChB,IAAM,EAAc,EAAS,EAAc,CACzC,MACA,MACA,WAAY,EACZ,QACF,CAAC,EACD,GAAI,EAEF,OADA,EAAoB,CAAW,EACxB,CAAE,YAAa,EAAM,CAEhC,CACF,CAEA,IAAM,EAAc,MAAM,EAAyB,CAAG,EAEtD,GAAI,EAAU,SAAW,EAEvB,OADA,EAAoB,CAAE,WAAY,UAAW,aAAc,EAAa,QAAS,CAAC,EAC3E,CAAE,YAAa,EAAK,EAG7B,IAAM,EAAuB,EAAsD,QAAQ,cAE3F,IAAK,IAAM,KAAY,EAAW,CAC5B,GAAqB,MAAM,EAAsB,EAAqB,CAAG,EACzE,EAAS,eAAe,MAAM,EAAsB,EAAS,cAAe,CAAG,EAEnF,IAAM,EACJ,OAAO,EAA2B,aAAgB,SAC9C,EAA2B,YAAc,IACzC,EAEA,EAAU,EAAmB,QAAQ,CAAY,EACnD,EAAQ,EAAS,OAAS,GAC1B,EACJ,GAAI,CACE,EAAQ,eACV,EAAQ,MAAM,EAAQ,aAAa,CAAE,WAAU,MAAK,KAAI,CAAC,GAG3D,EAAY,EAAQ,WAChB,MAAM,MAKF,EAAQ,aAAa,CACnB,WACA,QAAS,EAA2B,CAAO,EAC3C,QACA,MACA,IAAK,CAAE,GAAG,EAAK,GAAG,EAA2B,CAAG,CAAE,EAClD,kBACF,CAAC,CACL,EACC,EAAW,EAAS,CACnB,QACA,MACA,MACA,kBACF,CAAC,CACP,OAAS,EAAO,CAQd,OAPA,EAAoB,CAClB,WAAY,EAAS,GACrB,aAAc,EAAa,cAC3B,QACA,OAAQ,EAAe,CAAK,CAC9B,CAAC,EACD,MAAM,EAAsB,EAAK,CAAW,EACrC,CAAE,YAAa,EAAM,CAC9B,CAEA,IAAM,EAAc,MAAM,EAAgB,EAAK,EAA2B,yBAA2B,CAAC,CAAC,EACjG,EAAoC,CACxC,mBACA,kBAAmB,EAAO,gBAC1B,2BAA4B,CAC1B,gBAAiB,EAA2B,gBAC5C,wBAAyB,EAA2B,uBACtD,CACF,EACM,EAAkB,EAAiB,CAAE,YAAW,cAAa,QAAS,CAAa,CAAC,EACtF,EAAc,EAClB,GAAI,EAAgB,eAAiB,EAAa,SAChD,GAAI,CACF,IAAM,EAAsB,MAAM,EAAQ,OAAO,CAAE,WAAU,YAAW,cAAa,QAAS,CAAa,CAAC,EACxG,IACF,EAAc,CACZ,aAAc,EAAoB,cAAgB,EAAgB,aAClE,iBAAkB,EAAoB,iBACtC,OAAQ,EAAoB,MAC9B,EAEJ,OAAS,EAAO,CACd,EAAc,CACZ,aAAc,EAAa,cAC3B,OAAQ,EAAe,CAAK,CAC9B,CACF,CAgBF,GAbA,EAAoB,CAClB,WAAY,EAAS,GACrB,GAAG,EACH,WAAY,EAAU,OACtB,MAAO,EAAU,MACjB,OAAQ,EAAU,OAAO,MAAM,EAAG,EAAO,eAAe,GAAK,IAAA,GAC7D,QAAS,EAAY,QAAU,EAAU,OAAA,CAAQ,MAAM,EAAG,EAAO,eAAe,GAAK,IAAA,GACrF,YAAa,EAAU,YACvB,YAAa,EAAU,YACvB,YAAa,EAAY,OAAS,EAAI,EAAc,IAAA,EACtD,CAAC,EAED,MAAM,EAAsB,EAAK,CAAW,EACxC,EAAY,eAAiB,EAAa,SAAU,MAAO,CAAE,YAAa,EAAM,CACtF,CAEA,MAAO,CAAE,YAAa,EAAK,CAC7B,CAEA,SAAS,EAAsB,EAA0B,EAA2C,CAClG,OAAO,EAAO,eAAiB,EAAY,iBAAmB,WAChE,CAEA,SAAS,EACP,EACA,EAC2E,CAC3E,IAAM,EAAc,EAClB,EAAa,GACb,EAAa,MAAM,CAAC,EACpB,CAAE,IAAK,EAAQ,IAAK,SAAU,OAAQ,IAAK,EAAQ,GAAI,EACvD,EAAQ,OAAO,mBACjB,EACM,EAAa,EAAY,QAAU,IAAA,GAEzC,GAAI,EAAY,SAAW,EACzB,MAAO,CACL,WAAY,EAAQ,WACpB,aAAc,EAAa,YAC3B,aACA,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,EAGF,GAAI,EAAY,YAAc,EAAQ,OAAO,oBAC3C,MAAO,CACL,WAAY,EAAQ,WACpB,aAAc,EAAa,0BAC3B,aACA,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,EAGF,GACE,EAAY,OAAO,OAAS,EAAQ,OAAO,iBAC3C,EAAY,OAAO,OAAS,EAAQ,OAAO,gBAE3C,MAAO,CACL,WAAY,EAAQ,WACpB,aAAc,EAAa,iCAC3B,aACA,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,CAIJ,CAEA,SAAS,EAAe,EAAwB,CAC9C,OAAO,aAAiB,MAAQ,EAAM,QAAU,OAAO,CAAK,CAC9D,CAEA,SAAS,EAAkB,EAMH,CACtB,MAAO,CAAE,GAAI,EAAM,GAAI,MAAO,EAAM,MAAO,cAAe,EAAM,aAAc,CAChF,CAEA,eAAe,EACb,EAC+B,CAC/B,IAAM,EAAgB,MAAMA,EAAkB,EAAK,KAAK,EAAY,YAAY,CAAC,EAC3E,EAAmB,EAAc,IAAK,GAAa,EAAkB,CAAQ,CAAc,EAIjG,OAHI,EAAc,QAAQ,cACjB,OAAO,OAAO,EAAkB,CAAE,OAAQ,CAAE,cAAe,EAAc,OAAO,aAAc,CAAE,CAAC,EAEnG,CACT,CAEA,SAAS,EACP,EACA,EAMkB,CAClB,IAAM,EAAc,EAClB,EAAQ,GACR,EAAQ,MAAM,CAAC,EACf,CAAE,IAAK,EAAQ,IAAK,SAAU,OAAQ,MAAO,EAAQ,MAAO,IAAK,EAAQ,GAAI,EAC7E,EAAQ,gBACV,EAEA,MAAO,CACL,MAAO,EAAQ,MACf,OAAQ,EAAY,OACpB,OAAQ,EAAY,OACpB,OAAQ,EAAY,QAAU,IAAA,GAC9B,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,CACF,CAEA,SAAS,EAAiB,EAIC,CACzB,GAAI,EAAQ,UAAU,SAAW,EAC/B,MAAO,CAAE,aAAc,EAAa,cAAe,OAAQ,EAAQ,UAAU,MAAO,EAGtF,GAAI,EAAQ,UAAU,YAAc,EAAQ,QAAQ,iBAClD,MAAO,CAAE,aAAc,EAAa,oBAAqB,OAAQ,EAAQ,UAAU,MAAO,EAG5F,GACE,EAAQ,UAAU,aACjB,EAAQ,QAAQ,2BAA2B,iBAAmB,KAE/D,MAAO,CAAE,aAAc,EAAa,sBAAuB,OAAQ,EAAQ,UAAU,MAAO,EAG9F,GACE,EAAQ,UAAU,qBAClB,EAAQ,UAAU,OAAO,OAAS,EAAQ,QAAQ,mBAClD,EAAQ,UAAU,OAAO,OAAS,EAAQ,QAAQ,kBAElD,MAAO,CAAE,aAAc,EAAa,2BAA4B,OAAQ,EAAQ,UAAU,MAAO,EAGnG,IAAM,EAA0B,EAAQ,QAAQ,2BAA2B,yBAAyB,QAAU,EAK9G,OAJI,EAAQ,YAAY,OAAS,EACxB,CAAE,aAAc,EAAa,kCAAmC,EAGlE,CAAE,aAAc,EAAa,QAAS,CAC/C"}
+{"version":3,"file":"command.js","names":["readFileTestCases"],"sources":["../../src/presets/command.ts"],"sourcesContent":["import path from 'node:path';\n\nimport { z } from 'zod';\n\nimport { cleanWorkingDirectory, snapshotWorkingDirectory } from '../helpers/cleanWorkingDirectory.js';\nimport { copyTestCaseFileInput } from '../helpers/copyTestCaseFileInput.js';\nimport { findEntryPointFile } from '../helpers/findEntryPointFile.js';\nimport { findLanguageDefinitionByPath } from '../helpers/findLanguageDefinitionByPath.js';\nimport { judgeByStaticAnalysis } from '../helpers/judgeByStaticAnalysis.js';\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { readOutputFiles } from '../helpers/readOutputFiles.js';\nimport { runCustomRunner } from '../helpers/runCustomRunner.js';\nimport { passesIsolationCheckInDebugMode } from '../helpers/runIsolationCheckInDebugMode.js';\nimport {\n  getSandboxUserEnvOverrides,\n  makeAccessibleToSandboxUser,\n  wrapCommandWithSandboxUser,\n} from '../helpers/sandboxUser.js';\nimport { readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { readTestCases as readFileTestCases } from '../helpers/readTestCases.js';\nimport {\n  printDebugCwdBanner,\n  printDebugExpectationFailureBanner,\n  resolveCwds,\n  type ResolvedCwd,\n} from '../helpers/resolveCwds.js';\nimport { spawnSyncWithTimeout } from '../helpers/spawnSyncWithTimeout.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { ProblemMarkdownFrontMatter } from '../types/problem.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst BUILD_TIMEOUT_SECONDS = 10;\nconst JUDGE_DEFAULT_TIMEOUT_SECONDS = 2;\nconst MAX_STDOUT_LENGTH = 50_000;\n\nconst judgeParamsSchema = z.object({\n  language: z.union([z.string(), z.array(z.string())]).optional(),\n});\n\ntype JudgeParams = z.infer<typeof judgeParamsSchema>;\n\ninterface BaseCommandTestCase {\n  id: string;\n  input?: string;\n  fileInputPath?: string;\n}\n\ntype CommandJudgeCaseResult = Pick<TestCaseResult, 'decisionCode' | 'feedbackMarkdown' | 'stderr'>;\n\nexport interface CommandRunResult {\n  stdin: string;\n  stdout: string;\n  stderr: string;\n  status: number | undefined;\n  timeSeconds: number;\n  memoryBytes: number;\n  outputLimitExceeded?: boolean;\n}\n\ninterface CommandJudgeContext {\n  timeLimitSeconds: number;\n  outputLimitLength: number;\n  problemMarkdownFrontMatter: Pick<ProblemMarkdownFrontMatter, 'memoryLimitByte' | 'requiredOutputFilePaths'>;\n}\n\nexport interface CommandJudgeLimits {\n  buildTimeoutSeconds: number;\n  maxOutputLength: number;\n}\n\ninterface ResolvedCommandProblem<TTestCase extends BaseCommandTestCase> {\n  problemMarkdownFrontMatter: ProblemMarkdownFrontMatter;\n  testCases: readonly TTestCase[];\n  limits: CommandJudgeLimits;\n  timeLimitSeconds: number;\n}\n\nexport interface CommandJudgePresetOptions<\n  TTestCase extends BaseCommandTestCase = BaseCommandTestCase,\n  TRunResult extends CommandRunResult = CommandRunResult,\n> {\n  /**\n   * Set to false when the custom runner builds the submitted source itself.\n   * Defaults to true.\n   */\n  buildSubmission?: boolean;\n  limits?: CommandJudgeLimits;\n  runTimeoutSeconds?: number;\n  readTestCases?: (problemDir: string) => Promise<readonly TTestCase[]>;\n  /**\n   * Runs as the trusted harness user with the submission's `cwd`. Fixture files it creates there\n   * must be written with `createDirectoryWithoutFollowingSymlinks`/`writeFileWithoutFollowingSymlinks`\n   * (both exported): a submission can plant a symlink at a fixture path, and a plain `fs.writeFile`\n   * would follow it into a file only the harness can write.\n   */\n  resolveInput?: (context: { testCase: TTestCase; cwd: string; env: NodeJS.ProcessEnv }) => Promise<string> | string;\n  runCommand?: (context: {\n    testCase: TTestCase;\n    /**\n     * The command to run. Under `EXERCODE_SANDBOX_USER` delegation it is already wrapped so it\n     * executes as the sandbox user; spawn it as given (with the supplied `env`) instead of\n     * reconstructing it, or the submission runs as the trusted harness user.\n     *\n     * Its direct child is then a root-owned `sudo` whose descendants belong to the sandbox user, so\n     * the handler cannot signal them: enforce `timeLimitSeconds` with `startSandboxTimeoutWatchdog`\n     * and `killSandboxUserProcesses` (both exported) rather than `child.kill()` or an outer\n     * `timeout`. The preset terminates leftover sandbox processes after the handler returns.\n     */\n    command: readonly [string, ...string[]];\n    stdin: string;\n    cwd: string;\n    env: NodeJS.ProcessEnv;\n    timeLimitSeconds: number;\n  }) => Promise<TRunResult> | TRunResult;\n  test?: (context: {\n    testCase: TTestCase;\n    runResult: TRunResult;\n    outputFiles: NonNullable<TestCaseResult['outputFiles']>;\n    context: CommandJudgeContext;\n  }) => Promise<Partial<CommandJudgeCaseResult>> | Partial<CommandJudgeCaseResult> | undefined;\n}\n\n/**\n * A preset function for judging by executable command.\n *\n * Keep problem-specific logic in `resolveInput` and `test`.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { commandJudgePreset } from '@exercode/problem-utils/presets/command';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await commandJudgePreset(import.meta.dirname, {\n *   readTestCases: async () => [\n *     { id: '01', input: '1 2' },\n *   ],\n *   test: ({ runResult }) => {\n *     return runResult.stdout.trim() === '3'\n *       ? { decisionCode: DecisionCode.ACCEPTED }\n *       : { decisionCode: DecisionCode.WRONG_ANSWER };\n *   },\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/python '{ \"language\": \"python\" }'\n * ```\n *\n * Run without a cwd argument to judge each `<problemDir>/model_answers/*` directory\n * for debugging. A prominent banner is printed for each run.\n * ```bash\n * bun judge.ts\n * ```\n */\nexport async function commandJudgePreset<\n  TTestCase extends BaseCommandTestCase = BaseCommandTestCase,\n  TRunResult extends CommandRunResult = CommandRunResult,\n>(problemDir: string, options: CommandJudgePresetOptions<TTestCase, TRunResult>): Promise<void> {\n  const args = parseArgs(process.argv);\n  const params = judgeParamsSchema.parse(args.params);\n\n  const { cwds, isDebugMode } = await resolveCwds(problemDir, args.cwd);\n  const problemMarkdownFrontMatter = await readProblemMarkdownFrontMatter(problemDir);\n  const testCases = await (options.readTestCases ?? readCommandTestCases)(problemDir);\n  const limits = {\n    buildTimeoutSeconds: options.limits?.buildTimeoutSeconds ?? BUILD_TIMEOUT_SECONDS,\n    maxOutputLength: options.limits?.maxOutputLength ?? MAX_STDOUT_LENGTH,\n  };\n  const timeLimitSeconds =\n    typeof problemMarkdownFrontMatter.timeLimitMs === 'number'\n      ? problemMarkdownFrontMatter.timeLimitMs / 1000\n      : (options.runTimeoutSeconds ?? JUDGE_DEFAULT_TIMEOUT_SECONDS);\n  const problem: ResolvedCommandProblem<TTestCase> = {\n    problemMarkdownFrontMatter,\n    testCases,\n    limits,\n    timeLimitSeconds,\n  };\n\n  if (isDebugMode) {\n    const expectedMaxDurationMs = (limits.buildTimeoutSeconds + testCases.length * timeLimitSeconds) * 1000;\n    if (!(await passesIsolationCheckInDebugMode(problemDir, cwds, params, { expectedMaxDurationMs }))) {\n      process.exitCode = 1;\n      return;\n    }\n  }\n\n  for (const resolvedCwd of cwds) {\n    if (isDebugMode) printDebugCwdBanner(problemDir, resolvedCwd);\n    const result = await runCommandJudgeForCwd<TTestCase, TRunResult>(resolvedCwd.cwd, params, problem, options);\n    if (isDebugMode && !matchesExpectedResult(resolvedCwd, result)) {\n      process.exitCode = 1;\n      printDebugExpectationFailureBanner(problemDir, resolvedCwd);\n    }\n  }\n}\n\nasync function runCommandJudgeForCwd<\n  TTestCase extends BaseCommandTestCase,\n  TRunResult extends CommandRunResult = CommandRunResult,\n>(\n  cwd: string,\n  params: JudgeParams,\n  problem: ResolvedCommandProblem<TTestCase>,\n  options: CommandJudgePresetOptions<TTestCase, TRunResult>\n): Promise<{ allAccepted: boolean }> {\n  // The sandboxed submission must read its sources and write build/run outputs in its directory.\n  makeAccessibleToSandboxUser(cwd);\n\n  const { problemMarkdownFrontMatter, testCases, limits, timeLimitSeconds } = problem;\n  const prebuildTestCaseId = testCases[0]?.id ?? 'prebuild';\n\n  const staticAnalysisResult = await judgeByStaticAnalysis(cwd, problemMarkdownFrontMatter);\n  if (staticAnalysisResult) {\n    printTestCaseResult({ testCaseId: prebuildTestCaseId, ...staticAnalysisResult });\n    return { allAccepted: false };\n  }\n\n  const originalMainFilePath = await findEntryPointFile(cwd, params.language);\n  if (!originalMainFilePath) {\n    printTestCaseResult({\n      testCaseId: prebuildTestCaseId,\n      decisionCode: DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,\n      stderr: `main file not found${params.language ? `: language: ${params.language}` : ''}`,\n    });\n    return { allAccepted: false };\n  }\n\n  const languageDefinition = findLanguageDefinitionByPath(originalMainFilePath);\n  if (!languageDefinition) {\n    printTestCaseResult({\n      testCaseId: prebuildTestCaseId,\n      decisionCode: DecisionCode.WRONG_ANSWER,\n      stderr: 'unsupported language',\n    });\n    return { allAccepted: false };\n  }\n\n  // `CI` changes affects Chainlit. `FORCE_COLOR` affects Bun.\n  const env = { ...process.env, CI: '', FORCE_COLOR: '0' };\n\n  let mainFilePath = originalMainFilePath;\n  if (options.buildSubmission !== false) {\n    if (languageDefinition.prebuild) {\n      try {\n        await languageDefinition.prebuild(cwd);\n        const prebuiltMainFilePath = await findEntryPointFile(cwd, params.language);\n        if (prebuiltMainFilePath) mainFilePath = prebuiltMainFilePath;\n      } catch (error) {\n        printTestCaseResult({\n          testCaseId: prebuildTestCaseId,\n          decisionCode: DecisionCode.BUILD_ERROR,\n          stderr: error instanceof Error ? error.message : String(error),\n        });\n        return { allAccepted: false };\n      }\n    }\n\n    const buildCommand = languageDefinition.buildCommand?.(mainFilePath);\n    if (buildCommand) {\n      const buildResult = runBuild(buildCommand, {\n        cwd,\n        env,\n        testCaseId: prebuildTestCaseId,\n        limits,\n      });\n      if (buildResult) {\n        printTestCaseResult(buildResult);\n        return { allAccepted: false };\n      }\n    }\n  }\n\n  const cwdSnapshot = await snapshotWorkingDirectory(cwd);\n\n  if (testCases.length === 0) {\n    printTestCaseResult({ testCaseId: 'default', decisionCode: DecisionCode.ACCEPTED });\n    return { allAccepted: true };\n  }\n\n  const sharedFileInputPath = (testCases as { shared?: { fileInputPath?: string } }).shared?.fileInputPath;\n\n  for (const testCase of testCases) {\n    if (sharedFileInputPath) await copyTestCaseFileInput(sharedFileInputPath, cwd);\n    if (testCase.fileInputPath) await copyTestCaseFileInput(testCase.fileInputPath, cwd);\n\n    const command = languageDefinition.command(mainFilePath);\n    let stdin = testCase.input ?? '';\n    let runResult: TRunResult;\n    try {\n      if (options.resolveInput) {\n        stdin = await options.resolveInput({ testCase, cwd, env });\n      }\n\n      runResult = options.runCommand\n        ? await runCustomRunner(\n            () =>\n              // Hand custom runners a command that is already sandbox-wrapped, and the matching\n              // environment: they spawn it themselves, so an unwrapped command would run the\n              // submission as the trusted harness user and defeat the delegation boundary.\n              options.runCommand?.({\n                testCase,\n                command: wrapCommandWithSandboxUser(command),\n                stdin,\n                cwd,\n                env: { ...env, ...getSandboxUserEnvOverrides(env) },\n                timeLimitSeconds,\n              }) as Promise<TRunResult>\n          )\n        : (runCommand(command, {\n            stdin,\n            cwd,\n            env,\n            timeLimitSeconds,\n          }) as TRunResult);\n    } catch (error) {\n      printTestCaseResult({\n        testCaseId: testCase.id,\n        decisionCode: DecisionCode.RUNTIME_ERROR,\n        stdin,\n        stderr: errorToMessage(error),\n      });\n      await cleanWorkingDirectory(cwd, cwdSnapshot);\n      return { allAccepted: false };\n    }\n\n    const outputFiles = await readOutputFiles(cwd, problemMarkdownFrontMatter.requiredOutputFilePaths ?? []);\n    const judgeContext: CommandJudgeContext = {\n      timeLimitSeconds,\n      outputLimitLength: limits.maxOutputLength,\n      problemMarkdownFrontMatter: {\n        memoryLimitByte: problemMarkdownFrontMatter.memoryLimitByte,\n        requiredOutputFilePaths: problemMarkdownFrontMatter.requiredOutputFilePaths,\n      },\n    };\n    const baseJudgeResult = evaluateByLimits({ runResult, outputFiles, context: judgeContext });\n    let judgeResult = baseJudgeResult;\n    if (baseJudgeResult.decisionCode === DecisionCode.ACCEPTED) {\n      try {\n        const extendedJudgeResult = await options.test?.({ testCase, runResult, outputFiles, context: judgeContext });\n        if (extendedJudgeResult) {\n          judgeResult = {\n            decisionCode: extendedJudgeResult.decisionCode ?? baseJudgeResult.decisionCode,\n            feedbackMarkdown: extendedJudgeResult.feedbackMarkdown,\n            stderr: extendedJudgeResult.stderr,\n          };\n        }\n      } catch (error) {\n        judgeResult = {\n          decisionCode: DecisionCode.RUNTIME_ERROR,\n          stderr: errorToMessage(error),\n        };\n      }\n    }\n\n    printTestCaseResult({\n      testCaseId: testCase.id,\n      ...judgeResult,\n      exitStatus: runResult.status,\n      stdin: runResult.stdin,\n      stdout: runResult.stdout.slice(0, limits.maxOutputLength) || undefined,\n      stderr: (judgeResult.stderr ?? runResult.stderr).slice(0, limits.maxOutputLength) || undefined,\n      timeSeconds: runResult.timeSeconds,\n      memoryBytes: runResult.memoryBytes,\n      outputFiles: outputFiles.length > 0 ? outputFiles : undefined,\n    });\n\n    await cleanWorkingDirectory(cwd, cwdSnapshot);\n    if (judgeResult.decisionCode !== DecisionCode.ACCEPTED) return { allAccepted: false };\n  }\n\n  return { allAccepted: true };\n}\n\nfunction matchesExpectedResult(resolvedCwd: ResolvedCwd, result: { allAccepted: boolean }): boolean {\n  return result.allAccepted === (resolvedCwd.expectedResult === 'accepted');\n}\n\nfunction runBuild(\n  buildCommand: readonly [string, ...string[]],\n  context: { cwd: string; env: NodeJS.ProcessEnv; testCaseId: string; limits: CommandJudgeLimits }\n): (Omit<TestCaseResult, 'testCaseId'> & { testCaseId: string }) | undefined {\n  const spawnResult = spawnSyncWithTimeout(\n    buildCommand[0],\n    buildCommand.slice(1),\n    { cwd: context.cwd, encoding: 'utf8', env: context.env },\n    context.limits.buildTimeoutSeconds\n  );\n  const exitStatus = spawnResult.status ?? undefined;\n\n  if (spawnResult.status !== 0) {\n    return {\n      testCaseId: context.testCaseId,\n      decisionCode: DecisionCode.BUILD_ERROR,\n      exitStatus,\n      stdout: spawnResult.stdout.slice(0, context.limits.maxOutputLength),\n      stderr: spawnResult.stderr.slice(0, context.limits.maxOutputLength),\n      timeSeconds: spawnResult.timeSeconds,\n      memoryBytes: spawnResult.memoryBytes,\n    };\n  }\n\n  if (spawnResult.timeSeconds > context.limits.buildTimeoutSeconds) {\n    return {\n      testCaseId: context.testCaseId,\n      decisionCode: DecisionCode.BUILD_TIME_LIMIT_EXCEEDED,\n      exitStatus,\n      stdout: spawnResult.stdout.slice(0, context.limits.maxOutputLength),\n      stderr: spawnResult.stderr.slice(0, context.limits.maxOutputLength),\n      timeSeconds: spawnResult.timeSeconds,\n      memoryBytes: spawnResult.memoryBytes,\n    };\n  }\n\n  if (\n    spawnResult.stdout.length > context.limits.maxOutputLength ||\n    spawnResult.stderr.length > context.limits.maxOutputLength\n  ) {\n    return {\n      testCaseId: context.testCaseId,\n      decisionCode: DecisionCode.BUILD_OUTPUT_SIZE_LIMIT_EXCEEDED,\n      exitStatus,\n      stdout: spawnResult.stdout.slice(0, context.limits.maxOutputLength),\n      stderr: spawnResult.stderr.slice(0, context.limits.maxOutputLength),\n      timeSeconds: spawnResult.timeSeconds,\n      memoryBytes: spawnResult.memoryBytes,\n    };\n  }\n\n  return;\n}\n\nfunction errorToMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nfunction toCommandTestCase(value: {\n  id: string;\n  input?: string;\n  fileInputPath?: string;\n  fileOutputPath?: string;\n  output?: string;\n}): BaseCommandTestCase {\n  return { id: value.id, input: value.input, fileInputPath: value.fileInputPath };\n}\n\nasync function readCommandTestCases<TTestCase extends BaseCommandTestCase = BaseCommandTestCase>(\n  problemDir: string\n): Promise<readonly TTestCase[]> {\n  const fileTestCases = await readFileTestCases(path.join(problemDir, 'test_cases'));\n  const commandTestCases = fileTestCases.map((testCase) => toCommandTestCase(testCase) as TTestCase);\n  if (fileTestCases.shared?.fileInputPath) {\n    return Object.assign(commandTestCases, { shared: { fileInputPath: fileTestCases.shared.fileInputPath } });\n  }\n  return commandTestCases;\n}\n\nfunction runCommand(\n  command: readonly [string, ...string[]],\n  context: {\n    stdin: string;\n    cwd: string;\n    env: NodeJS.ProcessEnv;\n    timeLimitSeconds: number;\n  }\n): CommandRunResult {\n  const spawnResult = spawnSyncWithTimeout(\n    command[0],\n    command.slice(1),\n    { cwd: context.cwd, encoding: 'utf8', input: context.stdin, env: context.env },\n    context.timeLimitSeconds\n  );\n\n  return {\n    stdin: context.stdin,\n    stdout: spawnResult.stdout,\n    stderr: spawnResult.stderr,\n    status: spawnResult.status ?? undefined,\n    timeSeconds: spawnResult.timeSeconds,\n    memoryBytes: spawnResult.memoryBytes,\n  };\n}\n\nfunction evaluateByLimits(context: {\n  runResult: CommandRunResult;\n  outputFiles: NonNullable<TestCaseResult['outputFiles']>;\n  context: CommandJudgeContext;\n}): CommandJudgeCaseResult {\n  if (context.runResult.status !== 0) {\n    return { decisionCode: DecisionCode.RUNTIME_ERROR, stderr: context.runResult.stderr };\n  }\n\n  if (context.runResult.timeSeconds > context.context.timeLimitSeconds) {\n    return { decisionCode: DecisionCode.TIME_LIMIT_EXCEEDED, stderr: context.runResult.stderr };\n  }\n\n  if (\n    context.runResult.memoryBytes >\n    (context.context.problemMarkdownFrontMatter.memoryLimitByte ?? Number.POSITIVE_INFINITY)\n  ) {\n    return { decisionCode: DecisionCode.MEMORY_LIMIT_EXCEEDED, stderr: context.runResult.stderr };\n  }\n\n  if (\n    context.runResult.outputLimitExceeded ||\n    context.runResult.stdout.length > context.context.outputLimitLength ||\n    context.runResult.stderr.length > context.context.outputLimitLength\n  ) {\n    return { decisionCode: DecisionCode.OUTPUT_SIZE_LIMIT_EXCEEDED, stderr: context.runResult.stderr };\n  }\n\n  const requiredOutputFileCount = context.context.problemMarkdownFrontMatter.requiredOutputFilePaths?.length ?? 0;\n  if (context.outputFiles.length < requiredOutputFileCount) {\n    return { decisionCode: DecisionCode.MISSING_REQUIRED_OUTPUT_FILE_ERROR };\n  }\n\n  return { decisionCode: DecisionCode.ACCEPTED };\n}\n"],"mappings":"y0CAgCA,MAIM,EAAoB,EAAE,OAAO,CACjC,SAAU,EAAE,MAAM,CAAC,EAAE,OAAO,EAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAChE,CAAC,EAuHD,eAAsB,EAGpB,EAAoB,EAA0E,CAC9F,IAAM,EAAO,EAAU,QAAQ,IAAI,EAC7B,EAAS,EAAkB,MAAM,EAAK,MAAM,EAE5C,CAAE,OAAM,eAAgB,MAAM,EAAY,EAAY,EAAK,GAAG,EAC9D,EAA6B,MAAM,EAA+B,CAAU,EAC5E,EAAY,MAAO,EAAQ,eAAiB,EAAA,CAAsB,CAAU,EAC5E,EAAS,CACb,oBAAqB,EAAQ,QAAQ,qBAAuB,GAC5D,gBAAiB,EAAQ,QAAQ,iBAAmB,GACtD,EACM,EACJ,OAAO,EAA2B,aAAgB,SAC9C,EAA2B,YAAc,IACxC,EAAQ,mBAAqB,EAC9B,EAA6C,CACjD,6BACA,YACA,SACA,kBACF,EAEA,GAAI,EAAa,CACf,IAAM,GAAyB,EAAO,oBAAsB,EAAU,OAAS,GAAoB,IACnG,GAAI,CAAE,MAAM,EAAgC,EAAY,EAAM,EAAQ,CAAE,uBAAsB,CAAC,EAAI,CACjG,QAAQ,SAAW,EACnB,MACF,CACF,CAEA,IAAK,IAAM,KAAe,EAAM,CAC1B,GAAa,EAAoB,EAAY,CAAW,EAC5D,IAAM,EAAS,MAAM,EAA6C,EAAY,IAAK,EAAQ,EAAS,CAAO,EACvG,GAAe,CAAC,EAAsB,EAAa,CAAM,IAC3D,QAAQ,SAAW,EACnB,EAAmC,EAAY,CAAW,EAE9D,CACF,CAEA,eAAe,EAIb,EACA,EACA,EACA,EACmC,CAEnC,EAA4B,CAAG,EAE/B,GAAM,CAAE,6BAA4B,YAAW,SAAQ,oBAAqB,EACtE,EAAqB,EAAU,EAAE,EAAE,IAAM,WAEzC,EAAuB,MAAM,EAAsB,EAAK,CAA0B,EACxF,GAAI,EAEF,OADA,EAAoB,CAAE,WAAY,EAAoB,GAAG,CAAqB,CAAC,EACxE,CAAE,YAAa,EAAM,EAG9B,IAAM,EAAuB,MAAM,EAAmB,EAAK,EAAO,QAAQ,EAC1E,GAAI,CAAC,EAMH,OALA,EAAoB,CAClB,WAAY,EACZ,aAAc,EAAa,uCAC3B,OAAQ,sBAAsB,EAAO,SAAW,eAAe,EAAO,WAAa,IACrF,CAAC,EACM,CAAE,YAAa,EAAM,EAG9B,IAAM,EAAqB,EAA6B,CAAoB,EAC5E,GAAI,CAAC,EAMH,OALA,EAAoB,CAClB,WAAY,EACZ,aAAc,EAAa,aAC3B,OAAQ,sBACV,CAAC,EACM,CAAE,YAAa,EAAM,EAI9B,IAAM,EAAM,CAAE,GAAG,QAAQ,IAAK,GAAI,GAAI,YAAa,GAAI,EAEnD,EAAe,EACnB,GAAI,EAAQ,kBAAoB,GAAO,CACrC,GAAI,EAAmB,SACrB,GAAI,CACF,MAAM,EAAmB,SAAS,CAAG,EACrC,IAAM,EAAuB,MAAM,EAAmB,EAAK,EAAO,QAAQ,EACtE,IAAsB,EAAe,EAC3C,OAAS,EAAO,CAMd,OALA,EAAoB,CAClB,WAAY,EACZ,aAAc,EAAa,YAC3B,OAAQ,aAAiB,MAAQ,EAAM,QAAU,OAAO,CAAK,CAC/D,CAAC,EACM,CAAE,YAAa,EAAM,CAC9B,CAGF,IAAM,EAAe,EAAmB,eAAe,CAAY,EACnE,GAAI,EAAc,CAChB,IAAM,EAAc,EAAS,EAAc,CACzC,MACA,MACA,WAAY,EACZ,QACF,CAAC,EACD,GAAI,EAEF,OADA,EAAoB,CAAW,EACxB,CAAE,YAAa,EAAM,CAEhC,CACF,CAEA,IAAM,EAAc,MAAM,EAAyB,CAAG,EAEtD,GAAI,EAAU,SAAW,EAEvB,OADA,EAAoB,CAAE,WAAY,UAAW,aAAc,EAAa,QAAS,CAAC,EAC3E,CAAE,YAAa,EAAK,EAG7B,IAAM,EAAuB,EAAsD,QAAQ,cAE3F,IAAK,IAAM,KAAY,EAAW,CAC5B,GAAqB,MAAM,EAAsB,EAAqB,CAAG,EACzE,EAAS,eAAe,MAAM,EAAsB,EAAS,cAAe,CAAG,EAEnF,IAAM,EAAU,EAAmB,QAAQ,CAAY,EACnD,EAAQ,EAAS,OAAS,GAC1B,EACJ,GAAI,CACE,EAAQ,eACV,EAAQ,MAAM,EAAQ,aAAa,CAAE,WAAU,MAAK,KAAI,CAAC,GAG3D,EAAY,EAAQ,WAChB,MAAM,MAKF,EAAQ,aAAa,CACnB,WACA,QAAS,EAA2B,CAAO,EAC3C,QACA,MACA,IAAK,CAAE,GAAG,EAAK,GAAG,EAA2B,CAAG,CAAE,EAClD,kBACF,CAAC,CACL,EACC,EAAW,EAAS,CACnB,QACA,MACA,MACA,kBACF,CAAC,CACP,OAAS,EAAO,CAQd,OAPA,EAAoB,CAClB,WAAY,EAAS,GACrB,aAAc,EAAa,cAC3B,QACA,OAAQ,EAAe,CAAK,CAC9B,CAAC,EACD,MAAM,EAAsB,EAAK,CAAW,EACrC,CAAE,YAAa,EAAM,CAC9B,CAEA,IAAM,EAAc,MAAM,EAAgB,EAAK,EAA2B,yBAA2B,CAAC,CAAC,EACjG,EAAoC,CACxC,mBACA,kBAAmB,EAAO,gBAC1B,2BAA4B,CAC1B,gBAAiB,EAA2B,gBAC5C,wBAAyB,EAA2B,uBACtD,CACF,EACM,EAAkB,EAAiB,CAAE,YAAW,cAAa,QAAS,CAAa,CAAC,EACtF,EAAc,EAClB,GAAI,EAAgB,eAAiB,EAAa,SAChD,GAAI,CACF,IAAM,EAAsB,MAAM,EAAQ,OAAO,CAAE,WAAU,YAAW,cAAa,QAAS,CAAa,CAAC,EACxG,IACF,EAAc,CACZ,aAAc,EAAoB,cAAgB,EAAgB,aAClE,iBAAkB,EAAoB,iBACtC,OAAQ,EAAoB,MAC9B,EAEJ,OAAS,EAAO,CACd,EAAc,CACZ,aAAc,EAAa,cAC3B,OAAQ,EAAe,CAAK,CAC9B,CACF,CAgBF,GAbA,EAAoB,CAClB,WAAY,EAAS,GACrB,GAAG,EACH,WAAY,EAAU,OACtB,MAAO,EAAU,MACjB,OAAQ,EAAU,OAAO,MAAM,EAAG,EAAO,eAAe,GAAK,IAAA,GAC7D,QAAS,EAAY,QAAU,EAAU,OAAA,CAAQ,MAAM,EAAG,EAAO,eAAe,GAAK,IAAA,GACrF,YAAa,EAAU,YACvB,YAAa,EAAU,YACvB,YAAa,EAAY,OAAS,EAAI,EAAc,IAAA,EACtD,CAAC,EAED,MAAM,EAAsB,EAAK,CAAW,EACxC,EAAY,eAAiB,EAAa,SAAU,MAAO,CAAE,YAAa,EAAM,CACtF,CAEA,MAAO,CAAE,YAAa,EAAK,CAC7B,CAEA,SAAS,EAAsB,EAA0B,EAA2C,CAClG,OAAO,EAAO,eAAiB,EAAY,iBAAmB,WAChE,CAEA,SAAS,EACP,EACA,EAC2E,CAC3E,IAAM,EAAc,EAClB,EAAa,GACb,EAAa,MAAM,CAAC,EACpB,CAAE,IAAK,EAAQ,IAAK,SAAU,OAAQ,IAAK,EAAQ,GAAI,EACvD,EAAQ,OAAO,mBACjB,EACM,EAAa,EAAY,QAAU,IAAA,GAEzC,GAAI,EAAY,SAAW,EACzB,MAAO,CACL,WAAY,EAAQ,WACpB,aAAc,EAAa,YAC3B,aACA,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,EAGF,GAAI,EAAY,YAAc,EAAQ,OAAO,oBAC3C,MAAO,CACL,WAAY,EAAQ,WACpB,aAAc,EAAa,0BAC3B,aACA,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,EAGF,GACE,EAAY,OAAO,OAAS,EAAQ,OAAO,iBAC3C,EAAY,OAAO,OAAS,EAAQ,OAAO,gBAE3C,MAAO,CACL,WAAY,EAAQ,WACpB,aAAc,EAAa,iCAC3B,aACA,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,OAAQ,EAAY,OAAO,MAAM,EAAG,EAAQ,OAAO,eAAe,EAClE,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,CAIJ,CAEA,SAAS,EAAe,EAAwB,CAC9C,OAAO,aAAiB,MAAQ,EAAM,QAAU,OAAO,CAAK,CAC9D,CAEA,SAAS,EAAkB,EAMH,CACtB,MAAO,CAAE,GAAI,EAAM,GAAI,MAAO,EAAM,MAAO,cAAe,EAAM,aAAc,CAChF,CAEA,eAAe,EACb,EAC+B,CAC/B,IAAM,EAAgB,MAAMA,EAAkB,EAAK,KAAK,EAAY,YAAY,CAAC,EAC3E,EAAmB,EAAc,IAAK,GAAa,EAAkB,CAAQ,CAAc,EAIjG,OAHI,EAAc,QAAQ,cACjB,OAAO,OAAO,EAAkB,CAAE,OAAQ,CAAE,cAAe,EAAc,OAAO,aAAc,CAAE,CAAC,EAEnG,CACT,CAEA,SAAS,EACP,EACA,EAMkB,CAClB,IAAM,EAAc,EAClB,EAAQ,GACR,EAAQ,MAAM,CAAC,EACf,CAAE,IAAK,EAAQ,IAAK,SAAU,OAAQ,MAAO,EAAQ,MAAO,IAAK,EAAQ,GAAI,EAC7E,EAAQ,gBACV,EAEA,MAAO,CACL,MAAO,EAAQ,MACf,OAAQ,EAAY,OACpB,OAAQ,EAAY,OACpB,OAAQ,EAAY,QAAU,IAAA,GAC9B,YAAa,EAAY,YACzB,YAAa,EAAY,WAC3B,CACF,CAEA,SAAS,EAAiB,EAIC,CACzB,GAAI,EAAQ,UAAU,SAAW,EAC/B,MAAO,CAAE,aAAc,EAAa,cAAe,OAAQ,EAAQ,UAAU,MAAO,EAGtF,GAAI,EAAQ,UAAU,YAAc,EAAQ,QAAQ,iBAClD,MAAO,CAAE,aAAc,EAAa,oBAAqB,OAAQ,EAAQ,UAAU,MAAO,EAG5F,GACE,EAAQ,UAAU,aACjB,EAAQ,QAAQ,2BAA2B,iBAAmB,KAE/D,MAAO,CAAE,aAAc,EAAa,sBAAuB,OAAQ,EAAQ,UAAU,MAAO,EAG9F,GACE,EAAQ,UAAU,qBAClB,EAAQ,UAAU,OAAO,OAAS,EAAQ,QAAQ,mBAClD,EAAQ,UAAU,OAAO,OAAS,EAAQ,QAAQ,kBAElD,MAAO,CAAE,aAAc,EAAa,2BAA4B,OAAQ,EAAQ,UAAU,MAAO,EAGnG,IAAM,EAA0B,EAAQ,QAAQ,2BAA2B,yBAAyB,QAAU,EAK9G,OAJI,EAAQ,YAAY,OAAS,EACxB,CAAE,aAAc,EAAa,kCAAmC,EAGlE,CAAE,aAAc,EAAa,QAAS,CAC/C"}

package/dist/presets/evaluation.cjs

@@ -1,11 +1,11 @@
-"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("../_virtual/_rolldown/runtime.cjs"),t=require("../helpers/parseArgs.cjs"),n=require("../helpers/parseCsv.cjs"),r=require("../helpers/printTestCaseResult.cjs"),i=require("../types/decisionCode.cjs"),a=require("../helpers/printDebugBanner.cjs"),o=require("../helpers/checkProblemDirIsolation.cjs"),s=require("../helpers/judgeByStaticAnalysis.cjs"),c=require("../helpers/readOutputFiles.cjs"),l=require("../helpers/readProblemMarkdownFrontMatter.cjs"),u=require("../helpers/resolveCwds.cjs");let d=require("node:fs/promises");d=e.__toESM(d,1);let f=require("node:path");f=e.__toESM(f,1);const p={unterminated_quote:'引用符（`"`）が閉じられていません',unexpected_quote:'引用符（`"`）は値全体を囲む位置にしか書けません',text_after_quoted_field:`閉じ引用符の後に区切り文字以外の文字があります`},m={rmsle:{label:`RMSLE`,isHigherBetter:!1,validatePrediction:e=>D(e)??(Number(e)<0?`負の値です`:void 0),compute:(e,t)=>Math.sqrt(O(e.map((e,n)=>(Math.log1p(Number(e))-Math.log1p(Number(t[n])))**2)))},rmse:{label:`RMSE`,isHigherBetter:!1,validatePrediction:D,compute:(e,t)=>Math.sqrt(O(e.map((e,n)=>(Number(e)-Number(t[n]))**2)))},mae:{label:`MAE`,isHigherBetter:!1,validatePrediction:D,compute:(e,t)=>O(e.map((e,n)=>Math.abs(Number(e)-Number(t[n]))))},accuracy:{label:`Accuracy`,isHigherBetter:!0,validatePrediction:e=>e.trim()===``?`空です`:void 0,compute:(e,t)=>O(e.map((e,n)=>+(e.trim()===t[n]?.trim())))}};async function h(e,n){let i=t.parseArgs(process.argv),{cwds:s,isDebugMode:c}=await u.resolveCwds(e,i.cwd);if(c){let t=s.find(e=>e.expectedResult===`accepted`);if(t){if(!(await o.checkProblemDirIsolation(e,t,i.params)).passed){process.exitCode=1;return}}else a.printDebugBanner([`[DEBUG MODE] isolated problem directory check skipped`,``,`No accepted model answer is available for checking that the copied judge still accepts a valid submission.`])}let l=await x(f.default.join(e,n.answerFilePath),n);for(let t of s){c&&u.printDebugCwdBanner(e,t);let i=await g(e,t.cwd,l,n);r.printTestCaseResult({testCaseId:`evaluation`,...i}),c&&!E(t,i)&&(process.exitCode=1,u.printDebugExpectationFailureBanner(e,t))}}async function g(e,t,n,r){let a=await s.judgeByStaticAnalysis(t,await l.readProblemMarkdownFrontMatter(e));if(a)return a;let o=r.submissionFilePath??`submission.csv`,u;try{let e=f.default.join(t,o);if(!await c.isSafeSubmissionOutputPath(t,e))throw Error(`unsafe path`);u=await d.default.readFile(e,`utf8`)}catch{return{decisionCode:i.DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,feedbackMarkdown:`\`${o}\`が見つかりません。予測結果のCSVファイルを\`${o}\`という名前で提出してください。`}}let p=b(u,o,n,r);if(typeof p==`string`)return{decisionCode:i.DecisionCode.WRONG_ANSWER,feedbackMarkdown:p};let m=[...n.values()],h=r.metric.compute([...n.keys()].map(e=>p.get(e)),m);if(!Number.isFinite(h))return{decisionCode:i.DecisionCode.WRONG_ANSWER,feedbackMarkdown:`予測値が極端なため、${r.metric.label}を計算できませんでした。予測値を見直してください。`};let g=r.acceptableScore===void 0||(r.metric.isHigherBetter?h>=r.acceptableScore:h<=r.acceptableScore);return{decisionCode:g?i.DecisionCode.ACCEPTED:i.DecisionCode.WRONG_ANSWER,score:h,scoreLabel:r.metric.label,feedbackMarkdown:_(h,m.length,g,r)}}function _(e,t,n,r){let{label:i,isHigherBetter:a}=r.metric,o=v(e,r.acceptableScore),s=r.acceptableScore===void 0?``:`\n合格基準: ${i} ${a?`≧`:`≦`} ${y(r.acceptableScore,o)}（${n?`達成`:`未達成`}）`;return`| 指標 | スコア |
+"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("../_virtual/_rolldown/runtime.cjs"),t=require("../helpers/parseArgs.cjs"),n=require("../helpers/parseCsv.cjs"),r=require("../helpers/printTestCaseResult.cjs"),i=require("../types/decisionCode.cjs"),a=require("../helpers/judgeByStaticAnalysis.cjs"),o=require("../helpers/readOutputFiles.cjs"),s=require("../helpers/runIsolationCheckInDebugMode.cjs"),c=require("../helpers/readProblemMarkdownFrontMatter.cjs"),l=require("../helpers/resolveCwds.cjs");let u=require("node:fs/promises");u=e.__toESM(u,1);let d=require("node:path");d=e.__toESM(d,1);const f={unterminated_quote:'引用符（`"`）が閉じられていません',unexpected_quote:'引用符（`"`）は値全体を囲む位置にしか書けません',text_after_quoted_field:`閉じ引用符の後に区切り文字以外の文字があります`},p={rmsle:{label:`RMSLE`,isHigherBetter:!1,validatePrediction:e=>E(e)??(Number(e)<0?`負の値です`:void 0),compute:(e,t)=>Math.sqrt(D(e.map((e,n)=>(Math.log1p(Number(e))-Math.log1p(Number(t[n])))**2)))},rmse:{label:`RMSE`,isHigherBetter:!1,validatePrediction:E,compute:(e,t)=>Math.sqrt(D(e.map((e,n)=>(Number(e)-Number(t[n]))**2)))},mae:{label:`MAE`,isHigherBetter:!1,validatePrediction:E,compute:(e,t)=>D(e.map((e,n)=>Math.abs(Number(e)-Number(t[n]))))},accuracy:{label:`Accuracy`,isHigherBetter:!0,validatePrediction:e=>e.trim()===``?`空です`:void 0,compute:(e,t)=>D(e.map((e,n)=>+(e.trim()===t[n]?.trim())))}};async function m(e,n){let i=t.parseArgs(process.argv),{cwds:a,isDebugMode:o}=await l.resolveCwds(e,i.cwd);if(o&&!await s.passesIsolationCheckInDebugMode(e,a,i.params)){process.exitCode=1;return}let c=await b(d.default.join(e,n.answerFilePath),n);for(let t of a){o&&l.printDebugCwdBanner(e,t);let i=await h(e,t.cwd,c,n);r.printTestCaseResult({testCaseId:`evaluation`,...i}),o&&!T(t,i)&&(process.exitCode=1,l.printDebugExpectationFailureBanner(e,t))}}async function h(e,t,n,r){let s=await a.judgeByStaticAnalysis(t,await c.readProblemMarkdownFrontMatter(e));if(s)return s;let l=r.submissionFilePath??`submission.csv`,f;try{let e=d.default.join(t,l);if(!await o.isSafeSubmissionOutputPath(t,e))throw Error(`unsafe path`);f=await u.default.readFile(e,`utf8`)}catch{return{decisionCode:i.DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,feedbackMarkdown:`\`${l}\`が見つかりません。予測結果のCSVファイルを\`${l}\`という名前で提出してください。`}}let p=y(f,l,n,r);if(typeof p==`string`)return{decisionCode:i.DecisionCode.WRONG_ANSWER,feedbackMarkdown:p};let m=[...n.values()],h=r.metric.compute([...n.keys()].map(e=>p.get(e)),m);if(!Number.isFinite(h))return{decisionCode:i.DecisionCode.WRONG_ANSWER,feedbackMarkdown:`予測値が極端なため、${r.metric.label}を計算できませんでした。予測値を見直してください。`};let _=r.acceptableScore===void 0||(r.metric.isHigherBetter?h>=r.acceptableScore:h<=r.acceptableScore);return{decisionCode:_?i.DecisionCode.ACCEPTED:i.DecisionCode.WRONG_ANSWER,score:h,scoreLabel:r.metric.label,feedbackMarkdown:g(h,m.length,_,r)}}function g(e,t,n,r){let{label:i,isHigherBetter:a}=r.metric,o=_(e,r.acceptableScore),s=r.acceptableScore===void 0?``:`\n合格基準: ${i} ${a?`≧`:`≦`} ${v(r.acceptableScore,o)}（${n?`達成`:`未達成`}）`;return`| 指標 | スコア |
 | ---- | ------ |
-| ${i} | ${y(e,o)} |
+| ${i} | ${v(e,o)} |
 
 評価件数: ${t.toLocaleString(`en-US`)}件${s}
-`}function v(e,t){if(t===void 0||e===t)return 6;for(let n=6;n<17;n++)if(y(e,n)!==y(t,n))return n;return 17}function y(e,t){return Number(e.toPrecision(t)).toString()}function b(e,t,r,i){let a;try{a=n.parseCsvRecords(e)}catch(e){return`\`${t}\`をCSVとして読み込めませんでした。${e instanceof n.CsvParseError?`${e.lineNumber}行目付近: ${p[e.reason]}`:``}`}let o=S(a[0]?.cells??[],i);if(!o)return`\`${t}\`の1行目には\`${i.idColumn}\`列と\`${i.targetColumn}\`列のヘッダーが必要です。`;let s=new Map,c=[],l=0,u=e=>{l++,c.length<10&&c.push(e)};for(let{cells:e,lineNumber:t}of a.slice(1)){if(C(e))continue;let n=e[o.idIndex]?.trim()??``,a=e[o.targetIndex]??``;if(!r.has(n))u(`${t}行目: \`${i.idColumn}\`が\`${T(n)}\`の行は評価対象ではありません`);else if(s.has(n))u(`${t}行目: \`${i.idColumn}\`が\`${T(n)}\`の行が重複しています`);else{let e=i.metric.validatePrediction?.(a);e&&u(`${t}行目: \`${i.targetColumn}\`の値\`${T(a)}\`は${e}`),s.set(n,a)}}let d=[...r.keys()].filter(e=>!s.has(e));return d.length>0&&(l++,c.unshift(`\`${i.idColumn}\`が${w(d)}の行がありません（不足 ${d.length.toLocaleString(`en-US`)}件）`)),l>0?`\`${t}\`の内容に問題があります。
+`}function _(e,t){if(t===void 0||e===t)return 6;for(let n=6;n<17;n++)if(v(e,n)!==v(t,n))return n;return 17}function v(e,t){return Number(e.toPrecision(t)).toString()}function y(e,t,r,i){let a;try{a=n.parseCsvRecords(e)}catch(e){return`\`${t}\`をCSVとして読み込めませんでした。${e instanceof n.CsvParseError?`${e.lineNumber}行目付近: ${f[e.reason]}`:``}`}let o=x(a[0]?.cells??[],i);if(!o)return`\`${t}\`の1行目には\`${i.idColumn}\`列と\`${i.targetColumn}\`列のヘッダーが必要です。`;let s=new Map,c=[],l=0,u=e=>{l++,c.length<10&&c.push(e)};for(let{cells:e,lineNumber:t}of a.slice(1)){if(S(e))continue;let n=e[o.idIndex]?.trim()??``,a=e[o.targetIndex]??``;if(!r.has(n))u(`${t}行目: \`${i.idColumn}\`が\`${w(n)}\`の行は評価対象ではありません`);else if(s.has(n))u(`${t}行目: \`${i.idColumn}\`が\`${w(n)}\`の行が重複しています`);else{let e=i.metric.validatePrediction?.(a);e&&u(`${t}行目: \`${i.targetColumn}\`の値\`${w(a)}\`は${e}`),s.set(n,a)}}let d=[...r.keys()].filter(e=>!s.has(e));return d.length>0&&(l++,c.unshift(`\`${i.idColumn}\`が${C(d)}の行がありません（不足 ${d.length.toLocaleString(`en-US`)}件）`)),l>0?`\`${t}\`の内容に問題があります。
 
 ${c.map(e=>`- ${e}`).join(`
 `)}${l>c.length?`\n- ほか${(l-c.length).toLocaleString(`en-US`)}件`:``}
-`:s}async function x(e,t){let r=n.parseCsv(await d.default.readFile(e,`utf8`)),i=S(r[0]??[],t);if(!i)throw Error(`answer file must have ${t.idColumn} and ${t.targetColumn} columns: ${e}`);let a=new Map;for(let n of r.slice(1)){if(C(n))continue;let r=n[i.idIndex]?.trim()??``,o=n[i.targetIndex]??``;if(!r)throw Error(`empty ${t.idColumn} in answer file: ${e}`);if(a.has(r))throw Error(`duplicate ${t.idColumn} in answer file: ${r}`);let s=t.metric.validatePrediction?.(o);if(s)throw Error(`invalid ${t.targetColumn} in answer file for ${r}: ${s}`);a.set(r,o)}if(a.size===0)throw Error(`answer file has no rows: ${e}`);return a}function S(e,t){let n=e.map(e=>e.trim()),r=n.indexOf(t.idColumn),i=n.indexOf(t.targetColumn);return r===-1||i===-1?void 0:{idIndex:r,targetIndex:i}}function C(e){return e.every(e=>e.trim()===``)}function w(e){let t=e.slice(0,10).map(e=>`\`${T(e)}\``);return e.length>10?`${t.join(`, `)}, ...`:t.join(`, `)}function T(e){let t=e.replaceAll(/[\p{Cc}]/gu,` `).replaceAll("`",`'`);return t.trim()===``?`（空）`:t.length>50?`${t.slice(0,50)}…`:t}function E(e,t){return t.decisionCode===i.DecisionCode.ACCEPTED==(e.expectedResult===`accepted`)}function D(e){return Number.isFinite(Number(e))&&e.trim()!==``?void 0:`数値ではありません`}function O(e){return e.reduce((e,t)=>e+t,0)/e.length}exports.evaluationJudgePreset=h,exports.evaluationMetrics=m;
+`:s}async function b(e,t){let r=n.parseCsv(await u.default.readFile(e,`utf8`)),i=x(r[0]??[],t);if(!i)throw Error(`answer file must have ${t.idColumn} and ${t.targetColumn} columns: ${e}`);let a=new Map;for(let n of r.slice(1)){if(S(n))continue;let r=n[i.idIndex]?.trim()??``,o=n[i.targetIndex]??``;if(!r)throw Error(`empty ${t.idColumn} in answer file: ${e}`);if(a.has(r))throw Error(`duplicate ${t.idColumn} in answer file: ${r}`);let s=t.metric.validatePrediction?.(o);if(s)throw Error(`invalid ${t.targetColumn} in answer file for ${r}: ${s}`);a.set(r,o)}if(a.size===0)throw Error(`answer file has no rows: ${e}`);return a}function x(e,t){let n=e.map(e=>e.trim()),r=n.indexOf(t.idColumn),i=n.indexOf(t.targetColumn);return r===-1||i===-1?void 0:{idIndex:r,targetIndex:i}}function S(e){return e.every(e=>e.trim()===``)}function C(e){let t=e.slice(0,10).map(e=>`\`${w(e)}\``);return e.length>10?`${t.join(`, `)}, ...`:t.join(`, `)}function w(e){let t=e.replaceAll(/[\p{Cc}]/gu,` `).replaceAll("`",`'`);return t.trim()===``?`（空）`:t.length>50?`${t.slice(0,50)}…`:t}function T(e,t){return t.decisionCode===i.DecisionCode.ACCEPTED==(e.expectedResult===`accepted`)}function E(e){return Number.isFinite(Number(e))&&e.trim()!==``?void 0:`数値ではありません`}function D(e){return e.reduce((e,t)=>e+t,0)/e.length}exports.evaluationJudgePreset=m,exports.evaluationMetrics=p;
 //# sourceMappingURL=evaluation.cjs.map

package/dist/presets/evaluation.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation.cjs","names":["parseArgs","resolveCwds","checkProblemDirIsolation","path","judgeByStaticAnalysis","readProblemMarkdownFrontMatter","isSafeSubmissionOutputPath","fs","DecisionCode","parseCsvRecords","CsvParseError","parseCsv"],"sources":["../../src/presets/evaluation.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { checkProblemDirIsolation } from '../helpers/checkProblemDirIsolation.js';\nimport { judgeByStaticAnalysis } from '../helpers/judgeByStaticAnalysis.js';\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { CsvParseError, type CsvParseErrorReason, parseCsv, parseCsvRecords } from '../helpers/parseCsv.js';\nimport { printDebugBanner } from '../helpers/printDebugBanner.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { isSafeSubmissionOutputPath } from '../helpers/readOutputFiles.js';\nimport { readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport {\n  printDebugCwdBanner,\n  printDebugExpectationFailureBanner,\n  resolveCwds,\n  type ResolvedCwd,\n} from '../helpers/resolveCwds.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst DEFAULT_SUBMISSION_FILE_PATH = 'submission.csv';\nconst TEST_CASE_ID = 'evaluation';\nconst MAX_LISTED_IDS = 10;\n// Learner-controlled cells are echoed into the feedback, so clip them to keep the result line small.\nconst MAX_ECHOED_CELL_LENGTH = 50;\nconst CSV_PARSE_ERROR_MESSAGES: Record<CsvParseErrorReason, string> = {\n  unterminated_quote: '引用符（`\"`）が閉じられていません',\n  unexpected_quote: '引用符（`\"`）は値全体を囲む位置にしか書けません',\n  text_after_quoted_field: '閉じ引用符の後に区切り文字以外の文字があります',\n};\n\nexport interface EvaluationMetric {\n  /** Label shown with the score, e.g. `RMSLE`. */\n  label: string;\n  isHigherBetter: boolean;\n  /**\n   * Returns an error message when a predicted value cannot be scored (e.g. it is not a number).\n   * The message is appended to the row's line number and the offending value, so phrase it as a\n   * predicate such as `数値ではありません`. It also validates the ground truth when it is loaded.\n   */\n  validatePrediction?: (prediction: string) => string | undefined;\n  /** Computes the score from predictions and answers aligned by id. Both are raw CSV cell strings. */\n  compute: (predictions: readonly string[], answers: readonly string[]) => number;\n}\n\nexport interface EvaluationJudgePresetOptions {\n  /** CSV file holding the hidden ground truth, relative to the problem directory. */\n  answerFilePath: string;\n  /** CSV file the submission must contain. Defaults to `submission.csv`. */\n  submissionFilePath?: string;\n  /** Column that identifies each row in both CSV files. */\n  idColumn: string;\n  /** Column holding the answer in the ground truth and the prediction in the submission. */\n  targetColumn: string;\n  metric: EvaluationMetric;\n  /**\n   * Score required for acceptance: the submission is accepted when its score is at least this value\n   * for a higher-is-better metric and at most this value otherwise. Every valid submission is\n   * accepted when omitted.\n   */\n  acceptableScore?: number;\n}\n\nexport const evaluationMetrics = {\n  rmsle: {\n    label: 'RMSLE',\n    isHigherBetter: false,\n    validatePrediction: (prediction: string) =>\n      validateFiniteNumber(prediction) ?? (Number(prediction) < 0 ? '負の値です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(\n        mean(\n          predictions.map(\n            (prediction, index) => (Math.log1p(Number(prediction)) - Math.log1p(Number(answers[index]))) ** 2\n          )\n        )\n      ),\n  },\n  rmse: {\n    label: 'RMSE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(mean(predictions.map((prediction, index) => (Number(prediction) - Number(answers[index])) ** 2))),\n  },\n  mae: {\n    label: 'MAE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => Math.abs(Number(prediction) - Number(answers[index])))),\n  },\n  accuracy: {\n    label: 'Accuracy',\n    isHigherBetter: true,\n    validatePrediction: (prediction: string) => (prediction.trim() === '' ? '空です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => (prediction.trim() === answers[index]?.trim() ? 1 : 0))),\n  },\n} as const satisfies Record<string, EvaluationMetric>;\n\n/**\n * A preset for Kaggle-style model evaluation problems: the submission is a CSV of predictions for a\n * hidden test set, and the judge reports the metric value as `score` instead of comparing outputs.\n * The problem directory keeps the ground truth CSV, so it is never shipped to learners.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { evaluationJudgePreset, evaluationMetrics } from '@exercode/problem-utils/presets/evaluation';\n *\n * await evaluationJudgePreset(import.meta.dirname, {\n *   answerFilePath: 'evaluation/answer.csv',\n *   idColumn: 'Id',\n *   targetColumn: 'TradePrice',\n *   metric: evaluationMetrics.rmsle,\n *   acceptableScore: 0.5,\n * });\n * ```\n *\n * Run `bun judge.ts model_answers/default` to judge one submission, or `bun judge.ts` to judge every\n * `<problemDir>/model_answers/*` and `<problemDir>/model_answers.fails/*` directory for debugging.\n */\nexport async function evaluationJudgePreset(problemDir: string, options: EvaluationJudgePresetOptions): Promise<void> {\n  const args = parseArgs(process.argv);\n  const { cwds, isDebugMode } = await resolveCwds(problemDir, args.cwd);\n\n  if (isDebugMode) {\n    const acceptedCwd = cwds.find((cwd) => cwd.expectedResult === 'accepted');\n    if (acceptedCwd) {\n      const isolationCheckResult = await checkProblemDirIsolation(problemDir, acceptedCwd, args.params);\n      if (!isolationCheckResult.passed) {\n        process.exitCode = 1;\n        return;\n      }\n    } else {\n      printDebugBanner([\n        '[DEBUG MODE] isolated problem directory check skipped',\n        '',\n        'No accepted model answer is available for checking that the copied judge still accepts a valid submission.',\n      ]);\n    }\n  }\n\n  const answer = await readAnswer(path.join(problemDir, options.answerFilePath), options);\n  for (const resolvedCwd of cwds) {\n    if (isDebugMode) printDebugCwdBanner(problemDir, resolvedCwd);\n    const result = await judgeSubmission(problemDir, resolvedCwd.cwd, answer, options);\n    printTestCaseResult({ testCaseId: TEST_CASE_ID, ...result });\n    if (isDebugMode && !matchesExpectedResult(resolvedCwd, result)) {\n      process.exitCode = 1;\n      printDebugExpectationFailureBanner(problemDir, resolvedCwd);\n    }\n  }\n}\n\nasync function judgeSubmission(\n  problemDir: string,\n  cwd: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Promise<Omit<TestCaseResult, 'testCaseId'>> {\n  const staticAnalysisResult = await judgeByStaticAnalysis(cwd, await readProblemMarkdownFrontMatter(problemDir));\n  if (staticAnalysisResult) return staticAnalysisResult;\n\n  const submissionFilePath = options.submissionFilePath ?? DEFAULT_SUBMISSION_FILE_PATH;\n  let submissionText: string;\n  try {\n    const resolvedPath = path.join(cwd, submissionFilePath);\n    // The harness reads the file as the trusted user, so refuse symlinks escaping the submission.\n    if (!(await isSafeSubmissionOutputPath(cwd, resolvedPath))) throw new Error('unsafe path');\n    submissionText = await fs.readFile(resolvedPath, 'utf8');\n  } catch {\n    return {\n      decisionCode: DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,\n      feedbackMarkdown: `\\`${submissionFilePath}\\`が見つかりません。予測結果のCSVファイルを\\`${submissionFilePath}\\`という名前で提出してください。`,\n    };\n  }\n\n  const predictions = readPredictions(submissionText, submissionFilePath, answer, options);\n  if (typeof predictions === 'string') {\n    return { decisionCode: DecisionCode.WRONG_ANSWER, feedbackMarkdown: predictions };\n  }\n\n  const answerValues = [...answer.values()];\n  const score = options.metric.compute(\n    [...answer.keys()].map((id) => predictions.get(id) as string),\n    answerValues\n  );\n  if (!Number.isFinite(score)) {\n    return {\n      decisionCode: DecisionCode.WRONG_ANSWER,\n      feedbackMarkdown: `予測値が極端なため、${options.metric.label}を計算できませんでした。予測値を見直してください。`,\n    };\n  }\n  const isAccepted =\n    options.acceptableScore === undefined ||\n    (options.metric.isHigherBetter ? score >= options.acceptableScore : score <= options.acceptableScore);\n  return {\n    decisionCode: isAccepted ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER,\n    score,\n    scoreLabel: options.metric.label,\n    feedbackMarkdown: buildScoreMarkdown(score, answerValues.length, isAccepted, options),\n  };\n}\n\nfunction buildScoreMarkdown(\n  score: number,\n  rowCount: number,\n  isAccepted: boolean,\n  options: EvaluationJudgePresetOptions\n): string {\n  const { label, isHigherBetter } = options.metric;\n  const precision = findDistinguishingPrecision(score, options.acceptableScore);\n  const acceptanceLine =\n    options.acceptableScore === undefined\n      ? ''\n      : `\\n合格基準: ${label} ${isHigherBetter ? '≧' : '≦'} ${formatScore(options.acceptableScore, precision)}（${isAccepted ? '達成' : '未達成'}）`;\n  return `| 指標 | スコア |\n| ---- | ------ |\n| ${label} | ${formatScore(score, precision)} |\n\n評価件数: ${rowCount.toLocaleString('en-US')}件${acceptanceLine}\n`;\n}\n\n/**\n * Finds the number of significant digits at which the score and the threshold render differently\n * (or stop changing), so the feedback never shows a pair that contradicts the decision.\n */\nfunction findDistinguishingPrecision(score: number, threshold: number | undefined): number {\n  const minimumPrecision = 6;\n  if (threshold === undefined || score === threshold) return minimumPrecision;\n  for (let precision = minimumPrecision; precision < 17; precision++) {\n    if (formatScore(score, precision) !== formatScore(threshold, precision)) return precision;\n  }\n  return 17;\n}\n\nfunction formatScore(score: number, precision: number): string {\n  return Number(score.toPrecision(precision)).toString();\n}\n\nfunction readPredictions(\n  submissionText: string,\n  submissionFilePath: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Map<string, string> | string {\n  let records: ReturnType<typeof parseCsvRecords>;\n  try {\n    records = parseCsvRecords(submissionText);\n  } catch (error) {\n    const detail =\n      error instanceof CsvParseError ? `${error.lineNumber}行目付近: ${CSV_PARSE_ERROR_MESSAGES[error.reason]}` : '';\n    return `\\`${submissionFilePath}\\`をCSVとして読み込めませんでした。${detail}`;\n  }\n  const columns = findColumns(records[0]?.cells ?? [], options);\n  if (!columns) {\n    return `\\`${submissionFilePath}\\`の1行目には\\`${options.idColumn}\\`列と\\`${options.targetColumn}\\`列のヘッダーが必要です。`;\n  }\n\n  const predictions = new Map<string, string>();\n  const problems: string[] = [];\n  let problemCount = 0;\n  const reportProblem = (problem: string): void => {\n    problemCount++;\n    if (problems.length < MAX_LISTED_IDS) problems.push(problem);\n  };\n  for (const { cells: row, lineNumber } of records.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const prediction = row[columns.targetIndex] ?? '';\n    if (!answer.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行は評価対象ではありません`);\n    } else if (predictions.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行が重複しています`);\n    } else {\n      const validationError = options.metric.validatePrediction?.(prediction);\n      if (validationError) {\n        reportProblem(`${lineNumber}行目: \\`${options.targetColumn}\\`の値\\`${clip(prediction)}\\`は${validationError}`);\n      }\n      predictions.set(id, prediction);\n    }\n  }\n  const missingIds = [...answer.keys()].filter((id) => !predictions.has(id));\n  if (missingIds.length > 0) {\n    // Missing rows are the most actionable problem, so they must survive the listing cap.\n    problemCount++;\n    problems.unshift(\n      `\\`${options.idColumn}\\`が${listIds(missingIds)}の行がありません（不足 ${missingIds.length.toLocaleString('en-US')}件）`\n    );\n  }\n  if (problemCount > 0) {\n    return `\\`${submissionFilePath}\\`の内容に問題があります。\n\n${problems.map((problem) => `- ${problem}`).join('\\n')}${problemCount > problems.length ? `\\n- ほか${(problemCount - problems.length).toLocaleString('en-US')}件` : ''}\n`;\n  }\n  return predictions;\n}\n\nasync function readAnswer(answerFilePath: string, options: EvaluationJudgePresetOptions): Promise<Map<string, string>> {\n  const rows = parseCsv(await fs.readFile(answerFilePath, 'utf8'));\n  const columns = findColumns(rows[0] ?? [], options);\n  if (!columns) {\n    throw new Error(`answer file must have ${options.idColumn} and ${options.targetColumn} columns: ${answerFilePath}`);\n  }\n  const answer = new Map<string, string>();\n  for (const row of rows.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const value = row[columns.targetIndex] ?? '';\n    if (!id) throw new Error(`empty ${options.idColumn} in answer file: ${answerFilePath}`);\n    if (answer.has(id)) throw new Error(`duplicate ${options.idColumn} in answer file: ${id}`);\n    const validationError = options.metric.validatePrediction?.(value);\n    if (validationError)\n      throw new Error(`invalid ${options.targetColumn} in answer file for ${id}: ${validationError}`);\n    answer.set(id, value);\n  }\n  if (answer.size === 0) throw new Error(`answer file has no rows: ${answerFilePath}`);\n  return answer;\n}\n\nfunction findColumns(\n  header: readonly string[],\n  options: EvaluationJudgePresetOptions\n): { idIndex: number; targetIndex: number } | undefined {\n  const names = header.map((name) => name.trim());\n  const idIndex = names.indexOf(options.idColumn);\n  const targetIndex = names.indexOf(options.targetColumn);\n  return idIndex === -1 || targetIndex === -1 ? undefined : { idIndex, targetIndex };\n}\n\nfunction isBlankRow(row: readonly string[]): boolean {\n  return row.every((cell) => cell.trim() === '');\n}\n\nfunction listIds(ids: readonly string[]): string {\n  const listed = ids.slice(0, MAX_LISTED_IDS).map((id) => `\\`${clip(id)}\\``);\n  return ids.length > MAX_LISTED_IDS ? `${listed.join(', ')}, ...` : listed.join(', ');\n}\n\nfunction clip(value: string): string {\n  // Control characters and backticks would break the markdown list item and inline code the value is echoed in.\n  const printable = value.replaceAll(/[\\p{Cc}]/gu, ' ').replaceAll('`', \"'\");\n  if (printable.trim() === '') return '（空）';\n  return printable.length > MAX_ECHOED_CELL_LENGTH ? `${printable.slice(0, MAX_ECHOED_CELL_LENGTH)}…` : printable;\n}\n\nfunction matchesExpectedResult(resolvedCwd: ResolvedCwd, result: Pick<TestCaseResult, 'decisionCode'>): boolean {\n  return (result.decisionCode === DecisionCode.ACCEPTED) === (resolvedCwd.expectedResult === 'accepted');\n}\n\nfunction validateFiniteNumber(prediction: string): string | undefined {\n  return Number.isFinite(Number(prediction)) && prediction.trim() !== '' ? undefined : '数値ではありません';\n}\n\nfunction mean(values: readonly number[]): number {\n  return values.reduce((sum, value) => sum + value, 0) / values.length;\n}\n"],"mappings":"yqBAoBA,MAKM,EAAgE,CACpE,mBAAoB,qBACpB,iBAAkB,4BAClB,wBAAyB,yBAC3B,EAkCa,EAAoB,CAC/B,MAAO,CACL,MAAO,QACP,eAAgB,GAChB,mBAAqB,GACnB,EAAqB,CAAU,IAAM,OAAO,CAAU,EAAI,EAAI,QAAU,IAAA,IAC1E,SAAU,EAAgC,IACxC,KAAK,KACH,EACE,EAAY,KACT,EAAY,KAAW,KAAK,MAAM,OAAO,CAAU,CAAC,EAAI,KAAK,MAAM,OAAO,EAAQ,EAAM,CAAC,IAAM,CAClG,CACF,CACF,CACJ,EACA,KAAM,CACJ,MAAO,OACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,KAAK,KAAK,EAAK,EAAY,KAAK,EAAY,KAAW,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,IAAM,CAAC,CAAC,CAAC,CAC9G,EACA,IAAK,CACH,MAAO,MACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAU,KAAK,IAAI,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,CAAC,CAAC,CAAC,CACtG,EACA,SAAU,CACR,MAAO,WACP,eAAgB,GAChB,mBAAqB,GAAwB,EAAW,KAAK,IAAM,GAAK,MAAQ,IAAA,GAChF,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAW,IAAW,KAAK,IAAM,EAAQ,EAAM,EAAE,KAAK,EAAU,CAAC,CACvG,CACF,EAwBA,eAAsB,EAAsB,EAAoB,EAAsD,CACpH,IAAM,EAAOA,EAAAA,UAAU,QAAQ,IAAI,EAC7B,CAAE,OAAM,eAAgB,MAAMC,EAAAA,YAAY,EAAY,EAAK,GAAG,EAEpE,GAAI,EAAa,CACf,IAAM,EAAc,EAAK,KAAM,GAAQ,EAAI,iBAAmB,UAAU,EACxE,GAAI,EAEE,IAAA,EAAC,MAD8BC,EAAAA,yBAAyB,EAAY,EAAa,EAAK,MAAM,EAAA,CACtE,OAAQ,CAChC,QAAQ,SAAW,EACnB,MACF,OAEA,EAAA,iBAAiB,CACf,wDACA,GACA,4GACF,CAAC,CAEL,CAEA,IAAM,EAAS,MAAM,EAAWC,EAAAA,QAAK,KAAK,EAAY,EAAQ,cAAc,EAAG,CAAO,EACtF,IAAK,IAAM,KAAe,EAAM,CAC1B,GAAa,EAAA,oBAAoB,EAAY,CAAW,EAC5D,IAAM,EAAS,MAAM,EAAgB,EAAY,EAAY,IAAK,EAAQ,CAAO,EACjF,EAAA,oBAAoB,CAAE,WAAY,aAAc,GAAG,CAAO,CAAC,EACvD,GAAe,CAAC,EAAsB,EAAa,CAAM,IAC3D,QAAQ,SAAW,EACnB,EAAA,mCAAmC,EAAY,CAAW,EAE9D,CACF,CAEA,eAAe,EACb,EACA,EACA,EACA,EAC6C,CAC7C,IAAM,EAAuB,MAAMC,EAAAA,sBAAsB,EAAK,MAAMC,EAAAA,+BAA+B,CAAU,CAAC,EAC9G,GAAI,EAAsB,OAAO,EAEjC,IAAM,EAAqB,EAAQ,oBAAsB,iBACrD,EACJ,GAAI,CACF,IAAM,EAAeF,EAAAA,QAAK,KAAK,EAAK,CAAkB,EAEtD,GAAI,CAAE,MAAMG,EAAAA,2BAA2B,EAAK,CAAY,EAAI,MAAU,MAAM,aAAa,EACzF,EAAiB,MAAMC,EAAAA,QAAG,SAAS,EAAc,MAAM,CACzD,MAAQ,CACN,MAAO,CACL,aAAcC,EAAAA,aAAa,uCAC3B,iBAAkB,KAAK,EAAmB,4BAA4B,EAAmB,kBAC3F,CACF,CAEA,IAAM,EAAc,EAAgB,EAAgB,EAAoB,EAAQ,CAAO,EACvF,GAAI,OAAO,GAAgB,SACzB,MAAO,CAAE,aAAcA,EAAAA,aAAa,aAAc,iBAAkB,CAAY,EAGlF,IAAM,EAAe,CAAC,GAAG,EAAO,OAAO,CAAC,EAClC,EAAQ,EAAQ,OAAO,QAC3B,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,IAAK,GAAO,EAAY,IAAI,CAAE,CAAW,EAC5D,CACF,EACA,GAAI,CAAC,OAAO,SAAS,CAAK,EACxB,MAAO,CACL,aAAcA,EAAAA,aAAa,aAC3B,iBAAkB,aAAa,EAAQ,OAAO,MAAM,0BACtD,EAEF,IAAM,EACJ,EAAQ,kBAAoB,IAAA,KAC3B,EAAQ,OAAO,eAAiB,GAAS,EAAQ,gBAAkB,GAAS,EAAQ,iBACvF,MAAO,CACL,aAAc,EAAaA,EAAAA,aAAa,SAAWA,EAAAA,aAAa,aAChE,QACA,WAAY,EAAQ,OAAO,MAC3B,iBAAkB,EAAmB,EAAO,EAAa,OAAQ,EAAY,CAAO,CACtF,CACF,CAEA,SAAS,EACP,EACA,EACA,EACA,EACQ,CACR,GAAM,CAAE,QAAO,kBAAmB,EAAQ,OACpC,EAAY,EAA4B,EAAO,EAAQ,eAAe,EACtE,EACJ,EAAQ,kBAAoB,IAAA,GACxB,GACA,WAAW,EAAM,GAAG,EAAiB,IAAM,IAAI,GAAG,EAAY,EAAQ,gBAAiB,CAAS,EAAE,GAAG,EAAa,KAAO,MAAM,GACrI,MAAO;;IAEL,EAAM,KAAK,EAAY,EAAO,CAAS,EAAE;;QAErC,EAAS,eAAe,OAAO,EAAE,GAAG,EAAe;CAE3D,CAMA,SAAS,EAA4B,EAAe,EAAuC,CAEzF,GAAI,IAAc,IAAA,IAAa,IAAU,EAAW,MAAO,GAC3D,IAAK,IAAI,EAAY,EAAkB,EAAY,GAAI,IACrD,GAAI,EAAY,EAAO,CAAS,IAAM,EAAY,EAAW,CAAS,EAAG,OAAO,EAElF,MAAO,GACT,CAEA,SAAS,EAAY,EAAe,EAA2B,CAC7D,OAAO,OAAO,EAAM,YAAY,CAAS,CAAC,CAAC,CAAC,SAAS,CACvD,CAEA,SAAS,EACP,EACA,EACA,EACA,EAC8B,CAC9B,IAAI,EACJ,GAAI,CACF,EAAUC,EAAAA,gBAAgB,CAAc,CAC1C,OAAS,EAAO,CAGd,MAAO,KAAK,EAAmB,sBAD7B,aAAiBC,EAAAA,cAAgB,GAAG,EAAM,WAAW,QAAQ,EAAyB,EAAM,UAAY,IAE5G,CACA,IAAM,EAAU,EAAY,EAAQ,EAAE,EAAE,OAAS,CAAC,EAAG,CAAO,EAC5D,GAAI,CAAC,EACH,MAAO,KAAK,EAAmB,YAAY,EAAQ,SAAS,QAAQ,EAAQ,aAAa,gBAG3F,IAAM,EAAc,IAAI,IAClB,EAAqB,CAAC,EACxB,EAAe,EACb,EAAiB,GAA0B,CAC/C,IACI,EAAS,OAAS,IAAgB,EAAS,KAAK,CAAO,CAC7D,EACA,IAAK,GAAM,CAAE,MAAO,EAAK,gBAAgB,EAAQ,MAAM,CAAC,EAAG,CACzD,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAa,EAAI,EAAQ,cAAgB,GAC/C,GAAI,CAAC,EAAO,IAAI,CAAE,EAChB,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,iBAAiB,OACjF,GAAI,EAAY,IAAI,CAAE,EAC3B,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,aAAa,MAC7E,CACL,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAU,EAClE,GACF,EAAc,GAAG,EAAW,QAAQ,EAAQ,aAAa,QAAQ,EAAK,CAAU,EAAE,KAAK,GAAiB,EAE1G,EAAY,IAAI,EAAI,CAAU,CAChC,CACF,CACA,IAAM,EAAa,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,OAAQ,GAAO,CAAC,EAAY,IAAI,CAAE,CAAC,EAczE,OAbI,EAAW,OAAS,IAEtB,IACA,EAAS,QACP,KAAK,EAAQ,SAAS,KAAK,EAAQ,CAAU,EAAE,cAAc,EAAW,OAAO,eAAe,OAAO,EAAE,GACzG,GAEE,EAAe,EACV,KAAK,EAAmB;;EAEjC,EAAS,IAAK,GAAY,KAAK,GAAS,CAAC,CAAC,KAAK;CAAI,IAAI,EAAe,EAAS,OAAS,UAAU,EAAe,EAAS,OAAA,CAAQ,eAAe,OAAO,EAAE,GAAK,GAAG;EAG3J,CACT,CAEA,eAAe,EAAW,EAAwB,EAAqE,CACrH,IAAM,EAAOC,EAAAA,SAAS,MAAMJ,EAAAA,QAAG,SAAS,EAAgB,MAAM,CAAC,EACzD,EAAU,EAAY,EAAK,IAAM,CAAC,EAAG,CAAO,EAClD,GAAI,CAAC,EACH,MAAU,MAAM,yBAAyB,EAAQ,SAAS,OAAO,EAAQ,aAAa,YAAY,GAAgB,EAEpH,IAAM,EAAS,IAAI,IACnB,IAAK,IAAM,KAAO,EAAK,MAAM,CAAC,EAAG,CAC/B,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAQ,EAAI,EAAQ,cAAgB,GAC1C,GAAI,CAAC,EAAI,MAAU,MAAM,SAAS,EAAQ,SAAS,mBAAmB,GAAgB,EACtF,GAAI,EAAO,IAAI,CAAE,EAAG,MAAU,MAAM,aAAa,EAAQ,SAAS,mBAAmB,GAAI,EACzF,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAK,EACjE,GAAI,EACF,MAAU,MAAM,WAAW,EAAQ,aAAa,sBAAsB,EAAG,IAAI,GAAiB,EAChG,EAAO,IAAI,EAAI,CAAK,CACtB,CACA,GAAI,EAAO,OAAS,EAAG,MAAU,MAAM,4BAA4B,GAAgB,EACnF,OAAO,CACT,CAEA,SAAS,EACP,EACA,EACsD,CACtD,IAAM,EAAQ,EAAO,IAAK,GAAS,EAAK,KAAK,CAAC,EACxC,EAAU,EAAM,QAAQ,EAAQ,QAAQ,EACxC,EAAc,EAAM,QAAQ,EAAQ,YAAY,EACtD,OAAO,IAAY,IAAM,IAAgB,GAAK,IAAA,GAAY,CAAE,UAAS,aAAY,CACnF,CAEA,SAAS,EAAW,EAAiC,CACnD,OAAO,EAAI,MAAO,GAAS,EAAK,KAAK,IAAM,EAAE,CAC/C,CAEA,SAAS,EAAQ,EAAgC,CAC/C,IAAM,EAAS,EAAI,MAAM,EAAG,EAAc,CAAC,CAAC,IAAK,GAAO,KAAK,EAAK,CAAE,EAAE,GAAG,EACzE,OAAO,EAAI,OAAS,GAAiB,GAAG,EAAO,KAAK,IAAI,EAAE,OAAS,EAAO,KAAK,IAAI,CACrF,CAEA,SAAS,EAAK,EAAuB,CAEnC,IAAM,EAAY,EAAM,WAAW,aAAc,GAAG,CAAC,CAAC,WAAW,IAAK,GAAG,EAEzE,OADI,EAAU,KAAK,IAAM,GAAW,MAC7B,EAAU,OAAS,GAAyB,GAAG,EAAU,MAAM,EAAG,EAAsB,EAAE,GAAK,CACxG,CAEA,SAAS,EAAsB,EAA0B,EAAuD,CAC9G,OAAQ,EAAO,eAAiBC,EAAAA,aAAa,WAAe,EAAY,iBAAmB,WAC7F,CAEA,SAAS,EAAqB,EAAwC,CACpE,OAAO,OAAO,SAAS,OAAO,CAAU,CAAC,GAAK,EAAW,KAAK,IAAM,GAAK,IAAA,GAAY,WACvF,CAEA,SAAS,EAAK,EAAmC,CAC/C,OAAO,EAAO,QAAQ,EAAK,IAAU,EAAM,EAAO,CAAC,EAAI,EAAO,MAChE"}
+{"version":3,"file":"evaluation.cjs","names":["parseArgs","resolveCwds","passesIsolationCheckInDebugMode","path","judgeByStaticAnalysis","readProblemMarkdownFrontMatter","isSafeSubmissionOutputPath","fs","DecisionCode","parseCsvRecords","CsvParseError","parseCsv"],"sources":["../../src/presets/evaluation.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { judgeByStaticAnalysis } from '../helpers/judgeByStaticAnalysis.js';\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { CsvParseError, type CsvParseErrorReason, parseCsv, parseCsvRecords } from '../helpers/parseCsv.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { isSafeSubmissionOutputPath } from '../helpers/readOutputFiles.js';\nimport { readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { passesIsolationCheckInDebugMode } from '../helpers/runIsolationCheckInDebugMode.js';\nimport {\n  printDebugCwdBanner,\n  printDebugExpectationFailureBanner,\n  resolveCwds,\n  type ResolvedCwd,\n} from '../helpers/resolveCwds.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst DEFAULT_SUBMISSION_FILE_PATH = 'submission.csv';\nconst TEST_CASE_ID = 'evaluation';\nconst MAX_LISTED_IDS = 10;\n// Learner-controlled cells are echoed into the feedback, so clip them to keep the result line small.\nconst MAX_ECHOED_CELL_LENGTH = 50;\nconst CSV_PARSE_ERROR_MESSAGES: Record<CsvParseErrorReason, string> = {\n  unterminated_quote: '引用符（`\"`）が閉じられていません',\n  unexpected_quote: '引用符（`\"`）は値全体を囲む位置にしか書けません',\n  text_after_quoted_field: '閉じ引用符の後に区切り文字以外の文字があります',\n};\n\nexport interface EvaluationMetric {\n  /** Label shown with the score, e.g. `RMSLE`. */\n  label: string;\n  isHigherBetter: boolean;\n  /**\n   * Returns an error message when a predicted value cannot be scored (e.g. it is not a number).\n   * The message is appended to the row's line number and the offending value, so phrase it as a\n   * predicate such as `数値ではありません`. It also validates the ground truth when it is loaded.\n   */\n  validatePrediction?: (prediction: string) => string | undefined;\n  /** Computes the score from predictions and answers aligned by id. Both are raw CSV cell strings. */\n  compute: (predictions: readonly string[], answers: readonly string[]) => number;\n}\n\nexport interface EvaluationJudgePresetOptions {\n  /** CSV file holding the hidden ground truth, relative to the problem directory. */\n  answerFilePath: string;\n  /** CSV file the submission must contain. Defaults to `submission.csv`. */\n  submissionFilePath?: string;\n  /** Column that identifies each row in both CSV files. */\n  idColumn: string;\n  /** Column holding the answer in the ground truth and the prediction in the submission. */\n  targetColumn: string;\n  metric: EvaluationMetric;\n  /**\n   * Score required for acceptance: the submission is accepted when its score is at least this value\n   * for a higher-is-better metric and at most this value otherwise. Every valid submission is\n   * accepted when omitted.\n   */\n  acceptableScore?: number;\n}\n\nexport const evaluationMetrics = {\n  rmsle: {\n    label: 'RMSLE',\n    isHigherBetter: false,\n    validatePrediction: (prediction: string) =>\n      validateFiniteNumber(prediction) ?? (Number(prediction) < 0 ? '負の値です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(\n        mean(\n          predictions.map(\n            (prediction, index) => (Math.log1p(Number(prediction)) - Math.log1p(Number(answers[index]))) ** 2\n          )\n        )\n      ),\n  },\n  rmse: {\n    label: 'RMSE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(mean(predictions.map((prediction, index) => (Number(prediction) - Number(answers[index])) ** 2))),\n  },\n  mae: {\n    label: 'MAE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => Math.abs(Number(prediction) - Number(answers[index])))),\n  },\n  accuracy: {\n    label: 'Accuracy',\n    isHigherBetter: true,\n    validatePrediction: (prediction: string) => (prediction.trim() === '' ? '空です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => (prediction.trim() === answers[index]?.trim() ? 1 : 0))),\n  },\n} as const satisfies Record<string, EvaluationMetric>;\n\n/**\n * A preset for Kaggle-style model evaluation problems: the submission is a CSV of predictions for a\n * hidden test set, and the judge reports the metric value as `score` instead of comparing outputs.\n * The problem directory keeps the ground truth CSV, so it is never shipped to learners.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { evaluationJudgePreset, evaluationMetrics } from '@exercode/problem-utils/presets/evaluation';\n *\n * await evaluationJudgePreset(import.meta.dirname, {\n *   answerFilePath: 'evaluation/answer.csv',\n *   idColumn: 'Id',\n *   targetColumn: 'TradePrice',\n *   metric: evaluationMetrics.rmsle,\n *   acceptableScore: 0.5,\n * });\n * ```\n *\n * Run `bun judge.ts model_answers/default` to judge one submission, or `bun judge.ts` to judge every\n * `<problemDir>/model_answers/*` and `<problemDir>/model_answers.fails/*` directory for debugging.\n */\nexport async function evaluationJudgePreset(problemDir: string, options: EvaluationJudgePresetOptions): Promise<void> {\n  const args = parseArgs(process.argv);\n  const { cwds, isDebugMode } = await resolveCwds(problemDir, args.cwd);\n\n  if (isDebugMode && !(await passesIsolationCheckInDebugMode(problemDir, cwds, args.params))) {\n    process.exitCode = 1;\n    return;\n  }\n\n  const answer = await readAnswer(path.join(problemDir, options.answerFilePath), options);\n  for (const resolvedCwd of cwds) {\n    if (isDebugMode) printDebugCwdBanner(problemDir, resolvedCwd);\n    const result = await judgeSubmission(problemDir, resolvedCwd.cwd, answer, options);\n    printTestCaseResult({ testCaseId: TEST_CASE_ID, ...result });\n    if (isDebugMode && !matchesExpectedResult(resolvedCwd, result)) {\n      process.exitCode = 1;\n      printDebugExpectationFailureBanner(problemDir, resolvedCwd);\n    }\n  }\n}\n\nasync function judgeSubmission(\n  problemDir: string,\n  cwd: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Promise<Omit<TestCaseResult, 'testCaseId'>> {\n  const staticAnalysisResult = await judgeByStaticAnalysis(cwd, await readProblemMarkdownFrontMatter(problemDir));\n  if (staticAnalysisResult) return staticAnalysisResult;\n\n  const submissionFilePath = options.submissionFilePath ?? DEFAULT_SUBMISSION_FILE_PATH;\n  let submissionText: string;\n  try {\n    const resolvedPath = path.join(cwd, submissionFilePath);\n    // The harness reads the file as the trusted user, so refuse symlinks escaping the submission.\n    if (!(await isSafeSubmissionOutputPath(cwd, resolvedPath))) throw new Error('unsafe path');\n    submissionText = await fs.readFile(resolvedPath, 'utf8');\n  } catch {\n    return {\n      decisionCode: DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,\n      feedbackMarkdown: `\\`${submissionFilePath}\\`が見つかりません。予測結果のCSVファイルを\\`${submissionFilePath}\\`という名前で提出してください。`,\n    };\n  }\n\n  const predictions = readPredictions(submissionText, submissionFilePath, answer, options);\n  if (typeof predictions === 'string') {\n    return { decisionCode: DecisionCode.WRONG_ANSWER, feedbackMarkdown: predictions };\n  }\n\n  const answerValues = [...answer.values()];\n  const score = options.metric.compute(\n    [...answer.keys()].map((id) => predictions.get(id) as string),\n    answerValues\n  );\n  if (!Number.isFinite(score)) {\n    return {\n      decisionCode: DecisionCode.WRONG_ANSWER,\n      feedbackMarkdown: `予測値が極端なため、${options.metric.label}を計算できませんでした。予測値を見直してください。`,\n    };\n  }\n  const isAccepted =\n    options.acceptableScore === undefined ||\n    (options.metric.isHigherBetter ? score >= options.acceptableScore : score <= options.acceptableScore);\n  return {\n    decisionCode: isAccepted ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER,\n    score,\n    scoreLabel: options.metric.label,\n    feedbackMarkdown: buildScoreMarkdown(score, answerValues.length, isAccepted, options),\n  };\n}\n\nfunction buildScoreMarkdown(\n  score: number,\n  rowCount: number,\n  isAccepted: boolean,\n  options: EvaluationJudgePresetOptions\n): string {\n  const { label, isHigherBetter } = options.metric;\n  const precision = findDistinguishingPrecision(score, options.acceptableScore);\n  const acceptanceLine =\n    options.acceptableScore === undefined\n      ? ''\n      : `\\n合格基準: ${label} ${isHigherBetter ? '≧' : '≦'} ${formatScore(options.acceptableScore, precision)}（${isAccepted ? '達成' : '未達成'}）`;\n  return `| 指標 | スコア |\n| ---- | ------ |\n| ${label} | ${formatScore(score, precision)} |\n\n評価件数: ${rowCount.toLocaleString('en-US')}件${acceptanceLine}\n`;\n}\n\n/**\n * Finds the number of significant digits at which the score and the threshold render differently\n * (or stop changing), so the feedback never shows a pair that contradicts the decision.\n */\nfunction findDistinguishingPrecision(score: number, threshold: number | undefined): number {\n  const minimumPrecision = 6;\n  if (threshold === undefined || score === threshold) return minimumPrecision;\n  for (let precision = minimumPrecision; precision < 17; precision++) {\n    if (formatScore(score, precision) !== formatScore(threshold, precision)) return precision;\n  }\n  return 17;\n}\n\nfunction formatScore(score: number, precision: number): string {\n  return Number(score.toPrecision(precision)).toString();\n}\n\nfunction readPredictions(\n  submissionText: string,\n  submissionFilePath: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Map<string, string> | string {\n  let records: ReturnType<typeof parseCsvRecords>;\n  try {\n    records = parseCsvRecords(submissionText);\n  } catch (error) {\n    const detail =\n      error instanceof CsvParseError ? `${error.lineNumber}行目付近: ${CSV_PARSE_ERROR_MESSAGES[error.reason]}` : '';\n    return `\\`${submissionFilePath}\\`をCSVとして読み込めませんでした。${detail}`;\n  }\n  const columns = findColumns(records[0]?.cells ?? [], options);\n  if (!columns) {\n    return `\\`${submissionFilePath}\\`の1行目には\\`${options.idColumn}\\`列と\\`${options.targetColumn}\\`列のヘッダーが必要です。`;\n  }\n\n  const predictions = new Map<string, string>();\n  const problems: string[] = [];\n  let problemCount = 0;\n  const reportProblem = (problem: string): void => {\n    problemCount++;\n    if (problems.length < MAX_LISTED_IDS) problems.push(problem);\n  };\n  for (const { cells: row, lineNumber } of records.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const prediction = row[columns.targetIndex] ?? '';\n    if (!answer.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行は評価対象ではありません`);\n    } else if (predictions.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行が重複しています`);\n    } else {\n      const validationError = options.metric.validatePrediction?.(prediction);\n      if (validationError) {\n        reportProblem(`${lineNumber}行目: \\`${options.targetColumn}\\`の値\\`${clip(prediction)}\\`は${validationError}`);\n      }\n      predictions.set(id, prediction);\n    }\n  }\n  const missingIds = [...answer.keys()].filter((id) => !predictions.has(id));\n  if (missingIds.length > 0) {\n    // Missing rows are the most actionable problem, so they must survive the listing cap.\n    problemCount++;\n    problems.unshift(\n      `\\`${options.idColumn}\\`が${listIds(missingIds)}の行がありません（不足 ${missingIds.length.toLocaleString('en-US')}件）`\n    );\n  }\n  if (problemCount > 0) {\n    return `\\`${submissionFilePath}\\`の内容に問題があります。\n\n${problems.map((problem) => `- ${problem}`).join('\\n')}${problemCount > problems.length ? `\\n- ほか${(problemCount - problems.length).toLocaleString('en-US')}件` : ''}\n`;\n  }\n  return predictions;\n}\n\nasync function readAnswer(answerFilePath: string, options: EvaluationJudgePresetOptions): Promise<Map<string, string>> {\n  const rows = parseCsv(await fs.readFile(answerFilePath, 'utf8'));\n  const columns = findColumns(rows[0] ?? [], options);\n  if (!columns) {\n    throw new Error(`answer file must have ${options.idColumn} and ${options.targetColumn} columns: ${answerFilePath}`);\n  }\n  const answer = new Map<string, string>();\n  for (const row of rows.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const value = row[columns.targetIndex] ?? '';\n    if (!id) throw new Error(`empty ${options.idColumn} in answer file: ${answerFilePath}`);\n    if (answer.has(id)) throw new Error(`duplicate ${options.idColumn} in answer file: ${id}`);\n    const validationError = options.metric.validatePrediction?.(value);\n    if (validationError)\n      throw new Error(`invalid ${options.targetColumn} in answer file for ${id}: ${validationError}`);\n    answer.set(id, value);\n  }\n  if (answer.size === 0) throw new Error(`answer file has no rows: ${answerFilePath}`);\n  return answer;\n}\n\nfunction findColumns(\n  header: readonly string[],\n  options: EvaluationJudgePresetOptions\n): { idIndex: number; targetIndex: number } | undefined {\n  const names = header.map((name) => name.trim());\n  const idIndex = names.indexOf(options.idColumn);\n  const targetIndex = names.indexOf(options.targetColumn);\n  return idIndex === -1 || targetIndex === -1 ? undefined : { idIndex, targetIndex };\n}\n\nfunction isBlankRow(row: readonly string[]): boolean {\n  return row.every((cell) => cell.trim() === '');\n}\n\nfunction listIds(ids: readonly string[]): string {\n  const listed = ids.slice(0, MAX_LISTED_IDS).map((id) => `\\`${clip(id)}\\``);\n  return ids.length > MAX_LISTED_IDS ? `${listed.join(', ')}, ...` : listed.join(', ');\n}\n\nfunction clip(value: string): string {\n  // Control characters and backticks would break the markdown list item and inline code the value is echoed in.\n  const printable = value.replaceAll(/[\\p{Cc}]/gu, ' ').replaceAll('`', \"'\");\n  if (printable.trim() === '') return '（空）';\n  return printable.length > MAX_ECHOED_CELL_LENGTH ? `${printable.slice(0, MAX_ECHOED_CELL_LENGTH)}…` : printable;\n}\n\nfunction matchesExpectedResult(resolvedCwd: ResolvedCwd, result: Pick<TestCaseResult, 'decisionCode'>): boolean {\n  return (result.decisionCode === DecisionCode.ACCEPTED) === (resolvedCwd.expectedResult === 'accepted');\n}\n\nfunction validateFiniteNumber(prediction: string): string | undefined {\n  return Number.isFinite(Number(prediction)) && prediction.trim() !== '' ? undefined : '数値ではありません';\n}\n\nfunction mean(values: readonly number[]): number {\n  return values.reduce((sum, value) => sum + value, 0) / values.length;\n}\n"],"mappings":"goBAmBA,MAKM,EAAgE,CACpE,mBAAoB,qBACpB,iBAAkB,4BAClB,wBAAyB,yBAC3B,EAkCa,EAAoB,CAC/B,MAAO,CACL,MAAO,QACP,eAAgB,GAChB,mBAAqB,GACnB,EAAqB,CAAU,IAAM,OAAO,CAAU,EAAI,EAAI,QAAU,IAAA,IAC1E,SAAU,EAAgC,IACxC,KAAK,KACH,EACE,EAAY,KACT,EAAY,KAAW,KAAK,MAAM,OAAO,CAAU,CAAC,EAAI,KAAK,MAAM,OAAO,EAAQ,EAAM,CAAC,IAAM,CAClG,CACF,CACF,CACJ,EACA,KAAM,CACJ,MAAO,OACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,KAAK,KAAK,EAAK,EAAY,KAAK,EAAY,KAAW,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,IAAM,CAAC,CAAC,CAAC,CAC9G,EACA,IAAK,CACH,MAAO,MACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAU,KAAK,IAAI,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,CAAC,CAAC,CAAC,CACtG,EACA,SAAU,CACR,MAAO,WACP,eAAgB,GAChB,mBAAqB,GAAwB,EAAW,KAAK,IAAM,GAAK,MAAQ,IAAA,GAChF,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAW,IAAW,KAAK,IAAM,EAAQ,EAAM,EAAE,KAAK,EAAU,CAAC,CACvG,CACF,EAwBA,eAAsB,EAAsB,EAAoB,EAAsD,CACpH,IAAM,EAAOA,EAAAA,UAAU,QAAQ,IAAI,EAC7B,CAAE,OAAM,eAAgB,MAAMC,EAAAA,YAAY,EAAY,EAAK,GAAG,EAEpE,GAAI,GAAe,CAAE,MAAMC,EAAAA,gCAAgC,EAAY,EAAM,EAAK,MAAM,EAAI,CAC1F,QAAQ,SAAW,EACnB,MACF,CAEA,IAAM,EAAS,MAAM,EAAWC,EAAAA,QAAK,KAAK,EAAY,EAAQ,cAAc,EAAG,CAAO,EACtF,IAAK,IAAM,KAAe,EAAM,CAC1B,GAAa,EAAA,oBAAoB,EAAY,CAAW,EAC5D,IAAM,EAAS,MAAM,EAAgB,EAAY,EAAY,IAAK,EAAQ,CAAO,EACjF,EAAA,oBAAoB,CAAE,WAAY,aAAc,GAAG,CAAO,CAAC,EACvD,GAAe,CAAC,EAAsB,EAAa,CAAM,IAC3D,QAAQ,SAAW,EACnB,EAAA,mCAAmC,EAAY,CAAW,EAE9D,CACF,CAEA,eAAe,EACb,EACA,EACA,EACA,EAC6C,CAC7C,IAAM,EAAuB,MAAMC,EAAAA,sBAAsB,EAAK,MAAMC,EAAAA,+BAA+B,CAAU,CAAC,EAC9G,GAAI,EAAsB,OAAO,EAEjC,IAAM,EAAqB,EAAQ,oBAAsB,iBACrD,EACJ,GAAI,CACF,IAAM,EAAeF,EAAAA,QAAK,KAAK,EAAK,CAAkB,EAEtD,GAAI,CAAE,MAAMG,EAAAA,2BAA2B,EAAK,CAAY,EAAI,MAAU,MAAM,aAAa,EACzF,EAAiB,MAAMC,EAAAA,QAAG,SAAS,EAAc,MAAM,CACzD,MAAQ,CACN,MAAO,CACL,aAAcC,EAAAA,aAAa,uCAC3B,iBAAkB,KAAK,EAAmB,4BAA4B,EAAmB,kBAC3F,CACF,CAEA,IAAM,EAAc,EAAgB,EAAgB,EAAoB,EAAQ,CAAO,EACvF,GAAI,OAAO,GAAgB,SACzB,MAAO,CAAE,aAAcA,EAAAA,aAAa,aAAc,iBAAkB,CAAY,EAGlF,IAAM,EAAe,CAAC,GAAG,EAAO,OAAO,CAAC,EAClC,EAAQ,EAAQ,OAAO,QAC3B,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,IAAK,GAAO,EAAY,IAAI,CAAE,CAAW,EAC5D,CACF,EACA,GAAI,CAAC,OAAO,SAAS,CAAK,EACxB,MAAO,CACL,aAAcA,EAAAA,aAAa,aAC3B,iBAAkB,aAAa,EAAQ,OAAO,MAAM,0BACtD,EAEF,IAAM,EACJ,EAAQ,kBAAoB,IAAA,KAC3B,EAAQ,OAAO,eAAiB,GAAS,EAAQ,gBAAkB,GAAS,EAAQ,iBACvF,MAAO,CACL,aAAc,EAAaA,EAAAA,aAAa,SAAWA,EAAAA,aAAa,aAChE,QACA,WAAY,EAAQ,OAAO,MAC3B,iBAAkB,EAAmB,EAAO,EAAa,OAAQ,EAAY,CAAO,CACtF,CACF,CAEA,SAAS,EACP,EACA,EACA,EACA,EACQ,CACR,GAAM,CAAE,QAAO,kBAAmB,EAAQ,OACpC,EAAY,EAA4B,EAAO,EAAQ,eAAe,EACtE,EACJ,EAAQ,kBAAoB,IAAA,GACxB,GACA,WAAW,EAAM,GAAG,EAAiB,IAAM,IAAI,GAAG,EAAY,EAAQ,gBAAiB,CAAS,EAAE,GAAG,EAAa,KAAO,MAAM,GACrI,MAAO;;IAEL,EAAM,KAAK,EAAY,EAAO,CAAS,EAAE;;QAErC,EAAS,eAAe,OAAO,EAAE,GAAG,EAAe;CAE3D,CAMA,SAAS,EAA4B,EAAe,EAAuC,CAEzF,GAAI,IAAc,IAAA,IAAa,IAAU,EAAW,MAAO,GAC3D,IAAK,IAAI,EAAY,EAAkB,EAAY,GAAI,IACrD,GAAI,EAAY,EAAO,CAAS,IAAM,EAAY,EAAW,CAAS,EAAG,OAAO,EAElF,MAAO,GACT,CAEA,SAAS,EAAY,EAAe,EAA2B,CAC7D,OAAO,OAAO,EAAM,YAAY,CAAS,CAAC,CAAC,CAAC,SAAS,CACvD,CAEA,SAAS,EACP,EACA,EACA,EACA,EAC8B,CAC9B,IAAI,EACJ,GAAI,CACF,EAAUC,EAAAA,gBAAgB,CAAc,CAC1C,OAAS,EAAO,CAGd,MAAO,KAAK,EAAmB,sBAD7B,aAAiBC,EAAAA,cAAgB,GAAG,EAAM,WAAW,QAAQ,EAAyB,EAAM,UAAY,IAE5G,CACA,IAAM,EAAU,EAAY,EAAQ,EAAE,EAAE,OAAS,CAAC,EAAG,CAAO,EAC5D,GAAI,CAAC,EACH,MAAO,KAAK,EAAmB,YAAY,EAAQ,SAAS,QAAQ,EAAQ,aAAa,gBAG3F,IAAM,EAAc,IAAI,IAClB,EAAqB,CAAC,EACxB,EAAe,EACb,EAAiB,GAA0B,CAC/C,IACI,EAAS,OAAS,IAAgB,EAAS,KAAK,CAAO,CAC7D,EACA,IAAK,GAAM,CAAE,MAAO,EAAK,gBAAgB,EAAQ,MAAM,CAAC,EAAG,CACzD,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAa,EAAI,EAAQ,cAAgB,GAC/C,GAAI,CAAC,EAAO,IAAI,CAAE,EAChB,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,iBAAiB,OACjF,GAAI,EAAY,IAAI,CAAE,EAC3B,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,aAAa,MAC7E,CACL,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAU,EAClE,GACF,EAAc,GAAG,EAAW,QAAQ,EAAQ,aAAa,QAAQ,EAAK,CAAU,EAAE,KAAK,GAAiB,EAE1G,EAAY,IAAI,EAAI,CAAU,CAChC,CACF,CACA,IAAM,EAAa,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,OAAQ,GAAO,CAAC,EAAY,IAAI,CAAE,CAAC,EAczE,OAbI,EAAW,OAAS,IAEtB,IACA,EAAS,QACP,KAAK,EAAQ,SAAS,KAAK,EAAQ,CAAU,EAAE,cAAc,EAAW,OAAO,eAAe,OAAO,EAAE,GACzG,GAEE,EAAe,EACV,KAAK,EAAmB;;EAEjC,EAAS,IAAK,GAAY,KAAK,GAAS,CAAC,CAAC,KAAK;CAAI,IAAI,EAAe,EAAS,OAAS,UAAU,EAAe,EAAS,OAAA,CAAQ,eAAe,OAAO,EAAE,GAAK,GAAG;EAG3J,CACT,CAEA,eAAe,EAAW,EAAwB,EAAqE,CACrH,IAAM,EAAOC,EAAAA,SAAS,MAAMJ,EAAAA,QAAG,SAAS,EAAgB,MAAM,CAAC,EACzD,EAAU,EAAY,EAAK,IAAM,CAAC,EAAG,CAAO,EAClD,GAAI,CAAC,EACH,MAAU,MAAM,yBAAyB,EAAQ,SAAS,OAAO,EAAQ,aAAa,YAAY,GAAgB,EAEpH,IAAM,EAAS,IAAI,IACnB,IAAK,IAAM,KAAO,EAAK,MAAM,CAAC,EAAG,CAC/B,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAQ,EAAI,EAAQ,cAAgB,GAC1C,GAAI,CAAC,EAAI,MAAU,MAAM,SAAS,EAAQ,SAAS,mBAAmB,GAAgB,EACtF,GAAI,EAAO,IAAI,CAAE,EAAG,MAAU,MAAM,aAAa,EAAQ,SAAS,mBAAmB,GAAI,EACzF,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAK,EACjE,GAAI,EACF,MAAU,MAAM,WAAW,EAAQ,aAAa,sBAAsB,EAAG,IAAI,GAAiB,EAChG,EAAO,IAAI,EAAI,CAAK,CACtB,CACA,GAAI,EAAO,OAAS,EAAG,MAAU,MAAM,4BAA4B,GAAgB,EACnF,OAAO,CACT,CAEA,SAAS,EACP,EACA,EACsD,CACtD,IAAM,EAAQ,EAAO,IAAK,GAAS,EAAK,KAAK,CAAC,EACxC,EAAU,EAAM,QAAQ,EAAQ,QAAQ,EACxC,EAAc,EAAM,QAAQ,EAAQ,YAAY,EACtD,OAAO,IAAY,IAAM,IAAgB,GAAK,IAAA,GAAY,CAAE,UAAS,aAAY,CACnF,CAEA,SAAS,EAAW,EAAiC,CACnD,OAAO,EAAI,MAAO,GAAS,EAAK,KAAK,IAAM,EAAE,CAC/C,CAEA,SAAS,EAAQ,EAAgC,CAC/C,IAAM,EAAS,EAAI,MAAM,EAAG,EAAc,CAAC,CAAC,IAAK,GAAO,KAAK,EAAK,CAAE,EAAE,GAAG,EACzE,OAAO,EAAI,OAAS,GAAiB,GAAG,EAAO,KAAK,IAAI,EAAE,OAAS,EAAO,KAAK,IAAI,CACrF,CAEA,SAAS,EAAK,EAAuB,CAEnC,IAAM,EAAY,EAAM,WAAW,aAAc,GAAG,CAAC,CAAC,WAAW,IAAK,GAAG,EAEzE,OADI,EAAU,KAAK,IAAM,GAAW,MAC7B,EAAU,OAAS,GAAyB,GAAG,EAAU,MAAM,EAAG,EAAsB,EAAE,GAAK,CACxG,CAEA,SAAS,EAAsB,EAA0B,EAAuD,CAC9G,OAAQ,EAAO,eAAiBC,EAAAA,aAAa,WAAe,EAAY,iBAAmB,WAC7F,CAEA,SAAS,EAAqB,EAAwC,CACpE,OAAO,OAAO,SAAS,OAAO,CAAU,CAAC,GAAK,EAAW,KAAK,IAAM,GAAK,IAAA,GAAY,WACvF,CAEA,SAAS,EAAK,EAAmC,CAC/C,OAAO,EAAO,QAAQ,EAAK,IAAU,EAAM,EAAO,CAAC,EAAI,EAAO,MAChE"}

package/dist/presets/evaluation.js

@@ -1,11 +1,11 @@
-import{parseArgs as e}from"../helpers/parseArgs.js";import{CsvParseError as t,parseCsv as n,parseCsvRecords as r}from"../helpers/parseCsv.js";import{printTestCaseResult as i}from"../helpers/printTestCaseResult.js";import{DecisionCode as a}from"../types/decisionCode.js";import{printDebugBanner as o}from"../helpers/printDebugBanner.js";import{checkProblemDirIsolation as s}from"../helpers/checkProblemDirIsolation.js";import{judgeByStaticAnalysis as c}from"../helpers/judgeByStaticAnalysis.js";import{isSafeSubmissionOutputPath as l}from"../helpers/readOutputFiles.js";import{readProblemMarkdownFrontMatter as u}from"../helpers/readProblemMarkdownFrontMatter.js";import{printDebugCwdBanner as d,printDebugExpectationFailureBanner as f,resolveCwds as p}from"../helpers/resolveCwds.js";import m from"node:fs/promises";import h from"node:path";const g={unterminated_quote:'引用符（`"`）が閉じられていません',unexpected_quote:'引用符（`"`）は値全体を囲む位置にしか書けません',text_after_quoted_field:`閉じ引用符の後に区切り文字以外の文字があります`},_={rmsle:{label:`RMSLE`,isHigherBetter:!1,validatePrediction:e=>A(e)??(Number(e)<0?`負の値です`:void 0),compute:(e,t)=>Math.sqrt(j(e.map((e,n)=>(Math.log1p(Number(e))-Math.log1p(Number(t[n])))**2)))},rmse:{label:`RMSE`,isHigherBetter:!1,validatePrediction:A,compute:(e,t)=>Math.sqrt(j(e.map((e,n)=>(Number(e)-Number(t[n]))**2)))},mae:{label:`MAE`,isHigherBetter:!1,validatePrediction:A,compute:(e,t)=>j(e.map((e,n)=>Math.abs(Number(e)-Number(t[n]))))},accuracy:{label:`Accuracy`,isHigherBetter:!0,validatePrediction:e=>e.trim()===``?`空です`:void 0,compute:(e,t)=>j(e.map((e,n)=>+(e.trim()===t[n]?.trim())))}};async function v(t,n){let r=e(process.argv),{cwds:a,isDebugMode:c}=await p(t,r.cwd);if(c){let e=a.find(e=>e.expectedResult===`accepted`);if(e){if(!(await s(t,e,r.params)).passed){process.exitCode=1;return}}else o([`[DEBUG MODE] isolated problem directory check skipped`,``,`No accepted model answer is available for checking that the copied judge still accepts a valid submission.`])}let l=await w(h.join(t,n.answerFilePath),n);for(let e of a){c&&d(t,e);let r=await y(t,e.cwd,l,n);i({testCaseId:`evaluation`,...r}),c&&!k(e,r)&&(process.exitCode=1,f(t,e))}}async function y(e,t,n,r){let i=await c(t,await u(e));if(i)return i;let o=r.submissionFilePath??`submission.csv`,s;try{let e=h.join(t,o);if(!await l(t,e))throw Error(`unsafe path`);s=await m.readFile(e,`utf8`)}catch{return{decisionCode:a.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,feedbackMarkdown:`\`${o}\`が見つかりません。予測結果のCSVファイルを\`${o}\`という名前で提出してください。`}}let d=C(s,o,n,r);if(typeof d==`string`)return{decisionCode:a.WRONG_ANSWER,feedbackMarkdown:d};let f=[...n.values()],p=r.metric.compute([...n.keys()].map(e=>d.get(e)),f);if(!Number.isFinite(p))return{decisionCode:a.WRONG_ANSWER,feedbackMarkdown:`予測値が極端なため、${r.metric.label}を計算できませんでした。予測値を見直してください。`};let g=r.acceptableScore===void 0||(r.metric.isHigherBetter?p>=r.acceptableScore:p<=r.acceptableScore);return{decisionCode:g?a.ACCEPTED:a.WRONG_ANSWER,score:p,scoreLabel:r.metric.label,feedbackMarkdown:b(p,f.length,g,r)}}function b(e,t,n,r){let{label:i,isHigherBetter:a}=r.metric,o=x(e,r.acceptableScore),s=r.acceptableScore===void 0?``:`\n合格基準: ${i} ${a?`≧`:`≦`} ${S(r.acceptableScore,o)}（${n?`達成`:`未達成`}）`;return`| 指標 | スコア |
+import{parseArgs as e}from"../helpers/parseArgs.js";import{CsvParseError as t,parseCsv as n,parseCsvRecords as r}from"../helpers/parseCsv.js";import{printTestCaseResult as i}from"../helpers/printTestCaseResult.js";import{DecisionCode as a}from"../types/decisionCode.js";import{judgeByStaticAnalysis as o}from"../helpers/judgeByStaticAnalysis.js";import{isSafeSubmissionOutputPath as s}from"../helpers/readOutputFiles.js";import{passesIsolationCheckInDebugMode as c}from"../helpers/runIsolationCheckInDebugMode.js";import{readProblemMarkdownFrontMatter as l}from"../helpers/readProblemMarkdownFrontMatter.js";import{printDebugCwdBanner as u,printDebugExpectationFailureBanner as d,resolveCwds as f}from"../helpers/resolveCwds.js";import p from"node:fs/promises";import m from"node:path";const h={unterminated_quote:'引用符（`"`）が閉じられていません',unexpected_quote:'引用符（`"`）は値全体を囲む位置にしか書けません',text_after_quoted_field:`閉じ引用符の後に区切り文字以外の文字があります`},g={rmsle:{label:`RMSLE`,isHigherBetter:!1,validatePrediction:e=>k(e)??(Number(e)<0?`負の値です`:void 0),compute:(e,t)=>Math.sqrt(A(e.map((e,n)=>(Math.log1p(Number(e))-Math.log1p(Number(t[n])))**2)))},rmse:{label:`RMSE`,isHigherBetter:!1,validatePrediction:k,compute:(e,t)=>Math.sqrt(A(e.map((e,n)=>(Number(e)-Number(t[n]))**2)))},mae:{label:`MAE`,isHigherBetter:!1,validatePrediction:k,compute:(e,t)=>A(e.map((e,n)=>Math.abs(Number(e)-Number(t[n]))))},accuracy:{label:`Accuracy`,isHigherBetter:!0,validatePrediction:e=>e.trim()===``?`空です`:void 0,compute:(e,t)=>A(e.map((e,n)=>+(e.trim()===t[n]?.trim())))}};async function _(t,n){let r=e(process.argv),{cwds:a,isDebugMode:o}=await f(t,r.cwd);if(o&&!await c(t,a,r.params)){process.exitCode=1;return}let s=await C(m.join(t,n.answerFilePath),n);for(let e of a){o&&u(t,e);let r=await v(t,e.cwd,s,n);i({testCaseId:`evaluation`,...r}),o&&!O(e,r)&&(process.exitCode=1,d(t,e))}}async function v(e,t,n,r){let i=await o(t,await l(e));if(i)return i;let c=r.submissionFilePath??`submission.csv`,u;try{let e=m.join(t,c);if(!await s(t,e))throw Error(`unsafe path`);u=await p.readFile(e,`utf8`)}catch{return{decisionCode:a.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,feedbackMarkdown:`\`${c}\`が見つかりません。予測結果のCSVファイルを\`${c}\`という名前で提出してください。`}}let d=S(u,c,n,r);if(typeof d==`string`)return{decisionCode:a.WRONG_ANSWER,feedbackMarkdown:d};let f=[...n.values()],h=r.metric.compute([...n.keys()].map(e=>d.get(e)),f);if(!Number.isFinite(h))return{decisionCode:a.WRONG_ANSWER,feedbackMarkdown:`予測値が極端なため、${r.metric.label}を計算できませんでした。予測値を見直してください。`};let g=r.acceptableScore===void 0||(r.metric.isHigherBetter?h>=r.acceptableScore:h<=r.acceptableScore);return{decisionCode:g?a.ACCEPTED:a.WRONG_ANSWER,score:h,scoreLabel:r.metric.label,feedbackMarkdown:y(h,f.length,g,r)}}function y(e,t,n,r){let{label:i,isHigherBetter:a}=r.metric,o=b(e,r.acceptableScore),s=r.acceptableScore===void 0?``:`\n合格基準: ${i} ${a?`≧`:`≦`} ${x(r.acceptableScore,o)}（${n?`達成`:`未達成`}）`;return`| 指標 | スコア |
 | ---- | ------ |
-| ${i} | ${S(e,o)} |
+| ${i} | ${x(e,o)} |
 
 評価件数: ${t.toLocaleString(`en-US`)}件${s}
-`}function x(e,t){if(t===void 0||e===t)return 6;for(let n=6;n<17;n++)if(S(e,n)!==S(t,n))return n;return 17}function S(e,t){return Number(e.toPrecision(t)).toString()}function C(e,n,i,a){let o;try{o=r(e)}catch(e){return`\`${n}\`をCSVとして読み込めませんでした。${e instanceof t?`${e.lineNumber}行目付近: ${g[e.reason]}`:``}`}let s=T(o[0]?.cells??[],a);if(!s)return`\`${n}\`の1行目には\`${a.idColumn}\`列と\`${a.targetColumn}\`列のヘッダーが必要です。`;let c=new Map,l=[],u=0,d=e=>{u++,l.length<10&&l.push(e)};for(let{cells:e,lineNumber:t}of o.slice(1)){if(E(e))continue;let n=e[s.idIndex]?.trim()??``,r=e[s.targetIndex]??``;if(!i.has(n))d(`${t}行目: \`${a.idColumn}\`が\`${O(n)}\`の行は評価対象ではありません`);else if(c.has(n))d(`${t}行目: \`${a.idColumn}\`が\`${O(n)}\`の行が重複しています`);else{let e=a.metric.validatePrediction?.(r);e&&d(`${t}行目: \`${a.targetColumn}\`の値\`${O(r)}\`は${e}`),c.set(n,r)}}let f=[...i.keys()].filter(e=>!c.has(e));return f.length>0&&(u++,l.unshift(`\`${a.idColumn}\`が${D(f)}の行がありません（不足 ${f.length.toLocaleString(`en-US`)}件）`)),u>0?`\`${n}\`の内容に問題があります。
+`}function b(e,t){if(t===void 0||e===t)return 6;for(let n=6;n<17;n++)if(x(e,n)!==x(t,n))return n;return 17}function x(e,t){return Number(e.toPrecision(t)).toString()}function S(e,n,i,a){let o;try{o=r(e)}catch(e){return`\`${n}\`をCSVとして読み込めませんでした。${e instanceof t?`${e.lineNumber}行目付近: ${h[e.reason]}`:``}`}let s=w(o[0]?.cells??[],a);if(!s)return`\`${n}\`の1行目には\`${a.idColumn}\`列と\`${a.targetColumn}\`列のヘッダーが必要です。`;let c=new Map,l=[],u=0,d=e=>{u++,l.length<10&&l.push(e)};for(let{cells:e,lineNumber:t}of o.slice(1)){if(T(e))continue;let n=e[s.idIndex]?.trim()??``,r=e[s.targetIndex]??``;if(!i.has(n))d(`${t}行目: \`${a.idColumn}\`が\`${D(n)}\`の行は評価対象ではありません`);else if(c.has(n))d(`${t}行目: \`${a.idColumn}\`が\`${D(n)}\`の行が重複しています`);else{let e=a.metric.validatePrediction?.(r);e&&d(`${t}行目: \`${a.targetColumn}\`の値\`${D(r)}\`は${e}`),c.set(n,r)}}let f=[...i.keys()].filter(e=>!c.has(e));return f.length>0&&(u++,l.unshift(`\`${a.idColumn}\`が${E(f)}の行がありません（不足 ${f.length.toLocaleString(`en-US`)}件）`)),u>0?`\`${n}\`の内容に問題があります。
 
 ${l.map(e=>`- ${e}`).join(`
 `)}${u>l.length?`\n- ほか${(u-l.length).toLocaleString(`en-US`)}件`:``}
-`:c}async function w(e,t){let r=n(await m.readFile(e,`utf8`)),i=T(r[0]??[],t);if(!i)throw Error(`answer file must have ${t.idColumn} and ${t.targetColumn} columns: ${e}`);let a=new Map;for(let n of r.slice(1)){if(E(n))continue;let r=n[i.idIndex]?.trim()??``,o=n[i.targetIndex]??``;if(!r)throw Error(`empty ${t.idColumn} in answer file: ${e}`);if(a.has(r))throw Error(`duplicate ${t.idColumn} in answer file: ${r}`);let s=t.metric.validatePrediction?.(o);if(s)throw Error(`invalid ${t.targetColumn} in answer file for ${r}: ${s}`);a.set(r,o)}if(a.size===0)throw Error(`answer file has no rows: ${e}`);return a}function T(e,t){let n=e.map(e=>e.trim()),r=n.indexOf(t.idColumn),i=n.indexOf(t.targetColumn);return r===-1||i===-1?void 0:{idIndex:r,targetIndex:i}}function E(e){return e.every(e=>e.trim()===``)}function D(e){let t=e.slice(0,10).map(e=>`\`${O(e)}\``);return e.length>10?`${t.join(`, `)}, ...`:t.join(`, `)}function O(e){let t=e.replaceAll(/[\p{Cc}]/gu,` `).replaceAll("`",`'`);return t.trim()===``?`（空）`:t.length>50?`${t.slice(0,50)}…`:t}function k(e,t){return t.decisionCode===a.ACCEPTED==(e.expectedResult===`accepted`)}function A(e){return Number.isFinite(Number(e))&&e.trim()!==``?void 0:`数値ではありません`}function j(e){return e.reduce((e,t)=>e+t,0)/e.length}export{v as evaluationJudgePreset,_ as evaluationMetrics};
+`:c}async function C(e,t){let r=n(await p.readFile(e,`utf8`)),i=w(r[0]??[],t);if(!i)throw Error(`answer file must have ${t.idColumn} and ${t.targetColumn} columns: ${e}`);let a=new Map;for(let n of r.slice(1)){if(T(n))continue;let r=n[i.idIndex]?.trim()??``,o=n[i.targetIndex]??``;if(!r)throw Error(`empty ${t.idColumn} in answer file: ${e}`);if(a.has(r))throw Error(`duplicate ${t.idColumn} in answer file: ${r}`);let s=t.metric.validatePrediction?.(o);if(s)throw Error(`invalid ${t.targetColumn} in answer file for ${r}: ${s}`);a.set(r,o)}if(a.size===0)throw Error(`answer file has no rows: ${e}`);return a}function w(e,t){let n=e.map(e=>e.trim()),r=n.indexOf(t.idColumn),i=n.indexOf(t.targetColumn);return r===-1||i===-1?void 0:{idIndex:r,targetIndex:i}}function T(e){return e.every(e=>e.trim()===``)}function E(e){let t=e.slice(0,10).map(e=>`\`${D(e)}\``);return e.length>10?`${t.join(`, `)}, ...`:t.join(`, `)}function D(e){let t=e.replaceAll(/[\p{Cc}]/gu,` `).replaceAll("`",`'`);return t.trim()===``?`（空）`:t.length>50?`${t.slice(0,50)}…`:t}function O(e,t){return t.decisionCode===a.ACCEPTED==(e.expectedResult===`accepted`)}function k(e){return Number.isFinite(Number(e))&&e.trim()!==``?void 0:`数値ではありません`}function A(e){return e.reduce((e,t)=>e+t,0)/e.length}export{_ as evaluationJudgePreset,g as evaluationMetrics};
 //# sourceMappingURL=evaluation.js.map

package/dist/presets/evaluation.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation.js","names":[],"sources":["../../src/presets/evaluation.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { checkProblemDirIsolation } from '../helpers/checkProblemDirIsolation.js';\nimport { judgeByStaticAnalysis } from '../helpers/judgeByStaticAnalysis.js';\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { CsvParseError, type CsvParseErrorReason, parseCsv, parseCsvRecords } from '../helpers/parseCsv.js';\nimport { printDebugBanner } from '../helpers/printDebugBanner.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { isSafeSubmissionOutputPath } from '../helpers/readOutputFiles.js';\nimport { readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport {\n  printDebugCwdBanner,\n  printDebugExpectationFailureBanner,\n  resolveCwds,\n  type ResolvedCwd,\n} from '../helpers/resolveCwds.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst DEFAULT_SUBMISSION_FILE_PATH = 'submission.csv';\nconst TEST_CASE_ID = 'evaluation';\nconst MAX_LISTED_IDS = 10;\n// Learner-controlled cells are echoed into the feedback, so clip them to keep the result line small.\nconst MAX_ECHOED_CELL_LENGTH = 50;\nconst CSV_PARSE_ERROR_MESSAGES: Record<CsvParseErrorReason, string> = {\n  unterminated_quote: '引用符（`\"`）が閉じられていません',\n  unexpected_quote: '引用符（`\"`）は値全体を囲む位置にしか書けません',\n  text_after_quoted_field: '閉じ引用符の後に区切り文字以外の文字があります',\n};\n\nexport interface EvaluationMetric {\n  /** Label shown with the score, e.g. `RMSLE`. */\n  label: string;\n  isHigherBetter: boolean;\n  /**\n   * Returns an error message when a predicted value cannot be scored (e.g. it is not a number).\n   * The message is appended to the row's line number and the offending value, so phrase it as a\n   * predicate such as `数値ではありません`. It also validates the ground truth when it is loaded.\n   */\n  validatePrediction?: (prediction: string) => string | undefined;\n  /** Computes the score from predictions and answers aligned by id. Both are raw CSV cell strings. */\n  compute: (predictions: readonly string[], answers: readonly string[]) => number;\n}\n\nexport interface EvaluationJudgePresetOptions {\n  /** CSV file holding the hidden ground truth, relative to the problem directory. */\n  answerFilePath: string;\n  /** CSV file the submission must contain. Defaults to `submission.csv`. */\n  submissionFilePath?: string;\n  /** Column that identifies each row in both CSV files. */\n  idColumn: string;\n  /** Column holding the answer in the ground truth and the prediction in the submission. */\n  targetColumn: string;\n  metric: EvaluationMetric;\n  /**\n   * Score required for acceptance: the submission is accepted when its score is at least this value\n   * for a higher-is-better metric and at most this value otherwise. Every valid submission is\n   * accepted when omitted.\n   */\n  acceptableScore?: number;\n}\n\nexport const evaluationMetrics = {\n  rmsle: {\n    label: 'RMSLE',\n    isHigherBetter: false,\n    validatePrediction: (prediction: string) =>\n      validateFiniteNumber(prediction) ?? (Number(prediction) < 0 ? '負の値です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(\n        mean(\n          predictions.map(\n            (prediction, index) => (Math.log1p(Number(prediction)) - Math.log1p(Number(answers[index]))) ** 2\n          )\n        )\n      ),\n  },\n  rmse: {\n    label: 'RMSE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(mean(predictions.map((prediction, index) => (Number(prediction) - Number(answers[index])) ** 2))),\n  },\n  mae: {\n    label: 'MAE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => Math.abs(Number(prediction) - Number(answers[index])))),\n  },\n  accuracy: {\n    label: 'Accuracy',\n    isHigherBetter: true,\n    validatePrediction: (prediction: string) => (prediction.trim() === '' ? '空です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => (prediction.trim() === answers[index]?.trim() ? 1 : 0))),\n  },\n} as const satisfies Record<string, EvaluationMetric>;\n\n/**\n * A preset for Kaggle-style model evaluation problems: the submission is a CSV of predictions for a\n * hidden test set, and the judge reports the metric value as `score` instead of comparing outputs.\n * The problem directory keeps the ground truth CSV, so it is never shipped to learners.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { evaluationJudgePreset, evaluationMetrics } from '@exercode/problem-utils/presets/evaluation';\n *\n * await evaluationJudgePreset(import.meta.dirname, {\n *   answerFilePath: 'evaluation/answer.csv',\n *   idColumn: 'Id',\n *   targetColumn: 'TradePrice',\n *   metric: evaluationMetrics.rmsle,\n *   acceptableScore: 0.5,\n * });\n * ```\n *\n * Run `bun judge.ts model_answers/default` to judge one submission, or `bun judge.ts` to judge every\n * `<problemDir>/model_answers/*` and `<problemDir>/model_answers.fails/*` directory for debugging.\n */\nexport async function evaluationJudgePreset(problemDir: string, options: EvaluationJudgePresetOptions): Promise<void> {\n  const args = parseArgs(process.argv);\n  const { cwds, isDebugMode } = await resolveCwds(problemDir, args.cwd);\n\n  if (isDebugMode) {\n    const acceptedCwd = cwds.find((cwd) => cwd.expectedResult === 'accepted');\n    if (acceptedCwd) {\n      const isolationCheckResult = await checkProblemDirIsolation(problemDir, acceptedCwd, args.params);\n      if (!isolationCheckResult.passed) {\n        process.exitCode = 1;\n        return;\n      }\n    } else {\n      printDebugBanner([\n        '[DEBUG MODE] isolated problem directory check skipped',\n        '',\n        'No accepted model answer is available for checking that the copied judge still accepts a valid submission.',\n      ]);\n    }\n  }\n\n  const answer = await readAnswer(path.join(problemDir, options.answerFilePath), options);\n  for (const resolvedCwd of cwds) {\n    if (isDebugMode) printDebugCwdBanner(problemDir, resolvedCwd);\n    const result = await judgeSubmission(problemDir, resolvedCwd.cwd, answer, options);\n    printTestCaseResult({ testCaseId: TEST_CASE_ID, ...result });\n    if (isDebugMode && !matchesExpectedResult(resolvedCwd, result)) {\n      process.exitCode = 1;\n      printDebugExpectationFailureBanner(problemDir, resolvedCwd);\n    }\n  }\n}\n\nasync function judgeSubmission(\n  problemDir: string,\n  cwd: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Promise<Omit<TestCaseResult, 'testCaseId'>> {\n  const staticAnalysisResult = await judgeByStaticAnalysis(cwd, await readProblemMarkdownFrontMatter(problemDir));\n  if (staticAnalysisResult) return staticAnalysisResult;\n\n  const submissionFilePath = options.submissionFilePath ?? DEFAULT_SUBMISSION_FILE_PATH;\n  let submissionText: string;\n  try {\n    const resolvedPath = path.join(cwd, submissionFilePath);\n    // The harness reads the file as the trusted user, so refuse symlinks escaping the submission.\n    if (!(await isSafeSubmissionOutputPath(cwd, resolvedPath))) throw new Error('unsafe path');\n    submissionText = await fs.readFile(resolvedPath, 'utf8');\n  } catch {\n    return {\n      decisionCode: DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,\n      feedbackMarkdown: `\\`${submissionFilePath}\\`が見つかりません。予測結果のCSVファイルを\\`${submissionFilePath}\\`という名前で提出してください。`,\n    };\n  }\n\n  const predictions = readPredictions(submissionText, submissionFilePath, answer, options);\n  if (typeof predictions === 'string') {\n    return { decisionCode: DecisionCode.WRONG_ANSWER, feedbackMarkdown: predictions };\n  }\n\n  const answerValues = [...answer.values()];\n  const score = options.metric.compute(\n    [...answer.keys()].map((id) => predictions.get(id) as string),\n    answerValues\n  );\n  if (!Number.isFinite(score)) {\n    return {\n      decisionCode: DecisionCode.WRONG_ANSWER,\n      feedbackMarkdown: `予測値が極端なため、${options.metric.label}を計算できませんでした。予測値を見直してください。`,\n    };\n  }\n  const isAccepted =\n    options.acceptableScore === undefined ||\n    (options.metric.isHigherBetter ? score >= options.acceptableScore : score <= options.acceptableScore);\n  return {\n    decisionCode: isAccepted ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER,\n    score,\n    scoreLabel: options.metric.label,\n    feedbackMarkdown: buildScoreMarkdown(score, answerValues.length, isAccepted, options),\n  };\n}\n\nfunction buildScoreMarkdown(\n  score: number,\n  rowCount: number,\n  isAccepted: boolean,\n  options: EvaluationJudgePresetOptions\n): string {\n  const { label, isHigherBetter } = options.metric;\n  const precision = findDistinguishingPrecision(score, options.acceptableScore);\n  const acceptanceLine =\n    options.acceptableScore === undefined\n      ? ''\n      : `\\n合格基準: ${label} ${isHigherBetter ? '≧' : '≦'} ${formatScore(options.acceptableScore, precision)}（${isAccepted ? '達成' : '未達成'}）`;\n  return `| 指標 | スコア |\n| ---- | ------ |\n| ${label} | ${formatScore(score, precision)} |\n\n評価件数: ${rowCount.toLocaleString('en-US')}件${acceptanceLine}\n`;\n}\n\n/**\n * Finds the number of significant digits at which the score and the threshold render differently\n * (or stop changing), so the feedback never shows a pair that contradicts the decision.\n */\nfunction findDistinguishingPrecision(score: number, threshold: number | undefined): number {\n  const minimumPrecision = 6;\n  if (threshold === undefined || score === threshold) return minimumPrecision;\n  for (let precision = minimumPrecision; precision < 17; precision++) {\n    if (formatScore(score, precision) !== formatScore(threshold, precision)) return precision;\n  }\n  return 17;\n}\n\nfunction formatScore(score: number, precision: number): string {\n  return Number(score.toPrecision(precision)).toString();\n}\n\nfunction readPredictions(\n  submissionText: string,\n  submissionFilePath: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Map<string, string> | string {\n  let records: ReturnType<typeof parseCsvRecords>;\n  try {\n    records = parseCsvRecords(submissionText);\n  } catch (error) {\n    const detail =\n      error instanceof CsvParseError ? `${error.lineNumber}行目付近: ${CSV_PARSE_ERROR_MESSAGES[error.reason]}` : '';\n    return `\\`${submissionFilePath}\\`をCSVとして読み込めませんでした。${detail}`;\n  }\n  const columns = findColumns(records[0]?.cells ?? [], options);\n  if (!columns) {\n    return `\\`${submissionFilePath}\\`の1行目には\\`${options.idColumn}\\`列と\\`${options.targetColumn}\\`列のヘッダーが必要です。`;\n  }\n\n  const predictions = new Map<string, string>();\n  const problems: string[] = [];\n  let problemCount = 0;\n  const reportProblem = (problem: string): void => {\n    problemCount++;\n    if (problems.length < MAX_LISTED_IDS) problems.push(problem);\n  };\n  for (const { cells: row, lineNumber } of records.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const prediction = row[columns.targetIndex] ?? '';\n    if (!answer.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行は評価対象ではありません`);\n    } else if (predictions.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行が重複しています`);\n    } else {\n      const validationError = options.metric.validatePrediction?.(prediction);\n      if (validationError) {\n        reportProblem(`${lineNumber}行目: \\`${options.targetColumn}\\`の値\\`${clip(prediction)}\\`は${validationError}`);\n      }\n      predictions.set(id, prediction);\n    }\n  }\n  const missingIds = [...answer.keys()].filter((id) => !predictions.has(id));\n  if (missingIds.length > 0) {\n    // Missing rows are the most actionable problem, so they must survive the listing cap.\n    problemCount++;\n    problems.unshift(\n      `\\`${options.idColumn}\\`が${listIds(missingIds)}の行がありません（不足 ${missingIds.length.toLocaleString('en-US')}件）`\n    );\n  }\n  if (problemCount > 0) {\n    return `\\`${submissionFilePath}\\`の内容に問題があります。\n\n${problems.map((problem) => `- ${problem}`).join('\\n')}${problemCount > problems.length ? `\\n- ほか${(problemCount - problems.length).toLocaleString('en-US')}件` : ''}\n`;\n  }\n  return predictions;\n}\n\nasync function readAnswer(answerFilePath: string, options: EvaluationJudgePresetOptions): Promise<Map<string, string>> {\n  const rows = parseCsv(await fs.readFile(answerFilePath, 'utf8'));\n  const columns = findColumns(rows[0] ?? [], options);\n  if (!columns) {\n    throw new Error(`answer file must have ${options.idColumn} and ${options.targetColumn} columns: ${answerFilePath}`);\n  }\n  const answer = new Map<string, string>();\n  for (const row of rows.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const value = row[columns.targetIndex] ?? '';\n    if (!id) throw new Error(`empty ${options.idColumn} in answer file: ${answerFilePath}`);\n    if (answer.has(id)) throw new Error(`duplicate ${options.idColumn} in answer file: ${id}`);\n    const validationError = options.metric.validatePrediction?.(value);\n    if (validationError)\n      throw new Error(`invalid ${options.targetColumn} in answer file for ${id}: ${validationError}`);\n    answer.set(id, value);\n  }\n  if (answer.size === 0) throw new Error(`answer file has no rows: ${answerFilePath}`);\n  return answer;\n}\n\nfunction findColumns(\n  header: readonly string[],\n  options: EvaluationJudgePresetOptions\n): { idIndex: number; targetIndex: number } | undefined {\n  const names = header.map((name) => name.trim());\n  const idIndex = names.indexOf(options.idColumn);\n  const targetIndex = names.indexOf(options.targetColumn);\n  return idIndex === -1 || targetIndex === -1 ? undefined : { idIndex, targetIndex };\n}\n\nfunction isBlankRow(row: readonly string[]): boolean {\n  return row.every((cell) => cell.trim() === '');\n}\n\nfunction listIds(ids: readonly string[]): string {\n  const listed = ids.slice(0, MAX_LISTED_IDS).map((id) => `\\`${clip(id)}\\``);\n  return ids.length > MAX_LISTED_IDS ? `${listed.join(', ')}, ...` : listed.join(', ');\n}\n\nfunction clip(value: string): string {\n  // Control characters and backticks would break the markdown list item and inline code the value is echoed in.\n  const printable = value.replaceAll(/[\\p{Cc}]/gu, ' ').replaceAll('`', \"'\");\n  if (printable.trim() === '') return '（空）';\n  return printable.length > MAX_ECHOED_CELL_LENGTH ? `${printable.slice(0, MAX_ECHOED_CELL_LENGTH)}…` : printable;\n}\n\nfunction matchesExpectedResult(resolvedCwd: ResolvedCwd, result: Pick<TestCaseResult, 'decisionCode'>): boolean {\n  return (result.decisionCode === DecisionCode.ACCEPTED) === (resolvedCwd.expectedResult === 'accepted');\n}\n\nfunction validateFiniteNumber(prediction: string): string | undefined {\n  return Number.isFinite(Number(prediction)) && prediction.trim() !== '' ? undefined : '数値ではありません';\n}\n\nfunction mean(values: readonly number[]): number {\n  return values.reduce((sum, value) => sum + value, 0) / values.length;\n}\n"],"mappings":"y0BAoBA,MAKM,EAAgE,CACpE,mBAAoB,qBACpB,iBAAkB,4BAClB,wBAAyB,yBAC3B,EAkCa,EAAoB,CAC/B,MAAO,CACL,MAAO,QACP,eAAgB,GAChB,mBAAqB,GACnB,EAAqB,CAAU,IAAM,OAAO,CAAU,EAAI,EAAI,QAAU,IAAA,IAC1E,SAAU,EAAgC,IACxC,KAAK,KACH,EACE,EAAY,KACT,EAAY,KAAW,KAAK,MAAM,OAAO,CAAU,CAAC,EAAI,KAAK,MAAM,OAAO,EAAQ,EAAM,CAAC,IAAM,CAClG,CACF,CACF,CACJ,EACA,KAAM,CACJ,MAAO,OACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,KAAK,KAAK,EAAK,EAAY,KAAK,EAAY,KAAW,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,IAAM,CAAC,CAAC,CAAC,CAC9G,EACA,IAAK,CACH,MAAO,MACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAU,KAAK,IAAI,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,CAAC,CAAC,CAAC,CACtG,EACA,SAAU,CACR,MAAO,WACP,eAAgB,GAChB,mBAAqB,GAAwB,EAAW,KAAK,IAAM,GAAK,MAAQ,IAAA,GAChF,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAW,IAAW,KAAK,IAAM,EAAQ,EAAM,EAAE,KAAK,EAAU,CAAC,CACvG,CACF,EAwBA,eAAsB,EAAsB,EAAoB,EAAsD,CACpH,IAAM,EAAO,EAAU,QAAQ,IAAI,EAC7B,CAAE,OAAM,eAAgB,MAAM,EAAY,EAAY,EAAK,GAAG,EAEpE,GAAI,EAAa,CACf,IAAM,EAAc,EAAK,KAAM,GAAQ,EAAI,iBAAmB,UAAU,EACxE,GAAI,EAEE,IAAA,EAAC,MAD8B,EAAyB,EAAY,EAAa,EAAK,MAAM,EAAA,CACtE,OAAQ,CAChC,QAAQ,SAAW,EACnB,MACF,OAEA,EAAiB,CACf,wDACA,GACA,4GACF,CAAC,CAEL,CAEA,IAAM,EAAS,MAAM,EAAW,EAAK,KAAK,EAAY,EAAQ,cAAc,EAAG,CAAO,EACtF,IAAK,IAAM,KAAe,EAAM,CAC1B,GAAa,EAAoB,EAAY,CAAW,EAC5D,IAAM,EAAS,MAAM,EAAgB,EAAY,EAAY,IAAK,EAAQ,CAAO,EACjF,EAAoB,CAAE,WAAY,aAAc,GAAG,CAAO,CAAC,EACvD,GAAe,CAAC,EAAsB,EAAa,CAAM,IAC3D,QAAQ,SAAW,EACnB,EAAmC,EAAY,CAAW,EAE9D,CACF,CAEA,eAAe,EACb,EACA,EACA,EACA,EAC6C,CAC7C,IAAM,EAAuB,MAAM,EAAsB,EAAK,MAAM,EAA+B,CAAU,CAAC,EAC9G,GAAI,EAAsB,OAAO,EAEjC,IAAM,EAAqB,EAAQ,oBAAsB,iBACrD,EACJ,GAAI,CACF,IAAM,EAAe,EAAK,KAAK,EAAK,CAAkB,EAEtD,GAAI,CAAE,MAAM,EAA2B,EAAK,CAAY,EAAI,MAAU,MAAM,aAAa,EACzF,EAAiB,MAAM,EAAG,SAAS,EAAc,MAAM,CACzD,MAAQ,CACN,MAAO,CACL,aAAc,EAAa,uCAC3B,iBAAkB,KAAK,EAAmB,4BAA4B,EAAmB,kBAC3F,CACF,CAEA,IAAM,EAAc,EAAgB,EAAgB,EAAoB,EAAQ,CAAO,EACvF,GAAI,OAAO,GAAgB,SACzB,MAAO,CAAE,aAAc,EAAa,aAAc,iBAAkB,CAAY,EAGlF,IAAM,EAAe,CAAC,GAAG,EAAO,OAAO,CAAC,EAClC,EAAQ,EAAQ,OAAO,QAC3B,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,IAAK,GAAO,EAAY,IAAI,CAAE,CAAW,EAC5D,CACF,EACA,GAAI,CAAC,OAAO,SAAS,CAAK,EACxB,MAAO,CACL,aAAc,EAAa,aAC3B,iBAAkB,aAAa,EAAQ,OAAO,MAAM,0BACtD,EAEF,IAAM,EACJ,EAAQ,kBAAoB,IAAA,KAC3B,EAAQ,OAAO,eAAiB,GAAS,EAAQ,gBAAkB,GAAS,EAAQ,iBACvF,MAAO,CACL,aAAc,EAAa,EAAa,SAAW,EAAa,aAChE,QACA,WAAY,EAAQ,OAAO,MAC3B,iBAAkB,EAAmB,EAAO,EAAa,OAAQ,EAAY,CAAO,CACtF,CACF,CAEA,SAAS,EACP,EACA,EACA,EACA,EACQ,CACR,GAAM,CAAE,QAAO,kBAAmB,EAAQ,OACpC,EAAY,EAA4B,EAAO,EAAQ,eAAe,EACtE,EACJ,EAAQ,kBAAoB,IAAA,GACxB,GACA,WAAW,EAAM,GAAG,EAAiB,IAAM,IAAI,GAAG,EAAY,EAAQ,gBAAiB,CAAS,EAAE,GAAG,EAAa,KAAO,MAAM,GACrI,MAAO;;IAEL,EAAM,KAAK,EAAY,EAAO,CAAS,EAAE;;QAErC,EAAS,eAAe,OAAO,EAAE,GAAG,EAAe;CAE3D,CAMA,SAAS,EAA4B,EAAe,EAAuC,CAEzF,GAAI,IAAc,IAAA,IAAa,IAAU,EAAW,MAAO,GAC3D,IAAK,IAAI,EAAY,EAAkB,EAAY,GAAI,IACrD,GAAI,EAAY,EAAO,CAAS,IAAM,EAAY,EAAW,CAAS,EAAG,OAAO,EAElF,MAAO,GACT,CAEA,SAAS,EAAY,EAAe,EAA2B,CAC7D,OAAO,OAAO,EAAM,YAAY,CAAS,CAAC,CAAC,CAAC,SAAS,CACvD,CAEA,SAAS,EACP,EACA,EACA,EACA,EAC8B,CAC9B,IAAI,EACJ,GAAI,CACF,EAAU,EAAgB,CAAc,CAC1C,OAAS,EAAO,CAGd,MAAO,KAAK,EAAmB,sBAD7B,aAAiB,EAAgB,GAAG,EAAM,WAAW,QAAQ,EAAyB,EAAM,UAAY,IAE5G,CACA,IAAM,EAAU,EAAY,EAAQ,EAAE,EAAE,OAAS,CAAC,EAAG,CAAO,EAC5D,GAAI,CAAC,EACH,MAAO,KAAK,EAAmB,YAAY,EAAQ,SAAS,QAAQ,EAAQ,aAAa,gBAG3F,IAAM,EAAc,IAAI,IAClB,EAAqB,CAAC,EACxB,EAAe,EACb,EAAiB,GAA0B,CAC/C,IACI,EAAS,OAAS,IAAgB,EAAS,KAAK,CAAO,CAC7D,EACA,IAAK,GAAM,CAAE,MAAO,EAAK,gBAAgB,EAAQ,MAAM,CAAC,EAAG,CACzD,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAa,EAAI,EAAQ,cAAgB,GAC/C,GAAI,CAAC,EAAO,IAAI,CAAE,EAChB,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,iBAAiB,OACjF,GAAI,EAAY,IAAI,CAAE,EAC3B,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,aAAa,MAC7E,CACL,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAU,EAClE,GACF,EAAc,GAAG,EAAW,QAAQ,EAAQ,aAAa,QAAQ,EAAK,CAAU,EAAE,KAAK,GAAiB,EAE1G,EAAY,IAAI,EAAI,CAAU,CAChC,CACF,CACA,IAAM,EAAa,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,OAAQ,GAAO,CAAC,EAAY,IAAI,CAAE,CAAC,EAczE,OAbI,EAAW,OAAS,IAEtB,IACA,EAAS,QACP,KAAK,EAAQ,SAAS,KAAK,EAAQ,CAAU,EAAE,cAAc,EAAW,OAAO,eAAe,OAAO,EAAE,GACzG,GAEE,EAAe,EACV,KAAK,EAAmB;;EAEjC,EAAS,IAAK,GAAY,KAAK,GAAS,CAAC,CAAC,KAAK;CAAI,IAAI,EAAe,EAAS,OAAS,UAAU,EAAe,EAAS,OAAA,CAAQ,eAAe,OAAO,EAAE,GAAK,GAAG;EAG3J,CACT,CAEA,eAAe,EAAW,EAAwB,EAAqE,CACrH,IAAM,EAAO,EAAS,MAAM,EAAG,SAAS,EAAgB,MAAM,CAAC,EACzD,EAAU,EAAY,EAAK,IAAM,CAAC,EAAG,CAAO,EAClD,GAAI,CAAC,EACH,MAAU,MAAM,yBAAyB,EAAQ,SAAS,OAAO,EAAQ,aAAa,YAAY,GAAgB,EAEpH,IAAM,EAAS,IAAI,IACnB,IAAK,IAAM,KAAO,EAAK,MAAM,CAAC,EAAG,CAC/B,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAQ,EAAI,EAAQ,cAAgB,GAC1C,GAAI,CAAC,EAAI,MAAU,MAAM,SAAS,EAAQ,SAAS,mBAAmB,GAAgB,EACtF,GAAI,EAAO,IAAI,CAAE,EAAG,MAAU,MAAM,aAAa,EAAQ,SAAS,mBAAmB,GAAI,EACzF,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAK,EACjE,GAAI,EACF,MAAU,MAAM,WAAW,EAAQ,aAAa,sBAAsB,EAAG,IAAI,GAAiB,EAChG,EAAO,IAAI,EAAI,CAAK,CACtB,CACA,GAAI,EAAO,OAAS,EAAG,MAAU,MAAM,4BAA4B,GAAgB,EACnF,OAAO,CACT,CAEA,SAAS,EACP,EACA,EACsD,CACtD,IAAM,EAAQ,EAAO,IAAK,GAAS,EAAK,KAAK,CAAC,EACxC,EAAU,EAAM,QAAQ,EAAQ,QAAQ,EACxC,EAAc,EAAM,QAAQ,EAAQ,YAAY,EACtD,OAAO,IAAY,IAAM,IAAgB,GAAK,IAAA,GAAY,CAAE,UAAS,aAAY,CACnF,CAEA,SAAS,EAAW,EAAiC,CACnD,OAAO,EAAI,MAAO,GAAS,EAAK,KAAK,IAAM,EAAE,CAC/C,CAEA,SAAS,EAAQ,EAAgC,CAC/C,IAAM,EAAS,EAAI,MAAM,EAAG,EAAc,CAAC,CAAC,IAAK,GAAO,KAAK,EAAK,CAAE,EAAE,GAAG,EACzE,OAAO,EAAI,OAAS,GAAiB,GAAG,EAAO,KAAK,IAAI,EAAE,OAAS,EAAO,KAAK,IAAI,CACrF,CAEA,SAAS,EAAK,EAAuB,CAEnC,IAAM,EAAY,EAAM,WAAW,aAAc,GAAG,CAAC,CAAC,WAAW,IAAK,GAAG,EAEzE,OADI,EAAU,KAAK,IAAM,GAAW,MAC7B,EAAU,OAAS,GAAyB,GAAG,EAAU,MAAM,EAAG,EAAsB,EAAE,GAAK,CACxG,CAEA,SAAS,EAAsB,EAA0B,EAAuD,CAC9G,OAAQ,EAAO,eAAiB,EAAa,WAAe,EAAY,iBAAmB,WAC7F,CAEA,SAAS,EAAqB,EAAwC,CACpE,OAAO,OAAO,SAAS,OAAO,CAAU,CAAC,GAAK,EAAW,KAAK,IAAM,GAAK,IAAA,GAAY,WACvF,CAEA,SAAS,EAAK,EAAmC,CAC/C,OAAO,EAAO,QAAQ,EAAK,IAAU,EAAM,EAAO,CAAC,EAAI,EAAO,MAChE"}
+{"version":3,"file":"evaluation.js","names":[],"sources":["../../src/presets/evaluation.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { judgeByStaticAnalysis } from '../helpers/judgeByStaticAnalysis.js';\nimport { parseArgs } from '../helpers/parseArgs.js';\nimport { CsvParseError, type CsvParseErrorReason, parseCsv, parseCsvRecords } from '../helpers/parseCsv.js';\nimport { printTestCaseResult } from '../helpers/printTestCaseResult.js';\nimport { isSafeSubmissionOutputPath } from '../helpers/readOutputFiles.js';\nimport { readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { passesIsolationCheckInDebugMode } from '../helpers/runIsolationCheckInDebugMode.js';\nimport {\n  printDebugCwdBanner,\n  printDebugExpectationFailureBanner,\n  resolveCwds,\n  type ResolvedCwd,\n} from '../helpers/resolveCwds.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport type { TestCaseResult } from '../types/testCaseResult.js';\n\nconst DEFAULT_SUBMISSION_FILE_PATH = 'submission.csv';\nconst TEST_CASE_ID = 'evaluation';\nconst MAX_LISTED_IDS = 10;\n// Learner-controlled cells are echoed into the feedback, so clip them to keep the result line small.\nconst MAX_ECHOED_CELL_LENGTH = 50;\nconst CSV_PARSE_ERROR_MESSAGES: Record<CsvParseErrorReason, string> = {\n  unterminated_quote: '引用符（`\"`）が閉じられていません',\n  unexpected_quote: '引用符（`\"`）は値全体を囲む位置にしか書けません',\n  text_after_quoted_field: '閉じ引用符の後に区切り文字以外の文字があります',\n};\n\nexport interface EvaluationMetric {\n  /** Label shown with the score, e.g. `RMSLE`. */\n  label: string;\n  isHigherBetter: boolean;\n  /**\n   * Returns an error message when a predicted value cannot be scored (e.g. it is not a number).\n   * The message is appended to the row's line number and the offending value, so phrase it as a\n   * predicate such as `数値ではありません`. It also validates the ground truth when it is loaded.\n   */\n  validatePrediction?: (prediction: string) => string | undefined;\n  /** Computes the score from predictions and answers aligned by id. Both are raw CSV cell strings. */\n  compute: (predictions: readonly string[], answers: readonly string[]) => number;\n}\n\nexport interface EvaluationJudgePresetOptions {\n  /** CSV file holding the hidden ground truth, relative to the problem directory. */\n  answerFilePath: string;\n  /** CSV file the submission must contain. Defaults to `submission.csv`. */\n  submissionFilePath?: string;\n  /** Column that identifies each row in both CSV files. */\n  idColumn: string;\n  /** Column holding the answer in the ground truth and the prediction in the submission. */\n  targetColumn: string;\n  metric: EvaluationMetric;\n  /**\n   * Score required for acceptance: the submission is accepted when its score is at least this value\n   * for a higher-is-better metric and at most this value otherwise. Every valid submission is\n   * accepted when omitted.\n   */\n  acceptableScore?: number;\n}\n\nexport const evaluationMetrics = {\n  rmsle: {\n    label: 'RMSLE',\n    isHigherBetter: false,\n    validatePrediction: (prediction: string) =>\n      validateFiniteNumber(prediction) ?? (Number(prediction) < 0 ? '負の値です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(\n        mean(\n          predictions.map(\n            (prediction, index) => (Math.log1p(Number(prediction)) - Math.log1p(Number(answers[index]))) ** 2\n          )\n        )\n      ),\n  },\n  rmse: {\n    label: 'RMSE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      Math.sqrt(mean(predictions.map((prediction, index) => (Number(prediction) - Number(answers[index])) ** 2))),\n  },\n  mae: {\n    label: 'MAE',\n    isHigherBetter: false,\n    validatePrediction: validateFiniteNumber,\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => Math.abs(Number(prediction) - Number(answers[index])))),\n  },\n  accuracy: {\n    label: 'Accuracy',\n    isHigherBetter: true,\n    validatePrediction: (prediction: string) => (prediction.trim() === '' ? '空です' : undefined),\n    compute: (predictions: readonly string[], answers: readonly string[]) =>\n      mean(predictions.map((prediction, index) => (prediction.trim() === answers[index]?.trim() ? 1 : 0))),\n  },\n} as const satisfies Record<string, EvaluationMetric>;\n\n/**\n * A preset for Kaggle-style model evaluation problems: the submission is a CSV of predictions for a\n * hidden test set, and the judge reports the metric value as `score` instead of comparing outputs.\n * The problem directory keeps the ground truth CSV, so it is never shipped to learners.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { evaluationJudgePreset, evaluationMetrics } from '@exercode/problem-utils/presets/evaluation';\n *\n * await evaluationJudgePreset(import.meta.dirname, {\n *   answerFilePath: 'evaluation/answer.csv',\n *   idColumn: 'Id',\n *   targetColumn: 'TradePrice',\n *   metric: evaluationMetrics.rmsle,\n *   acceptableScore: 0.5,\n * });\n * ```\n *\n * Run `bun judge.ts model_answers/default` to judge one submission, or `bun judge.ts` to judge every\n * `<problemDir>/model_answers/*` and `<problemDir>/model_answers.fails/*` directory for debugging.\n */\nexport async function evaluationJudgePreset(problemDir: string, options: EvaluationJudgePresetOptions): Promise<void> {\n  const args = parseArgs(process.argv);\n  const { cwds, isDebugMode } = await resolveCwds(problemDir, args.cwd);\n\n  if (isDebugMode && !(await passesIsolationCheckInDebugMode(problemDir, cwds, args.params))) {\n    process.exitCode = 1;\n    return;\n  }\n\n  const answer = await readAnswer(path.join(problemDir, options.answerFilePath), options);\n  for (const resolvedCwd of cwds) {\n    if (isDebugMode) printDebugCwdBanner(problemDir, resolvedCwd);\n    const result = await judgeSubmission(problemDir, resolvedCwd.cwd, answer, options);\n    printTestCaseResult({ testCaseId: TEST_CASE_ID, ...result });\n    if (isDebugMode && !matchesExpectedResult(resolvedCwd, result)) {\n      process.exitCode = 1;\n      printDebugExpectationFailureBanner(problemDir, resolvedCwd);\n    }\n  }\n}\n\nasync function judgeSubmission(\n  problemDir: string,\n  cwd: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Promise<Omit<TestCaseResult, 'testCaseId'>> {\n  const staticAnalysisResult = await judgeByStaticAnalysis(cwd, await readProblemMarkdownFrontMatter(problemDir));\n  if (staticAnalysisResult) return staticAnalysisResult;\n\n  const submissionFilePath = options.submissionFilePath ?? DEFAULT_SUBMISSION_FILE_PATH;\n  let submissionText: string;\n  try {\n    const resolvedPath = path.join(cwd, submissionFilePath);\n    // The harness reads the file as the trusted user, so refuse symlinks escaping the submission.\n    if (!(await isSafeSubmissionOutputPath(cwd, resolvedPath))) throw new Error('unsafe path');\n    submissionText = await fs.readFile(resolvedPath, 'utf8');\n  } catch {\n    return {\n      decisionCode: DecisionCode.MISSING_REQUIRED_SUBMISSION_FILE_ERROR,\n      feedbackMarkdown: `\\`${submissionFilePath}\\`が見つかりません。予測結果のCSVファイルを\\`${submissionFilePath}\\`という名前で提出してください。`,\n    };\n  }\n\n  const predictions = readPredictions(submissionText, submissionFilePath, answer, options);\n  if (typeof predictions === 'string') {\n    return { decisionCode: DecisionCode.WRONG_ANSWER, feedbackMarkdown: predictions };\n  }\n\n  const answerValues = [...answer.values()];\n  const score = options.metric.compute(\n    [...answer.keys()].map((id) => predictions.get(id) as string),\n    answerValues\n  );\n  if (!Number.isFinite(score)) {\n    return {\n      decisionCode: DecisionCode.WRONG_ANSWER,\n      feedbackMarkdown: `予測値が極端なため、${options.metric.label}を計算できませんでした。予測値を見直してください。`,\n    };\n  }\n  const isAccepted =\n    options.acceptableScore === undefined ||\n    (options.metric.isHigherBetter ? score >= options.acceptableScore : score <= options.acceptableScore);\n  return {\n    decisionCode: isAccepted ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER,\n    score,\n    scoreLabel: options.metric.label,\n    feedbackMarkdown: buildScoreMarkdown(score, answerValues.length, isAccepted, options),\n  };\n}\n\nfunction buildScoreMarkdown(\n  score: number,\n  rowCount: number,\n  isAccepted: boolean,\n  options: EvaluationJudgePresetOptions\n): string {\n  const { label, isHigherBetter } = options.metric;\n  const precision = findDistinguishingPrecision(score, options.acceptableScore);\n  const acceptanceLine =\n    options.acceptableScore === undefined\n      ? ''\n      : `\\n合格基準: ${label} ${isHigherBetter ? '≧' : '≦'} ${formatScore(options.acceptableScore, precision)}（${isAccepted ? '達成' : '未達成'}）`;\n  return `| 指標 | スコア |\n| ---- | ------ |\n| ${label} | ${formatScore(score, precision)} |\n\n評価件数: ${rowCount.toLocaleString('en-US')}件${acceptanceLine}\n`;\n}\n\n/**\n * Finds the number of significant digits at which the score and the threshold render differently\n * (or stop changing), so the feedback never shows a pair that contradicts the decision.\n */\nfunction findDistinguishingPrecision(score: number, threshold: number | undefined): number {\n  const minimumPrecision = 6;\n  if (threshold === undefined || score === threshold) return minimumPrecision;\n  for (let precision = minimumPrecision; precision < 17; precision++) {\n    if (formatScore(score, precision) !== formatScore(threshold, precision)) return precision;\n  }\n  return 17;\n}\n\nfunction formatScore(score: number, precision: number): string {\n  return Number(score.toPrecision(precision)).toString();\n}\n\nfunction readPredictions(\n  submissionText: string,\n  submissionFilePath: string,\n  answer: ReadonlyMap<string, string>,\n  options: EvaluationJudgePresetOptions\n): Map<string, string> | string {\n  let records: ReturnType<typeof parseCsvRecords>;\n  try {\n    records = parseCsvRecords(submissionText);\n  } catch (error) {\n    const detail =\n      error instanceof CsvParseError ? `${error.lineNumber}行目付近: ${CSV_PARSE_ERROR_MESSAGES[error.reason]}` : '';\n    return `\\`${submissionFilePath}\\`をCSVとして読み込めませんでした。${detail}`;\n  }\n  const columns = findColumns(records[0]?.cells ?? [], options);\n  if (!columns) {\n    return `\\`${submissionFilePath}\\`の1行目には\\`${options.idColumn}\\`列と\\`${options.targetColumn}\\`列のヘッダーが必要です。`;\n  }\n\n  const predictions = new Map<string, string>();\n  const problems: string[] = [];\n  let problemCount = 0;\n  const reportProblem = (problem: string): void => {\n    problemCount++;\n    if (problems.length < MAX_LISTED_IDS) problems.push(problem);\n  };\n  for (const { cells: row, lineNumber } of records.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const prediction = row[columns.targetIndex] ?? '';\n    if (!answer.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行は評価対象ではありません`);\n    } else if (predictions.has(id)) {\n      reportProblem(`${lineNumber}行目: \\`${options.idColumn}\\`が\\`${clip(id)}\\`の行が重複しています`);\n    } else {\n      const validationError = options.metric.validatePrediction?.(prediction);\n      if (validationError) {\n        reportProblem(`${lineNumber}行目: \\`${options.targetColumn}\\`の値\\`${clip(prediction)}\\`は${validationError}`);\n      }\n      predictions.set(id, prediction);\n    }\n  }\n  const missingIds = [...answer.keys()].filter((id) => !predictions.has(id));\n  if (missingIds.length > 0) {\n    // Missing rows are the most actionable problem, so they must survive the listing cap.\n    problemCount++;\n    problems.unshift(\n      `\\`${options.idColumn}\\`が${listIds(missingIds)}の行がありません（不足 ${missingIds.length.toLocaleString('en-US')}件）`\n    );\n  }\n  if (problemCount > 0) {\n    return `\\`${submissionFilePath}\\`の内容に問題があります。\n\n${problems.map((problem) => `- ${problem}`).join('\\n')}${problemCount > problems.length ? `\\n- ほか${(problemCount - problems.length).toLocaleString('en-US')}件` : ''}\n`;\n  }\n  return predictions;\n}\n\nasync function readAnswer(answerFilePath: string, options: EvaluationJudgePresetOptions): Promise<Map<string, string>> {\n  const rows = parseCsv(await fs.readFile(answerFilePath, 'utf8'));\n  const columns = findColumns(rows[0] ?? [], options);\n  if (!columns) {\n    throw new Error(`answer file must have ${options.idColumn} and ${options.targetColumn} columns: ${answerFilePath}`);\n  }\n  const answer = new Map<string, string>();\n  for (const row of rows.slice(1)) {\n    if (isBlankRow(row)) continue;\n    const id = row[columns.idIndex]?.trim() ?? '';\n    const value = row[columns.targetIndex] ?? '';\n    if (!id) throw new Error(`empty ${options.idColumn} in answer file: ${answerFilePath}`);\n    if (answer.has(id)) throw new Error(`duplicate ${options.idColumn} in answer file: ${id}`);\n    const validationError = options.metric.validatePrediction?.(value);\n    if (validationError)\n      throw new Error(`invalid ${options.targetColumn} in answer file for ${id}: ${validationError}`);\n    answer.set(id, value);\n  }\n  if (answer.size === 0) throw new Error(`answer file has no rows: ${answerFilePath}`);\n  return answer;\n}\n\nfunction findColumns(\n  header: readonly string[],\n  options: EvaluationJudgePresetOptions\n): { idIndex: number; targetIndex: number } | undefined {\n  const names = header.map((name) => name.trim());\n  const idIndex = names.indexOf(options.idColumn);\n  const targetIndex = names.indexOf(options.targetColumn);\n  return idIndex === -1 || targetIndex === -1 ? undefined : { idIndex, targetIndex };\n}\n\nfunction isBlankRow(row: readonly string[]): boolean {\n  return row.every((cell) => cell.trim() === '');\n}\n\nfunction listIds(ids: readonly string[]): string {\n  const listed = ids.slice(0, MAX_LISTED_IDS).map((id) => `\\`${clip(id)}\\``);\n  return ids.length > MAX_LISTED_IDS ? `${listed.join(', ')}, ...` : listed.join(', ');\n}\n\nfunction clip(value: string): string {\n  // Control characters and backticks would break the markdown list item and inline code the value is echoed in.\n  const printable = value.replaceAll(/[\\p{Cc}]/gu, ' ').replaceAll('`', \"'\");\n  if (printable.trim() === '') return '（空）';\n  return printable.length > MAX_ECHOED_CELL_LENGTH ? `${printable.slice(0, MAX_ECHOED_CELL_LENGTH)}…` : printable;\n}\n\nfunction matchesExpectedResult(resolvedCwd: ResolvedCwd, result: Pick<TestCaseResult, 'decisionCode'>): boolean {\n  return (result.decisionCode === DecisionCode.ACCEPTED) === (resolvedCwd.expectedResult === 'accepted');\n}\n\nfunction validateFiniteNumber(prediction: string): string | undefined {\n  return Number.isFinite(Number(prediction)) && prediction.trim() !== '' ? undefined : '数値ではありません';\n}\n\nfunction mean(values: readonly number[]): number {\n  return values.reduce((sum, value) => sum + value, 0) / values.length;\n}\n"],"mappings":"kxBAmBA,MAKM,EAAgE,CACpE,mBAAoB,qBACpB,iBAAkB,4BAClB,wBAAyB,yBAC3B,EAkCa,EAAoB,CAC/B,MAAO,CACL,MAAO,QACP,eAAgB,GAChB,mBAAqB,GACnB,EAAqB,CAAU,IAAM,OAAO,CAAU,EAAI,EAAI,QAAU,IAAA,IAC1E,SAAU,EAAgC,IACxC,KAAK,KACH,EACE,EAAY,KACT,EAAY,KAAW,KAAK,MAAM,OAAO,CAAU,CAAC,EAAI,KAAK,MAAM,OAAO,EAAQ,EAAM,CAAC,IAAM,CAClG,CACF,CACF,CACJ,EACA,KAAM,CACJ,MAAO,OACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,KAAK,KAAK,EAAK,EAAY,KAAK,EAAY,KAAW,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,IAAM,CAAC,CAAC,CAAC,CAC9G,EACA,IAAK,CACH,MAAO,MACP,eAAgB,GAChB,mBAAoB,EACpB,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAU,KAAK,IAAI,OAAO,CAAU,EAAI,OAAO,EAAQ,EAAM,CAAC,CAAC,CAAC,CACtG,EACA,SAAU,CACR,MAAO,WACP,eAAgB,GAChB,mBAAqB,GAAwB,EAAW,KAAK,IAAM,GAAK,MAAQ,IAAA,GAChF,SAAU,EAAgC,IACxC,EAAK,EAAY,KAAK,EAAY,IAAW,IAAW,KAAK,IAAM,EAAQ,EAAM,EAAE,KAAK,EAAU,CAAC,CACvG,CACF,EAwBA,eAAsB,EAAsB,EAAoB,EAAsD,CACpH,IAAM,EAAO,EAAU,QAAQ,IAAI,EAC7B,CAAE,OAAM,eAAgB,MAAM,EAAY,EAAY,EAAK,GAAG,EAEpE,GAAI,GAAe,CAAE,MAAM,EAAgC,EAAY,EAAM,EAAK,MAAM,EAAI,CAC1F,QAAQ,SAAW,EACnB,MACF,CAEA,IAAM,EAAS,MAAM,EAAW,EAAK,KAAK,EAAY,EAAQ,cAAc,EAAG,CAAO,EACtF,IAAK,IAAM,KAAe,EAAM,CAC1B,GAAa,EAAoB,EAAY,CAAW,EAC5D,IAAM,EAAS,MAAM,EAAgB,EAAY,EAAY,IAAK,EAAQ,CAAO,EACjF,EAAoB,CAAE,WAAY,aAAc,GAAG,CAAO,CAAC,EACvD,GAAe,CAAC,EAAsB,EAAa,CAAM,IAC3D,QAAQ,SAAW,EACnB,EAAmC,EAAY,CAAW,EAE9D,CACF,CAEA,eAAe,EACb,EACA,EACA,EACA,EAC6C,CAC7C,IAAM,EAAuB,MAAM,EAAsB,EAAK,MAAM,EAA+B,CAAU,CAAC,EAC9G,GAAI,EAAsB,OAAO,EAEjC,IAAM,EAAqB,EAAQ,oBAAsB,iBACrD,EACJ,GAAI,CACF,IAAM,EAAe,EAAK,KAAK,EAAK,CAAkB,EAEtD,GAAI,CAAE,MAAM,EAA2B,EAAK,CAAY,EAAI,MAAU,MAAM,aAAa,EACzF,EAAiB,MAAM,EAAG,SAAS,EAAc,MAAM,CACzD,MAAQ,CACN,MAAO,CACL,aAAc,EAAa,uCAC3B,iBAAkB,KAAK,EAAmB,4BAA4B,EAAmB,kBAC3F,CACF,CAEA,IAAM,EAAc,EAAgB,EAAgB,EAAoB,EAAQ,CAAO,EACvF,GAAI,OAAO,GAAgB,SACzB,MAAO,CAAE,aAAc,EAAa,aAAc,iBAAkB,CAAY,EAGlF,IAAM,EAAe,CAAC,GAAG,EAAO,OAAO,CAAC,EAClC,EAAQ,EAAQ,OAAO,QAC3B,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,IAAK,GAAO,EAAY,IAAI,CAAE,CAAW,EAC5D,CACF,EACA,GAAI,CAAC,OAAO,SAAS,CAAK,EACxB,MAAO,CACL,aAAc,EAAa,aAC3B,iBAAkB,aAAa,EAAQ,OAAO,MAAM,0BACtD,EAEF,IAAM,EACJ,EAAQ,kBAAoB,IAAA,KAC3B,EAAQ,OAAO,eAAiB,GAAS,EAAQ,gBAAkB,GAAS,EAAQ,iBACvF,MAAO,CACL,aAAc,EAAa,EAAa,SAAW,EAAa,aAChE,QACA,WAAY,EAAQ,OAAO,MAC3B,iBAAkB,EAAmB,EAAO,EAAa,OAAQ,EAAY,CAAO,CACtF,CACF,CAEA,SAAS,EACP,EACA,EACA,EACA,EACQ,CACR,GAAM,CAAE,QAAO,kBAAmB,EAAQ,OACpC,EAAY,EAA4B,EAAO,EAAQ,eAAe,EACtE,EACJ,EAAQ,kBAAoB,IAAA,GACxB,GACA,WAAW,EAAM,GAAG,EAAiB,IAAM,IAAI,GAAG,EAAY,EAAQ,gBAAiB,CAAS,EAAE,GAAG,EAAa,KAAO,MAAM,GACrI,MAAO;;IAEL,EAAM,KAAK,EAAY,EAAO,CAAS,EAAE;;QAErC,EAAS,eAAe,OAAO,EAAE,GAAG,EAAe;CAE3D,CAMA,SAAS,EAA4B,EAAe,EAAuC,CAEzF,GAAI,IAAc,IAAA,IAAa,IAAU,EAAW,MAAO,GAC3D,IAAK,IAAI,EAAY,EAAkB,EAAY,GAAI,IACrD,GAAI,EAAY,EAAO,CAAS,IAAM,EAAY,EAAW,CAAS,EAAG,OAAO,EAElF,MAAO,GACT,CAEA,SAAS,EAAY,EAAe,EAA2B,CAC7D,OAAO,OAAO,EAAM,YAAY,CAAS,CAAC,CAAC,CAAC,SAAS,CACvD,CAEA,SAAS,EACP,EACA,EACA,EACA,EAC8B,CAC9B,IAAI,EACJ,GAAI,CACF,EAAU,EAAgB,CAAc,CAC1C,OAAS,EAAO,CAGd,MAAO,KAAK,EAAmB,sBAD7B,aAAiB,EAAgB,GAAG,EAAM,WAAW,QAAQ,EAAyB,EAAM,UAAY,IAE5G,CACA,IAAM,EAAU,EAAY,EAAQ,EAAE,EAAE,OAAS,CAAC,EAAG,CAAO,EAC5D,GAAI,CAAC,EACH,MAAO,KAAK,EAAmB,YAAY,EAAQ,SAAS,QAAQ,EAAQ,aAAa,gBAG3F,IAAM,EAAc,IAAI,IAClB,EAAqB,CAAC,EACxB,EAAe,EACb,EAAiB,GAA0B,CAC/C,IACI,EAAS,OAAS,IAAgB,EAAS,KAAK,CAAO,CAC7D,EACA,IAAK,GAAM,CAAE,MAAO,EAAK,gBAAgB,EAAQ,MAAM,CAAC,EAAG,CACzD,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAa,EAAI,EAAQ,cAAgB,GAC/C,GAAI,CAAC,EAAO,IAAI,CAAE,EAChB,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,iBAAiB,OACjF,GAAI,EAAY,IAAI,CAAE,EAC3B,EAAc,GAAG,EAAW,QAAQ,EAAQ,SAAS,OAAO,EAAK,CAAE,EAAE,aAAa,MAC7E,CACL,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAU,EAClE,GACF,EAAc,GAAG,EAAW,QAAQ,EAAQ,aAAa,QAAQ,EAAK,CAAU,EAAE,KAAK,GAAiB,EAE1G,EAAY,IAAI,EAAI,CAAU,CAChC,CACF,CACA,IAAM,EAAa,CAAC,GAAG,EAAO,KAAK,CAAC,CAAC,CAAC,OAAQ,GAAO,CAAC,EAAY,IAAI,CAAE,CAAC,EAczE,OAbI,EAAW,OAAS,IAEtB,IACA,EAAS,QACP,KAAK,EAAQ,SAAS,KAAK,EAAQ,CAAU,EAAE,cAAc,EAAW,OAAO,eAAe,OAAO,EAAE,GACzG,GAEE,EAAe,EACV,KAAK,EAAmB;;EAEjC,EAAS,IAAK,GAAY,KAAK,GAAS,CAAC,CAAC,KAAK;CAAI,IAAI,EAAe,EAAS,OAAS,UAAU,EAAe,EAAS,OAAA,CAAQ,eAAe,OAAO,EAAE,GAAK,GAAG;EAG3J,CACT,CAEA,eAAe,EAAW,EAAwB,EAAqE,CACrH,IAAM,EAAO,EAAS,MAAM,EAAG,SAAS,EAAgB,MAAM,CAAC,EACzD,EAAU,EAAY,EAAK,IAAM,CAAC,EAAG,CAAO,EAClD,GAAI,CAAC,EACH,MAAU,MAAM,yBAAyB,EAAQ,SAAS,OAAO,EAAQ,aAAa,YAAY,GAAgB,EAEpH,IAAM,EAAS,IAAI,IACnB,IAAK,IAAM,KAAO,EAAK,MAAM,CAAC,EAAG,CAC/B,GAAI,EAAW,CAAG,EAAG,SACrB,IAAM,EAAK,EAAI,EAAQ,QAAQ,EAAE,KAAK,GAAK,GACrC,EAAQ,EAAI,EAAQ,cAAgB,GAC1C,GAAI,CAAC,EAAI,MAAU,MAAM,SAAS,EAAQ,SAAS,mBAAmB,GAAgB,EACtF,GAAI,EAAO,IAAI,CAAE,EAAG,MAAU,MAAM,aAAa,EAAQ,SAAS,mBAAmB,GAAI,EACzF,IAAM,EAAkB,EAAQ,OAAO,qBAAqB,CAAK,EACjE,GAAI,EACF,MAAU,MAAM,WAAW,EAAQ,aAAa,sBAAsB,EAAG,IAAI,GAAiB,EAChG,EAAO,IAAI,EAAI,CAAK,CACtB,CACA,GAAI,EAAO,OAAS,EAAG,MAAU,MAAM,4BAA4B,GAAgB,EACnF,OAAO,CACT,CAEA,SAAS,EACP,EACA,EACsD,CACtD,IAAM,EAAQ,EAAO,IAAK,GAAS,EAAK,KAAK,CAAC,EACxC,EAAU,EAAM,QAAQ,EAAQ,QAAQ,EACxC,EAAc,EAAM,QAAQ,EAAQ,YAAY,EACtD,OAAO,IAAY,IAAM,IAAgB,GAAK,IAAA,GAAY,CAAE,UAAS,aAAY,CACnF,CAEA,SAAS,EAAW,EAAiC,CACnD,OAAO,EAAI,MAAO,GAAS,EAAK,KAAK,IAAM,EAAE,CAC/C,CAEA,SAAS,EAAQ,EAAgC,CAC/C,IAAM,EAAS,EAAI,MAAM,EAAG,EAAc,CAAC,CAAC,IAAK,GAAO,KAAK,EAAK,CAAE,EAAE,GAAG,EACzE,OAAO,EAAI,OAAS,GAAiB,GAAG,EAAO,KAAK,IAAI,EAAE,OAAS,EAAO,KAAK,IAAI,CACrF,CAEA,SAAS,EAAK,EAAuB,CAEnC,IAAM,EAAY,EAAM,WAAW,aAAc,GAAG,CAAC,CAAC,WAAW,IAAK,GAAG,EAEzE,OADI,EAAU,KAAK,IAAM,GAAW,MAC7B,EAAU,OAAS,GAAyB,GAAG,EAAU,MAAM,EAAG,EAAsB,EAAE,GAAK,CACxG,CAEA,SAAS,EAAsB,EAA0B,EAAuD,CAC9G,OAAQ,EAAO,eAAiB,EAAa,WAAe,EAAY,iBAAmB,WAC7F,CAEA,SAAS,EAAqB,EAAwC,CACpE,OAAO,OAAO,SAAS,OAAO,CAAU,CAAC,GAAK,EAAW,KAAK,IAAM,GAAK,IAAA,GAAY,WACvF,CAEA,SAAS,EAAK,EAAmC,CAC/C,OAAO,EAAO,QAAQ,EAAK,IAAU,EAAM,EAAO,CAAC,EAAI,EAAO,MAChE"}

package/dist/types/problem.cjs

@@ -1,2 +1,2 @@
-"use strict";let e=require("zod");const t=e.z.object({timeLimitMs:e.z.number().int().min(0).optional(),memoryLimitByte:e.z.number().int().min(0).optional(),requiredRegExpsInCode:e.z.array(e.z.string().min(1)).optional(),forbiddenRegExpsInCode:e.z.array(e.z.string().min(1)).optional(),forbiddenTextsInCode:e.z.array(e.z.string().min(1)).optional(),isManualScoringRequired:e.z.boolean().optional(),requiredEnvironmentVariables:e.z.array(e.z.string().min(1)).optional(),requiredOutputFilePaths:e.z.array(e.z.string().min(1)).optional(),requiredSubmissionFilePaths:e.z.array(e.z.string().min(1)).optional()});exports.problemMarkdownFrontMatterSchema=t;
+"use strict";let e=require("zod");const t=e.z.union([e.z.string().min(1),e.z.object({pattern:e.z.string().min(1),message:e.z.string().min(1).refine(e=>!/[\r\n]/.test(e),`message must be a single line`)})]),n=e.z.object({timeLimitMs:e.z.number().int().min(0).optional(),memoryLimitByte:e.z.number().int().min(0).optional(),requiredRegExpsInCode:e.z.array(t).optional(),forbiddenRegExpsInCode:e.z.array(t).optional(),forbiddenTextsInCode:e.z.array(t).optional(),isManualScoringRequired:e.z.boolean().optional(),requiredEnvironmentVariables:e.z.array(e.z.string().min(1)).optional(),requiredOutputFilePaths:e.z.array(e.z.string().min(1)).optional(),requiredSubmissionFilePaths:e.z.array(e.z.string().min(1)).optional()});function r(e){return typeof e==`string`?{pattern:e}:e}exports.codeRuleSchema=t,exports.normalizeCodeRule=r,exports.problemMarkdownFrontMatterSchema=n;
 //# sourceMappingURL=problem.cjs.map

package/dist/types/problem.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"problem.cjs","names":["z"],"sources":["../../src/types/problem.ts"],"sourcesContent":["import { z } from 'zod';\n\nexport const problemMarkdownFrontMatterSchema = z.object({\n  timeLimitMs: z.number().int().min(0).optional(),\n  memoryLimitByte: z.number().int().min(0).optional(),\n  requiredRegExpsInCode: z.array(z.string().min(1)).optional(),\n  forbiddenRegExpsInCode: z.array(z.string().min(1)).optional(),\n  forbiddenTextsInCode: z.array(z.string().min(1)).optional(),\n  isManualScoringRequired: z.boolean().optional(),\n  requiredEnvironmentVariables: z.array(z.string().min(1)).optional(),\n  requiredOutputFilePaths: z.array(z.string().min(1)).optional(),\n  requiredSubmissionFilePaths: z.array(z.string().min(1)).optional(),\n});\n\nexport type ProblemMarkdownFrontMatter = z.infer<typeof problemMarkdownFrontMatterSchema>;\n"],"mappings":"kCAEA,MAAa,EAAmCA,EAAAA,EAAE,OAAO,CACvD,YAAaA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC9C,gBAAiBA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAClD,sBAAuBA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC3D,uBAAwBA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC5D,qBAAsBA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC1D,wBAAyBA,EAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,EAC9C,6BAA8BA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAClE,wBAAyBA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC7D,4BAA6BA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC"}
+{"version":3,"file":"problem.cjs","names":["z"],"sources":["../../src/types/problem.ts"],"sourcesContent":["import { z } from 'zod';\n\n/**\n * A static-analysis rule on submitted code: either a bare pattern (its regular expression or text\n * is shown to learners on violation) or a pattern with a learner-facing message that replaces it.\n * The message is rendered as one Markdown list item, so it must be a single line.\n */\nexport const codeRuleSchema = z.union([\n  z.string().min(1),\n  z.object({\n    pattern: z.string().min(1),\n    message: z\n      .string()\n      .min(1)\n      .refine((message) => !/[\\r\\n]/.test(message), 'message must be a single line'),\n  }),\n]);\n\nexport type CodeRule = z.infer<typeof codeRuleSchema>;\n\nexport const problemMarkdownFrontMatterSchema = z.object({\n  timeLimitMs: z.number().int().min(0).optional(),\n  memoryLimitByte: z.number().int().min(0).optional(),\n  requiredRegExpsInCode: z.array(codeRuleSchema).optional(),\n  forbiddenRegExpsInCode: z.array(codeRuleSchema).optional(),\n  forbiddenTextsInCode: z.array(codeRuleSchema).optional(),\n  isManualScoringRequired: z.boolean().optional(),\n  requiredEnvironmentVariables: z.array(z.string().min(1)).optional(),\n  requiredOutputFilePaths: z.array(z.string().min(1)).optional(),\n  requiredSubmissionFilePaths: z.array(z.string().min(1)).optional(),\n});\n\nexport type ProblemMarkdownFrontMatter = z.infer<typeof problemMarkdownFrontMatterSchema>;\n\nexport function normalizeCodeRule(rule: CodeRule): { pattern: string; message?: string } {\n  return typeof rule === 'string' ? { pattern: rule } : rule;\n}\n"],"mappings":"kCAOA,MAAa,EAAiBA,EAAAA,EAAE,MAAM,CACpCA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,EAChBA,EAAAA,EAAE,OAAO,CACP,QAASA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,EACzB,QAASA,EAAAA,EACN,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,OAAQ,GAAY,CAAC,SAAS,KAAK,CAAO,EAAG,+BAA+B,CACjF,CAAC,CACH,CAAC,EAIY,EAAmCA,EAAAA,EAAE,OAAO,CACvD,YAAaA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC9C,gBAAiBA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAClD,sBAAuBA,EAAAA,EAAE,MAAM,CAAc,CAAC,CAAC,SAAS,EACxD,uBAAwBA,EAAAA,EAAE,MAAM,CAAc,CAAC,CAAC,SAAS,EACzD,qBAAsBA,EAAAA,EAAE,MAAM,CAAc,CAAC,CAAC,SAAS,EACvD,wBAAyBA,EAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,EAC9C,6BAA8BA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAClE,wBAAyBA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC7D,4BAA6BA,EAAAA,EAAE,MAAMA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,EAID,SAAgB,EAAkB,EAAuD,CACvF,OAAO,OAAO,GAAS,SAAW,CAAE,QAAS,CAAK,EAAI,CACxD"}

package/dist/types/problem.d.ts

@@ -1,13 +1,36 @@
 import { z } from 'zod';
+/**
+ * A static-analysis rule on submitted code: either a bare pattern (its regular expression or text
+ * is shown to learners on violation) or a pattern with a learner-facing message that replaces it.
+ * The message is rendered as one Markdown list item, so it must be a single line.
+ */
+export declare const codeRuleSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+    pattern: z.ZodString;
+    message: z.ZodString;
+}, z.core.$strip>]>;
+export type CodeRule = z.infer<typeof codeRuleSchema>;
 export declare const problemMarkdownFrontMatterSchema: z.ZodObject<{
     timeLimitMs: z.ZodOptional<z.ZodNumber>;
     memoryLimitByte: z.ZodOptional<z.ZodNumber>;
-    requiredRegExpsInCode: z.ZodOptional<z.ZodArray<z.ZodString>>;
-    forbiddenRegExpsInCode: z.ZodOptional<z.ZodArray<z.ZodString>>;
-    forbiddenTextsInCode: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    requiredRegExpsInCode: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+        pattern: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>]>>>;
+    forbiddenRegExpsInCode: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+        pattern: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>]>>>;
+    forbiddenTextsInCode: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+        pattern: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>]>>>;
     isManualScoringRequired: z.ZodOptional<z.ZodBoolean>;
     requiredEnvironmentVariables: z.ZodOptional<z.ZodArray<z.ZodString>>;
     requiredOutputFilePaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
     requiredSubmissionFilePaths: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export type ProblemMarkdownFrontMatter = z.infer<typeof problemMarkdownFrontMatterSchema>;
+export declare function normalizeCodeRule(rule: CodeRule): {
+    pattern: string;
+    message?: string;
+};

package/dist/types/problem.js

@@ -1,2 +1,2 @@
-import{z as e}from"zod";const t=e.object({timeLimitMs:e.number().int().min(0).optional(),memoryLimitByte:e.number().int().min(0).optional(),requiredRegExpsInCode:e.array(e.string().min(1)).optional(),forbiddenRegExpsInCode:e.array(e.string().min(1)).optional(),forbiddenTextsInCode:e.array(e.string().min(1)).optional(),isManualScoringRequired:e.boolean().optional(),requiredEnvironmentVariables:e.array(e.string().min(1)).optional(),requiredOutputFilePaths:e.array(e.string().min(1)).optional(),requiredSubmissionFilePaths:e.array(e.string().min(1)).optional()});export{t as problemMarkdownFrontMatterSchema};
+import{z as e}from"zod";const t=e.union([e.string().min(1),e.object({pattern:e.string().min(1),message:e.string().min(1).refine(e=>!/[\r\n]/.test(e),`message must be a single line`)})]),n=e.object({timeLimitMs:e.number().int().min(0).optional(),memoryLimitByte:e.number().int().min(0).optional(),requiredRegExpsInCode:e.array(t).optional(),forbiddenRegExpsInCode:e.array(t).optional(),forbiddenTextsInCode:e.array(t).optional(),isManualScoringRequired:e.boolean().optional(),requiredEnvironmentVariables:e.array(e.string().min(1)).optional(),requiredOutputFilePaths:e.array(e.string().min(1)).optional(),requiredSubmissionFilePaths:e.array(e.string().min(1)).optional()});function r(e){return typeof e==`string`?{pattern:e}:e}export{t as codeRuleSchema,r as normalizeCodeRule,n as problemMarkdownFrontMatterSchema};
 //# sourceMappingURL=problem.js.map

package/dist/types/problem.js.map

@@ -1 +1 @@
-{"version":3,"file":"problem.js","names":[],"sources":["../../src/types/problem.ts"],"sourcesContent":["import { z } from 'zod';\n\nexport const problemMarkdownFrontMatterSchema = z.object({\n  timeLimitMs: z.number().int().min(0).optional(),\n  memoryLimitByte: z.number().int().min(0).optional(),\n  requiredRegExpsInCode: z.array(z.string().min(1)).optional(),\n  forbiddenRegExpsInCode: z.array(z.string().min(1)).optional(),\n  forbiddenTextsInCode: z.array(z.string().min(1)).optional(),\n  isManualScoringRequired: z.boolean().optional(),\n  requiredEnvironmentVariables: z.array(z.string().min(1)).optional(),\n  requiredOutputFilePaths: z.array(z.string().min(1)).optional(),\n  requiredSubmissionFilePaths: z.array(z.string().min(1)).optional(),\n});\n\nexport type ProblemMarkdownFrontMatter = z.infer<typeof problemMarkdownFrontMatterSchema>;\n"],"mappings":"wBAEA,MAAa,EAAmC,EAAE,OAAO,CACvD,YAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC9C,gBAAiB,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAClD,sBAAuB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC3D,uBAAwB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC5D,qBAAsB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC1D,wBAAyB,EAAE,QAAQ,CAAC,CAAC,SAAS,EAC9C,6BAA8B,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAClE,wBAAyB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC7D,4BAA6B,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC"}
+{"version":3,"file":"problem.js","names":[],"sources":["../../src/types/problem.ts"],"sourcesContent":["import { z } from 'zod';\n\n/**\n * A static-analysis rule on submitted code: either a bare pattern (its regular expression or text\n * is shown to learners on violation) or a pattern with a learner-facing message that replaces it.\n * The message is rendered as one Markdown list item, so it must be a single line.\n */\nexport const codeRuleSchema = z.union([\n  z.string().min(1),\n  z.object({\n    pattern: z.string().min(1),\n    message: z\n      .string()\n      .min(1)\n      .refine((message) => !/[\\r\\n]/.test(message), 'message must be a single line'),\n  }),\n]);\n\nexport type CodeRule = z.infer<typeof codeRuleSchema>;\n\nexport const problemMarkdownFrontMatterSchema = z.object({\n  timeLimitMs: z.number().int().min(0).optional(),\n  memoryLimitByte: z.number().int().min(0).optional(),\n  requiredRegExpsInCode: z.array(codeRuleSchema).optional(),\n  forbiddenRegExpsInCode: z.array(codeRuleSchema).optional(),\n  forbiddenTextsInCode: z.array(codeRuleSchema).optional(),\n  isManualScoringRequired: z.boolean().optional(),\n  requiredEnvironmentVariables: z.array(z.string().min(1)).optional(),\n  requiredOutputFilePaths: z.array(z.string().min(1)).optional(),\n  requiredSubmissionFilePaths: z.array(z.string().min(1)).optional(),\n});\n\nexport type ProblemMarkdownFrontMatter = z.infer<typeof problemMarkdownFrontMatterSchema>;\n\nexport function normalizeCodeRule(rule: CodeRule): { pattern: string; message?: string } {\n  return typeof rule === 'string' ? { pattern: rule } : rule;\n}\n"],"mappings":"wBAOA,MAAa,EAAiB,EAAE,MAAM,CACpC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,EAChB,EAAE,OAAO,CACP,QAAS,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,EACzB,QAAS,EACN,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,OAAQ,GAAY,CAAC,SAAS,KAAK,CAAO,EAAG,+BAA+B,CACjF,CAAC,CACH,CAAC,EAIY,EAAmC,EAAE,OAAO,CACvD,YAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC9C,gBAAiB,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAClD,sBAAuB,EAAE,MAAM,CAAc,CAAC,CAAC,SAAS,EACxD,uBAAwB,EAAE,MAAM,CAAc,CAAC,CAAC,SAAS,EACzD,qBAAsB,EAAE,MAAM,CAAc,CAAC,CAAC,SAAS,EACvD,wBAAyB,EAAE,QAAQ,CAAC,CAAC,SAAS,EAC9C,6BAA8B,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAClE,wBAAyB,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC7D,4BAA6B,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,EAID,SAAgB,EAAkB,EAAuD,CACvF,OAAO,OAAO,GAAS,SAAW,CAAE,QAAS,CAAK,EAAI,CACxD"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exercode/problem-utils",
-  "version": "2.2.2",
+  "version": "2.3.1",
   "description": ":100: A set of utilities for judging programs on Exercode (https://exercode.willbooster.com/).",
   "keywords": [
     "exercode",