npm - @exercode/problem-utils - Versions diffs - 5.0.0 → 5.0.2 - Mend

@exercode/problem-utils 5.0.0 → 5.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (3) hide show

package/dist/cli/checkAllProblems.cjs.map +1 -1
package/dist/cli/checkAllProblems.js.map +1 -1
package/package.json +7 -7

package/dist/cli/checkAllProblems.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"checkAllProblems.cjs","names":["DecisionCode","path","forEachConcurrently","readProblemMarkdownFrontMatter","findDefaultStdioHarnessFiles","formatDefaultHarnessError","findModelAnswerDirs","readTestCases","getErrorMessage","judgesWithoutTestCases","MISSING_TEST_CASES_ERROR","findFailingModelAnswerDirs","agentChallengeSchema","fs","copyProblemDirToTemporaryRoot","runHarnessProcess","forciblyRemoveDirectory","TEST_CASE_RESULT_PREFIX","testCaseResultSchema","require_runtime","require_testCaseResult","require_defaultStdioHarness","require_readTestCases","require_temporaryProblemDirCopy","require_schemas","require_readProblemMarkdownFrontMatter","require_decisionCode","require_findModelAnswerDirs","require_runHarnessProcess","require_runSingleHarness","node_path","node_fs_promises","_willbooster_shared_lib","MAX_FAILURE_DETAIL_LENGTH","decisionCodeNames","name","code","checkAllProblems","options","parseCheckArgs","args","rootDir","toRelative","dir","allProblemDirs","findProblemDirs","problemDirs","relativeDir","problemDir","substring","failures","runs","agentChallengeCount","plan","planProblemCheck","failure","cliEntryPath","passedCount","label","run","failureDetail","executeCheckRun","frontMatterResult","planAgentChallengeCheck","defaultHarnessFileNames","modelAnswerDirs","fileExists","failingModelAnswerDirs","parsed","agentChallenge","dirent","issue","tempRoot","copiedProblemDir","error","truncate","harnessFailureDetail","removedTempRoot","summarizeHarnessFailure","removalFailureDetail","result","stderr","resultLines","stdout","line","testCaseResults","parsedResult","rejectedResult","hasRootDir","index","arg","value","visitDirectory","entries","entry","filePath","text"],"sources":["../../src/cli/checkAllProblems.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { forEachConcurrently, getErrorMessage } from '@willbooster/shared-lib';\n\nimport { findDefaultStdioHarnessFiles } from '../helpers/defaultStdioHarness.js';\nimport { findFailingModelAnswerDirs, findModelAnswerDirs } from '../helpers/findModelAnswerDirs.js';\nimport { judgesWithoutTestCases, readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { readTestCases } from '../helpers/readTestCases.js';\nimport { type HarnessProcessResult, runHarnessProcess } from '../helpers/runHarnessProcess.js';\nimport { copyProblemDirToTemporaryRoot, forciblyRemoveDirectory } from '../helpers/temporaryProblemDirCopy.js';\nimport { agentChallengeSchema } from '../learningMaterial/schemas.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport { TEST_CASE_RESULT_PREFIX, testCaseResultSchema } from '../types/testCaseResult.js';\nimport { formatDefaultHarnessError, MISSING_TEST_CASES_ERROR } from './runSingleHarness.js';\n\nconst RUN_TIMEOUT_MS = 600_000;\nconst MAX_RUN_OUTPUT_BYTES = 64 * 1024 * 1024;\nconst MAX_FAILURE_DETAIL_LENGTH = 1000;\n\n// DecisionCode is a plain const object (not a TypeScript enum), so it has no reverse mapping.\nconst decisionCodeNames = new Map<number, string>(Object.entries(DecisionCode).map(([name, code]) => [code, name]));\n\ninterface CheckOptions {\n rootDir: string;\n concurrency: number;\n only: string[];\n skip: string[];\n}\n\ninterface CheckRun {\n problemDir: string;\n answerDir: string;\n /** Whether all test cases must be accepted (`model_answers`) or at least one must fail (`model_answers.fails`). /\n expectation: 'accepted' \| 'rejected';\n}\n\n/\n Judge all model answers of all problems (directories containing `problem.md` or\n * `<id>.problem.md`) under a root directory: `model_answers/` must be fully accepted and\n `model_answers.fails/` must fail at least one test case. Agent challenges have no harness, so\n they are not judged but fail when their `agentChallenge` is invalid or their explanation is missing.\n /\nexport async function checkAllProblems(args: readonly string[]): Promise<number> {\n const options = parseCheckArgs(args);\n const rootDir = path.resolve(options.rootDir);\n // Normalize to forward slashes so --only / --skip substrings like courses/foo match on Windows.\n const toRelative = (dir: string): string => (path.relative(rootDir, dir) \|\| '.').replaceAll(path.sep, '/');\n\n const allProblemDirs = await findProblemDirs(rootDir);\n const problemDirs = allProblemDirs.filter((problemDir) => {\n const relativeDir = toRelative(problemDir);\n if (options.only.length > 0 && !options.only.some((substring) => relativeDir.includes(substring))) return false;\n return !options.skip.some((substring) => relativeDir.includes(substring));\n });\n if (problemDirs.length === 0) {\n console.error(\n allProblemDirs.length === 0\n ? `No problem directories (containing problem.md or <id>.problem.md) found under ${rootDir}.`\n : `All ${allProblemDirs.length} problem directories under ${rootDir} were excluded by --only/--skip.`\n );\n return 1;\n }\n\n const failures: string[] = [];\n const runs: CheckRun[] = [];\n let agentChallengeCount = 0;\n for (const problemDir of problemDirs) {\n const plan = await planProblemCheck(problemDir);\n if (plan.type === 'agentChallenge') {\n agentChallengeCount++;\n } else if (plan.type === 'failure') {\n failures.push(`${toRelative(problemDir)}: ${plan.message}`);\n } else {\n runs.push(...plan.runs);\n }\n }\n for (const failure of failures) console.error(`✗ ${failure}`);\n\n const cliEntryPath = path.resolve(process.argv[1] ?? '');\n let passedCount = 0;\n await forEachConcurrently(runs, options.concurrency, async (run) => {\n const label = `${toRelative(run.problemDir)} ${path.relative(run.problemDir, run.answerDir).replaceAll(path.sep, '/')}`;\n const failureDetail = await executeCheckRun(run, cliEntryPath);\n if (failureDetail === undefined) {\n passedCount++;\n console.info(`✓ ${label}`);\n } else {\n failures.push(`${label}: ${failureDetail}`);\n console.error(`✗ ${label}: ${failureDetail}`);\n }\n });\n\n console.info(\n `\\n${passedCount} passed, ${failures.length} failed (${runs.length} runs, ${problemDirs.length} problems${agentChallengeCount > 0 ? `, agent challenges skipped: ${agentChallengeCount}` : ''})`\n );\n return failures.length === 0 ? 0 : 1;\n}\n\ntype ProblemCheckPlan =\n { type: 'agentChallenge' } \| { type: 'failure'; message: string } \| { type: 'runs'; runs: CheckRun[] };\n\nasync function planProblemCheck(problemDir: string): Promise<ProblemCheckPlan> {\n // As before agent challenges existed, a front matter that fails to parse fails the problem only\n // when the test-case check below needs the front matter.\n const frontMatterResult = await readProblemMarkdownFrontMatter(problemDir).then(\n (frontMatter) => ({ frontMatter }),\n (error: unknown) => ({ error })\n );\n if ('frontMatter' in frontMatterResult && frontMatterResult.frontMatter.agentChallenge !== undefined) {\n return planAgentChallengeCheck(problemDir, frontMatterResult.frontMatter.agentChallenge);\n }\n\n const defaultHarnessFileNames = await findDefaultStdioHarnessFiles(problemDir);\n if (defaultHarnessFileNames.length > 0) {\n return { type: 'failure', message: formatDefaultHarnessError(defaultHarnessFileNames) };\n }\n\n const modelAnswerDirs = await findModelAnswerDirs(problemDir);\n if (modelAnswerDirs.length === 0) {\n return { type: 'failure', message: 'no model answers found under model_answers/' };\n }\n\n // Without test cases, stdioJudgePreset prints a single accepted sentinel result, so a standard\n // problem with an empty or missing test_cases/ would otherwise pass without being judged —\n // unless static-analysis rules or manual scoring make the problem judgeable without them.\n if (!(await fileExists(path.join(problemDir, 'judge.ts')))) {\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n if (testCases.length === 0) {\n if ('error' in frontMatterResult) {\n return {\n type: 'failure',\n message: `failed to read the problem markdown front matter: ${getErrorMessage(frontMatterResult.error)}`,\n };\n }\n if (!judgesWithoutTestCases(frontMatterResult.frontMatter)) {\n return { type: 'failure', message: MISSING_TEST_CASES_ERROR };\n }\n }\n }\n\n const failingModelAnswerDirs = await findFailingModelAnswerDirs(problemDir);\n return {\n type: 'runs',\n runs: [\n ...modelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'accepted' })),\n ...failingModelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'rejected' })),\n ],\n };\n}\n\n/* Skips a valid agent challenge and fails an invalid one instead of judging it as a stdio problem. /\nasync function planAgentChallengeCheck(problemDir: string, agentChallenge: unknown): Promise<ProblemCheckPlan> {\n const parsed = agentChallengeSchema.safeParse(agentChallenge);\n if (parsed.success) {\n const explanationDirents = await fs\n .readdir(path.join(problemDir, 'model_answers', 'explanation'), { recursive: true, withFileTypes: true })\n .catch(() => []);\n return explanationDirents.some((dirent) => dirent.isFile() && dirent.name.endsWith('.md'))\n ? { type: 'agentChallenge' }\n : { type: 'failure', message: 'no explanation found under model_answers/explanation/' };\n }\n const issues = parsed.error.issues.map((issue) => `${['agentChallenge', ...issue.path].join('.')}: ${issue.message}`);\n return { type: 'failure', message: `invalid agentChallenge (see validate-problem): ${issues.join('; ')}` };\n}\n\n/\n Run one answer directory through the problem's harness and return a failure detail, if any.\n * The problem directory is copied to a temporary location first so that judging (e.g. build\n * artifacts in answer directories) never modifies the checked repository.\n /\nasync function executeCheckRun(run: CheckRun, cliEntryPath: string): Promise<string \| undefined> {\n let tempRoot: string;\n let copiedProblemDir: string;\n try {\n ({ tempRoot, copiedProblemDir } = await copyProblemDirToTemporaryRoot(run.problemDir));\n } catch (error) {\n return truncate(`failed to copy the problem directory to a temporary location: ${getErrorMessage(error)}`);\n }\n\n let harnessFailureDetail;\n let removedTempRoot;\n try {\n const result = await runHarnessProcess(\n ['run', cliEntryPath, 'judge', path.relative(run.problemDir, run.answerDir)],\n {\n cwd: copiedProblemDir,\n env: process.env,\n timeoutMs: RUN_TIMEOUT_MS,\n maxOutputBytes: MAX_RUN_OUTPUT_BYTES,\n tempRoot,\n }\n );\n harnessFailureDetail = summarizeHarnessFailure(run, result);\n } catch (error) {\n harnessFailureDetail = truncate(`harness execution failed: ${getErrorMessage(error)}`);\n } finally {\n removedTempRoot = await forciblyRemoveDirectory(tempRoot);\n }\n if (removedTempRoot) return harnessFailureDetail;\n const removalFailureDetail = `failed to remove the temporary copy at ${tempRoot} (judged code may have left permission-locked files)`;\n return harnessFailureDetail === undefined\n ? removalFailureDetail\n : truncate(`${harnessFailureDetail}; ${removalFailureDetail}`);\n}\n\n/* Return why the harness run failed the check, or `undefined` when it passed. /\nfunction summarizeHarnessFailure(run: CheckRun, result: HarnessProcessResult): string \| undefined {\n const { stdout, stderr } = result;\n if (result.failureReason !== undefined) return truncate(result.failureReason);\n if (result.exitCode !== 0) {\n return truncate(`harness exited with ${result.exitCode ?? 'a signal'}${stderr.trim() ? `: ${stderr.trim()}` : ''}`);\n }\n\n const resultLines = stdout.split(/\\r?\\n/).filter((line) => line.startsWith(TEST_CASE_RESULT_PREFIX));\n const testCaseResults = [];\n for (const line of resultLines) {\n let parsedResult;\n try {\n parsedResult = testCaseResultSchema.safeParse(JSON.parse(line.slice(TEST_CASE_RESULT_PREFIX.length)));\n } catch {\n parsedResult = undefined;\n }\n if (!parsedResult?.success) return truncate(`malformed test case result line: ${line}`);\n testCaseResults.push(parsedResult.data);\n }\n if (testCaseResults.length === 0) return 'no test case results were printed';\n\n if (run.expectation === 'accepted') {\n const rejectedResult = testCaseResults.find((result) => result.decisionCode !== DecisionCode.ACCEPTED);\n if (rejectedResult) {\n return truncate(\n `${decisionCodeNames.get(rejectedResult.decisionCode) ?? rejectedResult.decisionCode} on test case ${rejectedResult.testCaseId}${rejectedResult.stderr?.trim() ? `: ${rejectedResult.stderr.trim()}` : ''}`\n );\n }\n return undefined;\n }\n return testCaseResults.every((result) => result.decisionCode === DecisionCode.ACCEPTED)\n ? 'expected at least one failing test case, but all test cases were accepted'\n : undefined;\n}\n\nfunction parseCheckArgs(args: readonly string[]): CheckOptions {\n const options: CheckOptions = {\n rootDir: '.',\n // Serial by default: judging decides TIME_LIMIT_EXCEEDED from wall-clock time, so parallel\n // runs on a small CI runner could fail timing-sensitive problems non-deterministically.\n concurrency: 1,\n only: [],\n skip: [],\n };\n let hasRootDir = false;\n for (let index = 0; index < args.length; index++) {\n const arg = args[index];\n if (arg === undefined) break;\n if (arg === '--concurrency' \|\| arg === '--only' \|\| arg === '--skip') {\n const value = args[++index];\n if (value === undefined) throw new Error(`${arg} requires a value`);\n if (arg === '--concurrency') {\n options.concurrency = Number(value);\n if (!Number.isInteger(options.concurrency) \|\| options.concurrency <= 0) {\n throw new Error(`--concurrency requires a positive integer, but got ${value}`);\n }\n } else if (arg === '--only') {\n options.only.push(value);\n } else {\n options.skip.push(value);\n }\n } else if (arg.startsWith('--')) {\n throw new Error(`Unknown option: ${arg}`);\n } else if (hasRootDir) {\n throw new Error(`Only one root directory can be specified, but got both ${options.rootDir} and ${arg}`);\n } else {\n options.rootDir = arg;\n hasRootDir = true;\n }\n }\n return options;\n}\n\n/* Find directories containing `problem.md` or `<id>.problem.md`, without descending into found problems. */\nasync function findProblemDirs(rootDir: string): Promise<string[]> {\n const problemDirs: string[] = [];\n await visitDirectory(rootDir, problemDirs);\n return problemDirs.toSorted();\n}\n\nasync function visitDirectory(dir: string, problemDirs: string[]): Promise<void> {\n // Traversal errors (e.g. an unreadable subtree) must fail the check: skipping them silently\n // could report a green result while covering only part of the repository.\n const entries = await fs.readdir(dir, { withFileTypes: true });\n if (entries.some((entry) => entry.isFile() && (entry.name === 'problem.md' \|\| entry.name.endsWith('.problem.md')))) {\n problemDirs.push(dir);\n return;\n }\n for (const entry of entries) {\n if (!entry.isDirectory() \|\| entry.name === 'node_modules' \|\| entry.name.startsWith('.')) continue;\n await visitDirectory(path.join(dir, entry.name), problemDirs);\n }\n}\n\nasync function fileExists(filePath: string): Promise<boolean> {\n try {\n await fs.stat(filePath);\n return true;\n } catch {\n return false;\n }\n}\n\nfunction truncate(text: string): string {\n return text.length <= MAX_FAILURE_DETAIL_LENGTH ? text : `${text.slice(0, MAX_FAILURE_DETAIL_LENGTH)}...`;\n}\n"],"mappings":"A,qtBAgBA,MAEMiC,EAA4B,IAG5BC,EAAoB,IAAI,IAAoB,OAAO,QAAQlC,EAAAA,YAAY,CAAC,CAAC,KAAK,CAACmC,EAAMC,KAAU,CAACA,EAAMD,CAAI,CAAC,CAAC,EAsBlH,eAAsBE,EAAiB,EAA0C,CAC/E,IAAMC,EAAUC,EAAeC,CAAI,EAC7BC,EAAUxC,EAAAA,QAAK,QAAQqC,EAAQ,OAAO,EAEtCI,EAAc,IAAyBzC,EAAAA,QAAK,SAASwC,EAASE,CAAG,GAAK,IAAA,CAAK,WAAW1C,EAAAA,QAAK,IAAK,GAAG,EAEnG2C,EAAiB,MAAMC,EAAgBJ,CAAO,EAC9CK,EAAcF,EAAe,OAAQ,GAAe,CACxD,IAAMG,EAAcL,EAAWM,CAAU,EAEzC,OADIV,EAAQ,KAAK,OAAS,GAAK,CAACA,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,EAAU,GACnG,CAACX,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,CAC1E,CAAC,EACD,GAAIH,EAAY,SAAW,EAMzB,OALA,QAAQ,MACNF,EAAe,SAAW,EACtB,iFAAiFH,EAAQ,GACzF,OAAOG,EAAe,OAAO,6BAA6BH,EAAQ,iCACxE,EACO,EAGT,IAAMS,EAAqB,CAAC,EACtBC,EAAmB,CAAC,EACtBC,EAAsB,EAC1B,IAAK,IAAMJ,KAAcF,EAAa,CACpC,IAAMO,EAAO,MAAMC,EAAiBN,CAAU,EAC1CK,EAAK,OAAS,iBAChB,IACSA,EAAK,OAAS,UACvBH,EAAS,KAAK,GAAGR,EAAWM,CAAU,EAAE,IAAIK,EAAK,SAAS,EAE1DF,EAAK,KAAK,GAAGE,EAAK,IAAI,CAE1B,CACA,IAAK,IAAME,KAAWL,EAAU,QAAQ,MAAM,KAAKK,GAAS,EAE5D,IAAMC,EAAevD,EAAAA,QAAK,QAAQ,QAAQ,KAAK,IAAM,EAAE,EACnDwD,EAAc,EAgBlB,OAfA,MAAA,EAAMvD,EAAAA,oBAAAA,CAAoBiD,EAAMb,EAAQ,YAAa,KAAO,IAAQ,CAClE,IAAMoB,EAAQ,GAAGhB,EAAWiB,EAAI,UAAU,EAAE,GAAG1D,EAAAA,QAAK,SAAS0D,EAAI,WAAYA,EAAI,SAAS,CAAC,CAAC,WAAW1D,EAAAA,QAAK,IAAK,GAAG,IAC9G2D,EAAgB,MAAMC,EAAgBF,EAAKH,CAAY,EACzDI,IAAkB,IAAA,IACpB,IACA,QAAQ,KAAK,KAAKF,GAAO,IAEzBR,EAAS,KAAK,GAAGQ,EAAM,IAAIE,GAAe,EAC1C,QAAQ,MAAM,KAAKF,EAAM,IAAIE,GAAe,EAEhD,CAAC,EAED,QAAQ,KACN,KAAKH,EAAY,WAAWP,EAAS,OAAO,WAAWC,EAAK,OAAO,SAASL,EAAY,OAAO,WAAWM,EAAsB,EAAI,+BAA+BA,IAAwB,GAAG,EAChM,EACOF,EAAS,SAAW,EAAI,EAAI,CACrC,CAKA,eAAeI,EAAiB,EAA+C,CAG7E,IAAMQ,EAAoB,MAAM3D,EAAAA,+BAA+B6C,CAAU,CAAC,CAAC,KACxE,IAAiB,CAAE,aAAY,GAC/B,IAAoB,CAAE,OAAM,EAC/B,EACA,GAAI,gBAAiBc,GAAqBA,EAAkB,YAAY,iBAAmB,IAAA,GACzF,OAAOC,EAAwBf,EAAYc,EAAkB,YAAY,cAAc,EAGzF,IAAME,EAA0B,MAAM5D,EAAAA,6BAA6B4C,CAAU,EAC7E,GAAIgB,EAAwB,OAAS,EACnC,MAAO,CAAE,KAAM,UAAW,QAAS3D,EAAAA,0BAA0B2D,CAAuB,CAAE,EAGxF,IAAMC,EAAkB,MAAM3D,EAAAA,oBAAoB0C,CAAU,EAC5D,GAAIiB,EAAgB,SAAW,EAC7B,MAAO,CAAE,KAAM,UAAW,QAAS,6CAA8C,EAMnF,GAAI,CAAE,MAAMC,EAAWjE,EAAAA,QAAK,KAAK+C,EAAY,UAAU,CAAC,IAElD,MADoBzC,EAAAA,cAAcN,EAAAA,QAAK,KAAK+C,EAAY,YAAY,CAAC,EAAA,CAC3D,SAAW,EAAG,CAC1B,GAAI,UAAWc,EACb,MAAO,CACL,KAAM,UACN,QAAS,sDAAA,EAAqDtD,EAAAA,gBAAAA,CAAgBsD,EAAkB,KAAK,GACvG,EAEF,GAAI,CAACrD,EAAAA,uBAAuBqD,EAAkB,WAAW,EACvD,MAAO,CAAE,KAAM,UAAW,QAASpD,EAAAA,wBAAyB,CAEhE,CAGF,IAAMyD,EAAyB,MAAMxD,EAAAA,2BAA2BqC,CAAU,EAC1E,MAAO,CACL,KAAM,OACN,KAAM,CACJ,GAAGiB,EAAgB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,EACpG,GAAGE,EAAuB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,CAC7G,CACF,CACF,CAGA,eAAeJ,EAAwB,EAAoB,EAAoD,CAC7G,IAAMK,EAASxD,EAAAA,qBAAqB,UAAUyD,CAAc,EAU5D,OATID,EAAO,SAIF,MAH0BvD,EAAAA,QAC9B,QAAQZ,EAAAA,QAAK,KAAK+C,EAAY,gBAAiB,aAAa,EAAG,CAAE,UAAW,GAAM,cAAe,EAAK,CAAC,CAAC,CACxG,UAAY,CAAC,CAAC,EAAA,CACS,KAAM,GAAWsB,EAAO,OAAO,GAAKA,EAAO,KAAK,SAAS,KAAK,CAAC,EACrF,CAAE,KAAM,gBAAiB,EACzB,CAAE,KAAM,UAAW,QAAS,uDAAwD,EAGnF,CAAE,KAAM,UAAW,QAAS,kDADpBF,EAAO,MAAM,OAAO,IAAK,GAAU,GAAG,CAAC,iBAAkB,GAAGG,EAAM,IAAI,CAAC,CAAC,KAAK,GAAG,EAAE,IAAIA,EAAM,SACjB,CAAC,CAAC,KAAK,IAAI,GAAI,CAC3G,CAOA,eAAeV,EAAgB,EAAe,EAAmD,CAC/F,IAAIW,EACAC,EACJ,GAAI,EACD,gCAAiC,MAAM3D,EAAAA,8BAA8B6C,EAAI,UAAU,EACtF,OAASe,EAAO,CACd,OAAOC,EAAS,kEAAA,EAAiEnE,EAAAA,gBAAAA,CAAgBkE,CAAK,GAAG,CAC3G,CAEA,IAAIE,EACAC,EACJ,GAAI,CAWF,EAAuBC,EAAwBnB,EAAK,MAV/B5C,EAAAA,kBACnB,CAAC,MAAOyC,EAAc,QAASvD,EAAAA,QAAK,SAAS0D,EAAI,WAAYA,EAAI,SAAS,CAAC,EAC3E,CACE,IAAKc,EACL,IAAK,QAAQ,IACb,UAAW,IACX,eAAgB,SAChB,UACF,CACF,CAC0D,CAC5D,OAASC,EAAO,CACd,EAAuBC,EAAS,8BAAA,EAA6BnE,EAAAA,gBAAAA,CAAgBkE,CAAK,GAAG,CACvF,QAAU,CACR,EAAkB,MAAM1D,EAAAA,wBAAwBwD,CAAQ,CAC1D,CACA,GAAIK,EAAiB,OAAOD,EAC5B,IAAMG,EAAuB,0CAA0CP,EAAS,sDAChF,OAAOI,IAAyB,IAAA,GAC5BG,EACAJ,EAAS,GAAGC,EAAqB,IAAIG,GAAsB,CACjE,CAGA,SAASD,EAAwB,EAAe,EAAkD,CAChG,GAAM,CAAE,SAAQ,UAAWE,EAC3B,GAAIA,EAAO,gBAAkB,IAAA,GAAW,OAAOL,EAASK,EAAO,aAAa,EAC5E,GAAIA,EAAO,WAAa,EACtB,OAAOL,EAAS,uBAAuBK,EAAO,UAAY,aAAaC,EAAO,KAAK,EAAI,KAAKA,EAAO,KAAK,IAAM,IAAI,EAGpH,IAAMC,EAAcC,EAAO,MAAM,OAAO,CAAC,CAAC,OAAQ,GAASC,EAAK,WAAWnE,EAAAA,uBAAuB,CAAC,EAC7FoE,EAAkB,CAAC,EACzB,IAAK,IAAMD,KAAQF,EAAa,CAC9B,IAAII,EACJ,GAAI,CACF,EAAepE,EAAAA,qBAAqB,UAAU,KAAK,MAAMkE,EAAK,MAAMnE,EAAAA,wBAAwB,MAAM,CAAC,CAAC,CACtG,MAAQ,CACN,EAAe,IAAA,EACjB,CACA,GAAI,CAACqE,GAAc,QAAS,OAAOX,EAAS,oCAAoCS,GAAM,EACtFC,EAAgB,KAAKC,EAAa,IAAI,CACxC,CACA,GAAID,EAAgB,SAAW,EAAG,MAAO,oCAEzC,GAAI1B,EAAI,cAAgB,WAAY,CAClC,IAAM4B,EAAiBF,EAAgB,KAAM,GAAWL,EAAO,eAAiBhF,EAAAA,aAAa,QAAQ,EAMrG,OALIuF,EACKZ,EACL,GAAGzC,EAAkB,IAAIqD,EAAe,YAAY,GAAKA,EAAe,aAAa,gBAAgBA,EAAe,aAAaA,EAAe,QAAQ,KAAK,EAAI,KAAKA,EAAe,OAAO,KAAK,IAAM,IACzM,EAEF,MACF,CACA,OAAOF,EAAgB,MAAO,GAAWL,EAAO,eAAiBhF,EAAAA,aAAa,QAAQ,EAClF,4EACA,IAAA,EACN,CAEA,SAASuC,EAAe,EAAuC,CAC7D,IAAMD,EAAwB,CAC5B,QAAS,IAGT,YAAa,EACb,KAAM,CAAC,EACP,KAAM,CAAC,CACT,EACIkD,EAAa,GACjB,IAAK,IAAIC,EAAQ,EAAGA,EAAQjD,EAAK,OAAQ,IAAS,CAChD,IAAMkD,EAAMlD,EAAKiD,GACjB,GAAIC,IAAQ,IAAA,GAAW,MACvB,GAAIA,IAAQ,iBAAmBA,IAAQ,UAAYA,IAAQ,SAAU,CACnE,IAAMC,EAAQnD,EAAK,EAAEiD,GACrB,GAAIE,IAAU,IAAA,GAAW,MAAU,MAAM,GAAGD,EAAI,kBAAkB,EAClE,GAAIA,IAAQ,gBAEV,IADA,EAAQ,YAAc,OAAOC,CAAK,EAC9B,CAAC,OAAO,UAAUrD,EAAQ,WAAW,GAAKA,EAAQ,aAAe,EACnE,MAAU,MAAM,sDAAsDqD,GAAO,CAAA,MAEtED,IAAQ,SACjBpD,EAAQ,KAAK,KAAKqD,CAAK,EAEvBrD,EAAQ,KAAK,KAAKqD,CAAK,CAE3B,MAAO,GAAID,EAAI,WAAW,IAAI,EAC5B,MAAU,MAAM,mBAAmBA,GAAK,OACnC,GAAIF,EACT,MAAU,MAAM,0DAA0DlD,EAAQ,QAAQ,OAAOoD,GAAK,MAEtG,GAAQ,QAAUA,EAClB,EAAa,EAEjB,CACA,OAAOpD,CACT,CAGA,eAAeO,EAAgB,EAAoC,CACjE,IAAMC,EAAwB,CAAC,EAE/B,OADA,MAAM8C,EAAenD,EAASK,CAAW,EAClCA,EAAY,SAAS,CAC9B,CAEA,eAAe8C,EAAe,EAAa,EAAsC,CAG/E,IAAMC,EAAU,MAAMhF,EAAAA,QAAG,QAAQ8B,EAAK,CAAE,cAAe,EAAK,CAAC,EAC7D,GAAIkD,EAAQ,KAAM,GAAUC,EAAM,OAAO,IAAMA,EAAM,OAAS,cAAgBA,EAAM,KAAK,SAAS,aAAa,EAAE,EAC/GhD,EAAY,KAAKH,CAAG,OAGtB,IAAK,IAAMmD,KAASD,EACbC,EAAM,YAAY,GAAKA,EAAM,OAAS,gBAAkB,GAAM,KAAK,WAAW,GAAG,GACtF,MAAMF,EAAe3F,EAAAA,QAAK,KAAK0C,EAAKmD,EAAM,IAAI,EAAGhD,CAAW,CAEhE,CAEA,eAAeoB,EAAW,EAAoC,CAC5D,GAAI,CAEF,OADA,MAAMrD,EAAAA,QAAG,KAAKkF,CAAQ,EACf,EACT,MAAQ,CACN,MAAO,EACT,CACF,CAEA,SAASpB,EAAS,EAAsB,CACtC,OAAOqB,EAAK,QAAU/D,EAA4B+D,EAAO,GAAGA,EAAK,MAAM,EAAG/D,CAAyB,EAAE,IACvG,C"}
1	+ {"version":3,"file":"checkAllProblems.cjs","names":["DecisionCode","path","forEachConcurrently","readProblemMarkdownFrontMatter","findDefaultStdioHarnessFiles","formatDefaultHarnessError","findModelAnswerDirs","readTestCases","getErrorMessage","judgesWithoutTestCases","MISSING_TEST_CASES_ERROR","findFailingModelAnswerDirs","agentChallengeSchema","fs","copyProblemDirToTemporaryRoot","runHarnessProcess","forciblyRemoveDirectory","TEST_CASE_RESULT_PREFIX","testCaseResultSchema","require_runtime","require_testCaseResult","require_defaultStdioHarness","require_readTestCases","require_temporaryProblemDirCopy","require_schemas","require_readProblemMarkdownFrontMatter","require_decisionCode","require_findModelAnswerDirs","require_runHarnessProcess","require_runSingleHarness","node_path","node_fs_promises","_willbooster_shared_lib","MAX_FAILURE_DETAIL_LENGTH","decisionCodeNames","name","code","checkAllProblems","options","parseCheckArgs","args","rootDir","toRelative","dir","allProblemDirs","findProblemDirs","problemDirs","relativeDir","problemDir","substring","failures","runs","agentChallengeCount","plan","planProblemCheck","failure","cliEntryPath","passedCount","label","run","failureDetail","executeCheckRun","frontMatterResult","planAgentChallengeCheck","defaultHarnessFileNames","modelAnswerDirs","fileExists","failingModelAnswerDirs","parsed","agentChallenge","dirent","issue","tempRoot","copiedProblemDir","error","truncate","harnessFailureDetail","removedTempRoot","summarizeHarnessFailure","removalFailureDetail","result","stderr","resultLines","stdout","line","testCaseResults","parsedResult","rejectedResult","hasRootDir","index","arg","value","visitDirectory","entries","entry","filePath","text"],"sources":["../../src/cli/checkAllProblems.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { forEachConcurrently, getErrorMessage } from '@willbooster/shared-lib';\n\nimport { findDefaultStdioHarnessFiles } from '../helpers/defaultStdioHarness.js';\nimport { findFailingModelAnswerDirs, findModelAnswerDirs } from '../helpers/findModelAnswerDirs.js';\nimport { judgesWithoutTestCases, readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { readTestCases } from '../helpers/readTestCases.js';\nimport { type HarnessProcessResult, runHarnessProcess } from '../helpers/runHarnessProcess.js';\nimport { copyProblemDirToTemporaryRoot, forciblyRemoveDirectory } from '../helpers/temporaryProblemDirCopy.js';\nimport { agentChallengeSchema } from '../learningMaterial/schemas.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport { TEST_CASE_RESULT_PREFIX, testCaseResultSchema } from '../types/testCaseResult.js';\nimport { formatDefaultHarnessError, MISSING_TEST_CASES_ERROR } from './runSingleHarness.js';\n\nconst RUN_TIMEOUT_MS = 600_000;\nconst MAX_RUN_OUTPUT_BYTES = 64 * 1024 * 1024;\nconst MAX_FAILURE_DETAIL_LENGTH = 1000;\n\n// DecisionCode is a plain const object (not a TypeScript enum), so it has no reverse mapping.\nconst decisionCodeNames = new Map<number, string>(Object.entries(DecisionCode).map(([name, code]) => [code, name]));\n\ninterface CheckOptions {\n rootDir: string;\n concurrency: number;\n only: string[];\n skip: string[];\n}\n\ninterface CheckRun {\n problemDir: string;\n answerDir: string;\n /** Whether all test cases must be accepted (`model_answers`) or at least one must fail (`model_answers.fails`). /\n expectation: 'accepted' \| 'rejected';\n}\n\n/\n Judge all model answers of all problems (directories containing `problem.md` or\n * `<id>.problem.md`) under a root directory: `model_answers/` must be fully accepted and\n `model_answers.fails/` must fail at least one test case. Agent challenges have no harness, so\n they are not judged but fail when their `agentChallenge` is invalid or their explanation is missing.\n /\nexport async function checkAllProblems(args: readonly string[]): Promise<number> {\n const options = parseCheckArgs(args);\n const rootDir = path.resolve(options.rootDir);\n // Normalize to forward slashes so --only / --skip substrings like courses/foo match on Windows.\n const toRelative = (dir: string): string => (path.relative(rootDir, dir) \|\| '.').replaceAll(path.sep, '/');\n\n const allProblemDirs = await findProblemDirs(rootDir);\n const problemDirs = allProblemDirs.filter((problemDir) => {\n const relativeDir = toRelative(problemDir);\n if (options.only.length > 0 && !options.only.some((substring) => relativeDir.includes(substring))) return false;\n return !options.skip.some((substring) => relativeDir.includes(substring));\n });\n if (problemDirs.length === 0) {\n console.error(\n allProblemDirs.length === 0\n ? `No problem directories (containing problem.md or <id>.problem.md) found under ${rootDir}.`\n : `All ${allProblemDirs.length} problem directories under ${rootDir} were excluded by --only/--skip.`\n );\n return 1;\n }\n\n const failures: string[] = [];\n const runs: CheckRun[] = [];\n let agentChallengeCount = 0;\n for (const problemDir of problemDirs) {\n const plan = await planProblemCheck(problemDir);\n if (plan.type === 'agentChallenge') {\n agentChallengeCount++;\n } else if (plan.type === 'failure') {\n failures.push(`${toRelative(problemDir)}: ${plan.message}`);\n } else {\n runs.push(...plan.runs);\n }\n }\n for (const failure of failures) console.error(`✗ ${failure}`);\n\n const cliEntryPath = path.resolve(process.argv[1] ?? '');\n let passedCount = 0;\n await forEachConcurrently(runs, options.concurrency, async (run) => {\n const label = `${toRelative(run.problemDir)} ${path.relative(run.problemDir, run.answerDir).replaceAll(path.sep, '/')}`;\n const failureDetail = await executeCheckRun(run, cliEntryPath);\n if (failureDetail === undefined) {\n passedCount++;\n console.info(`✓ ${label}`);\n } else {\n failures.push(`${label}: ${failureDetail}`);\n console.error(`✗ ${label}: ${failureDetail}`);\n }\n });\n\n console.info(\n `\\n${passedCount} passed, ${failures.length} failed (${runs.length} runs, ${problemDirs.length} problems${agentChallengeCount > 0 ? `, agent challenges skipped: ${agentChallengeCount}` : ''})`\n );\n return failures.length === 0 ? 0 : 1;\n}\n\ntype ProblemCheckPlan =\n \| { type: 'agentChallenge' }\n \| { type: 'failure'; message: string }\n \| { type: 'runs'; runs: CheckRun[] };\n\nasync function planProblemCheck(problemDir: string): Promise<ProblemCheckPlan> {\n // As before agent challenges existed, a front matter that fails to parse fails the problem only\n // when the test-case check below needs the front matter.\n const frontMatterResult = await readProblemMarkdownFrontMatter(problemDir).then(\n (frontMatter) => ({ frontMatter }),\n (error: unknown) => ({ error })\n );\n if ('frontMatter' in frontMatterResult && frontMatterResult.frontMatter.agentChallenge !== undefined) {\n return planAgentChallengeCheck(problemDir, frontMatterResult.frontMatter.agentChallenge);\n }\n\n const defaultHarnessFileNames = await findDefaultStdioHarnessFiles(problemDir);\n if (defaultHarnessFileNames.length > 0) {\n return { type: 'failure', message: formatDefaultHarnessError(defaultHarnessFileNames) };\n }\n\n const modelAnswerDirs = await findModelAnswerDirs(problemDir);\n if (modelAnswerDirs.length === 0) {\n return { type: 'failure', message: 'no model answers found under model_answers/' };\n }\n\n // Without test cases, stdioJudgePreset prints a single accepted sentinel result, so a standard\n // problem with an empty or missing test_cases/ would otherwise pass without being judged —\n // unless static-analysis rules or manual scoring make the problem judgeable without them.\n if (!(await fileExists(path.join(problemDir, 'judge.ts')))) {\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n if (testCases.length === 0) {\n if ('error' in frontMatterResult) {\n return {\n type: 'failure',\n message: `failed to read the problem markdown front matter: ${getErrorMessage(frontMatterResult.error)}`,\n };\n }\n if (!judgesWithoutTestCases(frontMatterResult.frontMatter)) {\n return { type: 'failure', message: MISSING_TEST_CASES_ERROR };\n }\n }\n }\n\n const failingModelAnswerDirs = await findFailingModelAnswerDirs(problemDir);\n return {\n type: 'runs',\n runs: [\n ...modelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'accepted' })),\n ...failingModelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'rejected' })),\n ],\n };\n}\n\n/* Skips a valid agent challenge and fails an invalid one instead of judging it as a stdio problem. /\nasync function planAgentChallengeCheck(problemDir: string, agentChallenge: unknown): Promise<ProblemCheckPlan> {\n const parsed = agentChallengeSchema.safeParse(agentChallenge);\n if (parsed.success) {\n const explanationDirents = await fs\n .readdir(path.join(problemDir, 'model_answers', 'explanation'), { recursive: true, withFileTypes: true })\n .catch(() => []);\n return explanationDirents.some((dirent) => dirent.isFile() && dirent.name.endsWith('.md'))\n ? { type: 'agentChallenge' }\n : { type: 'failure', message: 'no explanation found under model_answers/explanation/' };\n }\n const issues = parsed.error.issues.map((issue) => `${['agentChallenge', ...issue.path].join('.')}: ${issue.message}`);\n return { type: 'failure', message: `invalid agentChallenge (see validate-problem): ${issues.join('; ')}` };\n}\n\n/\n Run one answer directory through the problem's harness and return a failure detail, if any.\n * The problem directory is copied to a temporary location first so that judging (e.g. build\n * artifacts in answer directories) never modifies the checked repository.\n /\nasync function executeCheckRun(run: CheckRun, cliEntryPath: string): Promise<string \| undefined> {\n let tempRoot: string;\n let copiedProblemDir: string;\n try {\n ({ tempRoot, copiedProblemDir } = await copyProblemDirToTemporaryRoot(run.problemDir));\n } catch (error) {\n return truncate(`failed to copy the problem directory to a temporary location: ${getErrorMessage(error)}`);\n }\n\n let harnessFailureDetail;\n let removedTempRoot;\n try {\n const result = await runHarnessProcess(\n ['run', cliEntryPath, 'judge', path.relative(run.problemDir, run.answerDir)],\n {\n cwd: copiedProblemDir,\n env: process.env,\n timeoutMs: RUN_TIMEOUT_MS,\n maxOutputBytes: MAX_RUN_OUTPUT_BYTES,\n tempRoot,\n }\n );\n harnessFailureDetail = summarizeHarnessFailure(run, result);\n } catch (error) {\n harnessFailureDetail = truncate(`harness execution failed: ${getErrorMessage(error)}`);\n } finally {\n removedTempRoot = await forciblyRemoveDirectory(tempRoot);\n }\n if (removedTempRoot) return harnessFailureDetail;\n const removalFailureDetail = `failed to remove the temporary copy at ${tempRoot} (judged code may have left permission-locked files)`;\n return harnessFailureDetail === undefined\n ? removalFailureDetail\n : truncate(`${harnessFailureDetail}; ${removalFailureDetail}`);\n}\n\n/* Return why the harness run failed the check, or `undefined` when it passed. /\nfunction summarizeHarnessFailure(run: CheckRun, result: HarnessProcessResult): string \| undefined {\n const { stdout, stderr } = result;\n if (result.failureReason !== undefined) return truncate(result.failureReason);\n if (result.exitCode !== 0) {\n return truncate(`harness exited with ${result.exitCode ?? 'a signal'}${stderr.trim() ? `: ${stderr.trim()}` : ''}`);\n }\n\n const resultLines = stdout.split(/\\r?\\n/).filter((line) => line.startsWith(TEST_CASE_RESULT_PREFIX));\n const testCaseResults = [];\n for (const line of resultLines) {\n let parsedResult;\n try {\n parsedResult = testCaseResultSchema.safeParse(JSON.parse(line.slice(TEST_CASE_RESULT_PREFIX.length)));\n } catch {\n parsedResult = undefined;\n }\n if (!parsedResult?.success) return truncate(`malformed test case result line: ${line}`);\n testCaseResults.push(parsedResult.data);\n }\n if (testCaseResults.length === 0) return 'no test case results were printed';\n\n if (run.expectation === 'accepted') {\n const rejectedResult = testCaseResults.find((result) => result.decisionCode !== DecisionCode.ACCEPTED);\n if (rejectedResult) {\n return truncate(\n `${decisionCodeNames.get(rejectedResult.decisionCode) ?? rejectedResult.decisionCode} on test case ${rejectedResult.testCaseId}${rejectedResult.stderr?.trim() ? `: ${rejectedResult.stderr.trim()}` : ''}`\n );\n }\n return undefined;\n }\n return testCaseResults.every((result) => result.decisionCode === DecisionCode.ACCEPTED)\n ? 'expected at least one failing test case, but all test cases were accepted'\n : undefined;\n}\n\nfunction parseCheckArgs(args: readonly string[]): CheckOptions {\n const options: CheckOptions = {\n rootDir: '.',\n // Serial by default: judging decides TIME_LIMIT_EXCEEDED from wall-clock time, so parallel\n // runs on a small CI runner could fail timing-sensitive problems non-deterministically.\n concurrency: 1,\n only: [],\n skip: [],\n };\n let hasRootDir = false;\n for (let index = 0; index < args.length; index++) {\n const arg = args[index];\n if (arg === undefined) break;\n if (arg === '--concurrency' \|\| arg === '--only' \|\| arg === '--skip') {\n const value = args[++index];\n if (value === undefined) throw new Error(`${arg} requires a value`);\n if (arg === '--concurrency') {\n options.concurrency = Number(value);\n if (!Number.isInteger(options.concurrency) \|\| options.concurrency <= 0) {\n throw new Error(`--concurrency requires a positive integer, but got ${value}`);\n }\n } else if (arg === '--only') {\n options.only.push(value);\n } else {\n options.skip.push(value);\n }\n } else if (arg.startsWith('--')) {\n throw new Error(`Unknown option: ${arg}`);\n } else if (hasRootDir) {\n throw new Error(`Only one root directory can be specified, but got both ${options.rootDir} and ${arg}`);\n } else {\n options.rootDir = arg;\n hasRootDir = true;\n }\n }\n return options;\n}\n\n/* Find directories containing `problem.md` or `<id>.problem.md`, without descending into found problems. */\nasync function findProblemDirs(rootDir: string): Promise<string[]> {\n const problemDirs: string[] = [];\n await visitDirectory(rootDir, problemDirs);\n return problemDirs.toSorted();\n}\n\nasync function visitDirectory(dir: string, problemDirs: string[]): Promise<void> {\n // Traversal errors (e.g. an unreadable subtree) must fail the check: skipping them silently\n // could report a green result while covering only part of the repository.\n const entries = await fs.readdir(dir, { withFileTypes: true });\n if (entries.some((entry) => entry.isFile() && (entry.name === 'problem.md' \|\| entry.name.endsWith('.problem.md')))) {\n problemDirs.push(dir);\n return;\n }\n for (const entry of entries) {\n if (!entry.isDirectory() \|\| entry.name === 'node_modules' \|\| entry.name.startsWith('.')) continue;\n await visitDirectory(path.join(dir, entry.name), problemDirs);\n }\n}\n\nasync function fileExists(filePath: string): Promise<boolean> {\n try {\n await fs.stat(filePath);\n return true;\n } catch {\n return false;\n }\n}\n\nfunction truncate(text: string): string {\n return text.length <= MAX_FAILURE_DETAIL_LENGTH ? text : `${text.slice(0, MAX_FAILURE_DETAIL_LENGTH)}...`;\n}\n"],"mappings":"A,qtBAgBA,MAEMiC,EAA4B,IAG5BC,EAAoB,IAAI,IAAoB,OAAO,QAAQlC,EAAAA,YAAY,CAAC,CAAC,KAAK,CAACmC,EAAMC,KAAU,CAACA,EAAMD,CAAI,CAAC,CAAC,EAsBlH,eAAsBE,EAAiB,EAA0C,CAC/E,IAAMC,EAAUC,EAAeC,CAAI,EAC7BC,EAAUxC,EAAAA,QAAK,QAAQqC,EAAQ,OAAO,EAEtCI,EAAc,IAAyBzC,EAAAA,QAAK,SAASwC,EAASE,CAAG,GAAK,IAAA,CAAK,WAAW1C,EAAAA,QAAK,IAAK,GAAG,EAEnG2C,EAAiB,MAAMC,EAAgBJ,CAAO,EAC9CK,EAAcF,EAAe,OAAQ,GAAe,CACxD,IAAMG,EAAcL,EAAWM,CAAU,EAEzC,OADIV,EAAQ,KAAK,OAAS,GAAK,CAACA,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,EAAU,GACnG,CAACX,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,CAC1E,CAAC,EACD,GAAIH,EAAY,SAAW,EAMzB,OALA,QAAQ,MACNF,EAAe,SAAW,EACtB,iFAAiFH,EAAQ,GACzF,OAAOG,EAAe,OAAO,6BAA6BH,EAAQ,iCACxE,EACO,EAGT,IAAMS,EAAqB,CAAC,EACtBC,EAAmB,CAAC,EACtBC,EAAsB,EAC1B,IAAK,IAAMJ,KAAcF,EAAa,CACpC,IAAMO,EAAO,MAAMC,EAAiBN,CAAU,EAC1CK,EAAK,OAAS,iBAChB,IACSA,EAAK,OAAS,UACvBH,EAAS,KAAK,GAAGR,EAAWM,CAAU,EAAE,IAAIK,EAAK,SAAS,EAE1DF,EAAK,KAAK,GAAGE,EAAK,IAAI,CAE1B,CACA,IAAK,IAAME,KAAWL,EAAU,QAAQ,MAAM,KAAKK,GAAS,EAE5D,IAAMC,EAAevD,EAAAA,QAAK,QAAQ,QAAQ,KAAK,IAAM,EAAE,EACnDwD,EAAc,EAgBlB,OAfA,MAAA,EAAMvD,EAAAA,oBAAAA,CAAoBiD,EAAMb,EAAQ,YAAa,KAAO,IAAQ,CAClE,IAAMoB,EAAQ,GAAGhB,EAAWiB,EAAI,UAAU,EAAE,GAAG1D,EAAAA,QAAK,SAAS0D,EAAI,WAAYA,EAAI,SAAS,CAAC,CAAC,WAAW1D,EAAAA,QAAK,IAAK,GAAG,IAC9G2D,EAAgB,MAAMC,EAAgBF,EAAKH,CAAY,EACzDI,IAAkB,IAAA,IACpB,IACA,QAAQ,KAAK,KAAKF,GAAO,IAEzBR,EAAS,KAAK,GAAGQ,EAAM,IAAIE,GAAe,EAC1C,QAAQ,MAAM,KAAKF,EAAM,IAAIE,GAAe,EAEhD,CAAC,EAED,QAAQ,KACN,KAAKH,EAAY,WAAWP,EAAS,OAAO,WAAWC,EAAK,OAAO,SAASL,EAAY,OAAO,WAAWM,EAAsB,EAAI,+BAA+BA,IAAwB,GAAG,EAChM,EACOF,EAAS,SAAW,EAAI,EAAI,CACrC,CAOA,eAAeI,EAAiB,EAA+C,CAG7E,IAAMQ,EAAoB,MAAM3D,EAAAA,+BAA+B6C,CAAU,CAAC,CAAC,KACxE,IAAiB,CAAE,aAAY,GAC/B,IAAoB,CAAE,OAAM,EAC/B,EACA,GAAI,gBAAiBc,GAAqBA,EAAkB,YAAY,iBAAmB,IAAA,GACzF,OAAOC,EAAwBf,EAAYc,EAAkB,YAAY,cAAc,EAGzF,IAAME,EAA0B,MAAM5D,EAAAA,6BAA6B4C,CAAU,EAC7E,GAAIgB,EAAwB,OAAS,EACnC,MAAO,CAAE,KAAM,UAAW,QAAS3D,EAAAA,0BAA0B2D,CAAuB,CAAE,EAGxF,IAAMC,EAAkB,MAAM3D,EAAAA,oBAAoB0C,CAAU,EAC5D,GAAIiB,EAAgB,SAAW,EAC7B,MAAO,CAAE,KAAM,UAAW,QAAS,6CAA8C,EAMnF,GAAI,CAAE,MAAMC,EAAWjE,EAAAA,QAAK,KAAK+C,EAAY,UAAU,CAAC,IAElD,MADoBzC,EAAAA,cAAcN,EAAAA,QAAK,KAAK+C,EAAY,YAAY,CAAC,EAAA,CAC3D,SAAW,EAAG,CAC1B,GAAI,UAAWc,EACb,MAAO,CACL,KAAM,UACN,QAAS,sDAAA,EAAqDtD,EAAAA,gBAAAA,CAAgBsD,EAAkB,KAAK,GACvG,EAEF,GAAI,CAACrD,EAAAA,uBAAuBqD,EAAkB,WAAW,EACvD,MAAO,CAAE,KAAM,UAAW,QAASpD,EAAAA,wBAAyB,CAEhE,CAGF,IAAMyD,EAAyB,MAAMxD,EAAAA,2BAA2BqC,CAAU,EAC1E,MAAO,CACL,KAAM,OACN,KAAM,CACJ,GAAGiB,EAAgB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,EACpG,GAAGE,EAAuB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,CAC7G,CACF,CACF,CAGA,eAAeJ,EAAwB,EAAoB,EAAoD,CAC7G,IAAMK,EAASxD,EAAAA,qBAAqB,UAAUyD,CAAc,EAU5D,OATID,EAAO,SAIF,MAH0BvD,EAAAA,QAC9B,QAAQZ,EAAAA,QAAK,KAAK+C,EAAY,gBAAiB,aAAa,EAAG,CAAE,UAAW,GAAM,cAAe,EAAK,CAAC,CAAC,CACxG,UAAY,CAAC,CAAC,EAAA,CACS,KAAM,GAAWsB,EAAO,OAAO,GAAKA,EAAO,KAAK,SAAS,KAAK,CAAC,EACrF,CAAE,KAAM,gBAAiB,EACzB,CAAE,KAAM,UAAW,QAAS,uDAAwD,EAGnF,CAAE,KAAM,UAAW,QAAS,kDADpBF,EAAO,MAAM,OAAO,IAAK,GAAU,GAAG,CAAC,iBAAkB,GAAGG,EAAM,IAAI,CAAC,CAAC,KAAK,GAAG,EAAE,IAAIA,EAAM,SACjB,CAAC,CAAC,KAAK,IAAI,GAAI,CAC3G,CAOA,eAAeV,EAAgB,EAAe,EAAmD,CAC/F,IAAIW,EACAC,EACJ,GAAI,EACD,gCAAiC,MAAM3D,EAAAA,8BAA8B6C,EAAI,UAAU,EACtF,OAASe,EAAO,CACd,OAAOC,EAAS,kEAAA,EAAiEnE,EAAAA,gBAAAA,CAAgBkE,CAAK,GAAG,CAC3G,CAEA,IAAIE,EACAC,EACJ,GAAI,CAWF,EAAuBC,EAAwBnB,EAAK,MAV/B5C,EAAAA,kBACnB,CAAC,MAAOyC,EAAc,QAASvD,EAAAA,QAAK,SAAS0D,EAAI,WAAYA,EAAI,SAAS,CAAC,EAC3E,CACE,IAAKc,EACL,IAAK,QAAQ,IACb,UAAW,IACX,eAAgB,SAChB,UACF,CACF,CAC0D,CAC5D,OAASC,EAAO,CACd,EAAuBC,EAAS,8BAAA,EAA6BnE,EAAAA,gBAAAA,CAAgBkE,CAAK,GAAG,CACvF,QAAU,CACR,EAAkB,MAAM1D,EAAAA,wBAAwBwD,CAAQ,CAC1D,CACA,GAAIK,EAAiB,OAAOD,EAC5B,IAAMG,EAAuB,0CAA0CP,EAAS,sDAChF,OAAOI,IAAyB,IAAA,GAC5BG,EACAJ,EAAS,GAAGC,EAAqB,IAAIG,GAAsB,CACjE,CAGA,SAASD,EAAwB,EAAe,EAAkD,CAChG,GAAM,CAAE,SAAQ,UAAWE,EAC3B,GAAIA,EAAO,gBAAkB,IAAA,GAAW,OAAOL,EAASK,EAAO,aAAa,EAC5E,GAAIA,EAAO,WAAa,EACtB,OAAOL,EAAS,uBAAuBK,EAAO,UAAY,aAAaC,EAAO,KAAK,EAAI,KAAKA,EAAO,KAAK,IAAM,IAAI,EAGpH,IAAMC,EAAcC,EAAO,MAAM,OAAO,CAAC,CAAC,OAAQ,GAASC,EAAK,WAAWnE,EAAAA,uBAAuB,CAAC,EAC7FoE,EAAkB,CAAC,EACzB,IAAK,IAAMD,KAAQF,EAAa,CAC9B,IAAII,EACJ,GAAI,CACF,EAAepE,EAAAA,qBAAqB,UAAU,KAAK,MAAMkE,EAAK,MAAMnE,EAAAA,wBAAwB,MAAM,CAAC,CAAC,CACtG,MAAQ,CACN,EAAe,IAAA,EACjB,CACA,GAAI,CAACqE,GAAc,QAAS,OAAOX,EAAS,oCAAoCS,GAAM,EACtFC,EAAgB,KAAKC,EAAa,IAAI,CACxC,CACA,GAAID,EAAgB,SAAW,EAAG,MAAO,oCAEzC,GAAI1B,EAAI,cAAgB,WAAY,CAClC,IAAM4B,EAAiBF,EAAgB,KAAM,GAAWL,EAAO,eAAiBhF,EAAAA,aAAa,QAAQ,EAMrG,OALIuF,EACKZ,EACL,GAAGzC,EAAkB,IAAIqD,EAAe,YAAY,GAAKA,EAAe,aAAa,gBAAgBA,EAAe,aAAaA,EAAe,QAAQ,KAAK,EAAI,KAAKA,EAAe,OAAO,KAAK,IAAM,IACzM,EAEF,MACF,CACA,OAAOF,EAAgB,MAAO,GAAWL,EAAO,eAAiBhF,EAAAA,aAAa,QAAQ,EAClF,4EACA,IAAA,EACN,CAEA,SAASuC,EAAe,EAAuC,CAC7D,IAAMD,EAAwB,CAC5B,QAAS,IAGT,YAAa,EACb,KAAM,CAAC,EACP,KAAM,CAAC,CACT,EACIkD,EAAa,GACjB,IAAK,IAAIC,EAAQ,EAAGA,EAAQjD,EAAK,OAAQ,IAAS,CAChD,IAAMkD,EAAMlD,EAAKiD,GACjB,GAAIC,IAAQ,IAAA,GAAW,MACvB,GAAIA,IAAQ,iBAAmBA,IAAQ,UAAYA,IAAQ,SAAU,CACnE,IAAMC,EAAQnD,EAAK,EAAEiD,GACrB,GAAIE,IAAU,IAAA,GAAW,MAAU,MAAM,GAAGD,EAAI,kBAAkB,EAClE,GAAIA,IAAQ,gBAEV,IADA,EAAQ,YAAc,OAAOC,CAAK,EAC9B,CAAC,OAAO,UAAUrD,EAAQ,WAAW,GAAKA,EAAQ,aAAe,EACnE,MAAU,MAAM,sDAAsDqD,GAAO,CAAA,MAEtED,IAAQ,SACjBpD,EAAQ,KAAK,KAAKqD,CAAK,EAEvBrD,EAAQ,KAAK,KAAKqD,CAAK,CAE3B,MAAO,GAAID,EAAI,WAAW,IAAI,EAC5B,MAAU,MAAM,mBAAmBA,GAAK,OACnC,GAAIF,EACT,MAAU,MAAM,0DAA0DlD,EAAQ,QAAQ,OAAOoD,GAAK,MAEtG,GAAQ,QAAUA,EAClB,EAAa,EAEjB,CACA,OAAOpD,CACT,CAGA,eAAeO,EAAgB,EAAoC,CACjE,IAAMC,EAAwB,CAAC,EAE/B,OADA,MAAM8C,EAAenD,EAASK,CAAW,EAClCA,EAAY,SAAS,CAC9B,CAEA,eAAe8C,EAAe,EAAa,EAAsC,CAG/E,IAAMC,EAAU,MAAMhF,EAAAA,QAAG,QAAQ8B,EAAK,CAAE,cAAe,EAAK,CAAC,EAC7D,GAAIkD,EAAQ,KAAM,GAAUC,EAAM,OAAO,IAAMA,EAAM,OAAS,cAAgBA,EAAM,KAAK,SAAS,aAAa,EAAE,EAC/GhD,EAAY,KAAKH,CAAG,OAGtB,IAAK,IAAMmD,KAASD,EACbC,EAAM,YAAY,GAAKA,EAAM,OAAS,gBAAkB,GAAM,KAAK,WAAW,GAAG,GACtF,MAAMF,EAAe3F,EAAAA,QAAK,KAAK0C,EAAKmD,EAAM,IAAI,EAAGhD,CAAW,CAEhE,CAEA,eAAeoB,EAAW,EAAoC,CAC5D,GAAI,CAEF,OADA,MAAMrD,EAAAA,QAAG,KAAKkF,CAAQ,EACf,EACT,MAAQ,CACN,MAAO,EACT,CACF,CAEA,SAASpB,EAAS,EAAsB,CACtC,OAAOqB,EAAK,QAAU/D,EAA4B+D,EAAO,GAAGA,EAAK,MAAM,EAAG/D,CAAyB,EAAE,IACvG,C"}

package/dist/cli/checkAllProblems.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"checkAllProblems.js","names":["path","fs","MAX_FAILURE_DETAIL_LENGTH","decisionCodeNames","DecisionCode","name","code","checkAllProblems","options","parseCheckArgs","args","rootDir","toRelative","dir","allProblemDirs","findProblemDirs","problemDirs","relativeDir","problemDir","substring","failures","runs","agentChallengeCount","plan","planProblemCheck","failure","cliEntryPath","passedCount","forEachConcurrently","label","run","failureDetail","executeCheckRun","frontMatterResult","readProblemMarkdownFrontMatter","planAgentChallengeCheck","defaultHarnessFileNames","findDefaultStdioHarnessFiles","formatDefaultHarnessError","modelAnswerDirs","findModelAnswerDirs","fileExists","readTestCases","getErrorMessage","judgesWithoutTestCases","MISSING_TEST_CASES_ERROR","failingModelAnswerDirs","findFailingModelAnswerDirs","parsed","agentChallengeSchema","agentChallenge","dirent","issue","tempRoot","copiedProblemDir","copyProblemDirToTemporaryRoot","error","truncate","harnessFailureDetail","removedTempRoot","summarizeHarnessFailure","runHarnessProcess","forciblyRemoveDirectory","removalFailureDetail","result","stderr","resultLines","stdout","line","TEST_CASE_RESULT_PREFIX","testCaseResults","parsedResult","testCaseResultSchema","rejectedResult","hasRootDir","index","arg","value","visitDirectory","entries","entry","filePath","text"],"sources":["../../src/cli/checkAllProblems.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { forEachConcurrently, getErrorMessage } from '@willbooster/shared-lib';\n\nimport { findDefaultStdioHarnessFiles } from '../helpers/defaultStdioHarness.js';\nimport { findFailingModelAnswerDirs, findModelAnswerDirs } from '../helpers/findModelAnswerDirs.js';\nimport { judgesWithoutTestCases, readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { readTestCases } from '../helpers/readTestCases.js';\nimport { type HarnessProcessResult, runHarnessProcess } from '../helpers/runHarnessProcess.js';\nimport { copyProblemDirToTemporaryRoot, forciblyRemoveDirectory } from '../helpers/temporaryProblemDirCopy.js';\nimport { agentChallengeSchema } from '../learningMaterial/schemas.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport { TEST_CASE_RESULT_PREFIX, testCaseResultSchema } from '../types/testCaseResult.js';\nimport { formatDefaultHarnessError, MISSING_TEST_CASES_ERROR } from './runSingleHarness.js';\n\nconst RUN_TIMEOUT_MS = 600_000;\nconst MAX_RUN_OUTPUT_BYTES = 64 * 1024 * 1024;\nconst MAX_FAILURE_DETAIL_LENGTH = 1000;\n\n// DecisionCode is a plain const object (not a TypeScript enum), so it has no reverse mapping.\nconst decisionCodeNames = new Map<number, string>(Object.entries(DecisionCode).map(([name, code]) => [code, name]));\n\ninterface CheckOptions {\n rootDir: string;\n concurrency: number;\n only: string[];\n skip: string[];\n}\n\ninterface CheckRun {\n problemDir: string;\n answerDir: string;\n /** Whether all test cases must be accepted (`model_answers`) or at least one must fail (`model_answers.fails`). /\n expectation: 'accepted' \| 'rejected';\n}\n\n/\n Judge all model answers of all problems (directories containing `problem.md` or\n * `<id>.problem.md`) under a root directory: `model_answers/` must be fully accepted and\n `model_answers.fails/` must fail at least one test case. Agent challenges have no harness, so\n they are not judged but fail when their `agentChallenge` is invalid or their explanation is missing.\n /\nexport async function checkAllProblems(args: readonly string[]): Promise<number> {\n const options = parseCheckArgs(args);\n const rootDir = path.resolve(options.rootDir);\n // Normalize to forward slashes so --only / --skip substrings like courses/foo match on Windows.\n const toRelative = (dir: string): string => (path.relative(rootDir, dir) \|\| '.').replaceAll(path.sep, '/');\n\n const allProblemDirs = await findProblemDirs(rootDir);\n const problemDirs = allProblemDirs.filter((problemDir) => {\n const relativeDir = toRelative(problemDir);\n if (options.only.length > 0 && !options.only.some((substring) => relativeDir.includes(substring))) return false;\n return !options.skip.some((substring) => relativeDir.includes(substring));\n });\n if (problemDirs.length === 0) {\n console.error(\n allProblemDirs.length === 0\n ? `No problem directories (containing problem.md or <id>.problem.md) found under ${rootDir}.`\n : `All ${allProblemDirs.length} problem directories under ${rootDir} were excluded by --only/--skip.`\n );\n return 1;\n }\n\n const failures: string[] = [];\n const runs: CheckRun[] = [];\n let agentChallengeCount = 0;\n for (const problemDir of problemDirs) {\n const plan = await planProblemCheck(problemDir);\n if (plan.type === 'agentChallenge') {\n agentChallengeCount++;\n } else if (plan.type === 'failure') {\n failures.push(`${toRelative(problemDir)}: ${plan.message}`);\n } else {\n runs.push(...plan.runs);\n }\n }\n for (const failure of failures) console.error(`✗ ${failure}`);\n\n const cliEntryPath = path.resolve(process.argv[1] ?? '');\n let passedCount = 0;\n await forEachConcurrently(runs, options.concurrency, async (run) => {\n const label = `${toRelative(run.problemDir)} ${path.relative(run.problemDir, run.answerDir).replaceAll(path.sep, '/')}`;\n const failureDetail = await executeCheckRun(run, cliEntryPath);\n if (failureDetail === undefined) {\n passedCount++;\n console.info(`✓ ${label}`);\n } else {\n failures.push(`${label}: ${failureDetail}`);\n console.error(`✗ ${label}: ${failureDetail}`);\n }\n });\n\n console.info(\n `\\n${passedCount} passed, ${failures.length} failed (${runs.length} runs, ${problemDirs.length} problems${agentChallengeCount > 0 ? `, agent challenges skipped: ${agentChallengeCount}` : ''})`\n );\n return failures.length === 0 ? 0 : 1;\n}\n\ntype ProblemCheckPlan =\n { type: 'agentChallenge' } \| { type: 'failure'; message: string } \| { type: 'runs'; runs: CheckRun[] };\n\nasync function planProblemCheck(problemDir: string): Promise<ProblemCheckPlan> {\n // As before agent challenges existed, a front matter that fails to parse fails the problem only\n // when the test-case check below needs the front matter.\n const frontMatterResult = await readProblemMarkdownFrontMatter(problemDir).then(\n (frontMatter) => ({ frontMatter }),\n (error: unknown) => ({ error })\n );\n if ('frontMatter' in frontMatterResult && frontMatterResult.frontMatter.agentChallenge !== undefined) {\n return planAgentChallengeCheck(problemDir, frontMatterResult.frontMatter.agentChallenge);\n }\n\n const defaultHarnessFileNames = await findDefaultStdioHarnessFiles(problemDir);\n if (defaultHarnessFileNames.length > 0) {\n return { type: 'failure', message: formatDefaultHarnessError(defaultHarnessFileNames) };\n }\n\n const modelAnswerDirs = await findModelAnswerDirs(problemDir);\n if (modelAnswerDirs.length === 0) {\n return { type: 'failure', message: 'no model answers found under model_answers/' };\n }\n\n // Without test cases, stdioJudgePreset prints a single accepted sentinel result, so a standard\n // problem with an empty or missing test_cases/ would otherwise pass without being judged —\n // unless static-analysis rules or manual scoring make the problem judgeable without them.\n if (!(await fileExists(path.join(problemDir, 'judge.ts')))) {\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n if (testCases.length === 0) {\n if ('error' in frontMatterResult) {\n return {\n type: 'failure',\n message: `failed to read the problem markdown front matter: ${getErrorMessage(frontMatterResult.error)}`,\n };\n }\n if (!judgesWithoutTestCases(frontMatterResult.frontMatter)) {\n return { type: 'failure', message: MISSING_TEST_CASES_ERROR };\n }\n }\n }\n\n const failingModelAnswerDirs = await findFailingModelAnswerDirs(problemDir);\n return {\n type: 'runs',\n runs: [\n ...modelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'accepted' })),\n ...failingModelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'rejected' })),\n ],\n };\n}\n\n/* Skips a valid agent challenge and fails an invalid one instead of judging it as a stdio problem. /\nasync function planAgentChallengeCheck(problemDir: string, agentChallenge: unknown): Promise<ProblemCheckPlan> {\n const parsed = agentChallengeSchema.safeParse(agentChallenge);\n if (parsed.success) {\n const explanationDirents = await fs\n .readdir(path.join(problemDir, 'model_answers', 'explanation'), { recursive: true, withFileTypes: true })\n .catch(() => []);\n return explanationDirents.some((dirent) => dirent.isFile() && dirent.name.endsWith('.md'))\n ? { type: 'agentChallenge' }\n : { type: 'failure', message: 'no explanation found under model_answers/explanation/' };\n }\n const issues = parsed.error.issues.map((issue) => `${['agentChallenge', ...issue.path].join('.')}: ${issue.message}`);\n return { type: 'failure', message: `invalid agentChallenge (see validate-problem): ${issues.join('; ')}` };\n}\n\n/\n Run one answer directory through the problem's harness and return a failure detail, if any.\n * The problem directory is copied to a temporary location first so that judging (e.g. build\n * artifacts in answer directories) never modifies the checked repository.\n /\nasync function executeCheckRun(run: CheckRun, cliEntryPath: string): Promise<string \| undefined> {\n let tempRoot: string;\n let copiedProblemDir: string;\n try {\n ({ tempRoot, copiedProblemDir } = await copyProblemDirToTemporaryRoot(run.problemDir));\n } catch (error) {\n return truncate(`failed to copy the problem directory to a temporary location: ${getErrorMessage(error)}`);\n }\n\n let harnessFailureDetail;\n let removedTempRoot;\n try {\n const result = await runHarnessProcess(\n ['run', cliEntryPath, 'judge', path.relative(run.problemDir, run.answerDir)],\n {\n cwd: copiedProblemDir,\n env: process.env,\n timeoutMs: RUN_TIMEOUT_MS,\n maxOutputBytes: MAX_RUN_OUTPUT_BYTES,\n tempRoot,\n }\n );\n harnessFailureDetail = summarizeHarnessFailure(run, result);\n } catch (error) {\n harnessFailureDetail = truncate(`harness execution failed: ${getErrorMessage(error)}`);\n } finally {\n removedTempRoot = await forciblyRemoveDirectory(tempRoot);\n }\n if (removedTempRoot) return harnessFailureDetail;\n const removalFailureDetail = `failed to remove the temporary copy at ${tempRoot} (judged code may have left permission-locked files)`;\n return harnessFailureDetail === undefined\n ? removalFailureDetail\n : truncate(`${harnessFailureDetail}; ${removalFailureDetail}`);\n}\n\n/* Return why the harness run failed the check, or `undefined` when it passed. /\nfunction summarizeHarnessFailure(run: CheckRun, result: HarnessProcessResult): string \| undefined {\n const { stdout, stderr } = result;\n if (result.failureReason !== undefined) return truncate(result.failureReason);\n if (result.exitCode !== 0) {\n return truncate(`harness exited with ${result.exitCode ?? 'a signal'}${stderr.trim() ? `: ${stderr.trim()}` : ''}`);\n }\n\n const resultLines = stdout.split(/\\r?\\n/).filter((line) => line.startsWith(TEST_CASE_RESULT_PREFIX));\n const testCaseResults = [];\n for (const line of resultLines) {\n let parsedResult;\n try {\n parsedResult = testCaseResultSchema.safeParse(JSON.parse(line.slice(TEST_CASE_RESULT_PREFIX.length)));\n } catch {\n parsedResult = undefined;\n }\n if (!parsedResult?.success) return truncate(`malformed test case result line: ${line}`);\n testCaseResults.push(parsedResult.data);\n }\n if (testCaseResults.length === 0) return 'no test case results were printed';\n\n if (run.expectation === 'accepted') {\n const rejectedResult = testCaseResults.find((result) => result.decisionCode !== DecisionCode.ACCEPTED);\n if (rejectedResult) {\n return truncate(\n `${decisionCodeNames.get(rejectedResult.decisionCode) ?? rejectedResult.decisionCode} on test case ${rejectedResult.testCaseId}${rejectedResult.stderr?.trim() ? `: ${rejectedResult.stderr.trim()}` : ''}`\n );\n }\n return undefined;\n }\n return testCaseResults.every((result) => result.decisionCode === DecisionCode.ACCEPTED)\n ? 'expected at least one failing test case, but all test cases were accepted'\n : undefined;\n}\n\nfunction parseCheckArgs(args: readonly string[]): CheckOptions {\n const options: CheckOptions = {\n rootDir: '.',\n // Serial by default: judging decides TIME_LIMIT_EXCEEDED from wall-clock time, so parallel\n // runs on a small CI runner could fail timing-sensitive problems non-deterministically.\n concurrency: 1,\n only: [],\n skip: [],\n };\n let hasRootDir = false;\n for (let index = 0; index < args.length; index++) {\n const arg = args[index];\n if (arg === undefined) break;\n if (arg === '--concurrency' \|\| arg === '--only' \|\| arg === '--skip') {\n const value = args[++index];\n if (value === undefined) throw new Error(`${arg} requires a value`);\n if (arg === '--concurrency') {\n options.concurrency = Number(value);\n if (!Number.isInteger(options.concurrency) \|\| options.concurrency <= 0) {\n throw new Error(`--concurrency requires a positive integer, but got ${value}`);\n }\n } else if (arg === '--only') {\n options.only.push(value);\n } else {\n options.skip.push(value);\n }\n } else if (arg.startsWith('--')) {\n throw new Error(`Unknown option: ${arg}`);\n } else if (hasRootDir) {\n throw new Error(`Only one root directory can be specified, but got both ${options.rootDir} and ${arg}`);\n } else {\n options.rootDir = arg;\n hasRootDir = true;\n }\n }\n return options;\n}\n\n/* Find directories containing `problem.md` or `<id>.problem.md`, without descending into found problems. */\nasync function findProblemDirs(rootDir: string): Promise<string[]> {\n const problemDirs: string[] = [];\n await visitDirectory(rootDir, problemDirs);\n return problemDirs.toSorted();\n}\n\nasync function visitDirectory(dir: string, problemDirs: string[]): Promise<void> {\n // Traversal errors (e.g. an unreadable subtree) must fail the check: skipping them silently\n // could report a green result while covering only part of the repository.\n const entries = await fs.readdir(dir, { withFileTypes: true });\n if (entries.some((entry) => entry.isFile() && (entry.name === 'problem.md' \|\| entry.name.endsWith('.problem.md')))) {\n problemDirs.push(dir);\n return;\n }\n for (const entry of entries) {\n if (!entry.isDirectory() \|\| entry.name === 'node_modules' \|\| entry.name.startsWith('.')) continue;\n await visitDirectory(path.join(dir, entry.name), problemDirs);\n }\n}\n\nasync function fileExists(filePath: string): Promise<boolean> {\n try {\n await fs.stat(filePath);\n return true;\n } catch {\n return false;\n }\n}\n\nfunction truncate(text: string): string {\n return text.length <= MAX_FAILURE_DETAIL_LENGTH ? text : `${text.slice(0, MAX_FAILURE_DETAIL_LENGTH)}...`;\n}\n"],"mappings":"A,++BAgBA,MAEME,EAA4B,IAG5BC,EAAoB,IAAI,IAAoB,OAAO,QAAQC,CAAY,CAAC,CAAC,KAAK,CAACC,EAAMC,KAAU,CAACA,EAAMD,CAAI,CAAC,CAAC,EAsBlH,eAAsBE,EAAiB,EAA0C,CAC/E,IAAMC,EAAUC,EAAeC,CAAI,EAC7BC,EAAUX,EAAK,QAAQQ,EAAQ,OAAO,EAEtCI,EAAc,IAAyBZ,EAAK,SAASW,EAASE,CAAG,GAAK,IAAA,CAAK,WAAWb,EAAK,IAAK,GAAG,EAEnGc,EAAiB,MAAMC,EAAgBJ,CAAO,EAC9CK,EAAcF,EAAe,OAAQ,GAAe,CACxD,IAAMG,EAAcL,EAAWM,CAAU,EAEzC,OADIV,EAAQ,KAAK,OAAS,GAAK,CAACA,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,EAAU,GACnG,CAACX,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,CAC1E,CAAC,EACD,GAAIH,EAAY,SAAW,EAMzB,OALA,QAAQ,MACNF,EAAe,SAAW,EACtB,iFAAiFH,EAAQ,GACzF,OAAOG,EAAe,OAAO,6BAA6BH,EAAQ,iCACxE,EACO,EAGT,IAAMS,EAAqB,CAAC,EACtBC,EAAmB,CAAC,EACtBC,EAAsB,EAC1B,IAAK,IAAMJ,KAAcF,EAAa,CACpC,IAAMO,EAAO,MAAMC,EAAiBN,CAAU,EAC1CK,EAAK,OAAS,iBAChB,IACSA,EAAK,OAAS,UACvBH,EAAS,KAAK,GAAGR,EAAWM,CAAU,EAAE,IAAIK,EAAK,SAAS,EAE1DF,EAAK,KAAK,GAAGE,EAAK,IAAI,CAE1B,CACA,IAAK,IAAME,KAAWL,EAAU,QAAQ,MAAM,KAAKK,GAAS,EAE5D,IAAMC,EAAe1B,EAAK,QAAQ,QAAQ,KAAK,IAAM,EAAE,EACnD2B,EAAc,EAgBlB,OAfA,MAAMC,EAAoBP,EAAMb,EAAQ,YAAa,KAAO,IAAQ,CAClE,IAAMqB,EAAQ,GAAGjB,EAAWkB,EAAI,UAAU,EAAE,GAAG9B,EAAK,SAAS8B,EAAI,WAAYA,EAAI,SAAS,CAAC,CAAC,WAAW9B,EAAK,IAAK,GAAG,IAC9G+B,EAAgB,MAAMC,EAAgBF,EAAKJ,CAAY,EACzDK,IAAkB,IAAA,IACpB,IACA,QAAQ,KAAK,KAAKF,GAAO,IAEzBT,EAAS,KAAK,GAAGS,EAAM,IAAIE,GAAe,EAC1C,QAAQ,MAAM,KAAKF,EAAM,IAAIE,GAAe,EAEhD,CAAC,EAED,QAAQ,KACN,KAAKJ,EAAY,WAAWP,EAAS,OAAO,WAAWC,EAAK,OAAO,SAASL,EAAY,OAAO,WAAWM,EAAsB,EAAI,+BAA+BA,IAAwB,GAAG,EAChM,EACOF,EAAS,SAAW,EAAI,EAAI,CACrC,CAKA,eAAeI,EAAiB,EAA+C,CAG7E,IAAMS,EAAoB,MAAMC,EAA+BhB,CAAU,CAAC,CAAC,KACxE,IAAiB,CAAE,aAAY,GAC/B,IAAoB,CAAE,OAAM,EAC/B,EACA,GAAI,gBAAiBe,GAAqBA,EAAkB,YAAY,iBAAmB,IAAA,GACzF,OAAOE,EAAwBjB,EAAYe,EAAkB,YAAY,cAAc,EAGzF,IAAMG,EAA0B,MAAMC,EAA6BnB,CAAU,EAC7E,GAAIkB,EAAwB,OAAS,EACnC,MAAO,CAAE,KAAM,UAAW,QAASE,EAA0BF,CAAuB,CAAE,EAGxF,IAAMG,EAAkB,MAAMC,EAAoBtB,CAAU,EAC5D,GAAIqB,EAAgB,SAAW,EAC7B,MAAO,CAAE,KAAM,UAAW,QAAS,6CAA8C,EAMnF,GAAI,CAAE,MAAME,EAAWzC,EAAK,KAAKkB,EAAY,UAAU,CAAC,IAElD,MADoBwB,EAAc1C,EAAK,KAAKkB,EAAY,YAAY,CAAC,EAAA,CAC3D,SAAW,EAAG,CAC1B,GAAI,UAAWe,EACb,MAAO,CACL,KAAM,UACN,QAAS,qDAAqDU,EAAgBV,EAAkB,KAAK,GACvG,EAEF,GAAI,CAACW,EAAuBX,EAAkB,WAAW,EACvD,MAAO,CAAE,KAAM,UAAW,QAASY,CAAyB,CAEhE,CAGF,IAAMC,EAAyB,MAAMC,EAA2B7B,CAAU,EAC1E,MAAO,CACL,KAAM,OACN,KAAM,CACJ,GAAGqB,EAAgB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,EACpG,GAAGO,EAAuB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,CAC7G,CACF,CACF,CAGA,eAAeX,EAAwB,EAAoB,EAAoD,CAC7G,IAAMa,EAASC,EAAqB,UAAUC,CAAc,EAU5D,OATIF,EAAO,SAIF,MAH0B/C,EAC9B,QAAQD,EAAK,KAAKkB,EAAY,gBAAiB,aAAa,EAAG,CAAE,UAAW,GAAM,cAAe,EAAK,CAAC,CAAC,CACxG,UAAY,CAAC,CAAC,EAAA,CACS,KAAM,GAAWiC,EAAO,OAAO,GAAKA,EAAO,KAAK,SAAS,KAAK,CAAC,EACrF,CAAE,KAAM,gBAAiB,EACzB,CAAE,KAAM,UAAW,QAAS,uDAAwD,EAGnF,CAAE,KAAM,UAAW,QAAS,kDADpBH,EAAO,MAAM,OAAO,IAAK,GAAU,GAAG,CAAC,iBAAkB,GAAGI,EAAM,IAAI,CAAC,CAAC,KAAK,GAAG,EAAE,IAAIA,EAAM,SACjB,CAAC,CAAC,KAAK,IAAI,GAAI,CAC3G,CAOA,eAAepB,EAAgB,EAAe,EAAmD,CAC/F,IAAIqB,EACAC,EACJ,GAAI,EACD,gCAAiC,MAAMC,EAA8BzB,EAAI,UAAU,EACtF,OAAS0B,EAAO,CACd,OAAOC,EAAS,iEAAiEd,EAAgBa,CAAK,GAAG,CAC3G,CAEA,IAAIE,EACAC,EACJ,GAAI,CAWF,EAAuBC,EAAwB9B,EAAK,MAV/B+B,EACnB,CAAC,MAAOnC,EAAc,QAAS1B,EAAK,SAAS8B,EAAI,WAAYA,EAAI,SAAS,CAAC,EAC3E,CACE,IAAKwB,EACL,IAAK,QAAQ,IACb,UAAW,IACX,eAAgB,SAChB,UACF,CACF,CAC0D,CAC5D,OAASE,EAAO,CACd,EAAuBC,EAAS,6BAA6Bd,EAAgBa,CAAK,GAAG,CACvF,QAAU,CACR,EAAkB,MAAMM,EAAwBT,CAAQ,CAC1D,CACA,GAAIM,EAAiB,OAAOD,EAC5B,IAAMK,EAAuB,0CAA0CV,EAAS,sDAChF,OAAOK,IAAyB,IAAA,GAC5BK,EACAN,EAAS,GAAGC,EAAqB,IAAIK,GAAsB,CACjE,CAGA,SAASH,EAAwB,EAAe,EAAkD,CAChG,GAAM,CAAE,SAAQ,UAAWI,EAC3B,GAAIA,EAAO,gBAAkB,IAAA,GAAW,OAAOP,EAASO,EAAO,aAAa,EAC5E,GAAIA,EAAO,WAAa,EACtB,OAAOP,EAAS,uBAAuBO,EAAO,UAAY,aAAaC,EAAO,KAAK,EAAI,KAAKA,EAAO,KAAK,IAAM,IAAI,EAGpH,IAAMC,EAAcC,EAAO,MAAM,OAAO,CAAC,CAAC,OAAQ,GAASC,EAAK,WAAWC,CAAuB,CAAC,EAC7FC,EAAkB,CAAC,EACzB,IAAK,IAAMF,KAAQF,EAAa,CAC9B,IAAIK,EACJ,GAAI,CACF,EAAeC,EAAqB,UAAU,KAAK,MAAMJ,EAAK,MAAMC,EAAwB,MAAM,CAAC,CAAC,CACtG,MAAQ,CACN,EAAe,IAAA,EACjB,CACA,GAAI,CAACE,GAAc,QAAS,OAAOd,EAAS,oCAAoCW,GAAM,EACtFE,EAAgB,KAAKC,EAAa,IAAI,CACxC,CACA,GAAID,EAAgB,SAAW,EAAG,MAAO,oCAEzC,GAAIxC,EAAI,cAAgB,WAAY,CAClC,IAAM2C,EAAiBH,EAAgB,KAAM,GAAWN,EAAO,eAAiB5D,EAAa,QAAQ,EAMrG,OALIqE,EACKhB,EACL,GAAGtD,EAAkB,IAAIsE,EAAe,YAAY,GAAKA,EAAe,aAAa,gBAAgBA,EAAe,aAAaA,EAAe,QAAQ,KAAK,EAAI,KAAKA,EAAe,OAAO,KAAK,IAAM,IACzM,EAEF,MACF,CACA,OAAOH,EAAgB,MAAO,GAAWN,EAAO,eAAiB5D,EAAa,QAAQ,EAClF,4EACA,IAAA,EACN,CAEA,SAASK,EAAe,EAAuC,CAC7D,IAAMD,EAAwB,CAC5B,QAAS,IAGT,YAAa,EACb,KAAM,CAAC,EACP,KAAM,CAAC,CACT,EACIkE,EAAa,GACjB,IAAK,IAAIC,EAAQ,EAAGA,EAAQjE,EAAK,OAAQ,IAAS,CAChD,IAAMkE,EAAMlE,EAAKiE,GACjB,GAAIC,IAAQ,IAAA,GAAW,MACvB,GAAIA,IAAQ,iBAAmBA,IAAQ,UAAYA,IAAQ,SAAU,CACnE,IAAMC,EAAQnE,EAAK,EAAEiE,GACrB,GAAIE,IAAU,IAAA,GAAW,MAAU,MAAM,GAAGD,EAAI,kBAAkB,EAClE,GAAIA,IAAQ,gBAEV,IADA,EAAQ,YAAc,OAAOC,CAAK,EAC9B,CAAC,OAAO,UAAUrE,EAAQ,WAAW,GAAKA,EAAQ,aAAe,EACnE,MAAU,MAAM,sDAAsDqE,GAAO,CAAA,MAEtED,IAAQ,SACjBpE,EAAQ,KAAK,KAAKqE,CAAK,EAEvBrE,EAAQ,KAAK,KAAKqE,CAAK,CAE3B,MAAO,GAAID,EAAI,WAAW,IAAI,EAC5B,MAAU,MAAM,mBAAmBA,GAAK,OACnC,GAAIF,EACT,MAAU,MAAM,0DAA0DlE,EAAQ,QAAQ,OAAOoE,GAAK,MAEtG,GAAQ,QAAUA,EAClB,EAAa,EAEjB,CACA,OAAOpE,CACT,CAGA,eAAeO,EAAgB,EAAoC,CACjE,IAAMC,EAAwB,CAAC,EAE/B,OADA,MAAM8D,EAAenE,EAASK,CAAW,EAClCA,EAAY,SAAS,CAC9B,CAEA,eAAe8D,EAAe,EAAa,EAAsC,CAG/E,IAAMC,EAAU,MAAM9E,EAAG,QAAQY,EAAK,CAAE,cAAe,EAAK,CAAC,EAC7D,GAAIkE,EAAQ,KAAM,GAAUC,EAAM,OAAO,IAAMA,EAAM,OAAS,cAAgBA,EAAM,KAAK,SAAS,aAAa,EAAE,EAC/GhE,EAAY,KAAKH,CAAG,OAGtB,IAAK,IAAMmE,KAASD,EACbC,EAAM,YAAY,GAAKA,EAAM,OAAS,gBAAkB,GAAM,KAAK,WAAW,GAAG,GACtF,MAAMF,EAAe9E,EAAK,KAAKa,EAAKmE,EAAM,IAAI,EAAGhE,CAAW,CAEhE,CAEA,eAAeyB,EAAW,EAAoC,CAC5D,GAAI,CAEF,OADA,MAAMxC,EAAG,KAAKgF,CAAQ,EACf,EACT,MAAQ,CACN,MAAO,EACT,CACF,CAEA,SAASxB,EAAS,EAAsB,CACtC,OAAOyB,EAAK,QAAUhF,EAA4BgF,EAAO,GAAGA,EAAK,MAAM,EAAGhF,CAAyB,EAAE,IACvG,C"}
1	+ {"version":3,"file":"checkAllProblems.js","names":["path","fs","MAX_FAILURE_DETAIL_LENGTH","decisionCodeNames","DecisionCode","name","code","checkAllProblems","options","parseCheckArgs","args","rootDir","toRelative","dir","allProblemDirs","findProblemDirs","problemDirs","relativeDir","problemDir","substring","failures","runs","agentChallengeCount","plan","planProblemCheck","failure","cliEntryPath","passedCount","forEachConcurrently","label","run","failureDetail","executeCheckRun","frontMatterResult","readProblemMarkdownFrontMatter","planAgentChallengeCheck","defaultHarnessFileNames","findDefaultStdioHarnessFiles","formatDefaultHarnessError","modelAnswerDirs","findModelAnswerDirs","fileExists","readTestCases","getErrorMessage","judgesWithoutTestCases","MISSING_TEST_CASES_ERROR","failingModelAnswerDirs","findFailingModelAnswerDirs","parsed","agentChallengeSchema","agentChallenge","dirent","issue","tempRoot","copiedProblemDir","copyProblemDirToTemporaryRoot","error","truncate","harnessFailureDetail","removedTempRoot","summarizeHarnessFailure","runHarnessProcess","forciblyRemoveDirectory","removalFailureDetail","result","stderr","resultLines","stdout","line","TEST_CASE_RESULT_PREFIX","testCaseResults","parsedResult","testCaseResultSchema","rejectedResult","hasRootDir","index","arg","value","visitDirectory","entries","entry","filePath","text"],"sources":["../../src/cli/checkAllProblems.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { forEachConcurrently, getErrorMessage } from '@willbooster/shared-lib';\n\nimport { findDefaultStdioHarnessFiles } from '../helpers/defaultStdioHarness.js';\nimport { findFailingModelAnswerDirs, findModelAnswerDirs } from '../helpers/findModelAnswerDirs.js';\nimport { judgesWithoutTestCases, readProblemMarkdownFrontMatter } from '../helpers/readProblemMarkdownFrontMatter.js';\nimport { readTestCases } from '../helpers/readTestCases.js';\nimport { type HarnessProcessResult, runHarnessProcess } from '../helpers/runHarnessProcess.js';\nimport { copyProblemDirToTemporaryRoot, forciblyRemoveDirectory } from '../helpers/temporaryProblemDirCopy.js';\nimport { agentChallengeSchema } from '../learningMaterial/schemas.js';\nimport { DecisionCode } from '../types/decisionCode.js';\nimport { TEST_CASE_RESULT_PREFIX, testCaseResultSchema } from '../types/testCaseResult.js';\nimport { formatDefaultHarnessError, MISSING_TEST_CASES_ERROR } from './runSingleHarness.js';\n\nconst RUN_TIMEOUT_MS = 600_000;\nconst MAX_RUN_OUTPUT_BYTES = 64 * 1024 * 1024;\nconst MAX_FAILURE_DETAIL_LENGTH = 1000;\n\n// DecisionCode is a plain const object (not a TypeScript enum), so it has no reverse mapping.\nconst decisionCodeNames = new Map<number, string>(Object.entries(DecisionCode).map(([name, code]) => [code, name]));\n\ninterface CheckOptions {\n rootDir: string;\n concurrency: number;\n only: string[];\n skip: string[];\n}\n\ninterface CheckRun {\n problemDir: string;\n answerDir: string;\n /** Whether all test cases must be accepted (`model_answers`) or at least one must fail (`model_answers.fails`). /\n expectation: 'accepted' \| 'rejected';\n}\n\n/\n Judge all model answers of all problems (directories containing `problem.md` or\n * `<id>.problem.md`) under a root directory: `model_answers/` must be fully accepted and\n `model_answers.fails/` must fail at least one test case. Agent challenges have no harness, so\n they are not judged but fail when their `agentChallenge` is invalid or their explanation is missing.\n /\nexport async function checkAllProblems(args: readonly string[]): Promise<number> {\n const options = parseCheckArgs(args);\n const rootDir = path.resolve(options.rootDir);\n // Normalize to forward slashes so --only / --skip substrings like courses/foo match on Windows.\n const toRelative = (dir: string): string => (path.relative(rootDir, dir) \|\| '.').replaceAll(path.sep, '/');\n\n const allProblemDirs = await findProblemDirs(rootDir);\n const problemDirs = allProblemDirs.filter((problemDir) => {\n const relativeDir = toRelative(problemDir);\n if (options.only.length > 0 && !options.only.some((substring) => relativeDir.includes(substring))) return false;\n return !options.skip.some((substring) => relativeDir.includes(substring));\n });\n if (problemDirs.length === 0) {\n console.error(\n allProblemDirs.length === 0\n ? `No problem directories (containing problem.md or <id>.problem.md) found under ${rootDir}.`\n : `All ${allProblemDirs.length} problem directories under ${rootDir} were excluded by --only/--skip.`\n );\n return 1;\n }\n\n const failures: string[] = [];\n const runs: CheckRun[] = [];\n let agentChallengeCount = 0;\n for (const problemDir of problemDirs) {\n const plan = await planProblemCheck(problemDir);\n if (plan.type === 'agentChallenge') {\n agentChallengeCount++;\n } else if (plan.type === 'failure') {\n failures.push(`${toRelative(problemDir)}: ${plan.message}`);\n } else {\n runs.push(...plan.runs);\n }\n }\n for (const failure of failures) console.error(`✗ ${failure}`);\n\n const cliEntryPath = path.resolve(process.argv[1] ?? '');\n let passedCount = 0;\n await forEachConcurrently(runs, options.concurrency, async (run) => {\n const label = `${toRelative(run.problemDir)} ${path.relative(run.problemDir, run.answerDir).replaceAll(path.sep, '/')}`;\n const failureDetail = await executeCheckRun(run, cliEntryPath);\n if (failureDetail === undefined) {\n passedCount++;\n console.info(`✓ ${label}`);\n } else {\n failures.push(`${label}: ${failureDetail}`);\n console.error(`✗ ${label}: ${failureDetail}`);\n }\n });\n\n console.info(\n `\\n${passedCount} passed, ${failures.length} failed (${runs.length} runs, ${problemDirs.length} problems${agentChallengeCount > 0 ? `, agent challenges skipped: ${agentChallengeCount}` : ''})`\n );\n return failures.length === 0 ? 0 : 1;\n}\n\ntype ProblemCheckPlan =\n \| { type: 'agentChallenge' }\n \| { type: 'failure'; message: string }\n \| { type: 'runs'; runs: CheckRun[] };\n\nasync function planProblemCheck(problemDir: string): Promise<ProblemCheckPlan> {\n // As before agent challenges existed, a front matter that fails to parse fails the problem only\n // when the test-case check below needs the front matter.\n const frontMatterResult = await readProblemMarkdownFrontMatter(problemDir).then(\n (frontMatter) => ({ frontMatter }),\n (error: unknown) => ({ error })\n );\n if ('frontMatter' in frontMatterResult && frontMatterResult.frontMatter.agentChallenge !== undefined) {\n return planAgentChallengeCheck(problemDir, frontMatterResult.frontMatter.agentChallenge);\n }\n\n const defaultHarnessFileNames = await findDefaultStdioHarnessFiles(problemDir);\n if (defaultHarnessFileNames.length > 0) {\n return { type: 'failure', message: formatDefaultHarnessError(defaultHarnessFileNames) };\n }\n\n const modelAnswerDirs = await findModelAnswerDirs(problemDir);\n if (modelAnswerDirs.length === 0) {\n return { type: 'failure', message: 'no model answers found under model_answers/' };\n }\n\n // Without test cases, stdioJudgePreset prints a single accepted sentinel result, so a standard\n // problem with an empty or missing test_cases/ would otherwise pass without being judged —\n // unless static-analysis rules or manual scoring make the problem judgeable without them.\n if (!(await fileExists(path.join(problemDir, 'judge.ts')))) {\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n if (testCases.length === 0) {\n if ('error' in frontMatterResult) {\n return {\n type: 'failure',\n message: `failed to read the problem markdown front matter: ${getErrorMessage(frontMatterResult.error)}`,\n };\n }\n if (!judgesWithoutTestCases(frontMatterResult.frontMatter)) {\n return { type: 'failure', message: MISSING_TEST_CASES_ERROR };\n }\n }\n }\n\n const failingModelAnswerDirs = await findFailingModelAnswerDirs(problemDir);\n return {\n type: 'runs',\n runs: [\n ...modelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'accepted' })),\n ...failingModelAnswerDirs.map((answerDir): CheckRun => ({ problemDir, answerDir, expectation: 'rejected' })),\n ],\n };\n}\n\n/* Skips a valid agent challenge and fails an invalid one instead of judging it as a stdio problem. /\nasync function planAgentChallengeCheck(problemDir: string, agentChallenge: unknown): Promise<ProblemCheckPlan> {\n const parsed = agentChallengeSchema.safeParse(agentChallenge);\n if (parsed.success) {\n const explanationDirents = await fs\n .readdir(path.join(problemDir, 'model_answers', 'explanation'), { recursive: true, withFileTypes: true })\n .catch(() => []);\n return explanationDirents.some((dirent) => dirent.isFile() && dirent.name.endsWith('.md'))\n ? { type: 'agentChallenge' }\n : { type: 'failure', message: 'no explanation found under model_answers/explanation/' };\n }\n const issues = parsed.error.issues.map((issue) => `${['agentChallenge', ...issue.path].join('.')}: ${issue.message}`);\n return { type: 'failure', message: `invalid agentChallenge (see validate-problem): ${issues.join('; ')}` };\n}\n\n/\n Run one answer directory through the problem's harness and return a failure detail, if any.\n * The problem directory is copied to a temporary location first so that judging (e.g. build\n * artifacts in answer directories) never modifies the checked repository.\n /\nasync function executeCheckRun(run: CheckRun, cliEntryPath: string): Promise<string \| undefined> {\n let tempRoot: string;\n let copiedProblemDir: string;\n try {\n ({ tempRoot, copiedProblemDir } = await copyProblemDirToTemporaryRoot(run.problemDir));\n } catch (error) {\n return truncate(`failed to copy the problem directory to a temporary location: ${getErrorMessage(error)}`);\n }\n\n let harnessFailureDetail;\n let removedTempRoot;\n try {\n const result = await runHarnessProcess(\n ['run', cliEntryPath, 'judge', path.relative(run.problemDir, run.answerDir)],\n {\n cwd: copiedProblemDir,\n env: process.env,\n timeoutMs: RUN_TIMEOUT_MS,\n maxOutputBytes: MAX_RUN_OUTPUT_BYTES,\n tempRoot,\n }\n );\n harnessFailureDetail = summarizeHarnessFailure(run, result);\n } catch (error) {\n harnessFailureDetail = truncate(`harness execution failed: ${getErrorMessage(error)}`);\n } finally {\n removedTempRoot = await forciblyRemoveDirectory(tempRoot);\n }\n if (removedTempRoot) return harnessFailureDetail;\n const removalFailureDetail = `failed to remove the temporary copy at ${tempRoot} (judged code may have left permission-locked files)`;\n return harnessFailureDetail === undefined\n ? removalFailureDetail\n : truncate(`${harnessFailureDetail}; ${removalFailureDetail}`);\n}\n\n/* Return why the harness run failed the check, or `undefined` when it passed. /\nfunction summarizeHarnessFailure(run: CheckRun, result: HarnessProcessResult): string \| undefined {\n const { stdout, stderr } = result;\n if (result.failureReason !== undefined) return truncate(result.failureReason);\n if (result.exitCode !== 0) {\n return truncate(`harness exited with ${result.exitCode ?? 'a signal'}${stderr.trim() ? `: ${stderr.trim()}` : ''}`);\n }\n\n const resultLines = stdout.split(/\\r?\\n/).filter((line) => line.startsWith(TEST_CASE_RESULT_PREFIX));\n const testCaseResults = [];\n for (const line of resultLines) {\n let parsedResult;\n try {\n parsedResult = testCaseResultSchema.safeParse(JSON.parse(line.slice(TEST_CASE_RESULT_PREFIX.length)));\n } catch {\n parsedResult = undefined;\n }\n if (!parsedResult?.success) return truncate(`malformed test case result line: ${line}`);\n testCaseResults.push(parsedResult.data);\n }\n if (testCaseResults.length === 0) return 'no test case results were printed';\n\n if (run.expectation === 'accepted') {\n const rejectedResult = testCaseResults.find((result) => result.decisionCode !== DecisionCode.ACCEPTED);\n if (rejectedResult) {\n return truncate(\n `${decisionCodeNames.get(rejectedResult.decisionCode) ?? rejectedResult.decisionCode} on test case ${rejectedResult.testCaseId}${rejectedResult.stderr?.trim() ? `: ${rejectedResult.stderr.trim()}` : ''}`\n );\n }\n return undefined;\n }\n return testCaseResults.every((result) => result.decisionCode === DecisionCode.ACCEPTED)\n ? 'expected at least one failing test case, but all test cases were accepted'\n : undefined;\n}\n\nfunction parseCheckArgs(args: readonly string[]): CheckOptions {\n const options: CheckOptions = {\n rootDir: '.',\n // Serial by default: judging decides TIME_LIMIT_EXCEEDED from wall-clock time, so parallel\n // runs on a small CI runner could fail timing-sensitive problems non-deterministically.\n concurrency: 1,\n only: [],\n skip: [],\n };\n let hasRootDir = false;\n for (let index = 0; index < args.length; index++) {\n const arg = args[index];\n if (arg === undefined) break;\n if (arg === '--concurrency' \|\| arg === '--only' \|\| arg === '--skip') {\n const value = args[++index];\n if (value === undefined) throw new Error(`${arg} requires a value`);\n if (arg === '--concurrency') {\n options.concurrency = Number(value);\n if (!Number.isInteger(options.concurrency) \|\| options.concurrency <= 0) {\n throw new Error(`--concurrency requires a positive integer, but got ${value}`);\n }\n } else if (arg === '--only') {\n options.only.push(value);\n } else {\n options.skip.push(value);\n }\n } else if (arg.startsWith('--')) {\n throw new Error(`Unknown option: ${arg}`);\n } else if (hasRootDir) {\n throw new Error(`Only one root directory can be specified, but got both ${options.rootDir} and ${arg}`);\n } else {\n options.rootDir = arg;\n hasRootDir = true;\n }\n }\n return options;\n}\n\n/* Find directories containing `problem.md` or `<id>.problem.md`, without descending into found problems. */\nasync function findProblemDirs(rootDir: string): Promise<string[]> {\n const problemDirs: string[] = [];\n await visitDirectory(rootDir, problemDirs);\n return problemDirs.toSorted();\n}\n\nasync function visitDirectory(dir: string, problemDirs: string[]): Promise<void> {\n // Traversal errors (e.g. an unreadable subtree) must fail the check: skipping them silently\n // could report a green result while covering only part of the repository.\n const entries = await fs.readdir(dir, { withFileTypes: true });\n if (entries.some((entry) => entry.isFile() && (entry.name === 'problem.md' \|\| entry.name.endsWith('.problem.md')))) {\n problemDirs.push(dir);\n return;\n }\n for (const entry of entries) {\n if (!entry.isDirectory() \|\| entry.name === 'node_modules' \|\| entry.name.startsWith('.')) continue;\n await visitDirectory(path.join(dir, entry.name), problemDirs);\n }\n}\n\nasync function fileExists(filePath: string): Promise<boolean> {\n try {\n await fs.stat(filePath);\n return true;\n } catch {\n return false;\n }\n}\n\nfunction truncate(text: string): string {\n return text.length <= MAX_FAILURE_DETAIL_LENGTH ? text : `${text.slice(0, MAX_FAILURE_DETAIL_LENGTH)}...`;\n}\n"],"mappings":"A,++BAgBA,MAEME,EAA4B,IAG5BC,EAAoB,IAAI,IAAoB,OAAO,QAAQC,CAAY,CAAC,CAAC,KAAK,CAACC,EAAMC,KAAU,CAACA,EAAMD,CAAI,CAAC,CAAC,EAsBlH,eAAsBE,EAAiB,EAA0C,CAC/E,IAAMC,EAAUC,EAAeC,CAAI,EAC7BC,EAAUX,EAAK,QAAQQ,EAAQ,OAAO,EAEtCI,EAAc,IAAyBZ,EAAK,SAASW,EAASE,CAAG,GAAK,IAAA,CAAK,WAAWb,EAAK,IAAK,GAAG,EAEnGc,EAAiB,MAAMC,EAAgBJ,CAAO,EAC9CK,EAAcF,EAAe,OAAQ,GAAe,CACxD,IAAMG,EAAcL,EAAWM,CAAU,EAEzC,OADIV,EAAQ,KAAK,OAAS,GAAK,CAACA,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,EAAU,GACnG,CAACX,EAAQ,KAAK,KAAM,GAAcS,EAAY,SAASE,CAAS,CAAC,CAC1E,CAAC,EACD,GAAIH,EAAY,SAAW,EAMzB,OALA,QAAQ,MACNF,EAAe,SAAW,EACtB,iFAAiFH,EAAQ,GACzF,OAAOG,EAAe,OAAO,6BAA6BH,EAAQ,iCACxE,EACO,EAGT,IAAMS,EAAqB,CAAC,EACtBC,EAAmB,CAAC,EACtBC,EAAsB,EAC1B,IAAK,IAAMJ,KAAcF,EAAa,CACpC,IAAMO,EAAO,MAAMC,EAAiBN,CAAU,EAC1CK,EAAK,OAAS,iBAChB,IACSA,EAAK,OAAS,UACvBH,EAAS,KAAK,GAAGR,EAAWM,CAAU,EAAE,IAAIK,EAAK,SAAS,EAE1DF,EAAK,KAAK,GAAGE,EAAK,IAAI,CAE1B,CACA,IAAK,IAAME,KAAWL,EAAU,QAAQ,MAAM,KAAKK,GAAS,EAE5D,IAAMC,EAAe1B,EAAK,QAAQ,QAAQ,KAAK,IAAM,EAAE,EACnD2B,EAAc,EAgBlB,OAfA,MAAMC,EAAoBP,EAAMb,EAAQ,YAAa,KAAO,IAAQ,CAClE,IAAMqB,EAAQ,GAAGjB,EAAWkB,EAAI,UAAU,EAAE,GAAG9B,EAAK,SAAS8B,EAAI,WAAYA,EAAI,SAAS,CAAC,CAAC,WAAW9B,EAAK,IAAK,GAAG,IAC9G+B,EAAgB,MAAMC,EAAgBF,EAAKJ,CAAY,EACzDK,IAAkB,IAAA,IACpB,IACA,QAAQ,KAAK,KAAKF,GAAO,IAEzBT,EAAS,KAAK,GAAGS,EAAM,IAAIE,GAAe,EAC1C,QAAQ,MAAM,KAAKF,EAAM,IAAIE,GAAe,EAEhD,CAAC,EAED,QAAQ,KACN,KAAKJ,EAAY,WAAWP,EAAS,OAAO,WAAWC,EAAK,OAAO,SAASL,EAAY,OAAO,WAAWM,EAAsB,EAAI,+BAA+BA,IAAwB,GAAG,EAChM,EACOF,EAAS,SAAW,EAAI,EAAI,CACrC,CAOA,eAAeI,EAAiB,EAA+C,CAG7E,IAAMS,EAAoB,MAAMC,EAA+BhB,CAAU,CAAC,CAAC,KACxE,IAAiB,CAAE,aAAY,GAC/B,IAAoB,CAAE,OAAM,EAC/B,EACA,GAAI,gBAAiBe,GAAqBA,EAAkB,YAAY,iBAAmB,IAAA,GACzF,OAAOE,EAAwBjB,EAAYe,EAAkB,YAAY,cAAc,EAGzF,IAAMG,EAA0B,MAAMC,EAA6BnB,CAAU,EAC7E,GAAIkB,EAAwB,OAAS,EACnC,MAAO,CAAE,KAAM,UAAW,QAASE,EAA0BF,CAAuB,CAAE,EAGxF,IAAMG,EAAkB,MAAMC,EAAoBtB,CAAU,EAC5D,GAAIqB,EAAgB,SAAW,EAC7B,MAAO,CAAE,KAAM,UAAW,QAAS,6CAA8C,EAMnF,GAAI,CAAE,MAAME,EAAWzC,EAAK,KAAKkB,EAAY,UAAU,CAAC,IAElD,MADoBwB,EAAc1C,EAAK,KAAKkB,EAAY,YAAY,CAAC,EAAA,CAC3D,SAAW,EAAG,CAC1B,GAAI,UAAWe,EACb,MAAO,CACL,KAAM,UACN,QAAS,qDAAqDU,EAAgBV,EAAkB,KAAK,GACvG,EAEF,GAAI,CAACW,EAAuBX,EAAkB,WAAW,EACvD,MAAO,CAAE,KAAM,UAAW,QAASY,CAAyB,CAEhE,CAGF,IAAMC,EAAyB,MAAMC,EAA2B7B,CAAU,EAC1E,MAAO,CACL,KAAM,OACN,KAAM,CACJ,GAAGqB,EAAgB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,EACpG,GAAGO,EAAuB,IAAK,IAAyB,CAAE,aAAY,YAAW,YAAa,UAAW,EAAE,CAC7G,CACF,CACF,CAGA,eAAeX,EAAwB,EAAoB,EAAoD,CAC7G,IAAMa,EAASC,EAAqB,UAAUC,CAAc,EAU5D,OATIF,EAAO,SAIF,MAH0B/C,EAC9B,QAAQD,EAAK,KAAKkB,EAAY,gBAAiB,aAAa,EAAG,CAAE,UAAW,GAAM,cAAe,EAAK,CAAC,CAAC,CACxG,UAAY,CAAC,CAAC,EAAA,CACS,KAAM,GAAWiC,EAAO,OAAO,GAAKA,EAAO,KAAK,SAAS,KAAK,CAAC,EACrF,CAAE,KAAM,gBAAiB,EACzB,CAAE,KAAM,UAAW,QAAS,uDAAwD,EAGnF,CAAE,KAAM,UAAW,QAAS,kDADpBH,EAAO,MAAM,OAAO,IAAK,GAAU,GAAG,CAAC,iBAAkB,GAAGI,EAAM,IAAI,CAAC,CAAC,KAAK,GAAG,EAAE,IAAIA,EAAM,SACjB,CAAC,CAAC,KAAK,IAAI,GAAI,CAC3G,CAOA,eAAepB,EAAgB,EAAe,EAAmD,CAC/F,IAAIqB,EACAC,EACJ,GAAI,EACD,gCAAiC,MAAMC,EAA8BzB,EAAI,UAAU,EACtF,OAAS0B,EAAO,CACd,OAAOC,EAAS,iEAAiEd,EAAgBa,CAAK,GAAG,CAC3G,CAEA,IAAIE,EACAC,EACJ,GAAI,CAWF,EAAuBC,EAAwB9B,EAAK,MAV/B+B,EACnB,CAAC,MAAOnC,EAAc,QAAS1B,EAAK,SAAS8B,EAAI,WAAYA,EAAI,SAAS,CAAC,EAC3E,CACE,IAAKwB,EACL,IAAK,QAAQ,IACb,UAAW,IACX,eAAgB,SAChB,UACF,CACF,CAC0D,CAC5D,OAASE,EAAO,CACd,EAAuBC,EAAS,6BAA6Bd,EAAgBa,CAAK,GAAG,CACvF,QAAU,CACR,EAAkB,MAAMM,EAAwBT,CAAQ,CAC1D,CACA,GAAIM,EAAiB,OAAOD,EAC5B,IAAMK,EAAuB,0CAA0CV,EAAS,sDAChF,OAAOK,IAAyB,IAAA,GAC5BK,EACAN,EAAS,GAAGC,EAAqB,IAAIK,GAAsB,CACjE,CAGA,SAASH,EAAwB,EAAe,EAAkD,CAChG,GAAM,CAAE,SAAQ,UAAWI,EAC3B,GAAIA,EAAO,gBAAkB,IAAA,GAAW,OAAOP,EAASO,EAAO,aAAa,EAC5E,GAAIA,EAAO,WAAa,EACtB,OAAOP,EAAS,uBAAuBO,EAAO,UAAY,aAAaC,EAAO,KAAK,EAAI,KAAKA,EAAO,KAAK,IAAM,IAAI,EAGpH,IAAMC,EAAcC,EAAO,MAAM,OAAO,CAAC,CAAC,OAAQ,GAASC,EAAK,WAAWC,CAAuB,CAAC,EAC7FC,EAAkB,CAAC,EACzB,IAAK,IAAMF,KAAQF,EAAa,CAC9B,IAAIK,EACJ,GAAI,CACF,EAAeC,EAAqB,UAAU,KAAK,MAAMJ,EAAK,MAAMC,EAAwB,MAAM,CAAC,CAAC,CACtG,MAAQ,CACN,EAAe,IAAA,EACjB,CACA,GAAI,CAACE,GAAc,QAAS,OAAOd,EAAS,oCAAoCW,GAAM,EACtFE,EAAgB,KAAKC,EAAa,IAAI,CACxC,CACA,GAAID,EAAgB,SAAW,EAAG,MAAO,oCAEzC,GAAIxC,EAAI,cAAgB,WAAY,CAClC,IAAM2C,EAAiBH,EAAgB,KAAM,GAAWN,EAAO,eAAiB5D,EAAa,QAAQ,EAMrG,OALIqE,EACKhB,EACL,GAAGtD,EAAkB,IAAIsE,EAAe,YAAY,GAAKA,EAAe,aAAa,gBAAgBA,EAAe,aAAaA,EAAe,QAAQ,KAAK,EAAI,KAAKA,EAAe,OAAO,KAAK,IAAM,IACzM,EAEF,MACF,CACA,OAAOH,EAAgB,MAAO,GAAWN,EAAO,eAAiB5D,EAAa,QAAQ,EAClF,4EACA,IAAA,EACN,CAEA,SAASK,EAAe,EAAuC,CAC7D,IAAMD,EAAwB,CAC5B,QAAS,IAGT,YAAa,EACb,KAAM,CAAC,EACP,KAAM,CAAC,CACT,EACIkE,EAAa,GACjB,IAAK,IAAIC,EAAQ,EAAGA,EAAQjE,EAAK,OAAQ,IAAS,CAChD,IAAMkE,EAAMlE,EAAKiE,GACjB,GAAIC,IAAQ,IAAA,GAAW,MACvB,GAAIA,IAAQ,iBAAmBA,IAAQ,UAAYA,IAAQ,SAAU,CACnE,IAAMC,EAAQnE,EAAK,EAAEiE,GACrB,GAAIE,IAAU,IAAA,GAAW,MAAU,MAAM,GAAGD,EAAI,kBAAkB,EAClE,GAAIA,IAAQ,gBAEV,IADA,EAAQ,YAAc,OAAOC,CAAK,EAC9B,CAAC,OAAO,UAAUrE,EAAQ,WAAW,GAAKA,EAAQ,aAAe,EACnE,MAAU,MAAM,sDAAsDqE,GAAO,CAAA,MAEtED,IAAQ,SACjBpE,EAAQ,KAAK,KAAKqE,CAAK,EAEvBrE,EAAQ,KAAK,KAAKqE,CAAK,CAE3B,MAAO,GAAID,EAAI,WAAW,IAAI,EAC5B,MAAU,MAAM,mBAAmBA,GAAK,OACnC,GAAIF,EACT,MAAU,MAAM,0DAA0DlE,EAAQ,QAAQ,OAAOoE,GAAK,MAEtG,GAAQ,QAAUA,EAClB,EAAa,EAEjB,CACA,OAAOpE,CACT,CAGA,eAAeO,EAAgB,EAAoC,CACjE,IAAMC,EAAwB,CAAC,EAE/B,OADA,MAAM8D,EAAenE,EAASK,CAAW,EAClCA,EAAY,SAAS,CAC9B,CAEA,eAAe8D,EAAe,EAAa,EAAsC,CAG/E,IAAMC,EAAU,MAAM9E,EAAG,QAAQY,EAAK,CAAE,cAAe,EAAK,CAAC,EAC7D,GAAIkE,EAAQ,KAAM,GAAUC,EAAM,OAAO,IAAMA,EAAM,OAAS,cAAgBA,EAAM,KAAK,SAAS,aAAa,EAAE,EAC/GhE,EAAY,KAAKH,CAAG,OAGtB,IAAK,IAAMmE,KAASD,EACbC,EAAM,YAAY,GAAKA,EAAM,OAAS,gBAAkB,GAAM,KAAK,WAAW,GAAG,GACtF,MAAMF,EAAe9E,EAAK,KAAKa,EAAKmE,EAAM,IAAI,EAAGhE,CAAW,CAEhE,CAEA,eAAeyB,EAAW,EAAoC,CAC5D,GAAI,CAEF,OADA,MAAMxC,EAAG,KAAKgF,CAAQ,EACf,EACT,MAAQ,CACN,MAAO,EACT,CACF,CAEA,SAASxB,EAAS,EAAsB,CACtC,OAAOyB,EAAK,QAAUhF,EAA4BgF,EAAO,GAAGA,EAAK,MAAM,EAAGhF,CAAyB,EAAE,IACvG,C"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@exercode/problem-utils",
-  "version": "5.0.0",
+  "version": "5.0.2",
   "description": ":100: A set of utilities for judging programs on Exercode (https://exercode.willbooster.com/).",
   "keywords": [
     "exercode",
@@ -53,20 +53,20 @@
     "verify-full": "bun wb verify --full"
   },
   "dependencies": {
-    "@willbooster/shared-lib": "10.23.0",
+    "@willbooster/shared-lib": "10.24.0",
     "front-matter": "4.0.2",
     "yaml": "2.9.1",
     "zod": "4.6.7"
   },
   "devDependencies": {
     "@tsconfig/bun": "1.0.11",
-    "@types/bun": "1.4.3",
+    "@types/bun": "1.4.4",
     "@willbooster/oxlint-config": "1.4.9",
-    "@willbooster/wb": "22.29.3",
+    "@willbooster/wb": "23.0.1",
     "build-ts": "21.0.29",
-    "oxlint": "1.88.0",
-    "oxlint-tsgolint": "7.0.2004",
-    "undici-types": "8.12.2"
+    "oxlint": "1.89.0",
+    "oxlint-tsgolint": "7.0.2005",
+    "undici-types": "8.12.3"
   },
   "prettier": "@willbooster/prettier-config",
   "engines": {