@exercode/problem-utils-llm 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +6 -5
package/dist/index.cjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("./_virtual/_rolldown/runtime.cjs");let t=require("node:fs");t=e.__toESM(t,1);let n=require("node:path");n=e.__toESM(n,1);let r=require("@ai-sdk/amazon-bedrock"),
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("./_virtual/_rolldown/runtime.cjs");let t=require("node:fs");t=e.__toESM(t,1);let n=require("node:path");n=e.__toESM(n,1);let r=require("@willbooster/shared-lib"),i=require("@ai-sdk/amazon-bedrock"),a=require("@ai-sdk/google"),o=require("@ai-sdk/openai"),s=require("@ai-sdk/xai"),c=require("ai"),l=require("zod"),u=require("@exercode/problem-utils");const d={bedrock:i.bedrock,google:a.google,openai:o.openai,xai:s.xai},f=l.z.object({model:l.z.string().min(1)});async function p(e,i){let a=(0,u.parseArgs)(process.argv);if(!a.cwd)throw Error(`cwd argument required`);let o=f.parse(a.params),s=await(0,u.readTestCases)(n.default.join(e,`test_cases`)),l=await t.default.promises.readFile(n.default.join(a.cwd,`prompt.txt`),`utf8`);for(let e of s){let t=Date.now();try{let{text:n}=await(0,c.generateText)({model:h(o.model),...m(i.buildPrompt?.({prompt:l,testCase:e})??l.replaceAll(`{input}`,e.input??``))}),r=Date.now(),a={testCaseId:e.id,decisionCode:u.DecisionCode.ACCEPTED,stdin:e.input,stdout:n,timeSeconds:(r-t)/1e3,...await i.test({testCase:e,result:{output:n}})};if((0,u.printTestCaseResult)(a),a.decisionCode!==u.DecisionCode.ACCEPTED)break}catch(n){let i=Date.now();(0,u.printTestCaseResult)({testCaseId:e.id,decisionCode:u.DecisionCode.RUNTIME_ERROR,stdin:e.input,stderr:(0,r.getErrorMessage)(n),timeSeconds:(i-t)/1e3});break}}}function m(e){return typeof e==`string`?{prompt:e}:{messages:e,allowSystemInMessages:!0}}function h(e){let[t,n]=e.split(`/`);if(!t||!n)throw Error(`bad model: ${e}`);let r=d[t]?.(n);if(!r)throw Error(`model not found: ${e}`);return r}exports.llmJudgePreset=p;
|
|
2
2
|
//# sourceMappingURL=index.cjs.map
|
package/dist/index.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.cjs","names":["z","parseArgs","readTestCases","path","fs","generateText","DecisionCode"],"sources":["../src/index.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { xai } from '@ai-sdk/xai';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nimport {\n parseArgs,\n printTestCaseResult,\n readTestCases,\n DecisionCode,\n type TestCaseResult,\n} from '@exercode/problem-utils';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n bedrock,\n // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n google,\n // requires `OPENAI_API_KEY`\n openai,\n // requires `XAI_API_KEY`\n xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n model: z.string().min(1),\n});\n\n/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */\ninterface LlmTestCase {\n id: string;\n input?: string;\n output?: string;\n /** Present when `<id>.fin/` exists; the preset does not copy it anywhere (there is no program run). */\n fileInputPath?: string;\n /** Present when `<id>.fout/` exists, for `test` to read. */\n fileOutputPath?: string;\n}\n\ninterface LlmJudgePresetOptions {\n buildPrompt?: (context: { prompt: string; testCase: LlmTestCase }) => string | ModelMessage[];\n test: (context: {\n testCase: LlmTestCase;\n result: { output: string };\n }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils-llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n * test: (context) {\n * return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n * }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n const args = parseArgs(process.argv);\n if (!args.cwd) throw new Error('cwd argument required');\n const params = judgeParamsSchema.parse(args.params);\n\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n for (const testCase of testCases) {\n const startTimeMilliseconds = Date.now();\n try {\n const { text } = await generateText({\n model: toLanguageModel(params.model),\n ...toPromptOptions(\n options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? '')\n ),\n });\n\n const stopTimeMilliseconds = Date.now();\n\n const testCaseResult = {\n testCaseId: testCase.id,\n decisionCode: DecisionCode.ACCEPTED,\n stdin: testCase.input,\n stdout: text,\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n ...(await options.test({ testCase, result: { output: text } })),\n };\n\n printTestCaseResult(testCaseResult);\n\n if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n } catch (error) {\n const stopTimeMilliseconds = Date.now();\n\n printTestCaseResult({\n testCaseId: testCase.id,\n decisionCode: DecisionCode.RUNTIME_ERROR,\n stdin: testCase.input,\n stderr:
|
|
1
|
+
{"version":3,"file":"index.cjs","names":["z","parseArgs","readTestCases","path","fs","generateText","DecisionCode","getErrorMessage"],"sources":["../src/index.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\nimport { getErrorMessage } from '@willbooster/shared-lib';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { xai } from '@ai-sdk/xai';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nimport {\n parseArgs,\n printTestCaseResult,\n readTestCases,\n DecisionCode,\n type TestCaseResult,\n} from '@exercode/problem-utils';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n bedrock,\n // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n google,\n // requires `OPENAI_API_KEY`\n openai,\n // requires `XAI_API_KEY`\n xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n model: z.string().min(1),\n});\n\n/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */\ninterface LlmTestCase {\n id: string;\n input?: string;\n output?: string;\n /** Present when `<id>.fin/` exists; the preset does not copy it anywhere (there is no program run). */\n fileInputPath?: string;\n /** Present when `<id>.fout/` exists, for `test` to read. */\n fileOutputPath?: string;\n}\n\ninterface LlmJudgePresetOptions {\n buildPrompt?: (context: { prompt: string; testCase: LlmTestCase }) => string | ModelMessage[];\n test: (context: {\n testCase: LlmTestCase;\n result: { output: string };\n }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils-llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n * test: (context) {\n * return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n * }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n const args = parseArgs(process.argv);\n if (!args.cwd) throw new Error('cwd argument required');\n const params = judgeParamsSchema.parse(args.params);\n\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n for (const testCase of testCases) {\n const startTimeMilliseconds = Date.now();\n try {\n const { text } = await generateText({\n model: toLanguageModel(params.model),\n ...toPromptOptions(\n options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? '')\n ),\n });\n\n const stopTimeMilliseconds = Date.now();\n\n const testCaseResult = {\n testCaseId: testCase.id,\n decisionCode: DecisionCode.ACCEPTED,\n stdin: testCase.input,\n stdout: text,\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n ...(await options.test({ testCase, result: { output: text } })),\n };\n\n printTestCaseResult(testCaseResult);\n\n if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n } catch (error) {\n const stopTimeMilliseconds = Date.now();\n\n printTestCaseResult({\n testCaseId: testCase.id,\n decisionCode: DecisionCode.RUNTIME_ERROR,\n stdin: testCase.input,\n stderr: getErrorMessage(error),\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n });\n\n break;\n }\n }\n}\n\n/** `ai` rejects system messages inside `messages` unless told that a `buildPrompt` may place them there. */\nfunction toPromptOptions(\n built: string | ModelMessage[]\n): { prompt: string } | { messages: ModelMessage[]; allowSystemInMessages: true } {\n return typeof built === 'string' ? { prompt: built } : { messages: built, allowSystemInMessages: true };\n}\n\nfunction toLanguageModel(model: string): LanguageModel {\n const [providerId, modelId] = model.split('/');\n if (!providerId || !modelId) throw new Error(`bad model: ${model}`);\n const languageModel = providerByName[providerId]?.(modelId);\n if (!languageModel) throw new Error(`model not found: ${model}`);\n return languageModel;\n}\n"],"mappings":"8bAoBA,MAEM,EAA8F,CAElG,QAAA,EAAA,QAEA,OAAA,EAAA,OAEA,OAAA,EAAA,OAEA,IAAA,EAAA,GACF,EAEM,EAAoBA,EAAAA,EAAE,OAAO,CACjC,MAAOA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CACzB,CAAC,EA0CD,eAAsB,EAAe,EAAoB,EAA+C,CACtG,IAAM,GAAA,EAAOC,EAAAA,UAAAA,CAAU,QAAQ,IAAI,EACnC,GAAI,CAAC,EAAK,IAAK,MAAU,MAAM,uBAAuB,EACtD,IAAM,EAAS,EAAkB,MAAM,EAAK,MAAM,EAE5C,EAAY,MAAA,EAAMC,EAAAA,cAAAA,CAAcC,EAAAA,QAAK,KAAK,EAAY,YAAY,CAAC,EAEnE,EAAS,MAAMC,EAAAA,QAAG,SAAS,SAASD,EAAAA,QAAK,KAAK,EAAK,IAAK,YAAe,EAAG,MAAM,EAEtF,IAAK,IAAM,KAAY,EAAW,CAChC,IAAM,EAAwB,KAAK,IAAI,EACvC,GAAI,CACF,GAAM,CAAE,QAAS,MAAA,EAAME,EAAAA,aAAAA,CAAa,CAClC,MAAO,EAAgB,EAAO,KAAK,EACnC,GAAG,EACD,EAAQ,cAAc,CAAE,SAAQ,UAAS,CAAC,GAAK,EAAO,WAAW,UAAW,EAAS,OAAS,EAAE,CAClG,CACF,CAAC,EAEK,EAAuB,KAAK,IAAI,EAEhC,EAAiB,CACrB,WAAY,EAAS,GACrB,aAAcC,EAAAA,aAAa,SAC3B,MAAO,EAAS,MAChB,OAAQ,EACR,aAAc,EAAuB,GAAyB,IAC9D,GAAI,MAAM,EAAQ,KAAK,CAAE,WAAU,OAAQ,CAAE,OAAQ,CAAK,CAAE,CAAC,CAC/D,EAIA,IAFA,EAAA,EAAA,oBAAA,CAAoB,CAAc,EAE9B,EAAe,eAAiBA,EAAAA,aAAa,SAAU,KAC7D,OAAS,EAAO,CACd,IAAM,EAAuB,KAAK,IAAI,GAEtC,EAAA,EAAA,oBAAA,CAAoB,CAClB,WAAY,EAAS,GACrB,aAAcA,EAAAA,aAAa,cAC3B,MAAO,EAAS,MAChB,QAAA,EAAQC,EAAAA,gBAAAA,CAAgB,CAAK,EAC7B,aAAc,EAAuB,GAAyB,GAChE,CAAC,EAED,KACF,CACF,CACF,CAGA,SAAS,EACP,EACgF,CAChF,OAAO,OAAO,GAAU,SAAW,CAAE,OAAQ,CAAM,EAAI,CAAE,SAAU,EAAO,sBAAuB,EAAK,CACxG,CAEA,SAAS,EAAgB,EAA8B,CACrD,GAAM,CAAC,EAAY,GAAW,EAAM,MAAM,GAAG,EAC7C,GAAI,CAAC,GAAc,CAAC,EAAS,MAAU,MAAM,cAAc,GAAO,EAClE,IAAM,EAAgB,EAAe,EAAW,GAAG,CAAO,EAC1D,GAAI,CAAC,EAAe,MAAU,MAAM,oBAAoB,GAAO,EAC/D,OAAO,CACT"}
|
package/dist/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import e from"node:fs";import t from"node:path";import{
|
|
1
|
+
import e from"node:fs";import t from"node:path";import{getErrorMessage as n}from"@willbooster/shared-lib";import{bedrock as r}from"@ai-sdk/amazon-bedrock";import{google as i}from"@ai-sdk/google";import{openai as a}from"@ai-sdk/openai";import{xai as o}from"@ai-sdk/xai";import{generateText as s}from"ai";import{z as c}from"zod";import{DecisionCode as l,parseArgs as u,printTestCaseResult as d,readTestCases as f}from"@exercode/problem-utils";const p={bedrock:r,google:i,openai:a,xai:o},m=c.object({model:c.string().min(1)});async function h(r,i){let a=u(process.argv);if(!a.cwd)throw Error(`cwd argument required`);let o=m.parse(a.params),c=await f(t.join(r,`test_cases`)),p=await e.promises.readFile(t.join(a.cwd,`prompt.txt`),`utf8`);for(let e of c){let t=Date.now();try{let{text:n}=await s({model:_(o.model),...g(i.buildPrompt?.({prompt:p,testCase:e})??p.replaceAll(`{input}`,e.input??``))}),r=Date.now(),a={testCaseId:e.id,decisionCode:l.ACCEPTED,stdin:e.input,stdout:n,timeSeconds:(r-t)/1e3,...await i.test({testCase:e,result:{output:n}})};if(d(a),a.decisionCode!==l.ACCEPTED)break}catch(r){let i=Date.now();d({testCaseId:e.id,decisionCode:l.RUNTIME_ERROR,stdin:e.input,stderr:n(r),timeSeconds:(i-t)/1e3});break}}}function g(e){return typeof e==`string`?{prompt:e}:{messages:e,allowSystemInMessages:!0}}function _(e){let[t,n]=e.split(`/`);if(!t||!n)throw Error(`bad model: ${e}`);let r=p[t]?.(n);if(!r)throw Error(`model not found: ${e}`);return r}export{h as llmJudgePreset};
|
|
2
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":[],"sources":["../src/index.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { xai } from '@ai-sdk/xai';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nimport {\n parseArgs,\n printTestCaseResult,\n readTestCases,\n DecisionCode,\n type TestCaseResult,\n} from '@exercode/problem-utils';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n bedrock,\n // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n google,\n // requires `OPENAI_API_KEY`\n openai,\n // requires `XAI_API_KEY`\n xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n model: z.string().min(1),\n});\n\n/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */\ninterface LlmTestCase {\n id: string;\n input?: string;\n output?: string;\n /** Present when `<id>.fin/` exists; the preset does not copy it anywhere (there is no program run). */\n fileInputPath?: string;\n /** Present when `<id>.fout/` exists, for `test` to read. */\n fileOutputPath?: string;\n}\n\ninterface LlmJudgePresetOptions {\n buildPrompt?: (context: { prompt: string; testCase: LlmTestCase }) => string | ModelMessage[];\n test: (context: {\n testCase: LlmTestCase;\n result: { output: string };\n }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils-llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n * test: (context) {\n * return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n * }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n const args = parseArgs(process.argv);\n if (!args.cwd) throw new Error('cwd argument required');\n const params = judgeParamsSchema.parse(args.params);\n\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n for (const testCase of testCases) {\n const startTimeMilliseconds = Date.now();\n try {\n const { text } = await generateText({\n model: toLanguageModel(params.model),\n ...toPromptOptions(\n options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? '')\n ),\n });\n\n const stopTimeMilliseconds = Date.now();\n\n const testCaseResult = {\n testCaseId: testCase.id,\n decisionCode: DecisionCode.ACCEPTED,\n stdin: testCase.input,\n stdout: text,\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n ...(await options.test({ testCase, result: { output: text } })),\n };\n\n printTestCaseResult(testCaseResult);\n\n if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n } catch (error) {\n const stopTimeMilliseconds = Date.now();\n\n printTestCaseResult({\n testCaseId: testCase.id,\n decisionCode: DecisionCode.RUNTIME_ERROR,\n stdin: testCase.input,\n stderr:
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../src/index.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\nimport { getErrorMessage } from '@willbooster/shared-lib';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { xai } from '@ai-sdk/xai';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nimport {\n parseArgs,\n printTestCaseResult,\n readTestCases,\n DecisionCode,\n type TestCaseResult,\n} from '@exercode/problem-utils';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n bedrock,\n // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n google,\n // requires `OPENAI_API_KEY`\n openai,\n // requires `XAI_API_KEY`\n xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n model: z.string().min(1),\n});\n\n/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */\ninterface LlmTestCase {\n id: string;\n input?: string;\n output?: string;\n /** Present when `<id>.fin/` exists; the preset does not copy it anywhere (there is no program run). */\n fileInputPath?: string;\n /** Present when `<id>.fout/` exists, for `test` to read. */\n fileOutputPath?: string;\n}\n\ninterface LlmJudgePresetOptions {\n buildPrompt?: (context: { prompt: string; testCase: LlmTestCase }) => string | ModelMessage[];\n test: (context: {\n testCase: LlmTestCase;\n result: { output: string };\n }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils-llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n * test: (context) {\n * return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n * }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n const args = parseArgs(process.argv);\n if (!args.cwd) throw new Error('cwd argument required');\n const params = judgeParamsSchema.parse(args.params);\n\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n for (const testCase of testCases) {\n const startTimeMilliseconds = Date.now();\n try {\n const { text } = await generateText({\n model: toLanguageModel(params.model),\n ...toPromptOptions(\n options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? '')\n ),\n });\n\n const stopTimeMilliseconds = Date.now();\n\n const testCaseResult = {\n testCaseId: testCase.id,\n decisionCode: DecisionCode.ACCEPTED,\n stdin: testCase.input,\n stdout: text,\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n ...(await options.test({ testCase, result: { output: text } })),\n };\n\n printTestCaseResult(testCaseResult);\n\n if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n } catch (error) {\n const stopTimeMilliseconds = Date.now();\n\n printTestCaseResult({\n testCaseId: testCase.id,\n decisionCode: DecisionCode.RUNTIME_ERROR,\n stdin: testCase.input,\n stderr: getErrorMessage(error),\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n });\n\n break;\n }\n }\n}\n\n/** `ai` rejects system messages inside `messages` unless told that a `buildPrompt` may place them there. */\nfunction toPromptOptions(\n built: string | ModelMessage[]\n): { prompt: string } | { messages: ModelMessage[]; allowSystemInMessages: true } {\n return typeof built === 'string' ? { prompt: built } : { messages: built, allowSystemInMessages: true };\n}\n\nfunction toLanguageModel(model: string): LanguageModel {\n const [providerId, modelId] = model.split('/');\n if (!providerId || !modelId) throw new Error(`bad model: ${model}`);\n const languageModel = providerByName[providerId]?.(modelId);\n if (!languageModel) throw new Error(`model not found: ${model}`);\n return languageModel;\n}\n"],"mappings":"ybAoBA,MAEM,EAA8F,CAElG,UAEA,SAEA,SAEA,KACF,EAEM,EAAoB,EAAE,OAAO,CACjC,MAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CACzB,CAAC,EA0CD,eAAsB,EAAe,EAAoB,EAA+C,CACtG,IAAM,EAAO,EAAU,QAAQ,IAAI,EACnC,GAAI,CAAC,EAAK,IAAK,MAAU,MAAM,uBAAuB,EACtD,IAAM,EAAS,EAAkB,MAAM,EAAK,MAAM,EAE5C,EAAY,MAAM,EAAc,EAAK,KAAK,EAAY,YAAY,CAAC,EAEnE,EAAS,MAAM,EAAG,SAAS,SAAS,EAAK,KAAK,EAAK,IAAK,YAAe,EAAG,MAAM,EAEtF,IAAK,IAAM,KAAY,EAAW,CAChC,IAAM,EAAwB,KAAK,IAAI,EACvC,GAAI,CACF,GAAM,CAAE,QAAS,MAAM,EAAa,CAClC,MAAO,EAAgB,EAAO,KAAK,EACnC,GAAG,EACD,EAAQ,cAAc,CAAE,SAAQ,UAAS,CAAC,GAAK,EAAO,WAAW,UAAW,EAAS,OAAS,EAAE,CAClG,CACF,CAAC,EAEK,EAAuB,KAAK,IAAI,EAEhC,EAAiB,CACrB,WAAY,EAAS,GACrB,aAAc,EAAa,SAC3B,MAAO,EAAS,MAChB,OAAQ,EACR,aAAc,EAAuB,GAAyB,IAC9D,GAAI,MAAM,EAAQ,KAAK,CAAE,WAAU,OAAQ,CAAE,OAAQ,CAAK,CAAE,CAAC,CAC/D,EAIA,GAFA,EAAoB,CAAc,EAE9B,EAAe,eAAiB,EAAa,SAAU,KAC7D,OAAS,EAAO,CACd,IAAM,EAAuB,KAAK,IAAI,EAEtC,EAAoB,CAClB,WAAY,EAAS,GACrB,aAAc,EAAa,cAC3B,MAAO,EAAS,MAChB,OAAQ,EAAgB,CAAK,EAC7B,aAAc,EAAuB,GAAyB,GAChE,CAAC,EAED,KACF,CACF,CACF,CAGA,SAAS,EACP,EACgF,CAChF,OAAO,OAAO,GAAU,SAAW,CAAE,OAAQ,CAAM,EAAI,CAAE,SAAU,EAAO,sBAAuB,EAAK,CACxG,CAEA,SAAS,EAAgB,EAA8B,CACrD,GAAM,CAAC,EAAY,GAAW,EAAM,MAAM,GAAG,EAC7C,GAAI,CAAC,GAAc,CAAC,EAAS,MAAU,MAAM,cAAc,GAAO,EAClE,IAAM,EAAgB,EAAe,EAAW,GAAG,CAAO,EAC1D,GAAI,CAAC,EAAe,MAAU,MAAM,oBAAoB,GAAO,EAC/D,OAAO,CACT"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@exercode/problem-utils-llm",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.9",
|
|
4
4
|
"description": "Exercode llm judging utilities",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -39,8 +39,9 @@
|
|
|
39
39
|
"@ai-sdk/amazon-bedrock": "5.0.88",
|
|
40
40
|
"@ai-sdk/google": "4.0.76",
|
|
41
41
|
"@ai-sdk/openai": "4.0.71",
|
|
42
|
-
"@ai-sdk/xai": "
|
|
43
|
-
"@exercode/problem-utils": "4.
|
|
42
|
+
"@ai-sdk/xai": "5.0.4",
|
|
43
|
+
"@exercode/problem-utils": "4.3.0",
|
|
44
|
+
"@willbooster/shared-lib": "10.18.0",
|
|
44
45
|
"ai": "7.0.107",
|
|
45
46
|
"zod": "4.6.5"
|
|
46
47
|
},
|
|
@@ -49,8 +50,8 @@
|
|
|
49
50
|
"@types/bun": "1.4.2",
|
|
50
51
|
"@willbooster/oxfmt-config": "1.2.2",
|
|
51
52
|
"@willbooster/oxlint-config": "1.4.8",
|
|
52
|
-
"@willbooster/wb": "22.
|
|
53
|
-
"build-ts": "21.0.
|
|
53
|
+
"@willbooster/wb": "22.8.0",
|
|
54
|
+
"build-ts": "21.0.19",
|
|
54
55
|
"oxfmt": "0.68.0",
|
|
55
56
|
"oxlint": "1.83.0",
|
|
56
57
|
"oxlint-tsgolint": "7.0.2002",
|