@exercode/problem-utils-llm 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("./_virtual/_rolldown/runtime.cjs");let t=require("node:fs");t=e.__toESM(t,1);let n=require("node:path");n=e.__toESM(n,1);let r=require("@ai-sdk/amazon-bedrock"),i=require("@ai-sdk/google"),a=require("@ai-sdk/openai"),o=require("@ai-sdk/openai-compatible"),s=require("@ai-sdk/xai"),c=require("@exercode/problem-utils"),l=require("@willbooster/shared-lib"),u=require("ai"),d=require("zod");const f={bedrock:r.bedrock,google:i.google,openai:a.openai,xai:s.xai},p=d.z.object({model:d.z.string().min(1)}),m=d.z.record(d.z.string(),d.z.unknown());async function h(e,r){let i=(0,c.parseArgs)(process.argv);if(!i.cwd)throw Error(`cwd argument required`);let a=p.parse(i.params),o=await(0,c.readTestCases)(n.default.join(e,`test_cases`)),
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("./_virtual/_rolldown/runtime.cjs");let t=require("node:fs");t=e.__toESM(t,1);let n=require("node:path");n=e.__toESM(n,1);let r=require("@ai-sdk/amazon-bedrock"),i=require("@ai-sdk/google"),a=require("@ai-sdk/openai"),o=require("@ai-sdk/openai-compatible"),s=require("@ai-sdk/xai"),c=require("@exercode/problem-utils"),l=require("@willbooster/shared-lib"),u=require("ai"),d=require("zod");const f={bedrock:r.bedrock,google:i.google,openai:a.openai,xai:s.xai},p=d.z.object({model:d.z.string().min(1)}),m=d.z.record(d.z.string(),d.z.unknown());async function h(e,r){let i=(0,c.parseArgs)(process.argv);if(!i.cwd)throw Error(`cwd argument required`);let a=p.parse(i.params),o=r.model??a.model,s=await(0,c.readTestCases)(n.default.join(e,`test_cases`)),d=await t.default.promises.readFile(n.default.join(i.cwd,`prompt.txt`),`utf8`);for(let e of s){let t=Date.now();try{let{text:n}=await(0,u.generateText)({model:_(o,r.openAiCompatible)??y(o),...g(r.buildPrompt?.({prompt:d,testCase:e})??d.replaceAll(`{input}`,e.input??``))}),i=Date.now(),a={testCaseId:e.id,decisionCode:c.DecisionCode.ACCEPTED,stdin:e.input,stdout:n,timeSeconds:(i-t)/1e3,...await r.test({testCase:e,result:{output:n}})};if((0,c.printTestCaseResult)(a),a.decisionCode!==c.DecisionCode.ACCEPTED)break}catch(n){let r=Date.now();(0,c.printTestCaseResult)({testCaseId:e.id,decisionCode:c.DecisionCode.RUNTIME_ERROR,stdin:e.input,stderr:(0,l.getErrorMessage)(n),timeSeconds:(r-t)/1e3});break}}}function g(e){return typeof e==`string`?{prompt:e}:{messages:e,allowSystemInMessages:!0}}function _(e,t){let n=t?.baseURL??process.env.OPENAI_COMPATIBLE_BASE_URL;if(!n)return;let r=t?.extraBody??v();return(0,o.createOpenAICompatible)({name:`openAiCompatible`,baseURL:n,apiKey:t?.apiKey??process.env.OPENAI_COMPATIBLE_API_KEY,transformRequestBody:r&&(e=>({...e,...r,model:e.model}))}).chatModel(e)}function v(){let e=process.env.OPENAI_COMPATIBLE_EXTRA_BODY;return e?m.parse(JSON.parse(e)):void 0}function y(e){let[t,n]=e.split(`/`);if(!t||!n)throw Error(`bad model: ${e}`);let r=f[t]?.(n);if(!r)throw Error(`model not found: ${e}`);return r}exports.llmJudgePreset=h;
|
|
2
2
|
//# sourceMappingURL=index.cjs.map
|
package/dist/index.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.cjs","names":["z","parseArgs","readTestCases","path","fs","generateText","DecisionCode","getErrorMessage","createOpenAICompatible","require_runtime","node_fs","node_path","_ai_sdk_amazon_bedrock","_ai_sdk_google","_ai_sdk_openai","_ai_sdk_openai_compatible","_ai_sdk_xai","_exercode_problem_utils","_willbooster_shared_lib","ai","zod","providerByName","judgeParamsSchema","jsonObjectSchema","llmJudgePreset","args","params","testCases","problemDir","prompt","testCase","startTimeMilliseconds","toOpenAiCompatibleModel","
|
|
1
|
+
{"version":3,"file":"index.cjs","names":["z","parseArgs","readTestCases","path","fs","generateText","DecisionCode","getErrorMessage","createOpenAICompatible","require_runtime","node_fs","node_path","_ai_sdk_amazon_bedrock","_ai_sdk_google","_ai_sdk_openai","_ai_sdk_openai_compatible","_ai_sdk_xai","_exercode_problem_utils","_willbooster_shared_lib","ai","zod","providerByName","judgeParamsSchema","jsonObjectSchema","llmJudgePreset","args","params","model","options","testCases","problemDir","prompt","testCase","startTimeMilliseconds","toOpenAiCompatibleModel","toLanguageModel","toPromptOptions","stopTimeMilliseconds","testCaseResult","text","error","built","baseURL","extraBody","readExtraBodyFromEnv","body","json","providerId","modelId","languageModel"],"sources":["../src/index.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { createOpenAICompatible } from '@ai-sdk/openai-compatible';\nimport { xai } from '@ai-sdk/xai';\nimport {\n DecisionCode,\n parseArgs,\n printTestCaseResult,\n readTestCases,\n type TestCaseResult,\n} from '@exercode/problem-utils';\nimport { getErrorMessage } from '@willbooster/shared-lib';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n bedrock,\n // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n google,\n // requires `OPENAI_API_KEY`\n openai,\n // requires `XAI_API_KEY`\n xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n model: z.string().min(1),\n});\n\nconst jsonObjectSchema = z.record(z.string(), z.unknown());\n\n/**\n * Connection to an OpenAI-compatible Chat Completions endpoint. Each field falls back to its environment variable.\n * Never hard-code `apiKey` or a secret in `extraBody`: `judge.ts` is distributed with the problem.\n */\ninterface OpenAiCompatibleOptions {\n /** Defaults to `OPENAI_COMPATIBLE_BASE_URL`. The endpoint is used only when a base URL is resolved. */\n baseURL?: string;\n /** Defaults to `OPENAI_COMPATIBLE_API_KEY`. */\n apiKey?: string;\n /** Top-level request body fields except `model`. Defaults to the JSON object in `OPENAI_COMPATIBLE_EXTRA_BODY`. */\n extraBody?: Record<string, unknown>;\n}\n\n/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */\ninterface LlmTestCase {\n id: string;\n input?: string;\n output?: string;\n /** Present when `<id>.fin/` exists; the preset does not copy it anywhere (there is no program run). */\n fileInputPath?: string;\n /** Present when `<id>.fout/` exists, for `test` to read. */\n fileOutputPath?: string;\n}\n\ninterface LlmJudgePresetOptions {\n buildPrompt?: (context: { prompt: string; testCase: LlmTestCase }) => string | ModelMessage[];\n test: (context: {\n testCase: LlmTestCase;\n result: { output: string };\n }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n /** Overrides the `model` judge parameter, e.g., for a problem whose prompt must be judged by a specific model. */\n model?: string;\n /** Sends every test case to this endpoint, with the model passed verbatim as the model ID. */\n openAiCompatible?: OpenAiCompatibleOptions;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils-llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n * test: (context) {\n * return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n * }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n const args = parseArgs(process.argv);\n if (!args.cwd) throw new Error('cwd argument required');\n const params = judgeParamsSchema.parse(args.params);\n const model = options.model ?? params.model;\n\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n for (const testCase of testCases) {\n const startTimeMilliseconds = Date.now();\n try {\n const { text } = await generateText({\n model: toOpenAiCompatibleModel(model, options.openAiCompatible) ?? toLanguageModel(model),\n ...toPromptOptions(\n options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? '')\n ),\n });\n\n const stopTimeMilliseconds = Date.now();\n\n const testCaseResult = {\n testCaseId: testCase.id,\n decisionCode: DecisionCode.ACCEPTED,\n stdin: testCase.input,\n stdout: text,\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n ...(await options.test({ testCase, result: { output: text } })),\n };\n\n printTestCaseResult(testCaseResult);\n\n if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n } catch (error) {\n const stopTimeMilliseconds = Date.now();\n\n printTestCaseResult({\n testCaseId: testCase.id,\n decisionCode: DecisionCode.RUNTIME_ERROR,\n stdin: testCase.input,\n stderr: getErrorMessage(error),\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n });\n\n break;\n }\n }\n}\n\n/** `ai` rejects system messages inside `messages` unless told that a `buildPrompt` may place them there. */\nfunction toPromptOptions(\n built: string | ModelMessage[]\n): { prompt: string } | { messages: ModelMessage[]; allowSystemInMessages: true } {\n return typeof built === 'string' ? { prompt: built } : { messages: built, allowSystemInMessages: true };\n}\n\nfunction toOpenAiCompatibleModel(model: string, options?: OpenAiCompatibleOptions): LanguageModel | undefined {\n const baseURL = options?.baseURL ?? process.env.OPENAI_COMPATIBLE_BASE_URL;\n if (!baseURL) return undefined;\n\n const extraBody = options?.extraBody ?? readExtraBodyFromEnv();\n return createOpenAICompatible({\n name: 'openAiCompatible',\n baseURL,\n apiKey: options?.apiKey ?? process.env.OPENAI_COMPATIBLE_API_KEY,\n transformRequestBody: extraBody && ((body) => ({ ...body, ...extraBody, model: body.model })),\n }).chatModel(model);\n}\n\nfunction readExtraBodyFromEnv(): Record<string, unknown> | undefined {\n const json = process.env.OPENAI_COMPATIBLE_EXTRA_BODY;\n return json ? jsonObjectSchema.parse(JSON.parse(json)) : undefined;\n}\n\nfunction toLanguageModel(model: string): LanguageModel {\n const [providerId, modelId] = model.split('/');\n if (!providerId || !modelId) throw new Error(`bad model: ${model}`);\n const languageModel = providerByName[providerId]?.(modelId);\n if (!languageModel) throw new Error(`model not found: ${model}`);\n return languageModel;\n}\n"],"mappings":"A,qeAoBA,MAEMqB,EAA8F,CAElG,QAAAT,EAAA,QAEA,OAAAC,EAAA,OAEA,OAAAC,EAAA,OAEA,IAAAE,EAAA,GACF,EAEMM,EAAoBtB,EAAAA,EAAE,OAAO,CACjC,MAAOA,EAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CACzB,CAAC,EAEKuB,EAAmBvB,EAAAA,EAAE,OAAOA,EAAAA,EAAE,OAAO,EAAGA,EAAAA,EAAE,QAAQ,CAAC,EA2DzD,eAAsBwB,EAAe,EAAoB,EAA+C,CACtG,IAAMC,GAAA,EAAOxB,EAAAA,UAAAA,CAAU,QAAQ,IAAI,EACnC,GAAI,CAACwB,EAAK,IAAK,MAAU,MAAM,uBAAuB,EACtD,IAAMC,EAASJ,EAAkB,MAAMG,EAAK,MAAM,EAC5CE,EAAQC,EAAQ,OAASF,EAAO,MAEhCG,EAAY,MAAA,EAAM3B,EAAAA,cAAAA,CAAcC,EAAAA,QAAK,KAAK2B,EAAY,YAAY,CAAC,EAEnEC,EAAS,MAAM3B,EAAAA,QAAG,SAAS,SAASD,EAAAA,QAAK,KAAKsB,EAAK,IAAK,YAAe,EAAG,MAAM,EAEtF,IAAK,IAAMO,KAAYH,EAAW,CAChC,IAAMI,EAAwB,KAAK,IAAI,EACvC,GAAI,CACF,GAAM,CAAE,QAAS,MAAA,EAAM5B,EAAAA,aAAAA,CAAa,CAClC,MAAO6B,EAAwBP,EAAOC,EAAQ,gBAAgB,GAAKO,EAAgBR,CAAK,EACxF,GAAGS,EACDR,EAAQ,cAAc,CAAE,SAAQ,UAAS,CAAC,GAAKG,EAAO,WAAW,UAAWC,EAAS,OAAS,EAAE,CAClG,CACF,CAAC,EAEKK,EAAuB,KAAK,IAAI,EAEhCC,EAAiB,CACrB,WAAYN,EAAS,GACrB,aAAc1B,EAAAA,aAAa,SAC3B,MAAO0B,EAAS,MAChB,OAAQO,EACR,aAAcF,EAAuBJ,GAAyB,IAC9D,GAAI,MAAML,EAAQ,KAAK,CAAE,WAAU,OAAQ,CAAE,OAAQW,CAAK,CAAE,CAAC,CAC/D,EAIA,IAFA,EAAAtB,EAAA,oBAAA,CAAoBqB,CAAc,EAE9BA,EAAe,eAAiBhC,EAAAA,aAAa,SAAU,KAC7D,OAASkC,EAAO,CACd,IAAMH,EAAuB,KAAK,IAAI,GAEtC,EAAApB,EAAA,oBAAA,CAAoB,CAClB,WAAYe,EAAS,GACrB,aAAc1B,EAAAA,aAAa,cAC3B,MAAO0B,EAAS,MAChB,QAAA,EAAQzB,EAAAA,gBAAAA,CAAgBiC,CAAK,EAC7B,aAAcH,EAAuBJ,GAAyB,GAChE,CAAC,EAED,KACF,CACF,CACF,CAGA,SAASG,EACP,EACgF,CAChF,OAAO,OAAOK,GAAU,SAAW,CAAE,OAAQA,CAAM,EAAI,CAAE,SAAUA,EAAO,sBAAuB,EAAK,CACxG,CAEA,SAASP,EAAwB,EAAe,EAA8D,CAC5G,IAAMQ,EAAUd,GAAS,SAAW,QAAQ,IAAI,2BAChD,GAAI,CAACc,EAAS,OAEd,IAAMC,EAAYf,GAAS,WAAagB,EAAqB,EAC7D,OAAA,EAAOpC,EAAAA,uBAAAA,CAAuB,CAC5B,KAAM,mBACN,UACA,OAAQoB,GAAS,QAAU,QAAQ,IAAI,0BACvC,qBAAsBe,IAAe,IAAU,CAAE,GAAGE,EAAM,GAAGF,EAAW,MAAOE,EAAK,KAAM,GAC5F,CAAC,CAAC,CAAC,UAAUlB,CAAK,CACpB,CAEA,SAASiB,GAA4D,CACnE,IAAME,EAAO,QAAQ,IAAI,6BACzB,OAAOA,EAAOvB,EAAiB,MAAM,KAAK,MAAMuB,CAAI,CAAC,EAAI,IAAA,EAC3D,CAEA,SAASX,EAAgB,EAA8B,CACrD,GAAM,CAACY,EAAYC,GAAWrB,EAAM,MAAM,GAAG,EAC7C,GAAI,CAACoB,GAAc,CAACC,EAAS,MAAU,MAAM,cAAcrB,GAAO,EAClE,IAAMsB,EAAgB5B,EAAe0B,EAAW,GAAGC,CAAO,EAC1D,GAAI,CAACC,EAAe,MAAU,MAAM,oBAAoBtB,GAAO,EAC/D,OAAOsB,CACT,C"}
|
package/dist/index.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ interface OpenAiCompatibleOptions {
|
|
|
9
9
|
baseURL?: string;
|
|
10
10
|
/** Defaults to `OPENAI_COMPATIBLE_API_KEY`. */
|
|
11
11
|
apiKey?: string;
|
|
12
|
-
/** Top-level request body fields
|
|
12
|
+
/** Top-level request body fields except `model`. Defaults to the JSON object in `OPENAI_COMPATIBLE_EXTRA_BODY`. */
|
|
13
13
|
extraBody?: Record<string, unknown>;
|
|
14
14
|
}
|
|
15
15
|
/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */
|
|
@@ -33,7 +33,9 @@ interface LlmJudgePresetOptions {
|
|
|
33
33
|
output: string;
|
|
34
34
|
};
|
|
35
35
|
}) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;
|
|
36
|
-
/**
|
|
36
|
+
/** Overrides the `model` judge parameter, e.g., for a problem whose prompt must be judged by a specific model. */
|
|
37
|
+
model?: string;
|
|
38
|
+
/** Sends every test case to this endpoint, with the model passed verbatim as the model ID. */
|
|
37
39
|
openAiCompatible?: OpenAiCompatibleOptions;
|
|
38
40
|
}
|
|
39
41
|
/**
|
package/dist/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import e from"node:fs";import t from"node:path";import{bedrock as n}from"@ai-sdk/amazon-bedrock";import{google as r}from"@ai-sdk/google";import{openai as i}from"@ai-sdk/openai";import{createOpenAICompatible as a}from"@ai-sdk/openai-compatible";import{xai as o}from"@ai-sdk/xai";import{DecisionCode as s,parseArgs as c,printTestCaseResult as l,readTestCases as u}from"@exercode/problem-utils";import{getErrorMessage as d}from"@willbooster/shared-lib";import{generateText as f}from"ai";import{z as p}from"zod";const m={bedrock:n,google:r,openai:i,xai:o},h=p.object({model:p.string().min(1)}),g=p.record(p.string(),p.unknown());async function _(n,r){let i=c(process.argv);if(!i.cwd)throw Error(`cwd argument required`);let a=h.parse(i.params),o=await u(t.join(n,`test_cases`)),
|
|
1
|
+
import e from"node:fs";import t from"node:path";import{bedrock as n}from"@ai-sdk/amazon-bedrock";import{google as r}from"@ai-sdk/google";import{openai as i}from"@ai-sdk/openai";import{createOpenAICompatible as a}from"@ai-sdk/openai-compatible";import{xai as o}from"@ai-sdk/xai";import{DecisionCode as s,parseArgs as c,printTestCaseResult as l,readTestCases as u}from"@exercode/problem-utils";import{getErrorMessage as d}from"@willbooster/shared-lib";import{generateText as f}from"ai";import{z as p}from"zod";const m={bedrock:n,google:r,openai:i,xai:o},h=p.object({model:p.string().min(1)}),g=p.record(p.string(),p.unknown());async function _(n,r){let i=c(process.argv);if(!i.cwd)throw Error(`cwd argument required`);let a=h.parse(i.params),o=r.model??a.model,p=await u(t.join(n,`test_cases`)),m=await e.promises.readFile(t.join(i.cwd,`prompt.txt`),`utf8`);for(let e of p){let t=Date.now();try{let{text:n}=await f({model:y(o,r.openAiCompatible)??x(o),...v(r.buildPrompt?.({prompt:m,testCase:e})??m.replaceAll(`{input}`,e.input??``))}),i=Date.now(),a={testCaseId:e.id,decisionCode:s.ACCEPTED,stdin:e.input,stdout:n,timeSeconds:(i-t)/1e3,...await r.test({testCase:e,result:{output:n}})};if(l(a),a.decisionCode!==s.ACCEPTED)break}catch(n){let r=Date.now();l({testCaseId:e.id,decisionCode:s.RUNTIME_ERROR,stdin:e.input,stderr:d(n),timeSeconds:(r-t)/1e3});break}}}function v(e){return typeof e==`string`?{prompt:e}:{messages:e,allowSystemInMessages:!0}}function y(e,t){let n=t?.baseURL??process.env.OPENAI_COMPATIBLE_BASE_URL;if(!n)return;let r=t?.extraBody??b();return a({name:`openAiCompatible`,baseURL:n,apiKey:t?.apiKey??process.env.OPENAI_COMPATIBLE_API_KEY,transformRequestBody:r&&(e=>({...e,...r,model:e.model}))}).chatModel(e)}function b(){let e=process.env.OPENAI_COMPATIBLE_EXTRA_BODY;return e?g.parse(JSON.parse(e)):void 0}function x(e){let[t,n]=e.split(`/`);if(!t||!n)throw Error(`bad model: ${e}`);let r=m[t]?.(n);if(!r)throw Error(`model not found: ${e}`);return r}export{_ as llmJudgePreset};
|
|
2
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":["fs","path","providerByName","judgeParamsSchema","z","jsonObjectSchema","llmJudgePreset","args","parseArgs","params","testCases","readTestCases","problemDir","prompt","testCase","startTimeMilliseconds","generateText","toOpenAiCompatibleModel","
|
|
1
|
+
{"version":3,"file":"index.js","names":["fs","path","providerByName","judgeParamsSchema","z","jsonObjectSchema","llmJudgePreset","args","parseArgs","params","model","options","testCases","readTestCases","problemDir","prompt","testCase","startTimeMilliseconds","generateText","toOpenAiCompatibleModel","toLanguageModel","toPromptOptions","stopTimeMilliseconds","testCaseResult","DecisionCode","text","printTestCaseResult","error","getErrorMessage","built","baseURL","extraBody","readExtraBodyFromEnv","createOpenAICompatible","body","json","providerId","modelId","languageModel"],"sources":["../src/index.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\n\nimport { bedrock } from '@ai-sdk/amazon-bedrock';\nimport { google } from '@ai-sdk/google';\nimport { openai } from '@ai-sdk/openai';\nimport { createOpenAICompatible } from '@ai-sdk/openai-compatible';\nimport { xai } from '@ai-sdk/xai';\nimport {\n DecisionCode,\n parseArgs,\n printTestCaseResult,\n readTestCases,\n type TestCaseResult,\n} from '@exercode/problem-utils';\nimport { getErrorMessage } from '@willbooster/shared-lib';\nimport type { LanguageModel, ModelMessage } from 'ai';\nimport { generateText } from 'ai';\nimport { z } from 'zod';\n\nconst PROMPT_FILENAME = 'prompt.txt';\n\nconst providerByName: Record<string, typeof bedrock | typeof google | typeof openai | typeof xai> = {\n // requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`\n bedrock,\n // requires `GOOGLE_GENERATIVE_AI_API_KEY`\n google,\n // requires `OPENAI_API_KEY`\n openai,\n // requires `XAI_API_KEY`\n xai,\n} as const;\n\nconst judgeParamsSchema = z.object({\n model: z.string().min(1),\n});\n\nconst jsonObjectSchema = z.record(z.string(), z.unknown());\n\n/**\n * Connection to an OpenAI-compatible Chat Completions endpoint. Each field falls back to its environment variable.\n * Never hard-code `apiKey` or a secret in `extraBody`: `judge.ts` is distributed with the problem.\n */\ninterface OpenAiCompatibleOptions {\n /** Defaults to `OPENAI_COMPATIBLE_BASE_URL`. The endpoint is used only when a base URL is resolved. */\n baseURL?: string;\n /** Defaults to `OPENAI_COMPATIBLE_API_KEY`. */\n apiKey?: string;\n /** Top-level request body fields except `model`. Defaults to the JSON object in `OPENAI_COMPATIBLE_EXTRA_BODY`. */\n extraBody?: Record<string, unknown>;\n}\n\n/** A `test_cases/` entry; the preset uses `input` for the prompt and hands everything to `test`. */\ninterface LlmTestCase {\n id: string;\n input?: string;\n output?: string;\n /** Present when `<id>.fin/` exists; the preset does not copy it anywhere (there is no program run). */\n fileInputPath?: string;\n /** Present when `<id>.fout/` exists, for `test` to read. */\n fileOutputPath?: string;\n}\n\ninterface LlmJudgePresetOptions {\n buildPrompt?: (context: { prompt: string; testCase: LlmTestCase }) => string | ModelMessage[];\n test: (context: {\n testCase: LlmTestCase;\n result: { output: string };\n }) => Partial<TestCaseResult> | Promise<Partial<TestCaseResult>>;\n /** Overrides the `model` judge parameter, e.g., for a problem whose prompt must be judged by a specific model. */\n model?: string;\n /** Sends every test case to this endpoint, with the model passed verbatim as the model ID. */\n openAiCompatible?: OpenAiCompatibleOptions;\n}\n\n/**\n * A preset judge function for running and testing a user prompt in LLM.\n *\n * @example\n * Create `judge.ts`:\n * ```ts\n * import { llmJudgePreset } from '@exercode/problem-utils-llm';\n * import { DecisionCode } from '@exercode/problem-utils';\n *\n * await llmJudgePreset(import.meta.dirname, {\n * test: (context) {\n * return { decisionCode: context.result.output ? DecisionCode.ACCEPTED : DecisionCode.WRONG_ANSWER };\n * }\n * });\n * ```\n *\n * Run with the required parameters:\n * ```bash\n * bun judge.ts model_answers/java '{ \"model\": \"gemini-2.5-flash-lite\" }'\n * ```\n */\nexport async function llmJudgePreset(problemDir: string, options: LlmJudgePresetOptions): Promise<void> {\n const args = parseArgs(process.argv);\n if (!args.cwd) throw new Error('cwd argument required');\n const params = judgeParamsSchema.parse(args.params);\n const model = options.model ?? params.model;\n\n const testCases = await readTestCases(path.join(problemDir, 'test_cases'));\n\n const prompt = await fs.promises.readFile(path.join(args.cwd, PROMPT_FILENAME), 'utf8');\n\n for (const testCase of testCases) {\n const startTimeMilliseconds = Date.now();\n try {\n const { text } = await generateText({\n model: toOpenAiCompatibleModel(model, options.openAiCompatible) ?? toLanguageModel(model),\n ...toPromptOptions(\n options.buildPrompt?.({ prompt, testCase }) ?? prompt.replaceAll('{input}', testCase.input ?? '')\n ),\n });\n\n const stopTimeMilliseconds = Date.now();\n\n const testCaseResult = {\n testCaseId: testCase.id,\n decisionCode: DecisionCode.ACCEPTED,\n stdin: testCase.input,\n stdout: text,\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n ...(await options.test({ testCase, result: { output: text } })),\n };\n\n printTestCaseResult(testCaseResult);\n\n if (testCaseResult.decisionCode !== DecisionCode.ACCEPTED) break;\n } catch (error) {\n const stopTimeMilliseconds = Date.now();\n\n printTestCaseResult({\n testCaseId: testCase.id,\n decisionCode: DecisionCode.RUNTIME_ERROR,\n stdin: testCase.input,\n stderr: getErrorMessage(error),\n timeSeconds: (stopTimeMilliseconds - startTimeMilliseconds) / 1000,\n });\n\n break;\n }\n }\n}\n\n/** `ai` rejects system messages inside `messages` unless told that a `buildPrompt` may place them there. */\nfunction toPromptOptions(\n built: string | ModelMessage[]\n): { prompt: string } | { messages: ModelMessage[]; allowSystemInMessages: true } {\n return typeof built === 'string' ? { prompt: built } : { messages: built, allowSystemInMessages: true };\n}\n\nfunction toOpenAiCompatibleModel(model: string, options?: OpenAiCompatibleOptions): LanguageModel | undefined {\n const baseURL = options?.baseURL ?? process.env.OPENAI_COMPATIBLE_BASE_URL;\n if (!baseURL) return undefined;\n\n const extraBody = options?.extraBody ?? readExtraBodyFromEnv();\n return createOpenAICompatible({\n name: 'openAiCompatible',\n baseURL,\n apiKey: options?.apiKey ?? process.env.OPENAI_COMPATIBLE_API_KEY,\n transformRequestBody: extraBody && ((body) => ({ ...body, ...extraBody, model: body.model })),\n }).chatModel(model);\n}\n\nfunction readExtraBodyFromEnv(): Record<string, unknown> | undefined {\n const json = process.env.OPENAI_COMPATIBLE_EXTRA_BODY;\n return json ? jsonObjectSchema.parse(JSON.parse(json)) : undefined;\n}\n\nfunction toLanguageModel(model: string): LanguageModel {\n const [providerId, modelId] = model.split('/');\n if (!providerId || !modelId) throw new Error(`bad model: ${model}`);\n const languageModel = providerByName[providerId]?.(modelId);\n if (!languageModel) throw new Error(`model not found: ${model}`);\n return languageModel;\n}\n"],"mappings":"A,4fAoBA,MAEME,EAA8F,CAElG,UAEA,SAEA,SAEA,KACF,EAEMC,EAAoBC,EAAE,OAAO,CACjC,MAAOA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CACzB,CAAC,EAEKC,EAAmBD,EAAE,OAAOA,EAAE,OAAO,EAAGA,EAAE,QAAQ,CAAC,EA2DzD,eAAsBE,EAAe,EAAoB,EAA+C,CACtG,IAAMC,EAAOC,EAAU,QAAQ,IAAI,EACnC,GAAI,CAACD,EAAK,IAAK,MAAU,MAAM,uBAAuB,EACtD,IAAME,EAASN,EAAkB,MAAMI,EAAK,MAAM,EAC5CG,EAAQC,EAAQ,OAASF,EAAO,MAEhCG,EAAY,MAAMC,EAAcZ,EAAK,KAAKa,EAAY,YAAY,CAAC,EAEnEC,EAAS,MAAMf,EAAG,SAAS,SAASC,EAAK,KAAKM,EAAK,IAAK,YAAe,EAAG,MAAM,EAEtF,IAAK,IAAMS,KAAYJ,EAAW,CAChC,IAAMK,EAAwB,KAAK,IAAI,EACvC,GAAI,CACF,GAAM,CAAE,QAAS,MAAMC,EAAa,CAClC,MAAOC,EAAwBT,EAAOC,EAAQ,gBAAgB,GAAKS,EAAgBV,CAAK,EACxF,GAAGW,EACDV,EAAQ,cAAc,CAAE,SAAQ,UAAS,CAAC,GAAKI,EAAO,WAAW,UAAWC,EAAS,OAAS,EAAE,CAClG,CACF,CAAC,EAEKM,EAAuB,KAAK,IAAI,EAEhCC,EAAiB,CACrB,WAAYP,EAAS,GACrB,aAAcQ,EAAa,SAC3B,MAAOR,EAAS,MAChB,OAAQS,EACR,aAAcH,EAAuBL,GAAyB,IAC9D,GAAI,MAAMN,EAAQ,KAAK,CAAE,WAAU,OAAQ,CAAE,OAAQc,CAAK,CAAE,CAAC,CAC/D,EAIA,GAFAC,EAAoBH,CAAc,EAE9BA,EAAe,eAAiBC,EAAa,SAAU,KAC7D,OAASG,EAAO,CACd,IAAML,EAAuB,KAAK,IAAI,EAEtCI,EAAoB,CAClB,WAAYV,EAAS,GACrB,aAAcQ,EAAa,cAC3B,MAAOR,EAAS,MAChB,OAAQY,EAAgBD,CAAK,EAC7B,aAAcL,EAAuBL,GAAyB,GAChE,CAAC,EAED,KACF,CACF,CACF,CAGA,SAASI,EACP,EACgF,CAChF,OAAO,OAAOQ,GAAU,SAAW,CAAE,OAAQA,CAAM,EAAI,CAAE,SAAUA,EAAO,sBAAuB,EAAK,CACxG,CAEA,SAASV,EAAwB,EAAe,EAA8D,CAC5G,IAAMW,EAAUnB,GAAS,SAAW,QAAQ,IAAI,2BAChD,GAAI,CAACmB,EAAS,OAEd,IAAMC,EAAYpB,GAAS,WAAaqB,EAAqB,EAC7D,OAAOC,EAAuB,CAC5B,KAAM,mBACN,UACA,OAAQtB,GAAS,QAAU,QAAQ,IAAI,0BACvC,qBAAsBoB,IAAe,IAAU,CAAE,GAAGG,EAAM,GAAGH,EAAW,MAAOG,EAAK,KAAM,GAC5F,CAAC,CAAC,CAAC,UAAUxB,CAAK,CACpB,CAEA,SAASsB,GAA4D,CACnE,IAAMG,EAAO,QAAQ,IAAI,6BACzB,OAAOA,EAAO9B,EAAiB,MAAM,KAAK,MAAM8B,CAAI,CAAC,EAAI,IAAA,EAC3D,CAEA,SAASf,EAAgB,EAA8B,CACrD,GAAM,CAACgB,EAAYC,GAAW3B,EAAM,MAAM,GAAG,EAC7C,GAAI,CAAC0B,GAAc,CAACC,EAAS,MAAU,MAAM,cAAc3B,GAAO,EAClE,IAAM4B,EAAgBpC,EAAekC,EAAW,GAAGC,CAAO,EAC1D,GAAI,CAACC,EAAe,MAAU,MAAM,oBAAoB5B,GAAO,EAC/D,OAAO4B,CACT,C"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@exercode/problem-utils-llm",
|
|
3
|
-
"version": "1.1
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"description": "Exercode llm judging utilities",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "WillBooster Inc.",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"@ai-sdk/openai": "4.0.101",
|
|
46
46
|
"@ai-sdk/openai-compatible": "3.0.78",
|
|
47
47
|
"@ai-sdk/xai": "5.0.29",
|
|
48
|
-
"@exercode/problem-utils": "5.0
|
|
48
|
+
"@exercode/problem-utils": "5.1.0",
|
|
49
49
|
"@willbooster/shared-lib": "11.1.0",
|
|
50
50
|
"ai": "7.0.152",
|
|
51
51
|
"zod": "4.6.7"
|