@mzhub/promptc 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +224 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +111 -0
- package/dist/cli.js.map +1 -0
- package/dist/compiler/BootstrapFewShot.d.ts +13 -0
- package/dist/compiler/BootstrapFewShot.d.ts.map +1 -0
- package/dist/compiler/BootstrapFewShot.js +93 -0
- package/dist/compiler/BootstrapFewShot.js.map +1 -0
- package/dist/compiler/CandidatePool.d.ts +10 -0
- package/dist/compiler/CandidatePool.d.ts.map +1 -0
- package/dist/compiler/CandidatePool.js +29 -0
- package/dist/compiler/CandidatePool.js.map +1 -0
- package/dist/compiler/CompiledProgram.d.ts +43 -0
- package/dist/compiler/CompiledProgram.d.ts.map +1 -0
- package/dist/compiler/CompiledProgram.js +41 -0
- package/dist/compiler/CompiledProgram.js.map +1 -0
- package/dist/compiler/InstructionRewrite.d.ts +19 -0
- package/dist/compiler/InstructionRewrite.d.ts.map +1 -0
- package/dist/compiler/InstructionRewrite.js +117 -0
- package/dist/compiler/InstructionRewrite.js.map +1 -0
- package/dist/compiler/index.d.ts +8 -0
- package/dist/compiler/index.d.ts.map +1 -0
- package/dist/compiler/index.js +5 -0
- package/dist/compiler/index.js.map +1 -0
- package/dist/compiler/types.d.ts +41 -0
- package/dist/compiler/types.d.ts.map +1 -0
- package/dist/compiler/types.js +2 -0
- package/dist/compiler/types.js.map +1 -0
- package/dist/eval/exactMatch.d.ts +5 -0
- package/dist/eval/exactMatch.d.ts.map +1 -0
- package/dist/eval/exactMatch.js +58 -0
- package/dist/eval/exactMatch.js.map +1 -0
- package/dist/eval/index.d.ts +5 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +3 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/llmJudge.d.ts +9 -0
- package/dist/eval/llmJudge.d.ts.map +1 -0
- package/dist/eval/llmJudge.js +33 -0
- package/dist/eval/llmJudge.js.map +1 -0
- package/dist/eval/types.d.ts +2 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +2 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/program/ChainOfThought.d.ts +6 -0
- package/dist/program/ChainOfThought.d.ts.map +1 -0
- package/dist/program/ChainOfThought.js +44 -0
- package/dist/program/ChainOfThought.js.map +1 -0
- package/dist/program/Predict.d.ts +6 -0
- package/dist/program/Predict.d.ts.map +1 -0
- package/dist/program/Predict.js +33 -0
- package/dist/program/Predict.js.map +1 -0
- package/dist/program/Program.d.ts +33 -0
- package/dist/program/Program.d.ts.map +1 -0
- package/dist/program/Program.js +28 -0
- package/dist/program/Program.js.map +1 -0
- package/dist/program/index.d.ts +5 -0
- package/dist/program/index.d.ts.map +1 -0
- package/dist/program/index.js +4 -0
- package/dist/program/index.js.map +1 -0
- package/dist/providers/anthropic.d.ts +10 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +40 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/cerebras.d.ts +10 -0
- package/dist/providers/cerebras.d.ts.map +1 -0
- package/dist/providers/cerebras.js +39 -0
- package/dist/providers/cerebras.js.map +1 -0
- package/dist/providers/google.d.ts +10 -0
- package/dist/providers/google.d.ts.map +1 -0
- package/dist/providers/google.js +42 -0
- package/dist/providers/google.js.map +1 -0
- package/dist/providers/groq.d.ts +10 -0
- package/dist/providers/groq.d.ts.map +1 -0
- package/dist/providers/groq.js +42 -0
- package/dist/providers/groq.js.map +1 -0
- package/dist/providers/index.d.ts +11 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +31 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/ollama.d.ts +9 -0
- package/dist/providers/ollama.d.ts.map +1 -0
- package/dist/providers/ollama.js +39 -0
- package/dist/providers/ollama.js.map +1 -0
- package/dist/providers/openai.d.ts +10 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +42 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/types.d.ts +25 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/runtime/cache.d.ts +18 -0
- package/dist/runtime/cache.d.ts.map +1 -0
- package/dist/runtime/cache.js +45 -0
- package/dist/runtime/cache.js.map +1 -0
- package/dist/runtime/concurrency.d.ts +7 -0
- package/dist/runtime/concurrency.d.ts.map +1 -0
- package/dist/runtime/concurrency.js +14 -0
- package/dist/runtime/concurrency.js.map +1 -0
- package/dist/runtime/costTracker.d.ts +24 -0
- package/dist/runtime/costTracker.d.ts.map +1 -0
- package/dist/runtime/costTracker.js +37 -0
- package/dist/runtime/costTracker.js.map +1 -0
- package/dist/runtime/index.d.ts +9 -0
- package/dist/runtime/index.d.ts.map +1 -0
- package/dist/runtime/index.js +5 -0
- package/dist/runtime/index.js.map +1 -0
- package/dist/runtime/retry.d.ts +10 -0
- package/dist/runtime/retry.d.ts.map +1 -0
- package/dist/runtime/retry.js +39 -0
- package/dist/runtime/retry.js.map +1 -0
- package/dist/schema/defineSchema.d.ts +18 -0
- package/dist/schema/defineSchema.d.ts.map +1 -0
- package/dist/schema/defineSchema.js +27 -0
- package/dist/schema/defineSchema.js.map +1 -0
- package/dist/schema/index.d.ts +3 -0
- package/dist/schema/index.d.ts.map +1 -0
- package/dist/schema/index.js +2 -0
- package/dist/schema/index.js.map +1 -0
- package/examples/README.md +42 -0
- package/examples/load-compiled.ts +62 -0
- package/examples/multi-provider.ts +77 -0
- package/examples/name-extractor.ts +113 -0
- package/examples/qa-system.ts +98 -0
- package/package.json +62 -0
- package/src/cli.ts +122 -0
- package/src/compiler/BootstrapFewShot.ts +149 -0
- package/src/compiler/CandidatePool.ts +39 -0
- package/src/compiler/CompiledProgram.ts +112 -0
- package/src/compiler/InstructionRewrite.ts +200 -0
- package/src/compiler/index.ts +19 -0
- package/src/compiler/types.ts +46 -0
- package/src/eval/exactMatch.ts +65 -0
- package/src/eval/index.ts +4 -0
- package/src/eval/llmJudge.ts +45 -0
- package/src/eval/types.ts +4 -0
- package/src/index.ts +71 -0
- package/src/program/ChainOfThought.ts +59 -0
- package/src/program/Predict.ts +47 -0
- package/src/program/Program.ts +64 -0
- package/src/program/index.ts +4 -0
- package/src/providers/anthropic.ts +55 -0
- package/src/providers/cerebras.ts +53 -0
- package/src/providers/google.ts +57 -0
- package/src/providers/groq.ts +57 -0
- package/src/providers/index.ts +50 -0
- package/src/providers/ollama.ts +54 -0
- package/src/providers/openai.ts +57 -0
- package/src/providers/types.ts +27 -0
- package/src/runtime/cache.ts +65 -0
- package/src/runtime/concurrency.ts +21 -0
- package/src/runtime/costTracker.ts +58 -0
- package/src/runtime/index.ts +8 -0
- package/src/runtime/retry.ts +59 -0
- package/src/schema/defineSchema.ts +44 -0
- package/src/schema/index.ts +2 -0
- package/tests/candidatePool.test.ts +46 -0
- package/tests/evaluators.test.ts +69 -0
- package/tests/runtime.test.ts +106 -0
- package/tests/schema.test.ts +59 -0
- package/tsconfig.json +24 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { createConcurrencyManager } from "../runtime/concurrency.js";
|
|
2
|
+
import { CostTracker } from "../runtime/costTracker.js";
|
|
3
|
+
import { CandidatePool } from "./CandidatePool.js";
|
|
4
|
+
export class InstructionRewrite {
|
|
5
|
+
evaluator;
|
|
6
|
+
constructor(evaluator) {
|
|
7
|
+
this.evaluator = evaluator;
|
|
8
|
+
}
|
|
9
|
+
async compile(program, trainset, options) {
|
|
10
|
+
const { candidates: candidateCount = 5, instructionVariations = 5, concurrency = 5, examplesPerCandidate = 3, validationSplit = 0.3, seed, budget, provider, } = options;
|
|
11
|
+
const baseDescription = program["schema"].description;
|
|
12
|
+
const pool = new CandidatePool(trainset, seed);
|
|
13
|
+
const validationSet = pool.getValidationSet(validationSplit);
|
|
14
|
+
const limiter = createConcurrencyManager(concurrency);
|
|
15
|
+
const costTracker = new CostTracker();
|
|
16
|
+
const instructionVariants = await this.generateInstructionVariants(provider, baseDescription, instructionVariations);
|
|
17
|
+
const fewShotCandidates = pool.generateFewShotCandidates(candidateCount, examplesPerCandidate);
|
|
18
|
+
const allCandidates = [];
|
|
19
|
+
for (const instruction of instructionVariants) {
|
|
20
|
+
for (const examples of fewShotCandidates) {
|
|
21
|
+
allCandidates.push({ instructions: instruction, examples });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const results = [];
|
|
25
|
+
const evaluationPromises = allCandidates.map((candidate) => limiter.run(async () => {
|
|
26
|
+
if (budget?.maxTokens && costTracker.exceedsBudget(budget.maxTokens)) {
|
|
27
|
+
budget.onBudgetWarning?.(costTracker.total.totalTokens, budget.maxTokens);
|
|
28
|
+
return { score: -1, ...candidate };
|
|
29
|
+
}
|
|
30
|
+
let totalScore = 0;
|
|
31
|
+
let validCount = 0;
|
|
32
|
+
for (const testCase of validationSet) {
|
|
33
|
+
try {
|
|
34
|
+
const prediction = await program.run(testCase.input, {
|
|
35
|
+
instructions: candidate.instructions,
|
|
36
|
+
fewShotExamples: candidate.examples.map((ex) => ({
|
|
37
|
+
input: ex.input,
|
|
38
|
+
output: ex.output,
|
|
39
|
+
})),
|
|
40
|
+
});
|
|
41
|
+
costTracker.record(prediction.trace.usage);
|
|
42
|
+
const score = await this.evaluator(prediction.result, testCase.output);
|
|
43
|
+
totalScore += score;
|
|
44
|
+
validCount++;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
score: validCount > 0 ? totalScore / validCount : 0,
|
|
52
|
+
...candidate,
|
|
53
|
+
};
|
|
54
|
+
}));
|
|
55
|
+
const candidateResults = await Promise.all(evaluationPromises);
|
|
56
|
+
results.push(...candidateResults.filter((r) => r.score >= 0));
|
|
57
|
+
const winner = results.reduce((best, current) => (current.score > best.score ? current : best), {
|
|
58
|
+
score: -1,
|
|
59
|
+
instructions: baseDescription,
|
|
60
|
+
examples: [],
|
|
61
|
+
});
|
|
62
|
+
return {
|
|
63
|
+
meta: {
|
|
64
|
+
score: winner.score,
|
|
65
|
+
compiledAt: new Date().toISOString(),
|
|
66
|
+
strategy: "InstructionRewrite",
|
|
67
|
+
tokenUsage: costTracker.total,
|
|
68
|
+
},
|
|
69
|
+
config: {
|
|
70
|
+
instructions: winner.instructions,
|
|
71
|
+
fewShotExamples: winner.examples,
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
async generateInstructionVariants(provider, baseInstruction, count) {
|
|
76
|
+
const prompt = `You are an expert prompt engineer. Given a base instruction for an LLM task, generate ${count} different variations that might perform better.
|
|
77
|
+
|
|
78
|
+
Base instruction: "${baseInstruction}"
|
|
79
|
+
|
|
80
|
+
Requirements:
|
|
81
|
+
- Each variation should convey the same task but with different wording, tone, or structure
|
|
82
|
+
- Try variations like: more specific, more concise, role-based ("You are an expert..."), step-by-step, formal, casual
|
|
83
|
+
- Output as a JSON array of strings
|
|
84
|
+
|
|
85
|
+
Respond with ONLY a JSON array, no additional text:
|
|
86
|
+
["variation 1", "variation 2", ...]`;
|
|
87
|
+
const response = await provider.complete({
|
|
88
|
+
prompt,
|
|
89
|
+
temperature: 0.8,
|
|
90
|
+
maxTokens: 1024,
|
|
91
|
+
responseFormat: "json",
|
|
92
|
+
});
|
|
93
|
+
try {
|
|
94
|
+
const jsonMatch = response.content.match(/\[[\s\S]*\]/);
|
|
95
|
+
if (!jsonMatch) {
|
|
96
|
+
return [baseInstruction];
|
|
97
|
+
}
|
|
98
|
+
const variations = JSON.parse(jsonMatch[0]);
|
|
99
|
+
return [baseInstruction, ...variations.slice(0, count)];
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return [baseInstruction];
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
estimateCost(trainsetSize, options = {}) {
|
|
106
|
+
const { candidates = 5, instructionVariations = 5, validationSplit = 0.3, } = options;
|
|
107
|
+
const validationSize = Math.max(1, Math.floor(trainsetSize * validationSplit));
|
|
108
|
+
const totalCandidates = candidates * instructionVariations;
|
|
109
|
+
const estimatedCalls = totalCandidates * validationSize + 1;
|
|
110
|
+
const avgTokensPerCall = 500;
|
|
111
|
+
return {
|
|
112
|
+
estimatedCalls,
|
|
113
|
+
estimatedTokens: estimatedCalls * avgTokensPerCall,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=InstructionRewrite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"InstructionRewrite.js","sourceRoot":"","sources":["../../src/compiler/InstructionRewrite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAcnD,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,SAAuB;QAAvB,cAAS,GAAT,SAAS,CAAc;IAAG,CAAC;IAE/C,KAAK,CAAC,OAAO,CACX,OAA2B,EAC3B,QAA+B,EAC/B,OAAkC;QAElC,MAAM,EACJ,UAAU,EAAE,cAAc,GAAG,CAAC,EAC9B,qBAAqB,GAAG,CAAC,EACzB,WAAW,GAAG,CAAC,EACf,oBAAoB,GAAG,CAAC,EACxB,eAAe,GAAG,GAAG,EACrB,IAAI,EACJ,MAAM,EACN,QAAQ,GACT,GAAG,OAAO,CAAC;QAEZ,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QAEtC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAChE,QAAQ,EACR,eAAe,EACf,qBAAqB,CACtB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,yBAAyB,CACtD,cAAc,EACd,oBAAoB,CACrB,CAAC;QAEF,MAAM,aAAa,GAGd,EAAE,CAAC;QACR,KAAK,MAAM,WAAW,IAAI,mBAAmB,EAAE,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACzC,aAAa,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAIR,EAAE,CAAC;QAER,MAAM,kBAAkB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACzD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACrB,IAAI,MAAM,EAAE,SAAS,IAAI,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrE,MAAM,CAAC,eAAe,EAAE,CACtB,WAAW,CAAC,KAAK,CAAC,WAAW,EAC7B,MAAM,CAAC,SAAS,CACjB,CAAC;gBACF,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,CAAC;YACrC,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAY,EAAE;wBAC1D,YAAY,EAAE,SAAS,CAAC,YAAY;wBACpC,eAAe,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;4BAC/C,KAAK,EAAE,EAAE,CAAC,KAAgC;4BAC1C,MAAM,EAAE,EAAE,CAAC,MAAiC;yBAC7C,CAAC,CAAC;qBACJ,CAAC,CAAC;oBAEH,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAChC,UAAU,CAAC,MAAW,EACtB,QAAQ,CAAC,MAAM,CAChB,CAAC;oBACF,UAAU,IAAI,KAAK,CAAC;oBACpB,UAAU,EAAE,CAAC;gBACf,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;YACH,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBACnD,GAAG,SAAS;aACb,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAC3B,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAChE;YACE,KAAK,EAAE,CAAC,CAAC;YACT,YAAY,EAAE,eAAe;YAC7B,QAAQ,EAAE,EAA2B;SACtC,CACF,CAAC;QAEF,OAAO;YACL,IAAI,EAAE;gBACJ,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,QAAQ,EAAE,oBAAoB;gBAC9B,UAAU,EAAE,WAAW,CAAC,KAAK;aAC9B;YACD,MAAM,EAAE;gBACN,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,eAAe,EAAE,MAAM,CAAC,QAAQ;aACjC;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAAqB,EACrB,eAAuB,EACvB,KAAa;QAEb,MAAM,MAAM,GAAG,yFAAyF,KAAK;;qBAE5F,eAAe;;;;;;;;oCAQA,CAAC;QAEjC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC;YACvC,MAAM;YACN,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,eAAe,CAAC,CAAC;YAC3B,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAa,CAAC;YACxD,OAAO,CAAC,eAAe,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,eAAe,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,YAAY,CACV,YAAoB,EACpB,UAA8C,EAAE;QAKhD,MAAM,EACJ,UAAU,GAAG,CAAC,EACd,qBAAqB,GAAG,CAAC,EACzB,eAAe,GAAG,GAAG,GACtB,GAAG,OAAO,CAAC;QACZ,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,eAAe,CAAC,CAC3C,CAAC;QACF,MAAM,eAAe,GAAG,UAAU,GAAG,qBAAqB,CAAC;QAC3D,MAAM,cAAc,GAAG,eAAe,GAAG,cAAc,GAAG,CAAC,CAAC;QAC5D,MAAM,gBAAgB,GAAG,GAAG,CAAC;QAE7B,OAAO;YACL,cAAc;YACd,eAAe,EAAE,cAAc,GAAG,gBAAgB;SACnD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { BootstrapFewShot } from "./BootstrapFewShot.js";
|
|
2
|
+
export { InstructionRewrite } from "./InstructionRewrite.js";
|
|
3
|
+
export type { InstructionRewriteOptions } from "./InstructionRewrite.js";
|
|
4
|
+
export { CandidatePool } from "./CandidatePool.js";
|
|
5
|
+
export { createCompiledProgram, loadCompiledProgram, } from "./CompiledProgram.js";
|
|
6
|
+
export type { CompiledProgram, CompiledProgramMeta, SerializedCompiledProgram, } from "./CompiledProgram.js";
|
|
7
|
+
export type { Evaluator, Example, CompilationResult, CompileOptions, } from "./types.js";
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/compiler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,YAAY,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,eAAe,EACf,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,GACf,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { BootstrapFewShot } from "./BootstrapFewShot.js";
|
|
2
|
+
export { InstructionRewrite } from "./InstructionRewrite.js";
|
|
3
|
+
export { CandidatePool } from "./CandidatePool.js";
|
|
4
|
+
export { createCompiledProgram, loadCompiledProgram, } from "./CompiledProgram.js";
|
|
5
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/compiler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export type Evaluator<O> = (prediction: O, groundTruth: O) => number | Promise<number>;
|
|
2
|
+
export interface Example<I, O> {
|
|
3
|
+
input: I;
|
|
4
|
+
output: O;
|
|
5
|
+
}
|
|
6
|
+
export interface CompilationResult<I, O> {
|
|
7
|
+
meta: {
|
|
8
|
+
score: number;
|
|
9
|
+
compiledAt: string;
|
|
10
|
+
strategy: string;
|
|
11
|
+
tokenUsage: {
|
|
12
|
+
inputTokens: number;
|
|
13
|
+
outputTokens: number;
|
|
14
|
+
totalTokens: number;
|
|
15
|
+
calls: number;
|
|
16
|
+
};
|
|
17
|
+
};
|
|
18
|
+
config: {
|
|
19
|
+
instructions: string;
|
|
20
|
+
fewShotExamples: Array<Example<I, O>>;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export interface CompileOptions {
|
|
24
|
+
candidates?: number;
|
|
25
|
+
concurrency?: number;
|
|
26
|
+
examplesPerCandidate?: number;
|
|
27
|
+
validationSplit?: number;
|
|
28
|
+
seed?: number;
|
|
29
|
+
earlyStopThreshold?: number;
|
|
30
|
+
budget?: {
|
|
31
|
+
maxTokens?: number;
|
|
32
|
+
onBudgetWarning?: (used: number, max: number) => void;
|
|
33
|
+
};
|
|
34
|
+
onProgress?: (progress: {
|
|
35
|
+
candidatesEvaluated: number;
|
|
36
|
+
totalCandidates: number;
|
|
37
|
+
currentBestScore: number;
|
|
38
|
+
tokensUsed: number;
|
|
39
|
+
}) => void;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/compiler/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CACzB,UAAU,EAAE,CAAC,EACb,WAAW,EAAE,CAAC,KACX,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAE9B,MAAM,WAAW,OAAO,CAAC,CAAC,EAAE,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC;IACT,MAAM,EAAE,CAAC,CAAC;CACX;AAED,MAAM,WAAW,iBAAiB,CAAC,CAAC,EAAE,CAAC;IACrC,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE;YACV,WAAW,EAAE,MAAM,CAAC;YACpB,YAAY,EAAE,MAAM,CAAC;YACrB,WAAW,EAAE,MAAM,CAAC;YACpB,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC;IACF,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,MAAM,CAAC,EAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;KACvD,CAAC;IACF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QACtB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;KACpB,KAAK,IAAI,CAAC;CACZ"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/compiler/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Evaluator } from "./types.js";
|
|
2
|
+
export declare function exactMatch<O>(): Evaluator<O>;
|
|
3
|
+
export declare function partialMatch<O extends Record<string, unknown>>(): Evaluator<O>;
|
|
4
|
+
export declare function arrayOverlap<T>(): Evaluator<T[]>;
|
|
5
|
+
//# sourceMappingURL=exactMatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exactMatch.d.ts","sourceRoot":"","sources":["../../src/eval/exactMatch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAyB5C,wBAAgB,UAAU,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAI5C;AAED,wBAAgB,YAAY,CAC1B,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,SAAS,CAAC,CAAC,CAAC,CAchB;AAED,wBAAgB,YAAY,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,EAAE,CAAC,CAehD"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
function deepEqual(a, b) {
|
|
2
|
+
if (a === b)
|
|
3
|
+
return true;
|
|
4
|
+
if (typeof a !== typeof b)
|
|
5
|
+
return false;
|
|
6
|
+
if (a === null || b === null)
|
|
7
|
+
return a === b;
|
|
8
|
+
if (Array.isArray(a) && Array.isArray(b)) {
|
|
9
|
+
if (a.length !== b.length)
|
|
10
|
+
return false;
|
|
11
|
+
return a.every((item, i) => deepEqual(item, b[i]));
|
|
12
|
+
}
|
|
13
|
+
if (typeof a === "object" && typeof b === "object") {
|
|
14
|
+
const aObj = a;
|
|
15
|
+
const bObj = b;
|
|
16
|
+
const aKeys = Object.keys(aObj);
|
|
17
|
+
const bKeys = Object.keys(bObj);
|
|
18
|
+
if (aKeys.length !== bKeys.length)
|
|
19
|
+
return false;
|
|
20
|
+
return aKeys.every((key) => deepEqual(aObj[key], bObj[key]));
|
|
21
|
+
}
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
export function exactMatch() {
|
|
25
|
+
return (prediction, groundTruth) => {
|
|
26
|
+
return deepEqual(prediction, groundTruth) ? 1.0 : 0.0;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export function partialMatch() {
|
|
30
|
+
return (prediction, groundTruth) => {
|
|
31
|
+
const keys = Object.keys(groundTruth);
|
|
32
|
+
if (keys.length === 0)
|
|
33
|
+
return 1.0;
|
|
34
|
+
let matchCount = 0;
|
|
35
|
+
for (const key of keys) {
|
|
36
|
+
if (deepEqual(prediction[key], groundTruth[key])) {
|
|
37
|
+
matchCount++;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return matchCount / keys.length;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export function arrayOverlap() {
|
|
44
|
+
return (prediction, groundTruth) => {
|
|
45
|
+
if (groundTruth.length === 0)
|
|
46
|
+
return prediction.length === 0 ? 1.0 : 0.0;
|
|
47
|
+
const predSet = new Set(prediction.map((x) => JSON.stringify(x)));
|
|
48
|
+
const truthSet = new Set(groundTruth.map((x) => JSON.stringify(x)));
|
|
49
|
+
let intersection = 0;
|
|
50
|
+
for (const item of predSet) {
|
|
51
|
+
if (truthSet.has(item))
|
|
52
|
+
intersection++;
|
|
53
|
+
}
|
|
54
|
+
const union = new Set([...predSet, ...truthSet]).size;
|
|
55
|
+
return union > 0 ? intersection / union : 0;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=exactMatch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exactMatch.js","sourceRoot":"","sources":["../../src/eval/exactMatch.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,CAAU,EAAE,CAAU;IACvC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACxC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAChD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO,CAAC,UAAa,EAAE,WAAc,EAAU,EAAE;QAC/C,OAAO,SAAS,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACxD,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY;IAG1B,OAAO,CAAC,UAAa,EAAE,WAAc,EAAU,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAElC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACjD,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,CAAC,UAAe,EAAE,WAAgB,EAAU,EAAE;QACnD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAEzE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,YAAY,EAAE,CAAC;QACzC,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QACtD,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,YAAY,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { LLMProvider } from "../providers/types.js";
|
|
2
|
+
import type { Evaluator } from "./types.js";
|
|
3
|
+
export interface LLMJudgeConfig {
|
|
4
|
+
provider: LLMProvider;
|
|
5
|
+
criteria?: string;
|
|
6
|
+
model?: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function llmJudge<O>(config: LLMJudgeConfig): Evaluator<O>;
|
|
9
|
+
//# sourceMappingURL=llmJudge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llmJudge.d.ts","sourceRoot":"","sources":["../../src/eval/llmJudge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,CAmChE"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export function llmJudge(config) {
|
|
2
|
+
const { provider, criteria = "accuracy and completeness", model } = config;
|
|
3
|
+
return async (prediction, groundTruth) => {
|
|
4
|
+
const prompt = `You are evaluating the quality of an AI response.
|
|
5
|
+
|
|
6
|
+
EXPECTED OUTPUT:
|
|
7
|
+
${JSON.stringify(groundTruth, null, 2)}
|
|
8
|
+
|
|
9
|
+
ACTUAL OUTPUT:
|
|
10
|
+
${JSON.stringify(prediction, null, 2)}
|
|
11
|
+
|
|
12
|
+
EVALUATION CRITERIA: ${criteria}
|
|
13
|
+
|
|
14
|
+
Rate the actual output on a scale from 0.0 to 1.0, where:
|
|
15
|
+
- 1.0 = Perfect match or equivalent
|
|
16
|
+
- 0.5 = Partially correct
|
|
17
|
+
- 0.0 = Completely wrong
|
|
18
|
+
|
|
19
|
+
Respond with ONLY a number between 0.0 and 1.0, nothing else.`;
|
|
20
|
+
const response = await provider.complete({
|
|
21
|
+
prompt,
|
|
22
|
+
model,
|
|
23
|
+
temperature: 0,
|
|
24
|
+
maxTokens: 10,
|
|
25
|
+
});
|
|
26
|
+
const score = parseFloat(response.content.trim());
|
|
27
|
+
if (isNaN(score) || score < 0 || score > 1) {
|
|
28
|
+
return 0;
|
|
29
|
+
}
|
|
30
|
+
return score;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=llmJudge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llmJudge.js","sourceRoot":"","sources":["../../src/eval/llmJudge.ts"],"names":[],"mappings":"AASA,MAAM,UAAU,QAAQ,CAAI,MAAsB;IAChD,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,2BAA2B,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAE3E,OAAO,KAAK,EAAE,UAAa,EAAE,WAAc,EAAmB,EAAE;QAC9D,MAAM,MAAM,GAAG;;;EAGjB,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;;;EAGpC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;;uBAEd,QAAQ;;;;;;;8DAO+B,CAAC;QAE3D,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC;YACvC,MAAM;YACN,KAAK;YACL,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,EAAE;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CACzB,UAAU,EAAE,CAAC,EACb,WAAW,EAAE,CAAC,KACX,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":""}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { z } from "zod";
|
|
2
|
+
export { Schema, defineSchema } from "./schema/index.js";
|
|
3
|
+
export type { SchemaDefinition } from "./schema/index.js";
|
|
4
|
+
export { createProvider, OpenAIProvider, AnthropicProvider, GoogleProvider, OllamaProvider, GroqProvider, CerebrasProvider, } from "./providers/index.js";
|
|
5
|
+
export type { LLMProvider, CompletionParams, CompletionResult, ProviderConfig, ProviderName, } from "./providers/index.js";
|
|
6
|
+
export { Program, Predict, ChainOfThought } from "./program/index.js";
|
|
7
|
+
export type { ProgramConfig, ProgramOutput, ProgramTrace, } from "./program/index.js";
|
|
8
|
+
export { BootstrapFewShot, InstructionRewrite, CandidatePool, createCompiledProgram, loadCompiledProgram, } from "./compiler/index.js";
|
|
9
|
+
export type { Evaluator, Example, CompilationResult, CompileOptions, CompiledProgram, CompiledProgramMeta, SerializedCompiledProgram, InstructionRewriteOptions, } from "./compiler/index.js";
|
|
10
|
+
export { exactMatch, partialMatch, arrayOverlap, llmJudge, } from "./eval/index.js";
|
|
11
|
+
export type { LLMJudgeConfig } from "./eval/index.js";
|
|
12
|
+
export { createConcurrencyManager, CostTracker, estimateCost, withRetry, createRetryWrapper, PromptCache, createCache, } from "./runtime/index.js";
|
|
13
|
+
export type { ConcurrencyManager, TokenUsage, CostEstimate, RetryOptions, CacheOptions, } from "./runtime/index.js";
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,OAAO,EACL,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,YAAY,EACZ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,GACb,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,aAAa,EACb,aAAa,EACb,YAAY,GACb,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,mBAAmB,EACnB,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,QAAQ,GACT,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EACL,wBAAwB,EACxB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,kBAAkB,EAClB,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,YAAY,GACb,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { z } from "zod";
|
|
2
|
+
export { Schema, defineSchema } from "./schema/index.js";
|
|
3
|
+
export { createProvider, OpenAIProvider, AnthropicProvider, GoogleProvider, OllamaProvider, GroqProvider, CerebrasProvider, } from "./providers/index.js";
|
|
4
|
+
export { Program, Predict, ChainOfThought } from "./program/index.js";
|
|
5
|
+
export { BootstrapFewShot, InstructionRewrite, CandidatePool, createCompiledProgram, loadCompiledProgram, } from "./compiler/index.js";
|
|
6
|
+
export { exactMatch, partialMatch, arrayOverlap, llmJudge, } from "./eval/index.js";
|
|
7
|
+
export { createConcurrencyManager, CostTracker, estimateCost, withRetry, createRetryWrapper, PromptCache, createCache, } from "./runtime/index.js";
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGzD,OAAO,EACL,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,YAAY,EACZ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAOtE,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAY7B,OAAO,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,QAAQ,GACT,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,wBAAwB,EACxB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { z, ZodRawShape, ZodObject } from "zod";
|
|
2
|
+
import { Program, ProgramConfig, ProgramOutput } from "./Program.js";
|
|
3
|
+
export declare class ChainOfThought<I extends ZodRawShape, O extends ZodRawShape> extends Program<I, O> {
|
|
4
|
+
run(input: z.infer<ZodObject<I>>, config?: ProgramConfig): Promise<ProgramOutput<z.infer<ZodObject<O>>>>;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=ChainOfThought.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ChainOfThought.d.ts","sourceRoot":"","sources":["../../src/program/ChainOfThought.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAErE,qBAAa,cAAc,CACzB,CAAC,SAAS,WAAW,EACrB,CAAC,SAAS,WAAW,CACrB,SAAQ,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACf,GAAG,CACP,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,GAAE,aAAkB,GACzB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAgDjD"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { Program } from "./Program.js";
|
|
2
|
+
export class ChainOfThought extends Program {
|
|
3
|
+
async run(input, config = {}) {
|
|
4
|
+
const startTime = Date.now();
|
|
5
|
+
const validatedInput = this.schema.validateInput(input);
|
|
6
|
+
const instructions = config.instructions || this.schema.description;
|
|
7
|
+
const fewShotSection = this.buildFewShotString(config.fewShotExamples);
|
|
8
|
+
const outputKeys = this.schema.getOutputKeys();
|
|
9
|
+
const outputFormat = outputKeys
|
|
10
|
+
.map((k) => `"${k}": <value>`)
|
|
11
|
+
.join(",\n ");
|
|
12
|
+
const prompt = `${instructions}
|
|
13
|
+
|
|
14
|
+
${fewShotSection ? `Here are some examples:\n\n${fewShotSection}\n\n` : ""}Think through this step-by-step before providing your answer.
|
|
15
|
+
|
|
16
|
+
Input: ${JSON.stringify(validatedInput)}
|
|
17
|
+
|
|
18
|
+
Respond with valid JSON in this exact format:
|
|
19
|
+
{
|
|
20
|
+
"reasoning": "Your step-by-step thinking process...",
|
|
21
|
+
${outputFormat}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
Respond ONLY with valid JSON, no additional text.`;
|
|
25
|
+
const response = await this.provider.complete({
|
|
26
|
+
prompt,
|
|
27
|
+
responseFormat: "json",
|
|
28
|
+
});
|
|
29
|
+
const parsed = this.parseJsonResponse(response.content);
|
|
30
|
+
const reasoning = parsed.reasoning;
|
|
31
|
+
delete parsed.reasoning;
|
|
32
|
+
const result = this.schema.validateOutput(parsed);
|
|
33
|
+
return {
|
|
34
|
+
result,
|
|
35
|
+
trace: {
|
|
36
|
+
promptUsed: prompt,
|
|
37
|
+
reasoning,
|
|
38
|
+
usage: response.usage,
|
|
39
|
+
latencyMs: Date.now() - startTime,
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=ChainOfThought.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ChainOfThought.js","sourceRoot":"","sources":["../../src/program/ChainOfThought.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgC,MAAM,cAAc,CAAC;AAErE,MAAM,OAAO,cAGX,SAAQ,OAAa;IACrB,KAAK,CAAC,GAAG,CACP,KAA4B,EAC5B,SAAwB,EAAE;QAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAExD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM,cAAc,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,UAAU;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC;aAC7B,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnB,MAAM,MAAM,GAAG,GAAG,YAAY;;EAGhC,cAAc,CAAC,CAAC,CAAC,8BAA8B,cAAc,MAAM,CAAC,CAAC,CAAC,EACxE;;SAES,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC;;;;;IAKnC,YAAY;;;kDAGkC,CAAC;QAE/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC5C,MAAM;YACN,cAAc,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,SAA+B,CAAC;QACzD,OAAO,MAAM,CAAC,SAAS,CAAC;QAExB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE;gBACL,UAAU,EAAE,MAAM;gBAClB,SAAS;gBACT,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC;SACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { z, ZodRawShape, ZodObject } from "zod";
|
|
2
|
+
import { Program, ProgramConfig, ProgramOutput } from "./Program.js";
|
|
3
|
+
export declare class Predict<I extends ZodRawShape, O extends ZodRawShape> extends Program<I, O> {
|
|
4
|
+
run(input: z.infer<ZodObject<I>>, config?: ProgramConfig): Promise<ProgramOutput<z.infer<ZodObject<O>>>>;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=Predict.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Predict.d.ts","sourceRoot":"","sources":["../../src/program/Predict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAErE,qBAAa,OAAO,CAClB,CAAC,SAAS,WAAW,EACrB,CAAC,SAAS,WAAW,CACrB,SAAQ,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACf,GAAG,CACP,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,GAAE,aAAkB,GACzB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAoCjD"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Program } from "./Program.js";
|
|
2
|
+
export class Predict extends Program {
|
|
3
|
+
async run(input, config = {}) {
|
|
4
|
+
const startTime = Date.now();
|
|
5
|
+
const validatedInput = this.schema.validateInput(input);
|
|
6
|
+
const instructions = config.instructions || this.schema.description;
|
|
7
|
+
const fewShotSection = this.buildFewShotString(config.fewShotExamples);
|
|
8
|
+
const outputFormat = this.buildOutputFormat();
|
|
9
|
+
const prompt = `${instructions}
|
|
10
|
+
|
|
11
|
+
${fewShotSection ? `Here are some examples:\n\n${fewShotSection}\n\n` : ""}Now process this input and respond with valid JSON matching this format:
|
|
12
|
+
${outputFormat}
|
|
13
|
+
|
|
14
|
+
Input: ${JSON.stringify(validatedInput)}
|
|
15
|
+
|
|
16
|
+
Respond ONLY with valid JSON, no additional text.`;
|
|
17
|
+
const response = await this.provider.complete({
|
|
18
|
+
prompt,
|
|
19
|
+
responseFormat: "json",
|
|
20
|
+
});
|
|
21
|
+
const parsed = this.parseJsonResponse(response.content);
|
|
22
|
+
const result = this.schema.validateOutput(parsed);
|
|
23
|
+
return {
|
|
24
|
+
result,
|
|
25
|
+
trace: {
|
|
26
|
+
promptUsed: prompt,
|
|
27
|
+
usage: response.usage,
|
|
28
|
+
latencyMs: Date.now() - startTime,
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=Predict.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Predict.js","sourceRoot":"","sources":["../../src/program/Predict.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgC,MAAM,cAAc,CAAC;AAErE,MAAM,OAAO,OAGX,SAAQ,OAAa;IACrB,KAAK,CAAC,GAAG,CACP,KAA4B,EAC5B,SAAwB,EAAE;QAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAExD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM,cAAc,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE9C,MAAM,MAAM,GAAG,GAAG,YAAY;;EAGhC,cAAc,CAAC,CAAC,CAAC,8BAA8B,cAAc,MAAM,CAAC,CAAC,CAAC,EACxE;EACE,YAAY;;SAEL,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC;;kDAEW,CAAC;QAE/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC5C,MAAM;YACN,cAAc,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE;gBACL,UAAU,EAAE,MAAM;gBAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC;SACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { z, ZodRawShape, ZodObject } from "zod";
|
|
2
|
+
import { Schema } from "../schema/defineSchema.js";
|
|
3
|
+
import type { LLMProvider } from "../providers/types.js";
|
|
4
|
+
export interface ProgramConfig {
|
|
5
|
+
instructions?: string;
|
|
6
|
+
fewShotExamples?: Array<{
|
|
7
|
+
input: Record<string, unknown>;
|
|
8
|
+
output: Record<string, unknown>;
|
|
9
|
+
}>;
|
|
10
|
+
}
|
|
11
|
+
export interface ProgramTrace {
|
|
12
|
+
promptUsed: string;
|
|
13
|
+
reasoning?: string;
|
|
14
|
+
usage: {
|
|
15
|
+
inputTokens: number;
|
|
16
|
+
outputTokens: number;
|
|
17
|
+
};
|
|
18
|
+
latencyMs: number;
|
|
19
|
+
}
|
|
20
|
+
export interface ProgramOutput<O> {
|
|
21
|
+
result: O;
|
|
22
|
+
trace: ProgramTrace;
|
|
23
|
+
}
|
|
24
|
+
export declare abstract class Program<I extends ZodRawShape, O extends ZodRawShape> {
|
|
25
|
+
protected schema: Schema<I, O>;
|
|
26
|
+
protected provider: LLMProvider;
|
|
27
|
+
constructor(schema: Schema<I, O>, provider: LLMProvider);
|
|
28
|
+
abstract run(input: z.infer<ZodObject<I>>, config?: ProgramConfig): Promise<ProgramOutput<z.infer<ZodObject<O>>>>;
|
|
29
|
+
protected buildFewShotString(examples: ProgramConfig["fewShotExamples"]): string;
|
|
30
|
+
protected buildOutputFormat(): string;
|
|
31
|
+
protected parseJsonResponse(content: string): Record<string, unknown>;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=Program.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Program.d.ts","sourceRoot":"","sources":["../../src/program/Program.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,2BAA2B,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,KAAK,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACjC,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC;IACV,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,8BAAsB,OAAO,CAAC,CAAC,SAAS,WAAW,EAAE,CAAC,SAAS,WAAW;IAEtE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9B,SAAS,CAAC,QAAQ,EAAE,WAAW;gBADrB,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EACpB,QAAQ,EAAE,WAAW;IAGjC,QAAQ,CAAC,GAAG,CACV,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,CAAC,EAAE,aAAa,GACrB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhD,SAAS,CAAC,kBAAkB,CAC1B,QAAQ,EAAE,aAAa,CAAC,iBAAiB,CAAC,GACzC,MAAM;IAaT,SAAS,CAAC,iBAAiB,IAAI,MAAM;IAMrC,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAOtE"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export class Program {
|
|
2
|
+
schema;
|
|
3
|
+
provider;
|
|
4
|
+
constructor(schema, provider) {
|
|
5
|
+
this.schema = schema;
|
|
6
|
+
this.provider = provider;
|
|
7
|
+
}
|
|
8
|
+
buildFewShotString(examples) {
|
|
9
|
+
if (!examples || examples.length === 0)
|
|
10
|
+
return "";
|
|
11
|
+
return examples
|
|
12
|
+
.map((ex, i) => `Example ${i + 1}:\nInput: ${JSON.stringify(ex.input)}\nOutput: ${JSON.stringify(ex.output)}`)
|
|
13
|
+
.join("\n\n");
|
|
14
|
+
}
|
|
15
|
+
buildOutputFormat() {
|
|
16
|
+
const keys = this.schema.getOutputKeys();
|
|
17
|
+
const format = keys.map((k) => `"${k}": <value>`).join(",\n ");
|
|
18
|
+
return `{\n ${format}\n}`;
|
|
19
|
+
}
|
|
20
|
+
parseJsonResponse(content) {
|
|
21
|
+
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
|
22
|
+
if (!jsonMatch) {
|
|
23
|
+
throw new Error("No JSON found in response");
|
|
24
|
+
}
|
|
25
|
+
return JSON.parse(jsonMatch[0]);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=Program.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Program.js","sourceRoot":"","sources":["../../src/program/Program.ts"],"names":[],"mappings":"AAwBA,MAAM,OAAgB,OAAO;IAEf;IACA;IAFZ,YACY,MAAoB,EACpB,QAAqB;QADrB,WAAM,GAAN,MAAM,CAAc;QACpB,aAAQ,GAAR,QAAQ,CAAa;IAC9B,CAAC;IAOM,kBAAkB,CAC1B,QAA0C;QAE1C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElD,OAAO,QAAQ;aACZ,GAAG,CACF,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CACR,WAAW,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CACzC,EAAE,CAAC,KAAK,CACT,aAAa,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAC5C;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAES,iBAAiB;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChE,OAAO,QAAQ,MAAM,KAAK,CAAC;IAC7B,CAAC;IAES,iBAAiB,CAAC,OAAe;QACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/program/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/program/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { LLMProvider, CompletionParams, CompletionResult, ProviderConfig } from "./types.js";
|
|
2
|
+
export declare class AnthropicProvider implements LLMProvider {
|
|
3
|
+
name: string;
|
|
4
|
+
defaultModel: string;
|
|
5
|
+
private apiKey;
|
|
6
|
+
private baseUrl;
|
|
7
|
+
constructor(config: ProviderConfig);
|
|
8
|
+
complete(params: CompletionParams): Promise<CompletionResult>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=anthropic.d.ts.map
|