@mzhub/promptc 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +224 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +111 -0
- package/dist/cli.js.map +1 -0
- package/dist/compiler/BootstrapFewShot.d.ts +13 -0
- package/dist/compiler/BootstrapFewShot.d.ts.map +1 -0
- package/dist/compiler/BootstrapFewShot.js +93 -0
- package/dist/compiler/BootstrapFewShot.js.map +1 -0
- package/dist/compiler/CandidatePool.d.ts +10 -0
- package/dist/compiler/CandidatePool.d.ts.map +1 -0
- package/dist/compiler/CandidatePool.js +29 -0
- package/dist/compiler/CandidatePool.js.map +1 -0
- package/dist/compiler/CompiledProgram.d.ts +43 -0
- package/dist/compiler/CompiledProgram.d.ts.map +1 -0
- package/dist/compiler/CompiledProgram.js +41 -0
- package/dist/compiler/CompiledProgram.js.map +1 -0
- package/dist/compiler/InstructionRewrite.d.ts +19 -0
- package/dist/compiler/InstructionRewrite.d.ts.map +1 -0
- package/dist/compiler/InstructionRewrite.js +117 -0
- package/dist/compiler/InstructionRewrite.js.map +1 -0
- package/dist/compiler/index.d.ts +8 -0
- package/dist/compiler/index.d.ts.map +1 -0
- package/dist/compiler/index.js +5 -0
- package/dist/compiler/index.js.map +1 -0
- package/dist/compiler/types.d.ts +41 -0
- package/dist/compiler/types.d.ts.map +1 -0
- package/dist/compiler/types.js +2 -0
- package/dist/compiler/types.js.map +1 -0
- package/dist/eval/exactMatch.d.ts +5 -0
- package/dist/eval/exactMatch.d.ts.map +1 -0
- package/dist/eval/exactMatch.js +58 -0
- package/dist/eval/exactMatch.js.map +1 -0
- package/dist/eval/index.d.ts +5 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +3 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/llmJudge.d.ts +9 -0
- package/dist/eval/llmJudge.d.ts.map +1 -0
- package/dist/eval/llmJudge.js +33 -0
- package/dist/eval/llmJudge.js.map +1 -0
- package/dist/eval/types.d.ts +2 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +2 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/program/ChainOfThought.d.ts +6 -0
- package/dist/program/ChainOfThought.d.ts.map +1 -0
- package/dist/program/ChainOfThought.js +44 -0
- package/dist/program/ChainOfThought.js.map +1 -0
- package/dist/program/Predict.d.ts +6 -0
- package/dist/program/Predict.d.ts.map +1 -0
- package/dist/program/Predict.js +33 -0
- package/dist/program/Predict.js.map +1 -0
- package/dist/program/Program.d.ts +33 -0
- package/dist/program/Program.d.ts.map +1 -0
- package/dist/program/Program.js +28 -0
- package/dist/program/Program.js.map +1 -0
- package/dist/program/index.d.ts +5 -0
- package/dist/program/index.d.ts.map +1 -0
- package/dist/program/index.js +4 -0
- package/dist/program/index.js.map +1 -0
- package/dist/providers/anthropic.d.ts +10 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +40 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/cerebras.d.ts +10 -0
- package/dist/providers/cerebras.d.ts.map +1 -0
- package/dist/providers/cerebras.js +39 -0
- package/dist/providers/cerebras.js.map +1 -0
- package/dist/providers/google.d.ts +10 -0
- package/dist/providers/google.d.ts.map +1 -0
- package/dist/providers/google.js +42 -0
- package/dist/providers/google.js.map +1 -0
- package/dist/providers/groq.d.ts +10 -0
- package/dist/providers/groq.d.ts.map +1 -0
- package/dist/providers/groq.js +42 -0
- package/dist/providers/groq.js.map +1 -0
- package/dist/providers/index.d.ts +11 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +31 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/ollama.d.ts +9 -0
- package/dist/providers/ollama.d.ts.map +1 -0
- package/dist/providers/ollama.js +39 -0
- package/dist/providers/ollama.js.map +1 -0
- package/dist/providers/openai.d.ts +10 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +42 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/types.d.ts +25 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/runtime/cache.d.ts +18 -0
- package/dist/runtime/cache.d.ts.map +1 -0
- package/dist/runtime/cache.js +45 -0
- package/dist/runtime/cache.js.map +1 -0
- package/dist/runtime/concurrency.d.ts +7 -0
- package/dist/runtime/concurrency.d.ts.map +1 -0
- package/dist/runtime/concurrency.js +14 -0
- package/dist/runtime/concurrency.js.map +1 -0
- package/dist/runtime/costTracker.d.ts +24 -0
- package/dist/runtime/costTracker.d.ts.map +1 -0
- package/dist/runtime/costTracker.js +37 -0
- package/dist/runtime/costTracker.js.map +1 -0
- package/dist/runtime/index.d.ts +9 -0
- package/dist/runtime/index.d.ts.map +1 -0
- package/dist/runtime/index.js +5 -0
- package/dist/runtime/index.js.map +1 -0
- package/dist/runtime/retry.d.ts +10 -0
- package/dist/runtime/retry.d.ts.map +1 -0
- package/dist/runtime/retry.js +39 -0
- package/dist/runtime/retry.js.map +1 -0
- package/dist/schema/defineSchema.d.ts +18 -0
- package/dist/schema/defineSchema.d.ts.map +1 -0
- package/dist/schema/defineSchema.js +27 -0
- package/dist/schema/defineSchema.js.map +1 -0
- package/dist/schema/index.d.ts +3 -0
- package/dist/schema/index.d.ts.map +1 -0
- package/dist/schema/index.js +2 -0
- package/dist/schema/index.js.map +1 -0
- package/examples/README.md +42 -0
- package/examples/load-compiled.ts +62 -0
- package/examples/multi-provider.ts +77 -0
- package/examples/name-extractor.ts +113 -0
- package/examples/qa-system.ts +98 -0
- package/package.json +62 -0
- package/src/cli.ts +122 -0
- package/src/compiler/BootstrapFewShot.ts +149 -0
- package/src/compiler/CandidatePool.ts +39 -0
- package/src/compiler/CompiledProgram.ts +112 -0
- package/src/compiler/InstructionRewrite.ts +200 -0
- package/src/compiler/index.ts +19 -0
- package/src/compiler/types.ts +46 -0
- package/src/eval/exactMatch.ts +65 -0
- package/src/eval/index.ts +4 -0
- package/src/eval/llmJudge.ts +45 -0
- package/src/eval/types.ts +4 -0
- package/src/index.ts +71 -0
- package/src/program/ChainOfThought.ts +59 -0
- package/src/program/Predict.ts +47 -0
- package/src/program/Program.ts +64 -0
- package/src/program/index.ts +4 -0
- package/src/providers/anthropic.ts +55 -0
- package/src/providers/cerebras.ts +53 -0
- package/src/providers/google.ts +57 -0
- package/src/providers/groq.ts +57 -0
- package/src/providers/index.ts +50 -0
- package/src/providers/ollama.ts +54 -0
- package/src/providers/openai.ts +57 -0
- package/src/providers/types.ts +27 -0
- package/src/runtime/cache.ts +65 -0
- package/src/runtime/concurrency.ts +21 -0
- package/src/runtime/costTracker.ts +58 -0
- package/src/runtime/index.ts +8 -0
- package/src/runtime/retry.ts +59 -0
- package/src/schema/defineSchema.ts +44 -0
- package/src/schema/index.ts +2 -0
- package/tests/candidatePool.test.ts +46 -0
- package/tests/evaluators.test.ts +69 -0
- package/tests/runtime.test.ts +106 -0
- package/tests/schema.test.ts +59 -0
- package/tsconfig.json +24 -0
package/README.md
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# promptc
|
|
2
|
+
|
|
3
|
+
Type-safe LLM program compiler for JavaScript/TypeScript.
|
|
4
|
+
|
|
5
|
+
**promptc** brings DSPy-style prompt compilation to the JavaScript ecosystem. Define schemas, write programs, and let the compiler optimize your prompts automatically using evaluation data.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install promptc
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Install your preferred LLM provider (optional peer dependencies):
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
# Pick one or more
|
|
17
|
+
npm install openai
|
|
18
|
+
npm install @anthropic-ai/sdk
|
|
19
|
+
npm install @google/generative-ai
|
|
20
|
+
npm install ollama
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
```typescript
|
|
26
|
+
import {
|
|
27
|
+
defineSchema,
|
|
28
|
+
ChainOfThought,
|
|
29
|
+
BootstrapFewShot,
|
|
30
|
+
exactMatch,
|
|
31
|
+
createProvider,
|
|
32
|
+
z,
|
|
33
|
+
} from "promptc";
|
|
34
|
+
|
|
35
|
+
// 1. Define your schema
|
|
36
|
+
const NameExtractor = defineSchema({
|
|
37
|
+
description: "Extract proper names from text",
|
|
38
|
+
inputs: { text: z.string() },
|
|
39
|
+
outputs: { names: z.array(z.string()) },
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
// 2. Create a provider
|
|
43
|
+
const provider = createProvider("openai", {
|
|
44
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
// 3. Create a program
|
|
48
|
+
const pipeline = new ChainOfThought(NameExtractor, provider);
|
|
49
|
+
|
|
50
|
+
// 4. Prepare training data
|
|
51
|
+
const trainset = [
|
|
52
|
+
{
|
|
53
|
+
input: { text: "Bill Gates founded Microsoft." },
|
|
54
|
+
output: { names: ["Bill Gates"] },
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
input: { text: "Elon runs Tesla and SpaceX." },
|
|
58
|
+
output: { names: ["Elon"] },
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
input: { text: "Jeff Bezos started Amazon." },
|
|
62
|
+
output: { names: ["Jeff Bezos"] },
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
input: { text: "Satya Nadella is the CEO." },
|
|
66
|
+
output: { names: ["Satya Nadella"] },
|
|
67
|
+
},
|
|
68
|
+
{ input: { text: "Tim Cook leads Apple." }, output: { names: ["Tim Cook"] } },
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
// 5. Compile (finds optimal few-shot examples)
|
|
72
|
+
const compiler = new BootstrapFewShot(exactMatch());
|
|
73
|
+
const compiled = await compiler.compile(pipeline, trainset, {
|
|
74
|
+
candidates: 10,
|
|
75
|
+
concurrency: 5,
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
console.log("Best score:", compiled.meta.score);
|
|
79
|
+
console.log("Token usage:", compiled.meta.tokenUsage);
|
|
80
|
+
|
|
81
|
+
// 6. Save compiled config
|
|
82
|
+
import fs from "fs";
|
|
83
|
+
fs.writeFileSync("prompt.json", JSON.stringify(compiled, null, 2));
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Core Concepts
|
|
87
|
+
|
|
88
|
+
### Schema
|
|
89
|
+
|
|
90
|
+
Type-safe input/output contracts using Zod:
|
|
91
|
+
|
|
92
|
+
```typescript
|
|
93
|
+
const QASchema = defineSchema({
|
|
94
|
+
description: "Answer questions based on context",
|
|
95
|
+
inputs: {
|
|
96
|
+
question: z.string(),
|
|
97
|
+
context: z.string(),
|
|
98
|
+
},
|
|
99
|
+
outputs: {
|
|
100
|
+
answer: z.string(),
|
|
101
|
+
confidence: z.number().min(0).max(1),
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Programs
|
|
107
|
+
|
|
108
|
+
Execution strategies for LLM calls:
|
|
109
|
+
|
|
110
|
+
- **Predict** - Simple input → output
|
|
111
|
+
- **ChainOfThought** - Forces step-by-step reasoning before answering
|
|
112
|
+
|
|
113
|
+
```typescript
|
|
114
|
+
const simple = new Predict(schema, provider);
|
|
115
|
+
const reasoning = new ChainOfThought(schema, provider);
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Providers
|
|
119
|
+
|
|
120
|
+
Swap LLM backends with a single line:
|
|
121
|
+
|
|
122
|
+
```typescript
|
|
123
|
+
createProvider("openai", { apiKey: "..." });
|
|
124
|
+
createProvider("anthropic", { apiKey: "..." });
|
|
125
|
+
createProvider("google", { apiKey: "..." });
|
|
126
|
+
createProvider("groq", { apiKey: "..." });
|
|
127
|
+
createProvider("cerebras", { apiKey: "..." });
|
|
128
|
+
createProvider("ollama");
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Evaluators
|
|
132
|
+
|
|
133
|
+
Score predictions against ground truth:
|
|
134
|
+
|
|
135
|
+
```typescript
|
|
136
|
+
exactMatch(); // 1.0 if equal, 0.0 otherwise
|
|
137
|
+
partialMatch(); // Fraction of matching fields
|
|
138
|
+
arrayOverlap(); // Jaccard similarity for arrays
|
|
139
|
+
llmJudge({ provider }); // Use LLM to score
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### Compilers
|
|
143
|
+
|
|
144
|
+
#### BootstrapFewShot
|
|
145
|
+
|
|
146
|
+
Finds optimal few-shot examples:
|
|
147
|
+
|
|
148
|
+
```typescript
|
|
149
|
+
const compiler = new BootstrapFewShot(exactMatch());
|
|
150
|
+
const result = await compiler.compile(program, trainset, {
|
|
151
|
+
candidates: 20,
|
|
152
|
+
concurrency: 5,
|
|
153
|
+
earlyStopThreshold: 0.3, // Skip bad candidates early
|
|
154
|
+
});
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
#### InstructionRewrite
|
|
158
|
+
|
|
159
|
+
Uses LLM to generate and test instruction variations:
|
|
160
|
+
|
|
161
|
+
```typescript
|
|
162
|
+
const compiler = new InstructionRewrite(exactMatch());
|
|
163
|
+
const result = await compiler.compile(program, trainset, {
|
|
164
|
+
provider,
|
|
165
|
+
instructionVariations: 5,
|
|
166
|
+
candidates: 5,
|
|
167
|
+
});
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### CompiledProgram
|
|
171
|
+
|
|
172
|
+
Get a production-ready program with `.run()`:
|
|
173
|
+
|
|
174
|
+
````typescript
|
|
175
|
+
import { createCompiledProgram } from "promptc";
|
|
176
|
+
|
|
177
|
+
const compiled = createCompiledProgram(pipeline, result);
|
|
178
|
+
|
|
179
|
+
// Use in production
|
|
180
|
+
const output = await compiled.run({ text: "Some input" });
|
|
181
|
+
|
|
182
|
+
// Serialize to JSON
|
|
183
|
+
fs.writeFileSync("prompt.json", compiled.toJSON());
|
|
184
|
+
|
|
185
|
+
## Cost Management
|
|
186
|
+
|
|
187
|
+
```typescript
|
|
188
|
+
// Estimate before running
|
|
189
|
+
const estimate = compiler.estimateCost(trainset.length, { candidates: 20 });
|
|
190
|
+
console.log("Estimated calls:", estimate.estimatedCalls);
|
|
191
|
+
console.log("Estimated tokens:", estimate.estimatedTokens);
|
|
192
|
+
|
|
193
|
+
// Track during compilation
|
|
194
|
+
compiled.meta.tokenUsage.totalTokens;
|
|
195
|
+
compiled.meta.tokenUsage.calls;
|
|
196
|
+
````
|
|
197
|
+
|
|
198
|
+
## Compiled Output
|
|
199
|
+
|
|
200
|
+
The compiler returns a serializable artifact:
|
|
201
|
+
|
|
202
|
+
```json
|
|
203
|
+
{
|
|
204
|
+
"meta": {
|
|
205
|
+
"score": 0.87,
|
|
206
|
+
"compiledAt": "2024-12-29T12:00:00.000Z",
|
|
207
|
+
"strategy": "BootstrapFewShot",
|
|
208
|
+
"tokenUsage": {
|
|
209
|
+
"inputTokens": 45000,
|
|
210
|
+
"outputTokens": 12000,
|
|
211
|
+
"totalTokens": 57000,
|
|
212
|
+
"calls": 150
|
|
213
|
+
}
|
|
214
|
+
},
|
|
215
|
+
"config": {
|
|
216
|
+
"instructions": "Extract proper names from text",
|
|
217
|
+
"fewShotExamples": [...]
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## License
|
|
223
|
+
|
|
224
|
+
MIT
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { parseArgs } from "node:util";
|
|
3
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
4
|
+
import { resolve } from "node:path";
|
|
5
|
+
const { values, positionals } = parseArgs({
|
|
6
|
+
args: process.argv.slice(2),
|
|
7
|
+
options: {
|
|
8
|
+
config: { type: "string", short: "c" },
|
|
9
|
+
output: { type: "string", short: "o", default: "compiled-prompt.json" },
|
|
10
|
+
candidates: { type: "string", default: "10" },
|
|
11
|
+
concurrency: { type: "string", default: "5" },
|
|
12
|
+
provider: { type: "string", short: "p", default: "openai" },
|
|
13
|
+
model: { type: "string", short: "m" },
|
|
14
|
+
help: { type: "boolean", short: "h" },
|
|
15
|
+
},
|
|
16
|
+
allowPositionals: true,
|
|
17
|
+
});
|
|
18
|
+
function printHelp() {
|
|
19
|
+
console.log(`
|
|
20
|
+
promptc - Type-safe LLM program compiler
|
|
21
|
+
|
|
22
|
+
Usage:
|
|
23
|
+
promptc compile <trainset.json> [options]
|
|
24
|
+
promptc validate <compiled.json>
|
|
25
|
+
|
|
26
|
+
Commands:
|
|
27
|
+
compile Compile a program using a training set
|
|
28
|
+
validate Validate a compiled prompt JSON file
|
|
29
|
+
|
|
30
|
+
Options:
|
|
31
|
+
-c, --config <file> Path to config file
|
|
32
|
+
-o, --output <file> Output file (default: compiled-prompt.json)
|
|
33
|
+
-p, --provider <name> LLM provider: openai, anthropic, google, groq, cerebras, ollama
|
|
34
|
+
-m, --model <name> Model to use (overrides provider default)
|
|
35
|
+
--candidates <n> Number of candidates to try (default: 10)
|
|
36
|
+
--concurrency <n> Parallel evaluations (default: 5)
|
|
37
|
+
-h, --help Show this help
|
|
38
|
+
|
|
39
|
+
Examples:
|
|
40
|
+
promptc compile trainset.json -p openai -o prompt.json
|
|
41
|
+
promptc validate prompt.json
|
|
42
|
+
`);
|
|
43
|
+
}
|
|
44
|
+
async function validateCommand(filePath) {
|
|
45
|
+
if (!existsSync(filePath)) {
|
|
46
|
+
console.error(`Error: File not found: ${filePath}`);
|
|
47
|
+
process.exit(1);
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
const content = readFileSync(filePath, "utf-8");
|
|
51
|
+
const data = JSON.parse(content);
|
|
52
|
+
if (!data.meta || !data.config) {
|
|
53
|
+
console.error("Invalid format: missing 'meta' or 'config' fields");
|
|
54
|
+
process.exit(1);
|
|
55
|
+
}
|
|
56
|
+
console.log("✓ Valid compiled prompt file");
|
|
57
|
+
console.log(` Score: ${data.meta.score}`);
|
|
58
|
+
console.log(` Strategy: ${data.meta.strategy}`);
|
|
59
|
+
console.log(` Compiled: ${data.meta.compiledAt}`);
|
|
60
|
+
console.log(` Examples: ${data.config.fewShotExamples?.length || 0}`);
|
|
61
|
+
console.log(` Tokens used: ${data.meta.tokenUsage?.totalTokens || "N/A"}`);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
console.error("Invalid JSON file");
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async function main() {
|
|
69
|
+
if (values.help || positionals.length === 0) {
|
|
70
|
+
printHelp();
|
|
71
|
+
process.exit(0);
|
|
72
|
+
}
|
|
73
|
+
const command = positionals[0];
|
|
74
|
+
switch (command) {
|
|
75
|
+
case "validate":
|
|
76
|
+
if (!positionals[1]) {
|
|
77
|
+
console.error("Error: Please provide a file to validate");
|
|
78
|
+
process.exit(1);
|
|
79
|
+
}
|
|
80
|
+
await validateCommand(resolve(positionals[1]));
|
|
81
|
+
break;
|
|
82
|
+
case "compile":
|
|
83
|
+
console.log(`
|
|
84
|
+
Note: The 'compile' command requires a custom config file that defines:
|
|
85
|
+
- Your schema (using defineSchema)
|
|
86
|
+
- Your program (Predict or ChainOfThought)
|
|
87
|
+
- Your evaluator
|
|
88
|
+
|
|
89
|
+
Example config file (promptc.config.ts):
|
|
90
|
+
|
|
91
|
+
import { defineSchema, ChainOfThought, createProvider, exactMatch, z } from 'promptc';
|
|
92
|
+
|
|
93
|
+
export const schema = defineSchema({...});
|
|
94
|
+
export const provider = createProvider('openai', { apiKey: process.env.OPENAI_API_KEY });
|
|
95
|
+
export const program = new ChainOfThought(schema, provider);
|
|
96
|
+
export const evaluator = exactMatch();
|
|
97
|
+
|
|
98
|
+
Run with: promptc compile trainset.json -c promptc.config.ts
|
|
99
|
+
`);
|
|
100
|
+
break;
|
|
101
|
+
default:
|
|
102
|
+
console.error(`Unknown command: ${command}`);
|
|
103
|
+
printHelp();
|
|
104
|
+
process.exit(1);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
main().catch((error) => {
|
|
108
|
+
console.error("Error:", error.message);
|
|
109
|
+
process.exit(1);
|
|
110
|
+
});
|
|
111
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAiB,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;IACxC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE;QACtC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,sBAAsB,EAAE;QACvE,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;QAC7C,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;QAC7C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE;QAC3D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE;QACrC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACtC;IACD,gBAAgB,EAAE,IAAI;CACvB,CAAC,CAAC;AAEH,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;CAuBb,CAAC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEjC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,IAAI,KAAK,EAAE,CAAC,CAAC;IAC9E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,MAAM,CAAC,IAAI,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAE/B,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,UAAU;YACb,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpB,OAAO,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,eAAe,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM;QAER,KAAK,SAAS;YACZ,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;CAgBjB,CAAC,CAAC;YACG,MAAM;QAER;YACE,OAAO,CAAC,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;YAC7C,SAAS,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ZodRawShape } from "zod";
|
|
2
|
+
import { Program } from "../program/Program.js";
|
|
3
|
+
import type { Evaluator, Example, CompilationResult, CompileOptions } from "./types.js";
|
|
4
|
+
export declare class BootstrapFewShot<O> {
|
|
5
|
+
private evaluator;
|
|
6
|
+
constructor(evaluator: Evaluator<O>);
|
|
7
|
+
compile<I extends ZodRawShape, OShape extends ZodRawShape>(program: Program<I, OShape>, trainset: Example<unknown, O>[], options?: CompileOptions): Promise<CompilationResult<unknown, O>>;
|
|
8
|
+
estimateCost(trainsetSize: number, options?: CompileOptions): {
|
|
9
|
+
estimatedCalls: number;
|
|
10
|
+
estimatedTokens: number;
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=BootstrapFewShot.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BootstrapFewShot.d.ts","sourceRoot":"","sources":["../../src/compiler/BootstrapFewShot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,KAAK,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAIhD,OAAO,KAAK,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB,qBAAa,gBAAgB,CAAC,CAAC;IACjB,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAErC,OAAO,CAAC,CAAC,SAAS,WAAW,EAAE,MAAM,SAAS,WAAW,EAC7D,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,EAC3B,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAC/B,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IA6GzC,YAAY,CACV,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,cAAmB,GAC3B;QACD,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB;CAcF"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { createConcurrencyManager } from "../runtime/concurrency.js";
|
|
2
|
+
import { CostTracker } from "../runtime/costTracker.js";
|
|
3
|
+
import { CandidatePool } from "./CandidatePool.js";
|
|
4
|
+
export class BootstrapFewShot {
|
|
5
|
+
evaluator;
|
|
6
|
+
constructor(evaluator) {
|
|
7
|
+
this.evaluator = evaluator;
|
|
8
|
+
}
|
|
9
|
+
async compile(program, trainset, options = {}) {
|
|
10
|
+
const { candidates: candidateCount = 10, concurrency = 5, examplesPerCandidate = 3, validationSplit = 0.3, seed, earlyStopThreshold = 0, budget, onProgress, } = options;
|
|
11
|
+
const pool = new CandidatePool(trainset, seed);
|
|
12
|
+
const candidates = pool.generateFewShotCandidates(candidateCount, examplesPerCandidate);
|
|
13
|
+
const validationSet = pool.getValidationSet(validationSplit);
|
|
14
|
+
const limiter = createConcurrencyManager(concurrency);
|
|
15
|
+
const costTracker = new CostTracker();
|
|
16
|
+
const results = [];
|
|
17
|
+
const evaluationPromises = candidates.map((fewShotSet) => limiter.run(async () => {
|
|
18
|
+
if (budget?.maxTokens && costTracker.exceedsBudget(budget.maxTokens)) {
|
|
19
|
+
budget.onBudgetWarning?.(costTracker.total.totalTokens, budget.maxTokens);
|
|
20
|
+
return { score: -1, examples: fewShotSet };
|
|
21
|
+
}
|
|
22
|
+
let totalScore = 0;
|
|
23
|
+
let validCount = 0;
|
|
24
|
+
for (const testCase of validationSet) {
|
|
25
|
+
try {
|
|
26
|
+
const prediction = await program.run(testCase.input, {
|
|
27
|
+
fewShotExamples: fewShotSet.map((ex) => ({
|
|
28
|
+
input: ex.input,
|
|
29
|
+
output: ex.output,
|
|
30
|
+
})),
|
|
31
|
+
});
|
|
32
|
+
costTracker.record(prediction.trace.usage);
|
|
33
|
+
const score = await this.evaluator(prediction.result, testCase.output);
|
|
34
|
+
totalScore += score;
|
|
35
|
+
validCount++;
|
|
36
|
+
if (earlyStopThreshold > 0 && validCount >= 2) {
|
|
37
|
+
const avgSoFar = totalScore / validCount;
|
|
38
|
+
if (avgSoFar < earlyStopThreshold) {
|
|
39
|
+
break;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
score: validCount > 0 ? totalScore / validCount : 0,
|
|
49
|
+
examples: fewShotSet,
|
|
50
|
+
};
|
|
51
|
+
}));
|
|
52
|
+
const candidateResults = await Promise.all(evaluationPromises);
|
|
53
|
+
let currentBestScore = -1;
|
|
54
|
+
for (const result of candidateResults) {
|
|
55
|
+
if (result.score >= 0) {
|
|
56
|
+
results.push(result);
|
|
57
|
+
if (result.score > currentBestScore) {
|
|
58
|
+
currentBestScore = result.score;
|
|
59
|
+
}
|
|
60
|
+
onProgress?.({
|
|
61
|
+
candidatesEvaluated: results.length,
|
|
62
|
+
totalCandidates: candidateCount,
|
|
63
|
+
currentBestScore,
|
|
64
|
+
tokensUsed: costTracker.total.totalTokens,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const winner = results.reduce((best, current) => (current.score > best.score ? current : best), { score: -1, examples: [] });
|
|
69
|
+
return {
|
|
70
|
+
meta: {
|
|
71
|
+
score: winner.score,
|
|
72
|
+
compiledAt: new Date().toISOString(),
|
|
73
|
+
strategy: "BootstrapFewShot",
|
|
74
|
+
tokenUsage: costTracker.total,
|
|
75
|
+
},
|
|
76
|
+
config: {
|
|
77
|
+
instructions: program["schema"].description,
|
|
78
|
+
fewShotExamples: winner.examples,
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
estimateCost(trainsetSize, options = {}) {
|
|
83
|
+
const { candidates = 10, validationSplit = 0.3 } = options;
|
|
84
|
+
const validationSize = Math.max(1, Math.floor(trainsetSize * validationSplit));
|
|
85
|
+
const estimatedCalls = candidates * validationSize;
|
|
86
|
+
const avgTokensPerCall = 500;
|
|
87
|
+
return {
|
|
88
|
+
estimatedCalls,
|
|
89
|
+
estimatedTokens: estimatedCalls * avgTokensPerCall,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=BootstrapFewShot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BootstrapFewShot.js","sourceRoot":"","sources":["../../src/compiler/BootstrapFewShot.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAQnD,MAAM,OAAO,gBAAgB;IACP;IAApB,YAAoB,SAAuB;QAAvB,cAAS,GAAT,SAAS,CAAc;IAAG,CAAC;IAE/C,KAAK,CAAC,OAAO,CACX,OAA2B,EAC3B,QAA+B,EAC/B,UAA0B,EAAE;QAE5B,MAAM,EACJ,UAAU,EAAE,cAAc,GAAG,EAAE,EAC/B,WAAW,GAAG,CAAC,EACf,oBAAoB,GAAG,CAAC,EACxB,eAAe,GAAG,GAAG,EACrB,IAAI,EACJ,kBAAkB,GAAG,CAAC,EACtB,MAAM,EACN,UAAU,GACX,GAAG,OAAO,CAAC;QAEZ,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAC/C,cAAc,EACd,oBAAoB,CACrB,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QAEtC,MAAM,OAAO,GACX,EAAE,CAAC;QAEL,MAAM,kBAAkB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACvD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACrB,IAAI,MAAM,EAAE,SAAS,IAAI,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrE,MAAM,CAAC,eAAe,EAAE,CACtB,WAAW,CAAC,KAAK,CAAC,WAAW,EAC7B,MAAM,CAAC,SAAS,CACjB,CAAC;gBACF,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;YAC7C,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAY,EAAE;wBAC1D,eAAe,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;4BACvC,KAAK,EAAE,EAAE,CAAC,KAAgC;4BAC1C,MAAM,EAAE,EAAE,CAAC,MAAiC;yBAC7C,CAAC,CAAC;qBACJ,CAAC,CAAC;oBAEH,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAChC,UAAU,CAAC,MAAW,EACtB,QAAQ,CAAC,MAAM,CAChB,CAAC;oBACF,UAAU,IAAI,KAAK,CAAC;oBACpB,UAAU,EAAE,CAAC;oBAEb,IAAI,kBAAkB,GAAG,CAAC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;wBAC9C,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;wBACzC,IAAI,QAAQ,GAAG,kBAAkB,EAAE,CAAC;4BAClC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;YACH,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBACnD,QAAQ,EAAE,UAAU;aACrB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAE/D,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAC1B,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,IAAI,MAAM,CAAC,KAAK,GAAG,gBAAgB,EAAE,CAAC;oBACpC,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC;gBAClC,CAAC;gBACD,UAAU,EAAE,CAAC;oBACX,mBAAmB,EAAE,OAAO,CAAC,MAAM;oBACnC,eAAe,EAAE,cAAc;oBAC/B,gBAAgB;oBAChB,UAAU,EAAE,WAAW,CAAC,KAAK,CAAC,WAAW;iBAC1C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAC3B,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAChE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAA2B,EAAE,CACrD,CAAC;QAEF,OAAO;YACL,IAAI,EAAE;gBACJ,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,QAAQ,EAAE,kBAAkB;gBAC5B,UAAU,EAAE,WAAW,CAAC,KAAK;aAC9B;YACD,MAAM,EAAE;gBACN,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW;gBAC3C,eAAe,EAAE,MAAM,CAAC,QAAQ;aACjC;SACF,CAAC;IACJ,CAAC;IAED,YAAY,CACV,YAAoB,EACpB,UAA0B,EAAE;QAK5B,MAAM,EAAE,UAAU,GAAG,EAAE,EAAE,eAAe,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;QAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,eAAe,CAAC,CAC3C,CAAC;QACF,MAAM,cAAc,GAAG,UAAU,GAAG,cAAc,CAAC;QACnD,MAAM,gBAAgB,GAAG,GAAG,CAAC;QAE7B,OAAO;YACL,cAAc;YACd,eAAe,EAAE,cAAc,GAAG,gBAAgB;SACnD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Example } from "./types.js";
|
|
2
|
+
export declare class CandidatePool<I, O> {
|
|
3
|
+
private trainset;
|
|
4
|
+
private rng;
|
|
5
|
+
constructor(trainset: Example<I, O>[], seed?: number);
|
|
6
|
+
generateFewShotCandidates(count: number, examplesPerCandidate: number): Array<Example<I, O>[]>;
|
|
7
|
+
getValidationSet(splitRatio?: number): Example<I, O>[];
|
|
8
|
+
private seededRandom;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=CandidatePool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CandidatePool.d.ts","sourceRoot":"","sources":["../../src/compiler/CandidatePool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,qBAAa,aAAa,CAAC,CAAC,EAAE,CAAC;IAGjB,OAAO,CAAC,QAAQ;IAF5B,OAAO,CAAC,GAAG,CAAe;gBAEN,QAAQ,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM;IAI5D,yBAAyB,CACvB,KAAK,EAAE,MAAM,EACb,oBAAoB,EAAE,MAAM,GAC3B,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAazB,gBAAgB,CAAC,UAAU,GAAE,MAAY,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAM3D,OAAO,CAAC,YAAY;CAOrB"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export class CandidatePool {
|
|
2
|
+
trainset;
|
|
3
|
+
rng;
|
|
4
|
+
constructor(trainset, seed) {
|
|
5
|
+
this.trainset = trainset;
|
|
6
|
+
this.rng = seed !== undefined ? this.seededRandom(seed) : Math.random;
|
|
7
|
+
}
|
|
8
|
+
generateFewShotCandidates(count, examplesPerCandidate) {
|
|
9
|
+
const candidates = [];
|
|
10
|
+
for (let i = 0; i < count; i++) {
|
|
11
|
+
const shuffled = [...this.trainset].sort(() => this.rng() - 0.5);
|
|
12
|
+
candidates.push(shuffled.slice(0, Math.min(examplesPerCandidate, shuffled.length)));
|
|
13
|
+
}
|
|
14
|
+
return candidates;
|
|
15
|
+
}
|
|
16
|
+
getValidationSet(splitRatio = 0.3) {
|
|
17
|
+
const count = Math.max(1, Math.floor(this.trainset.length * splitRatio));
|
|
18
|
+
const shuffled = [...this.trainset].sort(() => this.rng() - 0.5);
|
|
19
|
+
return shuffled.slice(0, count);
|
|
20
|
+
}
|
|
21
|
+
seededRandom(seed) {
|
|
22
|
+
let s = seed;
|
|
23
|
+
return () => {
|
|
24
|
+
s = Math.sin(s) * 10000;
|
|
25
|
+
return s - Math.floor(s);
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=CandidatePool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CandidatePool.js","sourceRoot":"","sources":["../../src/compiler/CandidatePool.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,aAAa;IAGJ;IAFZ,GAAG,CAAe;IAE1B,YAAoB,QAAyB,EAAE,IAAa;QAAxC,aAAQ,GAAR,QAAQ,CAAiB;QAC3C,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACxE,CAAC;IAED,yBAAyB,CACvB,KAAa,EACb,oBAA4B;QAE5B,MAAM,UAAU,GAA2B,EAAE,CAAC;QAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;YACjE,UAAU,CAAC,IAAI,CACb,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CACnE,CAAC;QACJ,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,gBAAgB,CAAC,aAAqB,GAAG;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;QACjE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,IAAI,CAAC,GAAG,IAAI,CAAC;QACb,OAAO,GAAG,EAAE;YACV,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { ZodRawShape } from "zod";
|
|
2
|
+
import type { ProgramConfig, ProgramOutput } from "../program/Program.js";
|
|
3
|
+
import type { Example } from "./types.js";
|
|
4
|
+
export interface CompiledProgramMeta {
|
|
5
|
+
score: number;
|
|
6
|
+
compiledAt: string;
|
|
7
|
+
strategy: string;
|
|
8
|
+
tokenUsage: {
|
|
9
|
+
inputTokens: number;
|
|
10
|
+
outputTokens: number;
|
|
11
|
+
totalTokens: number;
|
|
12
|
+
calls: number;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export interface CompiledProgram<I, O> {
|
|
16
|
+
run(input: I, overrides?: Partial<ProgramConfig>): Promise<ProgramOutput<O>>;
|
|
17
|
+
meta: CompiledProgramMeta;
|
|
18
|
+
config: {
|
|
19
|
+
instructions: string;
|
|
20
|
+
fewShotExamples: Array<Example<unknown, unknown>>;
|
|
21
|
+
};
|
|
22
|
+
toJSON(): string;
|
|
23
|
+
}
|
|
24
|
+
export declare function createCompiledProgram<I extends ZodRawShape, O extends ZodRawShape, InputType = unknown, OutputType = unknown>(originalProgram: {
|
|
25
|
+
run(input: InputType, config?: ProgramConfig): Promise<ProgramOutput<OutputType>>;
|
|
26
|
+
}, compilationResult: {
|
|
27
|
+
meta: CompiledProgramMeta;
|
|
28
|
+
config: {
|
|
29
|
+
instructions: string;
|
|
30
|
+
fewShotExamples: Array<Example<unknown, unknown>>;
|
|
31
|
+
};
|
|
32
|
+
}): CompiledProgram<InputType, OutputType>;
|
|
33
|
+
export interface SerializedCompiledProgram {
|
|
34
|
+
meta: CompiledProgramMeta;
|
|
35
|
+
config: {
|
|
36
|
+
instructions: string;
|
|
37
|
+
fewShotExamples: Array<Example<unknown, unknown>>;
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export declare function loadCompiledProgram<InputType = unknown, OutputType = unknown>(jsonOrPath: string | SerializedCompiledProgram, program: {
|
|
41
|
+
run(input: InputType, config?: ProgramConfig): Promise<ProgramOutput<OutputType>>;
|
|
42
|
+
}): CompiledProgram<InputType, OutputType>;
|
|
43
|
+
//# sourceMappingURL=CompiledProgram.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CompiledProgram.d.ts","sourceRoot":"","sources":["../../src/compiler/CompiledProgram.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAa,MAAM,KAAK,CAAC;AAC7C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,eAAe,CAAC,CAAC,EAAE,CAAC;IACnC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;KACnD,CAAC;IACF,MAAM,IAAI,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,CACnC,CAAC,SAAS,WAAW,EACrB,CAAC,SAAS,WAAW,EACrB,SAAS,GAAG,OAAO,EACnB,UAAU,GAAG,OAAO,EAEpB,eAAe,EAAE;IACf,GAAG,CACD,KAAK,EAAE,SAAS,EAChB,MAAM,CAAC,EAAE,aAAa,GACrB,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;CACvC,EACD,iBAAiB,EAAE;IACjB,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;KACnD,CAAC;CACH,GACA,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CA8BxC;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;KACnD,CAAC;CACH;AAED,wBAAgB,mBAAmB,CAAC,SAAS,GAAG,OAAO,EAAE,UAAU,GAAG,OAAO,EAC3E,UAAU,EAAE,MAAM,GAAG,yBAAyB,EAC9C,OAAO,EAAE;IACP,GAAG,CACD,KAAK,EAAE,SAAS,EAChB,MAAM,CAAC,EAAE,aAAa,GACrB,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;CACvC,GACA,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAkBxC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export function createCompiledProgram(originalProgram, compilationResult) {
|
|
2
|
+
return {
|
|
3
|
+
async run(input, overrides) {
|
|
4
|
+
return originalProgram.run(input, {
|
|
5
|
+
instructions: overrides?.instructions ?? compilationResult.config.instructions,
|
|
6
|
+
fewShotExamples: overrides?.fewShotExamples ??
|
|
7
|
+
compilationResult.config.fewShotExamples.map((ex) => ({
|
|
8
|
+
input: ex.input,
|
|
9
|
+
output: ex.output,
|
|
10
|
+
})),
|
|
11
|
+
});
|
|
12
|
+
},
|
|
13
|
+
meta: compilationResult.meta,
|
|
14
|
+
config: compilationResult.config,
|
|
15
|
+
toJSON() {
|
|
16
|
+
return JSON.stringify({
|
|
17
|
+
meta: compilationResult.meta,
|
|
18
|
+
config: compilationResult.config,
|
|
19
|
+
}, null, 2);
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export function loadCompiledProgram(jsonOrPath, program) {
|
|
24
|
+
let data;
|
|
25
|
+
if (typeof jsonOrPath === "string") {
|
|
26
|
+
try {
|
|
27
|
+
data = JSON.parse(jsonOrPath);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
throw new Error("Invalid JSON string provided to loadCompiledProgram");
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
data = jsonOrPath;
|
|
35
|
+
}
|
|
36
|
+
if (!data.meta || !data.config) {
|
|
37
|
+
throw new Error("Invalid compiled program format: missing meta or config");
|
|
38
|
+
}
|
|
39
|
+
return createCompiledProgram(program, data);
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=CompiledProgram.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CompiledProgram.js","sourceRoot":"","sources":["../../src/compiler/CompiledProgram.ts"],"names":[],"mappings":"AA0BA,MAAM,UAAU,qBAAqB,CAMnC,eAKC,EACD,iBAMC;IAED,OAAO;QACL,KAAK,CAAC,GAAG,CACP,KAAgB,EAChB,SAAkC;YAElC,OAAO,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE;gBAChC,YAAY,EACV,SAAS,EAAE,YAAY,IAAI,iBAAiB,CAAC,MAAM,CAAC,YAAY;gBAClE,eAAe,EACb,SAAS,EAAE,eAAe;oBAC1B,iBAAiB,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;wBACpD,KAAK,EAAE,EAAE,CAAC,KAAgC;wBAC1C,MAAM,EAAE,EAAE,CAAC,MAAiC;qBAC7C,CAAC,CAAC;aACN,CAAC,CAAC;QACL,CAAC;QACD,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,MAAM,EAAE,iBAAiB,CAAC,MAAM;QAChC,MAAM;YACJ,OAAO,IAAI,CAAC,SAAS,CACnB;gBACE,IAAI,EAAE,iBAAiB,CAAC,IAAI;gBAC5B,MAAM,EAAE,iBAAiB,CAAC,MAAM;aACjC,EACD,IAAI,EACJ,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAUD,MAAM,UAAU,mBAAmB,CACjC,UAA8C,EAC9C,OAKC;IAED,IAAI,IAA+B,CAAC;IAEpC,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,OAAO,qBAAqB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ZodRawShape } from "zod";
|
|
2
|
+
import { Program } from "../program/Program.js";
|
|
3
|
+
import type { LLMProvider } from "../providers/types.js";
|
|
4
|
+
import type { Evaluator, Example, CompilationResult, CompileOptions } from "./types.js";
|
|
5
|
+
export interface InstructionRewriteOptions extends CompileOptions {
|
|
6
|
+
instructionVariations?: number;
|
|
7
|
+
provider: LLMProvider;
|
|
8
|
+
}
|
|
9
|
+
export declare class InstructionRewrite<O> {
|
|
10
|
+
private evaluator;
|
|
11
|
+
constructor(evaluator: Evaluator<O>);
|
|
12
|
+
compile<I extends ZodRawShape, OShape extends ZodRawShape>(program: Program<I, OShape>, trainset: Example<unknown, O>[], options: InstructionRewriteOptions): Promise<CompilationResult<unknown, O>>;
|
|
13
|
+
private generateInstructionVariants;
|
|
14
|
+
estimateCost(trainsetSize: number, options?: Partial<InstructionRewriteOptions>): {
|
|
15
|
+
estimatedCalls: number;
|
|
16
|
+
estimatedTokens: number;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=InstructionRewrite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"InstructionRewrite.d.ts","sourceRoot":"","sources":["../../src/compiler/InstructionRewrite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,KAAK,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAIhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC/D,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED,qBAAa,kBAAkB,CAAC,CAAC;IACnB,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAErC,OAAO,CAAC,CAAC,SAAS,WAAW,EAAE,MAAM,SAAS,WAAW,EAC7D,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,EAC3B,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAiH3B,2BAA2B;IAoCzC,YAAY,CACV,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,OAAO,CAAC,yBAAyB,CAAM,GAC/C;QACD,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB;CAmBF"}
|