flakelab 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/flakelab.mjs +0 -0
- package/dist/investigator/agent.js +80 -46
- package/dist/investigator/agent.js.map +1 -1
- package/dist/investigator/assessment.d.ts +17 -0
- package/dist/investigator/assessment.js +23 -0
- package/dist/investigator/assessment.js.map +1 -1
- package/dist/investigator/generation.d.ts +10 -0
- package/dist/investigator/generation.js +23 -0
- package/dist/investigator/generation.js.map +1 -0
- package/dist/investigator/planning.d.ts +3 -4
- package/dist/investigator/planning.js +30 -2
- package/dist/investigator/planning.js.map +1 -1
- package/package.json +20 -18
package/bin/flakelab.mjs
CHANGED
|
File without changes
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { generateText, Output } from "ai";
|
|
1
|
+
import { generateText, NoObjectGeneratedError, Output } from "ai";
|
|
2
2
|
import { evaluateExperiment } from "../discovery/evaluate.js";
|
|
3
|
-
import { applyInvestigationAssessment, investigationAssessmentSchema, validateExperimentEvidence, } from "./assessment.js";
|
|
3
|
+
import { applyInvestigationAssessment, generateValidInvestigationAssessment, investigationAssessmentSchema, validateExperimentEvidence, } from "./assessment.js";
|
|
4
4
|
import { InvestigationBudget } from "./budget.js";
|
|
5
|
+
import { RecoverableGenerationError } from "./generation.js";
|
|
5
6
|
import { InvestigationLedger } from "./ledger.js";
|
|
6
7
|
import { generateValidInvestigationPlan, investigationPlanSchema, } from "./planning.js";
|
|
7
8
|
import { readSafeTestContext } from "./safe-source.js";
|
|
@@ -84,23 +85,34 @@ export async function runInvestigation(options) {
|
|
|
84
85
|
const sources = await readSafeTestContext(options.projectRoot, options.test);
|
|
85
86
|
const planResult = await generateValidInvestigationPlan({
|
|
86
87
|
generate: async (prompt, temperature) => {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
88
|
+
try {
|
|
89
|
+
const result = await generateText({
|
|
90
|
+
model: options.model,
|
|
91
|
+
output: Output.object({ schema: investigationPlanSchema }),
|
|
92
|
+
prompt,
|
|
93
|
+
maxOutputTokens: options.outputTokenLimit,
|
|
94
|
+
maxRetries: 2,
|
|
95
|
+
timeout: { totalMs: budget.remainingMs() },
|
|
96
|
+
abortSignal: signal,
|
|
97
|
+
temperature,
|
|
98
|
+
});
|
|
99
|
+
return {
|
|
100
|
+
output: result.output,
|
|
101
|
+
usage: {
|
|
102
|
+
inputTokens: result.usage.inputTokens ?? 0,
|
|
103
|
+
outputTokens: result.usage.outputTokens ?? 0,
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
if (NoObjectGeneratedError.isInstance(error)) {
|
|
109
|
+
throw new RecoverableGenerationError(error.message, error.text, {
|
|
110
|
+
inputTokens: error.usage?.inputTokens ?? 0,
|
|
111
|
+
outputTokens: error.usage?.outputTokens ?? 0,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
104
116
|
},
|
|
105
117
|
initialPrompt: planningPrompt(options.test, sources, options.maximumDelayMs),
|
|
106
118
|
maxAttempts: Math.min(2, options.maxSteps - 1),
|
|
@@ -110,9 +122,9 @@ export async function runInvestigation(options) {
|
|
|
110
122
|
},
|
|
111
123
|
});
|
|
112
124
|
const plan = planResult.plan;
|
|
113
|
-
let modelSteps = planResult.
|
|
114
|
-
const planningInputTokens = planResult.
|
|
115
|
-
const planningOutputTokens = planResult.
|
|
125
|
+
let modelSteps = planResult.attempts.length;
|
|
126
|
+
const planningInputTokens = planResult.attempts.reduce((total, usage) => total + usage.inputTokens, 0);
|
|
127
|
+
const planningOutputTokens = planResult.attempts.reduce((total, usage) => total + usage.outputTokens, 0);
|
|
116
128
|
enforceCostBudget(planningInputTokens, planningOutputTokens, options, budget);
|
|
117
129
|
plan.experiments.forEach(() => {
|
|
118
130
|
budget.reserveExperiment(options.trialsPerExperiment);
|
|
@@ -127,42 +139,64 @@ export async function runInvestigation(options) {
|
|
|
127
139
|
})));
|
|
128
140
|
let state = createLedgerState(plan, results);
|
|
129
141
|
validateExperimentEvidence(state.evidence);
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
142
|
+
const ledgerState = { hypotheses: state.hypotheses, experiments: state.evidence };
|
|
143
|
+
const generateAssessment = async (prompt, temperature) => {
|
|
144
|
+
try {
|
|
145
|
+
const result = await generateText({
|
|
146
|
+
model: options.model,
|
|
147
|
+
output: Output.object({ schema: investigationAssessmentSchema }),
|
|
148
|
+
prompt,
|
|
149
|
+
maxOutputTokens: options.outputTokenLimit,
|
|
150
|
+
maxRetries: 2,
|
|
151
|
+
timeout: { totalMs: budget.remainingMs() },
|
|
152
|
+
abortSignal: signal,
|
|
153
|
+
temperature,
|
|
154
|
+
});
|
|
155
|
+
return {
|
|
156
|
+
output: result.output,
|
|
157
|
+
usage: {
|
|
158
|
+
inputTokens: result.usage.inputTokens ?? 0,
|
|
159
|
+
outputTokens: result.usage.outputTokens ?? 0,
|
|
160
|
+
},
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
catch (error) {
|
|
164
|
+
if (NoObjectGeneratedError.isInstance(error)) {
|
|
165
|
+
throw new RecoverableGenerationError(error.message, error.text, {
|
|
166
|
+
inputTokens: error.usage?.inputTokens ?? 0,
|
|
167
|
+
outputTokens: error.usage?.outputTokens ?? 0,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
const assessmentResult = await generateValidInvestigationAssessment({
|
|
174
|
+
generate: generateAssessment,
|
|
175
|
+
initialPrompt: assessmentPrompt(ledgerState),
|
|
176
|
+
maxAttempts: Math.min(2, options.maxSteps - modelSteps),
|
|
139
177
|
});
|
|
140
|
-
modelSteps +=
|
|
141
|
-
const
|
|
178
|
+
modelSteps += assessmentResult.attempts.length;
|
|
179
|
+
const assessmentAttempts = [...assessmentResult.attempts];
|
|
142
180
|
try {
|
|
143
|
-
applyInvestigationAssessment(state, assessmentResult.
|
|
181
|
+
applyInvestigationAssessment(state, assessmentResult.assessment);
|
|
144
182
|
}
|
|
145
183
|
catch (error) {
|
|
146
184
|
if (modelSteps >= options.maxSteps) {
|
|
147
185
|
throw error;
|
|
148
186
|
}
|
|
149
187
|
const validationError = error instanceof Error ? error.message : "invalid evidence assessment";
|
|
150
|
-
const repairResult = await
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
prompt: assessmentRepairPrompt({ hypotheses: state.hypotheses, experiments: state.evidence }, assessmentResult.output, validationError),
|
|
154
|
-
maxOutputTokens: options.outputTokenLimit,
|
|
155
|
-
maxRetries: 1,
|
|
156
|
-
timeout: { totalMs: budget.remainingMs() },
|
|
157
|
-
abortSignal: signal,
|
|
188
|
+
const repairResult = await generateValidInvestigationAssessment({
|
|
189
|
+
generate: generateAssessment,
|
|
190
|
+
initialPrompt: assessmentRepairPrompt(ledgerState, assessmentResult.assessment, validationError),
|
|
158
191
|
temperature: 0,
|
|
192
|
+
maxAttempts: Math.min(2, options.maxSteps - modelSteps),
|
|
159
193
|
});
|
|
160
|
-
|
|
194
|
+
assessmentAttempts.push(...repairResult.attempts);
|
|
161
195
|
state = createLedgerState(plan, results);
|
|
162
|
-
applyInvestigationAssessment(state, repairResult.
|
|
196
|
+
applyInvestigationAssessment(state, repairResult.assessment);
|
|
163
197
|
}
|
|
164
|
-
const inputTokens = planningInputTokens +
|
|
165
|
-
const outputTokens = planningOutputTokens +
|
|
198
|
+
const inputTokens = planningInputTokens + assessmentAttempts.reduce((total, usage) => total + usage.inputTokens, 0);
|
|
199
|
+
const outputTokens = planningOutputTokens + assessmentAttempts.reduce((total, usage) => total + usage.outputTokens, 0);
|
|
166
200
|
const cost = enforceCostBudget(inputTokens, outputTokens, options, budget);
|
|
167
201
|
return state.ledger.buildReport(options.test, options.modelId, sources.map((source) => source.path), {
|
|
168
202
|
inputTokens,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../src/investigator/agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../src/investigator/agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,EAAE,MAAM,IAAI,CAAA;AAEjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAG7D,OAAO,EACL,4BAA4B,EAE5B,oCAAoC,EACpC,6BAA6B,EAE7B,0BAA0B,GAC3B,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AACjD,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAA;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AACjD,OAAO,EACL,8BAA8B,EAC9B,uBAAuB,GAGxB,MAAM,eAAe,CAAA;AACtB,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAMtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AA+BlD,SAAS,aAAa,CACpB,WAAmB,EACnB,YAAoB,EACpB,OAA4B;IAE5B,OAAO,CACL,WAAW,GAAG,OAAO,CAAC,kBAAkB;UACtC,YAAY,GAAG,OAAO,CAAC,mBAAmB,CAC7C,GAAG,SAAS,CAAA;AACf,CAAC;AAED,SAAS,iBAAiB,CACxB,WAAmB,EACnB,YAAoB,EACpB,OAA4B,EAC5B,MAA2B;IAE3B,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAA;IAC9D,IAAI,IAAI,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAA;IAC1F,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,OAA4C,EAC5C,cAAsB;IAEtB,OAAO;QACL,qEAAqE;QACrE,sFAAsF;QACtF,4FAA4F;QAC5F,oBAAoB;QACpB,8FAA8F;QAC9F,4FAA4F;QAC5F,0FAA0F;QAC1F,yFAAyF;QACzF,gFAAgF;QAChF,wEAAwE;QACxE,qCAAqC;QACrC,iGAAiG;QACjG,kGAAkG;QAClG,qFAAqF;QACrF,sEAAsE;QACtE,uFAAuF;QACvF,4CAA4C;QAC5C,4HAA4H,cAAc,MAAM;QAChJ,+DAA+D;QAC/D,cAAc,IAAI,EAAE;QACpB,+BAA+B;QAC/B,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;KACvE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,WAAmB;IAC3C,OAAO;QACL,8EAA8E;QAC9E,uFAAuF;QACvF,wFAAwF;QACxF,yFAAyF;QACzF,uFAAuF;QACvF,2EAA2E;QAC3E,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;KACrC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC;AAED,SAAS,sBAAsB,CAC7B,WAAmB,EACnB,QAAiC,EACjC,eAAuB;IAEvB,OAAO;QACL,gBAAgB,CAAC,WAAW,CAAC;QAC7B,gFAAgF;QAChF,qBAAqB,eAAe,EAAE;QACtC,8BAA8B;QAC9B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,8EAA8E;KAC/E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAuB,EAAE,OAA2B;IAC7E,MAAM,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAA;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACpD,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAA;IAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAC1C,MAAM,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,eAAe,CAAC,CAAA;QACzD,OAAO,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IAC1E,CAAC,CAAC,CAAA;IACF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAA4B;IACjE,IAAI,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;IACvE,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAC/C,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,CAAA;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAA;IAChG,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5E,MAAM,UAAU,GAAG,MAAM,8BAA8B,CAAC;QACtD,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAA2B,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;oBAChC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;oBAC1D,MAAM;oBACN,eAAe,EAAE,OAAO,CAAC,gBAAgB;oBACzC,UAAU,EAAE,CAAC;oBACb,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,EAAE,EAAE;oBAC1C,WAAW,EAAE,MAAM;oBACnB,WAAW;iBACZ,CAAC,CAAA;gBACF,OAAO;oBACL,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,KAAK,EAAE;wBACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC;wBAC1C,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC;qBAC7C;iBACF,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,sBAAsB,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7C,MAAM,IAAI,0BAA0B,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;wBAC9D,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;wBAC1C,YAAY,EAAE,KAAK,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;qBAC7C,CAAC,CAAA;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAA;YACb,CAAC;QACH,CAAC;QACD,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC;QAC5E,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC9C,KAAK,EAAE;YACL,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC;KACF,CAAC,CAAA;IACF,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAA;IAC5B,IAAI,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAA;IAC3C,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CACpD,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,WAAW,EAC3C,CAAC,CACF,CAAA;IACD,MAAM,oBAAoB,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CACrD,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,EAC5C,CAAC,CACF,CAAA;IACD,iBAAiB,CAAC,mBAAmB,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;IAE7E,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;QAC5B,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAA;IACvD,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAC1E,kBAAkB,CAAC,OAAO,CAAC,OAAO,EAAE;QAClC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,MAAM,EAAE,iBAAiB,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;QAC9E,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,mBAAmB;KACpC,CAAC,CAAC,CAAC,CAAA;IACN,IAAI,KAAK,GAAG,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAC5C,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE1C,MAAM,WAAW,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAA;IACjF,MAAM,kBAAkB,GAAG,KAAK,EAC9B,MAAc,EACd,WAAmB,EACY,EAAE;QACjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAChC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,6BAA6B,EAAE,CAAC;gBAChE,MAAM;gBACN,eAAe,EAAE,OAAO,CAAC,gBAAgB;gBACzC,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,EAAE,EAAE;gBAC1C,WAAW,EAAE,MAAM;gBACnB,WAAW;aACZ,CAAC,CAAA;YACF,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE;oBACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC;oBAC1C,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC;iBAC7C;aACF,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,sBAAsB,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,0BAA0B,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;oBAC9D,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC1C,YAAY,EAAE,KAAK,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;iBAC7C,CAAC,CAAA;YACJ,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC,CAAA;IACD,MAAM,gBAAgB,GAAG,MAAM,oCAAoC,CAAC;QAClE,QAAQ,EAAE,kBAAkB;QAC5B,aAAa,EAAE,gBAAgB,CAAC,WAAW,CAAC;QAC5C,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC;KACxD,CAAC,CAAA;IACF,UAAU,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAA;IAC9C,MAAM,kBAAkB,GAAG,CAAC,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACzD,IAAI,CAAC;QACH,4BAA4B,CAAC,KAAK,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAA;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,KAAK,CAAA;QACb,CAAC;QACD,MAAM,eAAe,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAA;QAC9F,MAAM,YAAY,GAAG,MAAM,oCAAoC,CAAC;YAC9D,QAAQ,EAAE,kBAAkB;YAC5B,aAAa,EAAE,sBAAsB,CACnC,WAAW,EACX,gBAAgB,CAAC,UAAU,EAC3B,eAAe,CAChB;YACD,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC;SACxD,CAAC,CAAA;QACF,kBAAkB,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAA;QACjD,KAAK,GAAG,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QACxC,4BAA4B,CAAC,KAAK,EAAE,YAAY,CAAC,UAAU,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,GAAG,kBAAkB,CAAC,MAAM,CACjE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,WAAW,EAC3C,CAAC,CACF,CAAA;IACD,MAAM,YAAY,GAAG,oBAAoB,GAAG,kBAAkB,CAAC,MAAM,CACnE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,EAC5C,CAAC,CACF,CAAA;IACD,MAAM,IAAI,GAAG,iBAAiB,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1E,OAAO,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACnG,WAAW;QACX,YAAY;QACZ,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { InvestigationLedger } from "./ledger.js";
|
|
3
|
+
import { type ModelGenerationUsage } from "./generation.js";
|
|
3
4
|
import type { ExperimentEvidence, Hypothesis } from "./schema.js";
|
|
4
5
|
export declare const investigationAssessmentSchema: z.ZodObject<{
|
|
5
6
|
assessments: z.ZodArray<z.ZodObject<{
|
|
@@ -14,6 +15,21 @@ export declare const investigationAssessmentSchema: z.ZodObject<{
|
|
|
14
15
|
conclusion: z.ZodString;
|
|
15
16
|
}, z.core.$strip>;
|
|
16
17
|
export type InvestigationAssessment = z.infer<typeof investigationAssessmentSchema>;
|
|
18
|
+
export interface AssessmentGeneration {
|
|
19
|
+
output: InvestigationAssessment;
|
|
20
|
+
usage: ModelGenerationUsage;
|
|
21
|
+
}
|
|
22
|
+
interface GenerateValidAssessmentOptions {
|
|
23
|
+
generate: (prompt: string, temperature: number) => Promise<AssessmentGeneration>;
|
|
24
|
+
initialPrompt: string;
|
|
25
|
+
maxAttempts: number;
|
|
26
|
+
temperature?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface ValidAssessmentResult {
|
|
29
|
+
assessment: InvestigationAssessment;
|
|
30
|
+
attempts: ModelGenerationUsage[];
|
|
31
|
+
}
|
|
32
|
+
export declare function generateValidInvestigationAssessment(options: GenerateValidAssessmentOptions): Promise<ValidAssessmentResult>;
|
|
17
33
|
export interface AssessmentState {
|
|
18
34
|
evidence: ExperimentEvidence[];
|
|
19
35
|
hypotheses: Hypothesis[];
|
|
@@ -21,3 +37,4 @@ export interface AssessmentState {
|
|
|
21
37
|
}
|
|
22
38
|
export declare function validateExperimentEvidence(evidence: ExperimentEvidence[]): void;
|
|
23
39
|
export declare function applyInvestigationAssessment(state: AssessmentState, assessment: InvestigationAssessment): void;
|
|
40
|
+
export {};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { RecoverableGenerationError, schemaCorrectionPrompt, } from "./generation.js";
|
|
2
3
|
const assessmentItemSchema = z.object({
|
|
3
4
|
hypothesisId: z.string().regex(/^H\d+$/u),
|
|
4
5
|
status: z.enum(["rejected", "confirmed"]),
|
|
@@ -9,6 +10,28 @@ export const investigationAssessmentSchema = z.object({
|
|
|
9
10
|
conclusionHypothesisId: z.string().regex(/^H\d+$/u),
|
|
10
11
|
conclusion: z.string().min(30).max(2_000),
|
|
11
12
|
});
|
|
13
|
+
export async function generateValidInvestigationAssessment(options) {
|
|
14
|
+
const attempts = [];
|
|
15
|
+
let prompt = options.initialPrompt;
|
|
16
|
+
for (let attempt = 0; attempt < options.maxAttempts; attempt += 1) {
|
|
17
|
+
try {
|
|
18
|
+
const generated = await options.generate(prompt, attempt === 0 ? (options.temperature ?? 0.2) : 0);
|
|
19
|
+
attempts.push(generated.usage);
|
|
20
|
+
return { assessment: generated.output, attempts };
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
if (!(error instanceof RecoverableGenerationError)) {
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
attempts.push(error.usage);
|
|
27
|
+
if (attempt + 1 >= options.maxAttempts) {
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
prompt = schemaCorrectionPrompt(options.initialPrompt, error, "assessments (exactly 2), conclusionHypothesisId, and conclusion");
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
throw new Error("Investigation assessment generation exhausted its attempt budget");
|
|
34
|
+
}
|
|
12
35
|
function completedTrials(entry) {
|
|
13
36
|
return entry.result.passed + entry.result.failed;
|
|
14
37
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"assessment.js","sourceRoot":"","sources":["../../src/investigator/assessment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"assessment.js","sourceRoot":"","sources":["../../src/investigator/assessment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EAEL,0BAA0B,EAC1B,sBAAsB,GACvB,MAAM,iBAAiB,CAAA;AAGxB,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;IACzC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IACzC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;CAC1C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IACpD,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;IACnD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;CAC1C,CAAC,CAAA;AAqBF,MAAM,CAAC,KAAK,UAAU,oCAAoC,CACxD,OAAuC;IAEvC,MAAM,QAAQ,GAA2B,EAAE,CAAA;IAC3C,IAAI,MAAM,GAAG,OAAO,CAAC,aAAa,CAAA;IAClC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CACtC,MAAM,EACN,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CACjD,CAAA;YACD,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAC9B,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAA;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC,EAAE,CAAC;gBACnD,MAAM,KAAK,CAAA;YACb,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YAC1B,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,GAAG,sBAAsB,CAC7B,OAAO,CAAC,aAAa,EACrB,KAAK,EACL,iEAAiE,CAClE,CAAA;QACH,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;AACrF,CAAC;AAQD,SAAS,eAAe,CAAC,KAAyB;IAChD,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAA;AAClD,CAAC;AAED,SAAS,aAAa,CAAC,KAAyB;IAC9C,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,mBAAmB;YAC7C,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACzC,CAAC,CAAC,EAAE,CAAA;QACN,OAAO,cAAc,KAAK,CAAC,EAAE,aAAa,KAAK,CAAC,MAAM,CAAC,MAAM,uCAAuC,MAAM,EAAE,CAAA;IAC9G,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAChF,OAAO,cAAc,KAAK,CAAC,EAAE,kEAAkE,CAAA;IACjG,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,QAA8B;IACvE,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAA;IAClF,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6DAA6D,CAAC,CAAA;IACxF,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAA8B;IACtD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;IACjF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAA;IAClF,CAAC;IACD,OAAO,SAAS,CAAC,CAAC,CAAC,CAAA;AACrB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAyB,EAAE,QAA4B;IACvE,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU;WACrC,KAAK,CAAC,MAAM,CAAC,SAAS;WACtB,KAAK,CAAC,MAAM,CAAC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAA;AAC/D,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAA8B,EAC9B,YAAoB,EACpB,MAAgC,EAChC,QAA4B;IAE5B,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAC9C,KAAK,CAAC,YAAY,KAAK,YAAY,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;IAC7E,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAC9C,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAA;IAClF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,cAAc,YAAY,cAAc,MAAM,2BAA2B,CAAC,CAAA;IAC5F,CAAC;IACD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC1C,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,KAAsB,EACtB,UAAmC;IAEnC,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC1C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAA;IACpF,IAAI,WAAW,CAAC,IAAI,KAAK,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAA;IACpF,CAAC;IACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IACjD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAoB,CAAA;IACxD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,oBAAoB,CACtC,KAAK,CAAC,QAAQ,EACd,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,EACX,QAAQ,CACT,CAAA;QACD,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;QACxD,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;IACpF,CAAC;IACD,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAA;IACtF,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;IACpE,CAAC;IACD,KAAK,CAAC,MAAM,CAAC,QAAQ,CACnB,UAAU,CAAC,sBAAsB,EACjC,UAAU,CAAC,UAAU,EACrB,kBAAkB,CACnB,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface ModelGenerationUsage {
|
|
2
|
+
inputTokens: number;
|
|
3
|
+
outputTokens: number;
|
|
4
|
+
}
|
|
5
|
+
export declare class RecoverableGenerationError extends Error {
|
|
6
|
+
readonly generatedText: string | undefined;
|
|
7
|
+
readonly usage: ModelGenerationUsage;
|
|
8
|
+
constructor(message: string, generatedText: string | undefined, usage: ModelGenerationUsage);
|
|
9
|
+
}
|
|
10
|
+
export declare function schemaCorrectionPrompt(initialPrompt: string, failure: RecoverableGenerationError, requirements: string): string;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export class RecoverableGenerationError extends Error {
|
|
2
|
+
generatedText;
|
|
3
|
+
usage;
|
|
4
|
+
constructor(message, generatedText, usage) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "RecoverableGenerationError";
|
|
7
|
+
this.generatedText = generatedText;
|
|
8
|
+
this.usage = usage;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
export function schemaCorrectionPrompt(initialPrompt, failure, requirements) {
|
|
12
|
+
const generated = failure.generatedText?.slice(0, 4_000) ?? "(provider output unavailable)";
|
|
13
|
+
return [
|
|
14
|
+
initialPrompt,
|
|
15
|
+
"Your previous JSON response did not match the required output schema.",
|
|
16
|
+
`Schema error: ${failure.message}`,
|
|
17
|
+
`Required shape: ${requirements}`,
|
|
18
|
+
"Previous invalid JSON:",
|
|
19
|
+
generated,
|
|
20
|
+
"Return only one corrected JSON object.",
|
|
21
|
+
].join("\n");
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=generation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generation.js","sourceRoot":"","sources":["../../src/investigator/generation.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,0BAA2B,SAAQ,KAAK;IAC1C,aAAa,CAAoB;IACjC,KAAK,CAAsB;IAEpC,YACE,OAAe,EACf,aAAiC,EACjC,KAA2B;QAE3B,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,4BAA4B,CAAA;QACxC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;QAClC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACpB,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CACpC,aAAqB,EACrB,OAAmC,EACnC,YAAoB;IAEpB,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,+BAA+B,CAAA;IAC3F,OAAO;QACL,aAAa;QACb,uEAAuE;QACvE,iBAAiB,OAAO,CAAC,OAAO,EAAE;QAClC,mBAAmB,YAAY,EAAE;QACjC,wBAAwB;QACxB,SAAS;QACT,wCAAwC;KACzC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { type ModelGenerationUsage } from "./generation.js";
|
|
2
3
|
export declare const investigationPlanSchema: z.ZodObject<{
|
|
3
4
|
hypotheses: z.ZodArray<z.ZodObject<{
|
|
4
5
|
statement: z.ZodString;
|
|
@@ -86,10 +87,7 @@ export declare const investigationPlanSchema: z.ZodObject<{
|
|
|
86
87
|
export type InvestigationPlan = z.infer<typeof investigationPlanSchema>;
|
|
87
88
|
export interface PlanGeneration {
|
|
88
89
|
output: InvestigationPlan;
|
|
89
|
-
usage:
|
|
90
|
-
inputTokens: number;
|
|
91
|
-
outputTokens: number;
|
|
92
|
-
};
|
|
90
|
+
usage: ModelGenerationUsage;
|
|
93
91
|
}
|
|
94
92
|
interface PlanRules {
|
|
95
93
|
maxExperiments: number;
|
|
@@ -102,6 +100,7 @@ interface GenerateValidPlanOptions {
|
|
|
102
100
|
rules: PlanRules;
|
|
103
101
|
}
|
|
104
102
|
export interface ValidPlanResult {
|
|
103
|
+
attempts: ModelGenerationUsage[];
|
|
105
104
|
generations: PlanGeneration[];
|
|
106
105
|
plan: InvestigationPlan;
|
|
107
106
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { RecoverableGenerationError, schemaCorrectionPrompt, } from "./generation.js";
|
|
2
3
|
import { experimentConditionSchema } from "./schema.js";
|
|
3
4
|
const plannedHypothesisSchema = z.object({
|
|
4
5
|
statement: z.string().min(8).max(500),
|
|
@@ -58,15 +59,42 @@ function repairPrompt(initialPrompt, invalidPlan, validationError) {
|
|
|
58
59
|
"Keep exactly three experiments, cover every hypothesis, and do not repeat a condition kind.",
|
|
59
60
|
].join("\n");
|
|
60
61
|
}
|
|
62
|
+
function recoverSchemaFailure(error, options, attempt, attempts) {
|
|
63
|
+
if (!(error instanceof RecoverableGenerationError)) {
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
attempts.push(error.usage);
|
|
67
|
+
if (attempt + 1 >= options.maxAttempts) {
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
return schemaCorrectionPrompt(options.initialPrompt, error, "hypotheses (exactly 2) and experiments (exactly 3)");
|
|
71
|
+
}
|
|
72
|
+
function asGenerationError(error) {
|
|
73
|
+
if (error instanceof Error) {
|
|
74
|
+
return error;
|
|
75
|
+
}
|
|
76
|
+
return new Error("Investigation plan generation failed without an error message", {
|
|
77
|
+
cause: error,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
61
80
|
export async function generateValidInvestigationPlan(options) {
|
|
81
|
+
const attempts = [];
|
|
62
82
|
const generations = [];
|
|
63
83
|
let prompt = options.initialPrompt;
|
|
64
84
|
for (let attempt = 0; attempt < options.maxAttempts; attempt += 1) {
|
|
65
|
-
|
|
85
|
+
let generated;
|
|
86
|
+
try {
|
|
87
|
+
generated = await options.generate(prompt, attempt === 0 ? 0.2 : 0);
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
prompt = recoverSchemaFailure(asGenerationError(error), options, attempt, attempts);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
attempts.push(generated.usage);
|
|
66
94
|
generations.push(generated);
|
|
67
95
|
try {
|
|
68
96
|
validateInvestigationPlan(generated.output, options.rules);
|
|
69
|
-
return { generations, plan: generated.output };
|
|
97
|
+
return { attempts, generations, plan: generated.output };
|
|
70
98
|
}
|
|
71
99
|
catch (error) {
|
|
72
100
|
if (attempt + 1 >= options.maxAttempts) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"planning.js","sourceRoot":"","sources":["../../src/investigator/planning.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAA;AAEvD,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;CACvC,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACtD,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC/C,SAAS,EAAE,yBAAyB;KACrC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;CACd,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"planning.js","sourceRoot":"","sources":["../../src/investigator/planning.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAEL,0BAA0B,EAC1B,sBAAsB,GACvB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAA;AAEvD,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;CACvC,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACtD,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC/C,SAAS,EAAE,yBAAyB;KACrC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;CACd,CAAC,CAAA;AA2BF,SAAS,cAAc,CAAC,IAAuB;IAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,IAAuB,EAAE,KAAgB;IACjF,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;IAC/E,CAAC;IACD,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IACD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAA;IACzF,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAA;IACzE,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAChF,KAAK,CAAC,eAAe,KAAK,KAAK,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,eAAe;WACpC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,wBAAwB;WACjD,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,qBAAqB;WAC9C,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,qBAAqB,CAAC;WACjD,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,CAAA;IACpD,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,kBAAkB;WACxC,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC,cAAc,CAAC,CAAA;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAA;IACpF,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CACb,8EAA8E;cAC5E,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClC,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CACnB,aAAqB,EACrB,WAA8B,EAC9B,eAAuB;IAEvB,OAAO;QACL,aAAa;QACb,kFAAkF;QAClF,qBAAqB,eAAe,EAAE;QACtC,wBAAwB;QACxB,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,2FAA2F;QAC3F,6FAA6F;KAC9F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAY,EACZ,OAAiC,EACjC,OAAe,EACf,QAAgC;IAEhC,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC,EAAE,CAAC;QACnD,MAAM,KAAK,CAAA;IACb,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAC1B,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,MAAM,KAAK,CAAA;IACb,CAAC;IACD,OAAO,sBAAsB,CAC3B,OAAO,CAAC,aAAa,EACrB,KAAK,EACL,oDAAoD,CACrD,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAI,KAAQ;IACpC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,+DAA+D,EAAE;QAChF,KAAK,EAAE,KAAK;KACb,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,OAAiC;IAEjC,MAAM,QAAQ,GAA2B,EAAE,CAAA;IAC3C,MAAM,WAAW,GAAqB,EAAE,CAAA;IACxC,IAAI,MAAM,GAAG,OAAO,CAAC,aAAa,CAAA;IAClC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAClE,IAAI,SAAyB,CAAA;QAC7B,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;YACnF,SAAQ;QACV,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAC9B,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAC3B,IAAI,CAAC;YACH,yBAAyB,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;YAC1D,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,CAAA;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,4BAA4B,CAAA;YACrF,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACzE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;AAC/E,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "flakelab",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.11",
|
|
4
4
|
"description": "Find, reproduce, and prove fixes for flaky Playwright tests.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
"engines": {
|
|
14
14
|
"node": ">=20"
|
|
15
15
|
},
|
|
16
|
+
"packageManager": "pnpm@11.6.0",
|
|
16
17
|
"bin": {
|
|
17
18
|
"flakelab": "./bin/flakelab.mjs"
|
|
18
19
|
},
|
|
@@ -24,6 +25,23 @@
|
|
|
24
25
|
"publishConfig": {
|
|
25
26
|
"access": "public"
|
|
26
27
|
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "node scripts/clean-build.mjs && tsc -p tsconfig.build.json && node scripts/build-report-client.mjs",
|
|
30
|
+
"build:report-client": "node scripts/build-report-client.mjs",
|
|
31
|
+
"ci:select": "tsx src/ci/select-command.ts",
|
|
32
|
+
"ci:summary": "tsx src/ci/summary-command.ts",
|
|
33
|
+
"flakelab": "tsx src/cli.ts",
|
|
34
|
+
"lint": "eslint . --max-warnings=0",
|
|
35
|
+
"lint:fix": "eslint . --fix --max-warnings=0",
|
|
36
|
+
"prepack": "pnpm build",
|
|
37
|
+
"test": "pnpm build:report-client && playwright test --pass-with-no-tests",
|
|
38
|
+
"test:headed": "pnpm build:report-client && playwright test --headed",
|
|
39
|
+
"typecheck": "tsc --noEmit",
|
|
40
|
+
"quality": "pnpm typecheck && pnpm lint && pnpm test",
|
|
41
|
+
"verify:solari-parallel": "tsx src/verify-solari-parallel.ts",
|
|
42
|
+
"verify:solari": "tsx src/verify-solari.ts",
|
|
43
|
+
"verify:bisect": "tsx src/verify-bisect.ts"
|
|
44
|
+
},
|
|
27
45
|
"dependencies": {
|
|
28
46
|
"@ai-sdk/groq": "^4.0.35",
|
|
29
47
|
"@solarisdk/browser": "^0.1.2",
|
|
@@ -54,21 +72,5 @@
|
|
|
54
72
|
"typescript": "^5.9.2",
|
|
55
73
|
"typescript-eslint": "^8.69.0",
|
|
56
74
|
"vite": "^8.2.2"
|
|
57
|
-
},
|
|
58
|
-
"scripts": {
|
|
59
|
-
"build": "node scripts/clean-build.mjs && tsc -p tsconfig.build.json && node scripts/build-report-client.mjs",
|
|
60
|
-
"build:report-client": "node scripts/build-report-client.mjs",
|
|
61
|
-
"ci:select": "tsx src/ci/select-command.ts",
|
|
62
|
-
"ci:summary": "tsx src/ci/summary-command.ts",
|
|
63
|
-
"flakelab": "tsx src/cli.ts",
|
|
64
|
-
"lint": "eslint . --max-warnings=0",
|
|
65
|
-
"lint:fix": "eslint . --fix --max-warnings=0",
|
|
66
|
-
"test": "pnpm build:report-client && playwright test --pass-with-no-tests",
|
|
67
|
-
"test:headed": "pnpm build:report-client && playwright test --headed",
|
|
68
|
-
"typecheck": "tsc --noEmit",
|
|
69
|
-
"quality": "pnpm typecheck && pnpm lint && pnpm test",
|
|
70
|
-
"verify:solari-parallel": "tsx src/verify-solari-parallel.ts",
|
|
71
|
-
"verify:solari": "tsx src/verify-solari.ts",
|
|
72
|
-
"verify:bisect": "tsx src/verify-bisect.ts"
|
|
73
75
|
}
|
|
74
|
-
}
|
|
76
|
+
}
|