@cursor/july 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ab.d.ts +8 -95
- package/dist/ab.d.ts.map +1 -1
- package/dist/ab.js +9 -150
- package/dist/bin/agent-serve.js +14 -6
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +4 -4
- package/dist/docs/assets/{app.DabPG-io.js → app.DMboGKUD.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.O6eP7QZr.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.jmyr0bU0.js → VPLocalSearchBox.DkEqEnJU.js} +1 -1
- package/dist/docs/assets/chunks/{theme.DysN9-VN.js → theme.ASugdMNm.js} +2 -2
- package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
- package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
- package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
- package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
- package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
- package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
- package/dist/docs/building-with-agents.html +4 -4
- package/dist/docs/concepts.html +4 -4
- package/dist/docs/deployment.html +58 -17
- package/dist/docs/evals.html +4 -4
- package/dist/docs/example-agents/approval-buddy.html +34 -0
- package/dist/docs/example-agents/benny.html +31 -0
- package/dist/docs/example-agents/bugbot.html +35 -0
- package/dist/docs/example-agents/codebase-wiki.html +32 -0
- package/dist/docs/example-agents/codeowners-review.html +32 -0
- package/dist/docs/example-agents/concierge.html +47 -0
- package/dist/docs/example-agents/fsd.html +39 -0
- package/dist/docs/example-agents/index.html +26 -0
- package/dist/docs/example-agents/knowledge-base.html +35 -0
- package/dist/docs/example-agents/security-reviewer.html +43 -0
- package/dist/docs/example-agents/slack-agent.html +29 -0
- package/dist/docs/example-agents/weather-agent.html +48 -0
- package/dist/docs/guides/agent-to-agent.html +4 -4
- package/dist/docs/guides/cloud-runtime.html +5 -5
- package/dist/docs/guides/github.html +4 -4
- package/dist/docs/guides/human-in-the-loop.html +4 -4
- package/dist/docs/guides/slack.html +4 -4
- package/dist/docs/guides/webhooks.html +4 -4
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +4 -4
- package/dist/docs/index.html +7 -7
- package/dist/docs/quickstart.html +4 -4
- package/dist/docs/reference/agent-config.html +4 -4
- package/dist/docs/reference/channels.html +4 -4
- package/dist/docs/reference/cli.html +52 -30
- package/dist/docs/reference/connections.html +4 -4
- package/dist/docs/reference/hooks.html +4 -4
- package/dist/docs/reference/http-api.html +4 -4
- package/dist/docs/reference/instructions.html +4 -4
- package/dist/docs/reference/playground.html +4 -4
- package/dist/docs/reference/project-layout.html +4 -4
- package/dist/docs/reference/schedules.html +4 -4
- package/dist/docs/reference/sessions.html +4 -4
- package/dist/docs/reference/skills.html +4 -4
- package/dist/docs/reference/subagents.html +4 -4
- package/dist/docs/reference/tools.html +4 -4
- package/dist/docs/scaffolding-agents.html +4 -4
- package/dist/docs/storage.html +4 -4
- package/dist/docs/troubleshooting.html +4 -4
- package/dist/evals.d.ts +4 -61
- package/dist/evals.d.ts.map +1 -1
- package/dist/evals.js +3 -66
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/internal/ab-collector.d.ts +7 -5
- package/dist/internal/ab-collector.d.ts.map +1 -1
- package/dist/internal/ab-collector.js +3 -14
- package/dist/internal/ab-snapshot.d.ts +2 -4
- package/dist/internal/ab-snapshot.d.ts.map +1 -1
- package/dist/internal/cli-ax.d.ts +18 -3
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +80 -42
- package/dist/internal/cli-deploy.js +1 -1
- package/dist/internal/discovery.js +3 -3
- package/dist/internal/eval-run-store.d.ts +33 -30
- package/dist/internal/eval-run-store.d.ts.map +1 -1
- package/dist/internal/eval-run-store.js +84 -96
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +3 -2
- package/dist/internal/playground-proxy.d.ts +5 -5
- package/dist/internal/playground-proxy.js +3 -3
- package/dist/internal/resolve-prod-target.d.ts +30 -0
- package/dist/internal/resolve-prod-target.d.ts.map +1 -1
- package/dist/internal/resolve-prod-target.js +74 -2
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +5 -3
- package/dist/internal/session-engine.d.ts +1 -2
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +14 -31
- package/dist/internal/storage-coordinator.d.ts +16 -15
- package/dist/internal/storage-coordinator.d.ts.map +1 -1
- package/dist/internal/storage-coordinator.js +73 -80
- package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
- package/dist/playground/assets/index-7GJIUFQj.css +1 -0
- package/dist/playground/assets/index-mux9aj5P.js +319 -0
- package/dist/playground/index.html +2 -2
- package/dist/storage.d.ts +51 -10
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +27 -10
- package/docs/README.md +34 -5
- package/docs/deployment.md +352 -149
- package/docs/example-agents/approval-buddy.md +270 -0
- package/docs/example-agents/benny.md +186 -0
- package/docs/example-agents/bugbot.md +231 -0
- package/docs/example-agents/codebase-wiki.md +174 -0
- package/docs/example-agents/codeowners-review.md +195 -0
- package/docs/example-agents/concierge.md +205 -0
- package/docs/example-agents/fsd.md +330 -0
- package/docs/example-agents/index.md +102 -0
- package/docs/example-agents/knowledge-base.md +171 -0
- package/docs/example-agents/security-reviewer.md +296 -0
- package/docs/example-agents/slack-agent.md +146 -0
- package/docs/example-agents/weather-agent.md +302 -0
- package/docs/reference/cli.md +546 -147
- package/package.json +1 -1
- package/src/ab.ts +9 -261
- package/src/bin/agent-serve.ts +14 -6
- package/src/evals.ts +4 -118
- package/src/index.ts +2 -0
- package/src/internal/ab-collector.ts +12 -22
- package/src/internal/ab-snapshot.ts +2 -4
- package/src/internal/cli-ax.ts +101 -45
- package/src/internal/cli-deploy.ts +1 -1
- package/src/internal/discovery.ts +2 -2
- package/src/internal/eval-run-store.ts +86 -100
- package/src/internal/init-project.ts +3 -2
- package/src/internal/playground-proxy.ts +5 -5
- package/src/internal/resolve-prod-target.ts +101 -3
- package/src/internal/server.ts +5 -3
- package/src/internal/session-engine.ts +9 -29
- package/src/internal/storage-coordinator.ts +109 -101
- package/src/storage.ts +79 -14
- package/dist/docs/assets/chunks/@localSearchIndexroot.QwK5BtEH.js +0 -1
- package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
- package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
- package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
- package/dist/internal/json-dir-store.d.ts +0 -32
- package/dist/internal/json-dir-store.d.ts.map +0 -1
- package/dist/internal/json-dir-store.js +0 -100
- package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
- package/dist/playground/assets/index-BEauYlII.css +0 -1
- package/dist/playground/assets/index-BtM0wEGg.js +0 -319
- package/src/internal/json-dir-store.ts +0 -109
|
@@ -48,6 +48,29 @@ export interface ResolvedProdTarget {
|
|
|
48
48
|
expiresAt: string;
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Thrown when a hosted deployment cannot be resolved. `notFound` is true
|
|
53
|
+
* when the deployment does not exist, so callers can suggest deploying
|
|
54
|
+
* (or running locally) instead of surfacing a bare API error.
|
|
55
|
+
*/
|
|
56
|
+
export class ProdTargetError extends Error {
|
|
57
|
+
constructor(
|
|
58
|
+
message: string,
|
|
59
|
+
readonly options: { slug: string; notFound?: boolean } = { slug: "" }
|
|
60
|
+
) {
|
|
61
|
+
super(message);
|
|
62
|
+
this.name = "ProdTargetError";
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
get slug(): string {
|
|
66
|
+
return this.options.slug;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
get notFound(): boolean {
|
|
70
|
+
return this.options.notFound === true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
51
74
|
/**
|
|
52
75
|
* Look up a running deployment and return its engineAccess URL/headers.
|
|
53
76
|
* Throws {@link Error} with a CLI-ready message on failure.
|
|
@@ -65,8 +88,10 @@ export async function resolveProdTarget(
|
|
|
65
88
|
err,
|
|
66
89
|
});
|
|
67
90
|
if (typeof auth === "number") {
|
|
91
|
+
// Not "--prod": `eval` resolves the hosted deployment by default,
|
|
92
|
+
// without any flag.
|
|
68
93
|
throw new Error(
|
|
69
|
-
`Could not resolve Cursor credentials/team for
|
|
94
|
+
`Could not resolve Cursor credentials/team for the hosted deployment (see above).`
|
|
70
95
|
);
|
|
71
96
|
}
|
|
72
97
|
|
|
@@ -83,8 +108,9 @@ export async function resolveProdTarget(
|
|
|
83
108
|
});
|
|
84
109
|
} catch (error) {
|
|
85
110
|
if (error instanceof DeployApiError) {
|
|
86
|
-
throw new
|
|
87
|
-
`Deployment "${slug}" (team ${auth.teamId}): ${error.message}
|
|
111
|
+
throw new ProdTargetError(
|
|
112
|
+
`Deployment "${slug}" (team ${auth.teamId}): ${error.message}`,
|
|
113
|
+
{ slug, notFound: error.status === 404 }
|
|
88
114
|
);
|
|
89
115
|
}
|
|
90
116
|
throw error;
|
|
@@ -143,6 +169,78 @@ export async function resolveProdTarget(
|
|
|
143
169
|
};
|
|
144
170
|
}
|
|
145
171
|
|
|
172
|
+
export interface ResolvedProdAliasPlayground {
|
|
173
|
+
/** Hosted playground URL on the API alias (SPA owns alias-token sign-in). */
|
|
174
|
+
playgroundUrl: string;
|
|
175
|
+
slug: string;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Look up a running deployment and return its stable alias playground URL.
|
|
180
|
+
* Does not mint `engineAccess` — the browser signs in with the alias token.
|
|
181
|
+
* Throws {@link Error} with a CLI-ready message on failure.
|
|
182
|
+
*/
|
|
183
|
+
export async function resolveProdAliasPlaygroundUrl(
|
|
184
|
+
options: ResolveProdTargetOptions & { sessionId?: string }
|
|
185
|
+
): Promise<ResolvedProdAliasPlayground> {
|
|
186
|
+
const err =
|
|
187
|
+
options.err ?? ((text: string) => void process.stderr.write(text));
|
|
188
|
+
const slug = resolveProdSlug(options);
|
|
189
|
+
const auth = await resolveDeployAuth({
|
|
190
|
+
team: options.team,
|
|
191
|
+
apiKey: options.apiKey,
|
|
192
|
+
fetchImpl: options.fetchImpl,
|
|
193
|
+
err,
|
|
194
|
+
});
|
|
195
|
+
if (typeof auth === "number") {
|
|
196
|
+
throw new Error(
|
|
197
|
+
`Could not resolve Cursor credentials/team for the hosted deployment (see above).`
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
let detail: DeploymentDetail;
|
|
202
|
+
try {
|
|
203
|
+
detail = await auth.client.getDeployment({
|
|
204
|
+
teamId: auth.teamId,
|
|
205
|
+
slug,
|
|
206
|
+
});
|
|
207
|
+
} catch (error) {
|
|
208
|
+
if (error instanceof DeployApiError) {
|
|
209
|
+
throw new Error(
|
|
210
|
+
`Deployment "${slug}" (team ${auth.teamId}): ${error.message}`
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const statusOk = detail.status === "running" || detail.status === "degraded";
|
|
217
|
+
if (!statusOk) {
|
|
218
|
+
const lastError =
|
|
219
|
+
detail.lastError === null || detail.lastError === undefined
|
|
220
|
+
? ""
|
|
221
|
+
: ` ${sanitizeCustomerFacingDetail(detail.lastError)}`;
|
|
222
|
+
throw new Error(
|
|
223
|
+
`Deployment "${slug}" is ${detail.status}, not running.${lastError}\n` +
|
|
224
|
+
`Deploy or wait: \`${CLI} deploy --slug ${slug}\` / \`${CLI} deployment ${slug}\``
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const aliasBase = auth.client.aliasUrl(detail.engineAlias).replace(/\/$/, "");
|
|
229
|
+
const base = `${aliasBase}/playground`;
|
|
230
|
+
const sessionId =
|
|
231
|
+
options.sessionId !== undefined && options.sessionId.trim() !== ""
|
|
232
|
+
? options.sessionId.trim()
|
|
233
|
+
: undefined;
|
|
234
|
+
const playgroundUrl =
|
|
235
|
+
sessionId === undefined
|
|
236
|
+
? base
|
|
237
|
+
: `${base}?sessionId=${encodeURIComponent(sessionId)}`;
|
|
238
|
+
|
|
239
|
+
err(`${stderrPalette().dim(`Using hosted deployment "${slug}".`)}\n`);
|
|
240
|
+
|
|
241
|
+
return { playgroundUrl, slug };
|
|
242
|
+
}
|
|
243
|
+
|
|
146
244
|
/** Strip obvious hosting internals from customer-facing error fragments. */
|
|
147
245
|
function sanitizeCustomerFacingDetail(message: string): string {
|
|
148
246
|
return message
|
package/src/internal/server.ts
CHANGED
|
@@ -362,9 +362,8 @@ export async function startServer(
|
|
|
362
362
|
const evalRuns = new EvalRunStore(
|
|
363
363
|
project.rootDir,
|
|
364
364
|
logger,
|
|
365
|
-
//
|
|
366
|
-
|
|
367
|
-
engine.storage?.asEvalRunPersistence()
|
|
365
|
+
// Playground eval history mirrors onto the defineStorage evals table.
|
|
366
|
+
engine.storage?.evalRuns()
|
|
368
367
|
);
|
|
369
368
|
// Eval history restores with the same startup step as sessions (reads
|
|
370
369
|
// also hydrate lazily, but doing it here surfaces store problems at
|
|
@@ -738,6 +737,9 @@ export async function startServer(
|
|
|
738
737
|
for (const runtime of runtimes) {
|
|
739
738
|
runtime.scheduleRunner.stop();
|
|
740
739
|
runtime.reminderRunner.stop();
|
|
740
|
+
// Before the engines close their storage coordinators, so the
|
|
741
|
+
// interrupted-batch write can still reach the sink.
|
|
742
|
+
runtime.evalRuns.close();
|
|
741
743
|
}
|
|
742
744
|
for (const stop of channelStopHooks) {
|
|
743
745
|
try {
|
|
@@ -204,8 +204,7 @@ export class SessionEngine {
|
|
|
204
204
|
* Runtime for `agent/storage.ts` (`defineStorage`), when
|
|
205
205
|
* authored. All durable state changes funnel through it (session
|
|
206
206
|
* records, event chunks, A/B samples/snapshots); the eval run store
|
|
207
|
-
* attaches via {@link StorageCoordinator.
|
|
208
|
-
* serve start.
|
|
207
|
+
* attaches via {@link StorageCoordinator.evalRuns} at serve start.
|
|
209
208
|
*/
|
|
210
209
|
readonly storage: StorageCoordinator | undefined;
|
|
211
210
|
/** In-flight lazy restores, deduped per channel + continuation key. */
|
|
@@ -261,18 +260,12 @@ export class SessionEngine {
|
|
|
261
260
|
projectRoot: options.project.rootDir,
|
|
262
261
|
logger: this.logger,
|
|
263
262
|
});
|
|
264
|
-
// Domain-specific hooks win; the defineStorage sink is the fallback.
|
|
265
|
-
const persistSamples =
|
|
266
|
-
options.project.abConfig?.persistSamples ??
|
|
267
|
-
this.storage?.asABSamplePersistence();
|
|
268
263
|
this.abCollector = new ABCollector(
|
|
269
264
|
options.project.abs,
|
|
270
265
|
this.logger,
|
|
271
266
|
async (sessionId) => (await this.logs.get(sessionId)).snapshot(0),
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
persistSamples,
|
|
275
|
-
}
|
|
267
|
+
// Samples flow to the defineStorage `abs` table (no-op without one).
|
|
268
|
+
{ persistSample: (sample) => this.storage?.abSample(sample) }
|
|
276
269
|
);
|
|
277
270
|
// Peer and Cursor-account transports are symbolic until the serve host
|
|
278
271
|
// resolves them; the host-side MCP registry starts with the concrete
|
|
@@ -1592,6 +1585,7 @@ export class SessionEngine {
|
|
|
1592
1585
|
return restoredSnapshot;
|
|
1593
1586
|
}
|
|
1594
1587
|
}
|
|
1588
|
+
const abTable = this.project.storage?.abs;
|
|
1595
1589
|
const snapshot = await buildABSnapshot({
|
|
1596
1590
|
experiments: this.project.abs,
|
|
1597
1591
|
agentName: this.project.name,
|
|
@@ -1602,27 +1596,13 @@ export class SessionEngine {
|
|
|
1602
1596
|
logger: this.logger,
|
|
1603
1597
|
config: {
|
|
1604
1598
|
maxPlaygroundSessions,
|
|
1605
|
-
durableSamples:
|
|
1606
|
-
durableSnapshots:
|
|
1599
|
+
durableSamples: abTable !== undefined,
|
|
1600
|
+
durableSnapshots: abTable?.putSnapshot !== undefined,
|
|
1607
1601
|
},
|
|
1608
1602
|
});
|
|
1609
|
-
// Side effect on the read path: playground polls GET /v1/abs ~every 4s
|
|
1610
|
-
//
|
|
1611
|
-
|
|
1612
|
-
if (persistSnapshots !== undefined) {
|
|
1613
|
-
try {
|
|
1614
|
-
await persistSnapshots.save(snapshot, {
|
|
1615
|
-
projectRoot: this.project.rootDir,
|
|
1616
|
-
});
|
|
1617
|
-
} catch (error) {
|
|
1618
|
-
this.logger(
|
|
1619
|
-
`[agentkit] ab persistSnapshots.save threw: ${describeError(error)}`
|
|
1620
|
-
);
|
|
1621
|
-
}
|
|
1622
|
-
} else {
|
|
1623
|
-
// defineStorage fallback (throttled inside the coordinator).
|
|
1624
|
-
this.storage?.abSnapshot(snapshot);
|
|
1625
|
-
}
|
|
1603
|
+
// Side effect on the read path: playground polls GET /v1/abs ~every 4s;
|
|
1604
|
+
// the coordinator throttles the actual table write.
|
|
1605
|
+
this.storage?.abSnapshot(snapshot);
|
|
1626
1606
|
return snapshot;
|
|
1627
1607
|
}
|
|
1628
1608
|
|
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
* failures — a throwing sink is logged and its write dropped; storage
|
|
11
11
|
* must never stall or fail a turn.
|
|
12
12
|
*
|
|
13
|
-
* Scaling shape: sessions
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* Scaling shape: sessions and reminders reduce to keyed puts on the KV
|
|
14
|
+
* sink; eval runs and A/B metrics go to their dedicated tables
|
|
15
|
+
* (`evals` / `abs`). Everything shares one bounded, serialized queue.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import type { ABMetricSample
|
|
19
|
-
import type {
|
|
18
|
+
import type { ABMetricSample } from "../ab.js";
|
|
19
|
+
import type { EvalRunSnapshot } from "../evals.js";
|
|
20
20
|
import {
|
|
21
21
|
type ResolvedStoragePolicy,
|
|
22
22
|
resolveStoragePolicy,
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
import type { JsonValue, SessionEvent, SessionRecord } from "../types.js";
|
|
28
28
|
import type { ABSnapshot } from "./ab-snapshot.js";
|
|
29
29
|
import { describeError } from "./describe-error.js";
|
|
30
|
+
import type { EvalRunStorage } from "./eval-run-store.js";
|
|
30
31
|
import { isReminderRecord, type ReminderRecord } from "./reminder-store.js";
|
|
31
32
|
|
|
32
33
|
/** Event types that close a unit of work — flush point for turn-end batching. */
|
|
@@ -62,10 +63,12 @@ const DEBOUNCE_MAX_BATCH = 200;
|
|
|
62
63
|
*/
|
|
63
64
|
const AB_SNAPSHOT_MIN_INTERVAL_MS = 60_000;
|
|
64
65
|
|
|
65
|
-
/** One queued sink call. */
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
/** One queued sink call (KV put/delete or a dedicated-table write). */
|
|
67
|
+
interface StorageOp {
|
|
68
|
+
/** Human-readable op label for failure logs, e.g. `put(agentkit/v1/...)`. */
|
|
69
|
+
label: string;
|
|
70
|
+
run: (ctx: StorageContext) => void | Promise<void>;
|
|
71
|
+
}
|
|
69
72
|
|
|
70
73
|
export interface StorageCoordinatorOptions {
|
|
71
74
|
definition: StorageDefinition;
|
|
@@ -182,18 +185,20 @@ export class StorageCoordinator {
|
|
|
182
185
|
const ops: StorageOp[] = [];
|
|
183
186
|
const first = buffer.events[0];
|
|
184
187
|
if (first !== undefined) {
|
|
185
|
-
ops.push(
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
188
|
+
ops.push(
|
|
189
|
+
this.putOp(
|
|
190
|
+
storageKeys.sessionEvents(this.agentName, sessionId, first.index),
|
|
191
|
+
buffer.events as unknown as JsonValue
|
|
192
|
+
)
|
|
193
|
+
);
|
|
190
194
|
}
|
|
191
195
|
if (buffer.record !== undefined) {
|
|
192
|
-
ops.push(
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
196
|
+
ops.push(
|
|
197
|
+
this.putOp(
|
|
198
|
+
storageKeys.session(this.agentName, sessionId),
|
|
199
|
+
buffer.record as unknown as JsonValue
|
|
200
|
+
)
|
|
201
|
+
);
|
|
197
202
|
ops.push(...this.continuationOps(buffer.record));
|
|
198
203
|
}
|
|
199
204
|
if (ops.length > 0) {
|
|
@@ -214,25 +219,39 @@ export class StorageCoordinator {
|
|
|
214
219
|
this.continuationIndex.set(record.sessionId, next);
|
|
215
220
|
const ops: StorageOp[] = [];
|
|
216
221
|
if (previous != null) {
|
|
217
|
-
ops.push(
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
previous
|
|
223
|
-
),
|
|
224
|
-
});
|
|
222
|
+
ops.push(
|
|
223
|
+
this.deleteOp(
|
|
224
|
+
storageKeys.continuation(this.agentName, record.channelId, previous)
|
|
225
|
+
)
|
|
226
|
+
);
|
|
225
227
|
}
|
|
226
228
|
if (next != null) {
|
|
227
|
-
ops.push(
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
229
|
+
ops.push(
|
|
230
|
+
this.putOp(
|
|
231
|
+
storageKeys.continuation(this.agentName, record.channelId, next),
|
|
232
|
+
{ sessionId: record.sessionId }
|
|
233
|
+
)
|
|
234
|
+
);
|
|
232
235
|
}
|
|
233
236
|
return ops;
|
|
234
237
|
}
|
|
235
238
|
|
|
239
|
+
/** KV upsert op on the sink's `put`. */
|
|
240
|
+
private putOp(key: string, value: JsonValue): StorageOp {
|
|
241
|
+
return {
|
|
242
|
+
label: `put(${key})`,
|
|
243
|
+
run: (ctx) => this.definition.put(key, value, ctx),
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** KV delete op on the sink's `delete` (skipped when the hook is absent). */
|
|
248
|
+
private deleteOp(key: string): StorageOp {
|
|
249
|
+
return {
|
|
250
|
+
label: `delete(${key})`,
|
|
251
|
+
run: (ctx) => this.definition.delete?.(key, ctx),
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
|
|
236
255
|
// ==========================================================================
|
|
237
256
|
// Sessions (reads / restore)
|
|
238
257
|
// ==========================================================================
|
|
@@ -351,11 +370,10 @@ export class StorageCoordinator {
|
|
|
351
370
|
}
|
|
352
371
|
this.enqueue(
|
|
353
372
|
[
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
},
|
|
373
|
+
this.putOp(
|
|
374
|
+
storageKeys.reminder(this.agentName, record.id),
|
|
375
|
+
record as unknown as JsonValue
|
|
376
|
+
),
|
|
359
377
|
],
|
|
360
378
|
"policy"
|
|
361
379
|
);
|
|
@@ -375,31 +393,29 @@ export class StorageCoordinator {
|
|
|
375
393
|
}
|
|
376
394
|
|
|
377
395
|
// ==========================================================================
|
|
378
|
-
// Evals
|
|
396
|
+
// Evals (dedicated `evals` table)
|
|
379
397
|
// ==========================================================================
|
|
380
398
|
|
|
381
399
|
/**
|
|
382
|
-
* Adapter
|
|
383
|
-
*
|
|
384
|
-
*
|
|
400
|
+
* Adapter over the sink's dedicated `evals` table for the playground
|
|
401
|
+
* eval-run store. Undefined when the table is not configured — eval
|
|
402
|
+
* history then stays in process memory.
|
|
385
403
|
*/
|
|
386
|
-
|
|
404
|
+
evalRuns(): EvalRunStorage | undefined {
|
|
405
|
+
const table = this.definition.evals;
|
|
406
|
+
if (table === undefined) {
|
|
407
|
+
return undefined;
|
|
408
|
+
}
|
|
387
409
|
return {
|
|
388
|
-
load: async () => {
|
|
389
|
-
const entries = await this.tryList(
|
|
390
|
-
storageKeys.evalRunPrefix(this.agentName)
|
|
391
|
-
);
|
|
392
|
-
return entries
|
|
393
|
-
.map((entry) => entry.value as unknown as EvalRunSnapshot)
|
|
394
|
-
.filter((run) => run != null);
|
|
395
|
-
},
|
|
396
410
|
save: (run: EvalRunSnapshot) => {
|
|
411
|
+
// Clone at enqueue time: the store mutates snapshots in place as
|
|
412
|
+
// cases finish, and delivery is async.
|
|
413
|
+
const snapshot = structuredClone(run);
|
|
397
414
|
this.enqueue(
|
|
398
415
|
[
|
|
399
416
|
{
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
value: run as unknown as JsonValue,
|
|
417
|
+
label: `evals.put(${run.runId})`,
|
|
418
|
+
run: (ctx) => table.put(snapshot, ctx),
|
|
403
419
|
},
|
|
404
420
|
],
|
|
405
421
|
"policy"
|
|
@@ -409,86 +425,82 @@ export class StorageCoordinator {
|
|
|
409
425
|
this.enqueue(
|
|
410
426
|
[
|
|
411
427
|
{
|
|
412
|
-
|
|
413
|
-
|
|
428
|
+
label: `evals.delete(${runId})`,
|
|
429
|
+
run: (ctx) => table.delete(runId, ctx),
|
|
414
430
|
},
|
|
415
431
|
],
|
|
416
432
|
"policy"
|
|
417
433
|
);
|
|
418
434
|
},
|
|
435
|
+
list: async () => {
|
|
436
|
+
try {
|
|
437
|
+
const runs = await table.list(this.context("restore"));
|
|
438
|
+
return Array.isArray(runs) ? runs : [];
|
|
439
|
+
} catch (error) {
|
|
440
|
+
this.logger(
|
|
441
|
+
`[agentkit] storage evals.list() failed: ${describeError(error)}`
|
|
442
|
+
);
|
|
443
|
+
return [];
|
|
444
|
+
}
|
|
445
|
+
},
|
|
419
446
|
};
|
|
420
447
|
}
|
|
421
448
|
|
|
422
449
|
// ==========================================================================
|
|
423
|
-
// A/Bs
|
|
450
|
+
// A/Bs (dedicated `abs` table)
|
|
424
451
|
// ==========================================================================
|
|
425
452
|
|
|
426
|
-
/**
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
),
|
|
442
|
-
value: sample as unknown as JsonValue,
|
|
443
|
-
},
|
|
444
|
-
],
|
|
445
|
-
"policy"
|
|
446
|
-
);
|
|
447
|
-
},
|
|
448
|
-
};
|
|
453
|
+
/** Append one metric sample to the `abs` table (no-op without it). */
|
|
454
|
+
abSample(sample: ABMetricSample): void {
|
|
455
|
+
const table = this.definition.abs;
|
|
456
|
+
if (this.closed || table === undefined) {
|
|
457
|
+
return;
|
|
458
|
+
}
|
|
459
|
+
this.enqueue(
|
|
460
|
+
[
|
|
461
|
+
{
|
|
462
|
+
label: `abs.putSample(${sample.sessionId})`,
|
|
463
|
+
run: (ctx) => table.putSample(sample, ctx),
|
|
464
|
+
},
|
|
465
|
+
],
|
|
466
|
+
"policy"
|
|
467
|
+
);
|
|
449
468
|
}
|
|
450
469
|
|
|
451
470
|
/**
|
|
452
471
|
* Refresh the persisted aggregate A/B snapshot, throttled to once per
|
|
453
472
|
* {@link AB_SNAPSHOT_MIN_INTERVAL_MS} (the playground recomputes the
|
|
454
|
-
* fold on every `GET /v1/abs` poll).
|
|
473
|
+
* fold on every `GET /v1/abs` poll). No-op without `abs.putSnapshot`.
|
|
455
474
|
*/
|
|
456
475
|
abSnapshot(snapshot: ABSnapshot): void {
|
|
476
|
+
const putSnapshot = this.definition.abs?.putSnapshot;
|
|
457
477
|
const now = Date.now();
|
|
458
478
|
if (
|
|
459
479
|
this.closed ||
|
|
480
|
+
putSnapshot === undefined ||
|
|
460
481
|
now - this.lastAbSnapshotAt < AB_SNAPSHOT_MIN_INTERVAL_MS
|
|
461
482
|
) {
|
|
462
483
|
return;
|
|
463
484
|
}
|
|
464
485
|
this.lastAbSnapshotAt = now;
|
|
465
486
|
this.enqueue(
|
|
466
|
-
[
|
|
467
|
-
{
|
|
468
|
-
op: "put",
|
|
469
|
-
key: storageKeys.abSnapshot(this.agentName),
|
|
470
|
-
value: snapshot as unknown as JsonValue,
|
|
471
|
-
},
|
|
472
|
-
],
|
|
487
|
+
[{ label: "abs.putSnapshot", run: (ctx) => putSnapshot(snapshot, ctx) }],
|
|
473
488
|
"policy"
|
|
474
489
|
);
|
|
475
490
|
}
|
|
476
491
|
|
|
477
492
|
/** Latest persisted aggregate A/B snapshot; errors read as absent. */
|
|
478
493
|
async getLatestAbSnapshot(): Promise<ABSnapshot | undefined> {
|
|
479
|
-
const
|
|
480
|
-
if (
|
|
494
|
+
const getSnapshot = this.definition.abs?.getSnapshot;
|
|
495
|
+
if (getSnapshot === undefined) {
|
|
481
496
|
return undefined;
|
|
482
497
|
}
|
|
483
498
|
try {
|
|
484
|
-
const value = await
|
|
485
|
-
|
|
486
|
-
this.context("restore")
|
|
487
|
-
);
|
|
488
|
-
return value == null ? undefined : (value as unknown as ABSnapshot);
|
|
499
|
+
const value = await getSnapshot(this.context("restore"));
|
|
500
|
+
return value ?? undefined;
|
|
489
501
|
} catch (error) {
|
|
490
502
|
this.logger(
|
|
491
|
-
`[agentkit] storage
|
|
503
|
+
`[agentkit] storage abs.getSnapshot() failed: ${describeError(error)}`
|
|
492
504
|
);
|
|
493
505
|
return undefined;
|
|
494
506
|
}
|
|
@@ -593,14 +605,10 @@ export class StorageCoordinator {
|
|
|
593
605
|
this.queue = this.queue.then(async () => {
|
|
594
606
|
for (const op of ops) {
|
|
595
607
|
try {
|
|
596
|
-
|
|
597
|
-
await this.definition.put(op.key, op.value, ctx);
|
|
598
|
-
} else {
|
|
599
|
-
await this.definition.delete?.(op.key, ctx);
|
|
600
|
-
}
|
|
608
|
+
await op.run(ctx);
|
|
601
609
|
} catch (error) {
|
|
602
610
|
this.logger(
|
|
603
|
-
`[agentkit] storage ${op.
|
|
611
|
+
`[agentkit] storage ${op.label} failed: ${describeError(error)}`
|
|
604
612
|
);
|
|
605
613
|
} finally {
|
|
606
614
|
this.pending -= 1;
|