@cursor/july 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ab.d.ts +8 -95
- package/dist/ab.d.ts.map +1 -1
- package/dist/ab.js +9 -150
- package/dist/bin/agent-serve.js +14 -6
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +4 -4
- package/dist/docs/assets/{app.DF-Yx4OP.js → app.DMboGKUD.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.O6eP7QZr.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.DSeJSudL.js → VPLocalSearchBox.DkEqEnJU.js} +1 -1
- package/dist/docs/assets/chunks/{theme.DZVLTXOC.js → theme.ASugdMNm.js} +2 -2
- package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
- package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
- package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
- package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
- package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
- package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
- package/dist/docs/building-with-agents.html +4 -4
- package/dist/docs/concepts.html +4 -4
- package/dist/docs/deployment.html +58 -17
- package/dist/docs/evals.html +4 -4
- package/dist/docs/example-agents/approval-buddy.html +34 -0
- package/dist/docs/example-agents/benny.html +31 -0
- package/dist/docs/example-agents/bugbot.html +35 -0
- package/dist/docs/example-agents/codebase-wiki.html +32 -0
- package/dist/docs/example-agents/codeowners-review.html +32 -0
- package/dist/docs/example-agents/concierge.html +47 -0
- package/dist/docs/example-agents/fsd.html +39 -0
- package/dist/docs/example-agents/index.html +26 -0
- package/dist/docs/example-agents/knowledge-base.html +35 -0
- package/dist/docs/example-agents/security-reviewer.html +43 -0
- package/dist/docs/example-agents/slack-agent.html +29 -0
- package/dist/docs/example-agents/weather-agent.html +48 -0
- package/dist/docs/guides/agent-to-agent.html +4 -4
- package/dist/docs/guides/cloud-runtime.html +5 -5
- package/dist/docs/guides/github.html +4 -4
- package/dist/docs/guides/human-in-the-loop.html +4 -4
- package/dist/docs/guides/slack.html +4 -4
- package/dist/docs/guides/webhooks.html +4 -4
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +4 -4
- package/dist/docs/index.html +7 -7
- package/dist/docs/quickstart.html +4 -4
- package/dist/docs/reference/agent-config.html +4 -4
- package/dist/docs/reference/channels.html +4 -4
- package/dist/docs/reference/cli.html +52 -30
- package/dist/docs/reference/connections.html +4 -4
- package/dist/docs/reference/hooks.html +4 -4
- package/dist/docs/reference/http-api.html +4 -4
- package/dist/docs/reference/instructions.html +4 -4
- package/dist/docs/reference/playground.html +4 -4
- package/dist/docs/reference/project-layout.html +4 -4
- package/dist/docs/reference/schedules.html +4 -4
- package/dist/docs/reference/sessions.html +4 -4
- package/dist/docs/reference/skills.html +4 -4
- package/dist/docs/reference/subagents.html +4 -4
- package/dist/docs/reference/tools.html +4 -4
- package/dist/docs/scaffolding-agents.html +4 -4
- package/dist/docs/storage.html +4 -4
- package/dist/docs/troubleshooting.html +4 -4
- package/dist/evals.d.ts +4 -61
- package/dist/evals.d.ts.map +1 -1
- package/dist/evals.js +3 -66
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/internal/ab-collector.d.ts +7 -5
- package/dist/internal/ab-collector.d.ts.map +1 -1
- package/dist/internal/ab-collector.js +3 -14
- package/dist/internal/ab-snapshot.d.ts +2 -4
- package/dist/internal/ab-snapshot.d.ts.map +1 -1
- package/dist/internal/cli-ax.d.ts +18 -3
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +80 -42
- package/dist/internal/cli-deploy.d.ts.map +1 -1
- package/dist/internal/cli-deploy.js +5 -2
- package/dist/internal/cursor-event-relay.d.ts +4 -1
- package/dist/internal/cursor-event-relay.d.ts.map +1 -1
- package/dist/internal/cursor-event-relay.js +6 -0
- package/dist/internal/cursor-relay-core.d.ts +36 -1
- package/dist/internal/cursor-relay-core.d.ts.map +1 -1
- package/dist/internal/cursor-relay-core.js +89 -8
- package/dist/internal/cursor-slack-relay.d.ts +2 -2
- package/dist/internal/cursor-slack-relay.d.ts.map +1 -1
- package/dist/internal/cursor-slack-relay.js +4 -0
- package/dist/internal/discovery.js +3 -3
- package/dist/internal/eval-run-store.d.ts +33 -30
- package/dist/internal/eval-run-store.d.ts.map +1 -1
- package/dist/internal/eval-run-store.js +84 -96
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +3 -2
- package/dist/internal/playground-proxy.d.ts +5 -5
- package/dist/internal/playground-proxy.js +3 -3
- package/dist/internal/resolve-prod-target.d.ts +30 -0
- package/dist/internal/resolve-prod-target.d.ts.map +1 -1
- package/dist/internal/resolve-prod-target.js +74 -2
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +5 -3
- package/dist/internal/session-engine.d.ts +1 -2
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +14 -31
- package/dist/internal/storage-coordinator.d.ts +16 -15
- package/dist/internal/storage-coordinator.d.ts.map +1 -1
- package/dist/internal/storage-coordinator.js +73 -80
- package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
- package/dist/playground/assets/index-7GJIUFQj.css +1 -0
- package/dist/playground/assets/index-mux9aj5P.js +319 -0
- package/dist/playground/index.html +2 -2
- package/dist/storage.d.ts +51 -10
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +27 -10
- package/docs/README.md +34 -5
- package/docs/deployment.md +352 -149
- package/docs/example-agents/approval-buddy.md +270 -0
- package/docs/example-agents/benny.md +186 -0
- package/docs/example-agents/bugbot.md +231 -0
- package/docs/example-agents/codebase-wiki.md +174 -0
- package/docs/example-agents/codeowners-review.md +195 -0
- package/docs/example-agents/concierge.md +205 -0
- package/docs/example-agents/fsd.md +330 -0
- package/docs/example-agents/index.md +102 -0
- package/docs/example-agents/knowledge-base.md +171 -0
- package/docs/example-agents/security-reviewer.md +296 -0
- package/docs/example-agents/slack-agent.md +146 -0
- package/docs/example-agents/weather-agent.md +302 -0
- package/docs/reference/cli.md +546 -147
- package/package.json +1 -1
- package/src/ab.ts +9 -261
- package/src/bin/agent-serve.ts +14 -6
- package/src/evals.ts +4 -118
- package/src/index.ts +2 -0
- package/src/internal/ab-collector.ts +12 -22
- package/src/internal/ab-snapshot.ts +2 -4
- package/src/internal/cli-ax.ts +101 -45
- package/src/internal/cli-deploy.ts +5 -2
- package/src/internal/cursor-event-relay.ts +10 -1
- package/src/internal/cursor-relay-core.ts +115 -9
- package/src/internal/cursor-slack-relay.ts +6 -2
- package/src/internal/discovery.ts +2 -2
- package/src/internal/eval-run-store.ts +86 -100
- package/src/internal/init-project.ts +3 -2
- package/src/internal/playground-proxy.ts +5 -5
- package/src/internal/resolve-prod-target.ts +101 -3
- package/src/internal/server.ts +5 -3
- package/src/internal/session-engine.ts +9 -29
- package/src/internal/storage-coordinator.ts +109 -101
- package/src/storage.ts +79 -14
- package/dist/docs/assets/chunks/@localSearchIndexroot.0XrheBv2.js +0 -1
- package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
- package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
- package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
- package/dist/internal/json-dir-store.d.ts +0 -32
- package/dist/internal/json-dir-store.d.ts.map +0 -1
- package/dist/internal/json-dir-store.js +0 -100
- package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
- package/dist/playground/assets/index-Ct_Usdm8.css +0 -1
- package/dist/playground/assets/index-Dsyb-Vaj.js +0 -319
- package/src/internal/json-dir-store.ts +0 -109
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cursor/july",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "(early alpha) Filesystem-first framework for defining Cursor agents as markdown and TypeScript and serving them over channels with the Cursor SDK.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE.md",
|
|
6
6
|
"repository": {
|
package/src/ab.ts
CHANGED
|
@@ -10,9 +10,8 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Assignments and foldable metrics live in each session's `events.ndjson`
|
|
12
12
|
* (survive serve restart). Optional project defaults in `agent/ab.config.ts`
|
|
13
|
-
* ({@link defineABConfig})
|
|
14
|
-
*
|
|
15
|
-
* eval `persistRuns`.
|
|
13
|
+
* ({@link defineABConfig}) cap the playground fold window; durable
|
|
14
|
+
* sample/snapshot export lives on the `defineStorage` `abs` table.
|
|
16
15
|
*
|
|
17
16
|
* ```ts
|
|
18
17
|
* import { defineAB, splitBySessionHash } from "@anysphere/agent-serve/ab";
|
|
@@ -31,20 +30,8 @@
|
|
|
31
30
|
* ```
|
|
32
31
|
*/
|
|
33
32
|
|
|
34
|
-
import { randomUUID } from "node:crypto";
|
|
35
|
-
import { appendFile, mkdir } from "node:fs/promises";
|
|
36
|
-
import { join } from "node:path";
|
|
37
|
-
import type { ABSnapshot } from "./internal/ab-snapshot.js";
|
|
38
33
|
import { normalizePositiveInt } from "./internal/bounded-int.js";
|
|
39
34
|
import { brandDefinition } from "./internal/brand.js";
|
|
40
|
-
import {
|
|
41
|
-
assertNonEmptyDir,
|
|
42
|
-
deleteFileInDir,
|
|
43
|
-
listJsonBasenames,
|
|
44
|
-
readJsonDir,
|
|
45
|
-
resolveProjectRelativeDir,
|
|
46
|
-
writeJsonFile,
|
|
47
|
-
} from "./internal/json-dir-store.js";
|
|
48
35
|
import type { SessionEvent, SessionInfo } from "./types.js";
|
|
49
36
|
|
|
50
37
|
export type { ABSnapshot } from "./internal/ab-snapshot.js";
|
|
@@ -370,7 +357,7 @@ export function mergeABInstructions(
|
|
|
370
357
|
}
|
|
371
358
|
|
|
372
359
|
// ---------------------------------------------------------------------------
|
|
373
|
-
// Project config (`agent/ab.config.ts`) — fold window
|
|
360
|
+
// Project config (`agent/ab.config.ts`) — playground fold window
|
|
374
361
|
// ---------------------------------------------------------------------------
|
|
375
362
|
|
|
376
363
|
/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
|
|
@@ -379,128 +366,36 @@ export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
|
|
|
379
366
|
/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
|
|
380
367
|
export const AB_MAX_PLAYGROUND_SESSIONS = 10_000;
|
|
381
368
|
|
|
382
|
-
/** Default snapshot files kept when {@link ABConfigFile.persistSnapshots} is set. */
|
|
383
|
-
export const AB_DEFAULT_RETAIN_SNAPSHOTS = 20;
|
|
384
|
-
|
|
385
|
-
/** Hard ceiling for {@link ABConfigFile.retainSnapshots}. */
|
|
386
|
-
export const AB_MAX_RETAIN_SNAPSHOTS = 500;
|
|
387
|
-
|
|
388
|
-
/** Context passed to A/B persistence hooks. */
|
|
389
|
-
export interface ABPersistContext {
|
|
390
|
-
/** Absolute agent project root (directory that contains `agent/`). */
|
|
391
|
-
projectRoot: string;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
/**
|
|
395
|
-
* Durable sink for {@link ABMetricSample} rows (same payload as
|
|
396
|
-
* {@link ABConfig.onSample}). Session `events.ndjson` remains the source of
|
|
397
|
-
* truth for assignment + fold; this is for analytics export.
|
|
398
|
-
*/
|
|
399
|
-
export interface ABSamplePersistence {
|
|
400
|
-
save(sample: ABMetricSample, ctx: ABPersistContext): void | Promise<void>;
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
/**
|
|
404
|
-
* Durable store for playground-style aggregate snapshots (`GET /v1/abs`).
|
|
405
|
-
* Without this, aggregates are always recomputed from session logs (nothing
|
|
406
|
-
* is kept after sessions are pruned).
|
|
407
|
-
*
|
|
408
|
-
* **Runtime note:** `serve` currently only calls {@link save} (after each
|
|
409
|
-
* `GET /v1/abs`). {@link load} / {@link delete} are for author tooling or a
|
|
410
|
-
* custom store; they are not wired into the playground hydrate path.
|
|
411
|
-
*/
|
|
412
|
-
export interface ABSnapshotPersistence {
|
|
413
|
-
load(ctx: ABPersistContext): ABSnapshot[] | Promise<ABSnapshot[]>;
|
|
414
|
-
save(snapshot: ABSnapshot, ctx: ABPersistContext): void | Promise<void>;
|
|
415
|
-
delete?(snapshotId: string, ctx: ABPersistContext): void | Promise<void>;
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/** Lets {@link defineABConfig} push `retainSnapshots` into {@link persistABSnapshotsToDir}. */
|
|
419
|
-
const abSnapshotRetainState = new WeakMap<
|
|
420
|
-
ABSnapshotPersistence,
|
|
421
|
-
{ retain: number }
|
|
422
|
-
>();
|
|
423
|
-
|
|
424
369
|
/**
|
|
425
370
|
* Project-wide A/B defaults (`agent/ab.config.ts`).
|
|
426
371
|
*
|
|
427
372
|
* **Assignments and foldable metrics** already persist in each session's
|
|
428
|
-
* `events.ndjson` under `--state-root
|
|
429
|
-
*
|
|
430
|
-
*
|
|
373
|
+
* `events.ndjson` under `--state-root`; durable sample/snapshot export
|
|
374
|
+
* lives on the `defineStorage` `abs` table (`agent/storage.ts`). This
|
|
375
|
+
* config only caps how many sessions the playground / `GET /v1/abs` folds.
|
|
431
376
|
*/
|
|
432
377
|
export interface ABConfigFile {
|
|
433
378
|
/**
|
|
434
379
|
* Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
|
|
435
|
-
* tab
|
|
436
|
-
* session logs, change assignment, or limit {@link persistSamples}.
|
|
380
|
+
* tab**. Does not prune session logs or change assignment.
|
|
437
381
|
*
|
|
438
382
|
* Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
|
|
439
383
|
* {@link AB_MAX_PLAYGROUND_SESSIONS}.
|
|
440
384
|
*/
|
|
441
385
|
maxPlaygroundSessions?: number;
|
|
442
|
-
/**
|
|
443
|
-
* Snapshot-file budget when {@link persistSnapshots} is set.
|
|
444
|
-
* Default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}; hard-capped at
|
|
445
|
-
* {@link AB_MAX_RETAIN_SNAPSHOTS}.
|
|
446
|
-
*
|
|
447
|
-
* When using {@link persistABSnapshotsToDir}, this value controls on-disk
|
|
448
|
-
* pruning (overriding the helper's `{ retain }` default). Custom stores
|
|
449
|
-
* should apply the same budget themselves.
|
|
450
|
-
*/
|
|
451
|
-
retainSnapshots?: number;
|
|
452
|
-
/**
|
|
453
|
-
* Optional sink for cumulative metric samples (in addition to each
|
|
454
|
-
* experiment's {@link ABConfig.onSample}).
|
|
455
|
-
*
|
|
456
|
-
* ```ts
|
|
457
|
-
* persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
|
|
458
|
-
* ```
|
|
459
|
-
*/
|
|
460
|
-
persistSamples?: ABSamplePersistence;
|
|
461
|
-
/**
|
|
462
|
-
* Optional store for aggregate snapshots written when `GET /v1/abs` runs.
|
|
463
|
-
*
|
|
464
|
-
* ```ts
|
|
465
|
-
* persistSnapshots: persistABSnapshotsToDir(".agent-serve/ab-snapshots"),
|
|
466
|
-
* ```
|
|
467
|
-
*/
|
|
468
|
-
persistSnapshots?: ABSnapshotPersistence;
|
|
469
386
|
}
|
|
470
387
|
|
|
471
388
|
/**
|
|
472
389
|
* Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
|
|
473
390
|
*
|
|
474
|
-
* Session event logs remain the assignment/metrics source of truth;
|
|
475
|
-
*
|
|
391
|
+
* Session event logs remain the assignment/metrics source of truth;
|
|
392
|
+
* durable exports go through the `defineStorage` `abs` table.
|
|
476
393
|
*/
|
|
477
394
|
export function defineABConfig(config: ABConfigFile): ABConfigFile {
|
|
478
|
-
const persistSamples = normalizeSamplePersistence(config.persistSamples);
|
|
479
|
-
const persistSnapshots = normalizeSnapshotPersistence(
|
|
480
|
-
config.persistSnapshots
|
|
481
|
-
);
|
|
482
|
-
const helperRetain =
|
|
483
|
-
persistSnapshots !== undefined
|
|
484
|
-
? abSnapshotRetainState.get(persistSnapshots)
|
|
485
|
-
: undefined;
|
|
486
|
-
let retainSnapshots: number | undefined;
|
|
487
|
-
if (config.retainSnapshots !== undefined) {
|
|
488
|
-
retainSnapshots = normalizeABRetainSnapshots(config.retainSnapshots);
|
|
489
|
-
if (helperRetain !== undefined) {
|
|
490
|
-
helperRetain.retain = retainSnapshots;
|
|
491
|
-
}
|
|
492
|
-
} else if (persistSnapshots !== undefined) {
|
|
493
|
-
retainSnapshots =
|
|
494
|
-
helperRetain?.retain ??
|
|
495
|
-
normalizeABRetainSnapshots(AB_DEFAULT_RETAIN_SNAPSHOTS);
|
|
496
|
-
}
|
|
497
395
|
return {
|
|
498
396
|
maxPlaygroundSessions: normalizeABMaxPlaygroundSessions(
|
|
499
397
|
config.maxPlaygroundSessions ?? AB_DEFAULT_MAX_PLAYGROUND_SESSIONS
|
|
500
398
|
),
|
|
501
|
-
...(retainSnapshots === undefined ? {} : { retainSnapshots }),
|
|
502
|
-
...(persistSamples === undefined ? {} : { persistSamples }),
|
|
503
|
-
...(persistSnapshots === undefined ? {} : { persistSnapshots }),
|
|
504
399
|
};
|
|
505
400
|
}
|
|
506
401
|
|
|
@@ -526,150 +421,3 @@ export function resolveABMaxPlaygroundSessions(value: unknown): number {
|
|
|
526
421
|
return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
|
|
527
422
|
}
|
|
528
423
|
}
|
|
529
|
-
|
|
530
|
-
/** Validate {@link ABConfigFile.retainSnapshots}. */
|
|
531
|
-
export function normalizeABRetainSnapshots(value: unknown): number {
|
|
532
|
-
return normalizePositiveInt(value, {
|
|
533
|
-
label: "ab.config.ts: retainSnapshots",
|
|
534
|
-
max: AB_MAX_RETAIN_SNAPSHOTS,
|
|
535
|
-
});
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
function normalizeSamplePersistence(
|
|
539
|
-
value: ABSamplePersistence | undefined
|
|
540
|
-
): ABSamplePersistence | undefined {
|
|
541
|
-
if (value === undefined) {
|
|
542
|
-
return undefined;
|
|
543
|
-
}
|
|
544
|
-
if (
|
|
545
|
-
typeof value !== "object" ||
|
|
546
|
-
value === null ||
|
|
547
|
-
typeof value.save !== "function"
|
|
548
|
-
) {
|
|
549
|
-
throw new Error(
|
|
550
|
-
"ab.config.ts: persistSamples must define save(sample, ctx)"
|
|
551
|
-
);
|
|
552
|
-
}
|
|
553
|
-
return value;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
function normalizeSnapshotPersistence(
|
|
557
|
-
value: ABSnapshotPersistence | undefined
|
|
558
|
-
): ABSnapshotPersistence | undefined {
|
|
559
|
-
if (value === undefined) {
|
|
560
|
-
return undefined;
|
|
561
|
-
}
|
|
562
|
-
if (
|
|
563
|
-
typeof value !== "object" ||
|
|
564
|
-
value === null ||
|
|
565
|
-
typeof value.load !== "function" ||
|
|
566
|
-
typeof value.save !== "function"
|
|
567
|
-
) {
|
|
568
|
-
throw new Error(
|
|
569
|
-
"ab.config.ts: persistSnapshots must define load(ctx) and save(snapshot, ctx)"
|
|
570
|
-
);
|
|
571
|
-
}
|
|
572
|
-
if (value.delete !== undefined && typeof value.delete !== "function") {
|
|
573
|
-
throw new Error("ab.config.ts: persistSnapshots.delete must be a function");
|
|
574
|
-
}
|
|
575
|
-
return value;
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
/**
|
|
579
|
-
* Append-only NDJSON {@link ABSamplePersistence} under `dir` (relative to
|
|
580
|
-
* the agent project root).
|
|
581
|
-
*
|
|
582
|
-
* ```ts
|
|
583
|
-
* // agent/ab.config.ts
|
|
584
|
-
* export default defineABConfig({
|
|
585
|
-
* persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
|
|
586
|
-
* });
|
|
587
|
-
* ```
|
|
588
|
-
*/
|
|
589
|
-
export function persistABSamplesToDir(dir: string): ABSamplePersistence {
|
|
590
|
-
const relativeDir = assertNonEmptyDir(dir, "persistABSamplesToDir");
|
|
591
|
-
const resolveDir = (projectRoot: string): string =>
|
|
592
|
-
resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
|
|
593
|
-
|
|
594
|
-
return {
|
|
595
|
-
async save(sample, ctx) {
|
|
596
|
-
const root = resolveDir(ctx.projectRoot);
|
|
597
|
-
await mkdir(root, { recursive: true });
|
|
598
|
-
await appendFile(
|
|
599
|
-
join(root, "samples.ndjson"),
|
|
600
|
-
`${JSON.stringify(sample)}\n`,
|
|
601
|
-
"utf8"
|
|
602
|
-
);
|
|
603
|
-
},
|
|
604
|
-
};
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
/**
|
|
608
|
-
* Filesystem {@link ABSnapshotPersistence}: one JSON file per snapshot under
|
|
609
|
-
* `dir`, pruned to `retain` files (default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}).
|
|
610
|
-
*
|
|
611
|
-
* Prefer setting {@link ABConfigFile.retainSnapshots} in `defineABConfig` —
|
|
612
|
-
* that value is applied to this helper's prune budget. Pass `{ retain }` here
|
|
613
|
-
* only when constructing the helper outside `defineABConfig`, or as a default
|
|
614
|
-
* that `retainSnapshots` may override.
|
|
615
|
-
*/
|
|
616
|
-
export function persistABSnapshotsToDir(
|
|
617
|
-
dir: string,
|
|
618
|
-
options?: { retain?: number }
|
|
619
|
-
): ABSnapshotPersistence {
|
|
620
|
-
const relativeDir = assertNonEmptyDir(dir, "persistABSnapshotsToDir");
|
|
621
|
-
const retainState = {
|
|
622
|
-
retain: normalizeABRetainSnapshots(
|
|
623
|
-
options?.retain ?? AB_DEFAULT_RETAIN_SNAPSHOTS
|
|
624
|
-
),
|
|
625
|
-
};
|
|
626
|
-
const resolveDir = (projectRoot: string): string =>
|
|
627
|
-
resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
|
|
628
|
-
let saveChain: Promise<void> = Promise.resolve();
|
|
629
|
-
|
|
630
|
-
const persist: ABSnapshotPersistence = {
|
|
631
|
-
async load(ctx) {
|
|
632
|
-
return readJsonDir<ABSnapshot>(resolveDir(ctx.projectRoot), {
|
|
633
|
-
sortBasenames: "desc",
|
|
634
|
-
});
|
|
635
|
-
},
|
|
636
|
-
async save(snapshot, ctx) {
|
|
637
|
-
const run = async (): Promise<void> => {
|
|
638
|
-
const root = resolveDir(ctx.projectRoot);
|
|
639
|
-
const snapshotId =
|
|
640
|
-
snapshot.config?.snapshotId ??
|
|
641
|
-
`absnap_${randomUUID().replace(/-/g, "")}`;
|
|
642
|
-
const toWrite: ABSnapshot = {
|
|
643
|
-
...snapshot,
|
|
644
|
-
config: {
|
|
645
|
-
...snapshot.config,
|
|
646
|
-
snapshotId,
|
|
647
|
-
},
|
|
648
|
-
};
|
|
649
|
-
await writeJsonFile({
|
|
650
|
-
root,
|
|
651
|
-
fileName: `${snapshotId}.json`,
|
|
652
|
-
value: toWrite,
|
|
653
|
-
});
|
|
654
|
-
const names = (await listJsonBasenames(root)).sort().reverse();
|
|
655
|
-
for (const stale of names.slice(retainState.retain)) {
|
|
656
|
-
await deleteFileInDir({ root, fileName: stale });
|
|
657
|
-
}
|
|
658
|
-
};
|
|
659
|
-
const next = saveChain.then(run, run);
|
|
660
|
-
saveChain = next.then(
|
|
661
|
-
() => undefined,
|
|
662
|
-
() => undefined
|
|
663
|
-
);
|
|
664
|
-
await next;
|
|
665
|
-
},
|
|
666
|
-
async delete(snapshotId, ctx) {
|
|
667
|
-
await deleteFileInDir({
|
|
668
|
-
root: resolveDir(ctx.projectRoot),
|
|
669
|
-
fileName: `${snapshotId}.json`,
|
|
670
|
-
});
|
|
671
|
-
},
|
|
672
|
-
};
|
|
673
|
-
abSnapshotRetainState.set(persist, retainState);
|
|
674
|
-
return persist;
|
|
675
|
-
}
|
package/src/bin/agent-serve.ts
CHANGED
|
@@ -90,7 +90,7 @@ Usage:
|
|
|
90
90
|
${CLI} init [dir] [--json]
|
|
91
91
|
${CLI} eval [--dir <path>] [evalId...] [--list] [--json]
|
|
92
92
|
[--tag <tag>]... [--verbose] [--timeout-ms <n>]
|
|
93
|
-
[--no-stream] [--url <baseUrl>
|
|
93
|
+
[--no-stream] [--local | --url <baseUrl>] [--slug <slug>]
|
|
94
94
|
[--team <id>]
|
|
95
95
|
${CLI} info [--dir <path>] [--json]
|
|
96
96
|
${CLI} validate [--dir <path>]
|
|
@@ -155,9 +155,11 @@ Commands:
|
|
|
155
155
|
machine-readable and never blocks on login. No directory:
|
|
156
156
|
print the setup guide.
|
|
157
157
|
eval Run filesystem evals under evals/**/*.eval.ts (streams on a TTY).
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
158
|
+
Runs against the team's hosted deployment by default (slug from
|
|
159
|
+
--slug or the directory name); pass --local for an ephemeral
|
|
160
|
+
local server, or --url for a running one. Pass evalId args to
|
|
161
|
+
select: one datapoint (weather/nyc), a whole file (weather →
|
|
162
|
+
all cases in that file), several of either, or omit to run all.
|
|
161
163
|
info Print the discovered agent surface (tools, skills, channels, ...).
|
|
162
164
|
validate Check the project(s) and exit non-zero on error diagnostics.
|
|
163
165
|
login Sign the host in to Cursor: browser sign-in mints a named,
|
|
@@ -192,7 +194,7 @@ Agent loop:
|
|
|
192
194
|
${CLI} validate --dir .
|
|
193
195
|
${CLI} info --dir . --json
|
|
194
196
|
${CLI} run --dir . --message "..."
|
|
195
|
-
${CLI} eval --dir .
|
|
197
|
+
${CLI} eval --local --dir .
|
|
196
198
|
${CLI} dev .
|
|
197
199
|
|
|
198
200
|
Options:
|
|
@@ -231,7 +233,10 @@ Options:
|
|
|
231
233
|
--prod chat/resume/run/call/eval/logs/sessions/session/playground:
|
|
232
234
|
use the team's hosted deployment. Slug from --slug or the
|
|
233
235
|
--dir basename. Mutually exclusive with --url. Requires
|
|
234
|
-
\`${CLI} login\`.
|
|
236
|
+
\`${CLI} login\`. (eval already defaults to the
|
|
237
|
+
deployment; there --prod is just being explicit.)
|
|
238
|
+
--local eval: run against an ephemeral local server instead of
|
|
239
|
+
the hosted deployment. Mutually exclusive with --url.
|
|
235
240
|
--once logs: print current progress / log buffer once and exit
|
|
236
241
|
(default is to follow).
|
|
237
242
|
--print playground: print the URL only; do not open a browser.
|
|
@@ -347,6 +352,7 @@ async function main(argv: string[]): Promise<number> {
|
|
|
347
352
|
"no-docs": { type: "boolean", default: false },
|
|
348
353
|
url: { type: "string" },
|
|
349
354
|
prod: { type: "boolean", default: false },
|
|
355
|
+
local: { type: "boolean", default: false },
|
|
350
356
|
message: { type: "string", multiple: true },
|
|
351
357
|
input: { type: "string" },
|
|
352
358
|
"messages-file": { type: "string" },
|
|
@@ -628,6 +634,7 @@ async function main(argv: string[]): Promise<number> {
|
|
|
628
634
|
tags: values.tag,
|
|
629
635
|
url: values.url,
|
|
630
636
|
prod: values.prod,
|
|
637
|
+
local: values.local,
|
|
631
638
|
team: values.team,
|
|
632
639
|
apiKey: values["api-key"],
|
|
633
640
|
slug: values.slug,
|
|
@@ -1045,6 +1052,7 @@ function axFromOptions(
|
|
|
1045
1052
|
toolInput: partial.toolInput,
|
|
1046
1053
|
url: partial.url,
|
|
1047
1054
|
prod: partial.prod,
|
|
1055
|
+
local: partial.local,
|
|
1048
1056
|
team: partial.team,
|
|
1049
1057
|
apiKey: partial.apiKey,
|
|
1050
1058
|
fetchImpl: partial.fetchImpl,
|
package/src/evals.ts
CHANGED
|
@@ -48,13 +48,6 @@
|
|
|
48
48
|
|
|
49
49
|
import { normalizePositiveInt } from "./internal/bounded-int.js";
|
|
50
50
|
import { brandDefinition } from "./internal/brand.js";
|
|
51
|
-
import {
|
|
52
|
-
assertNonEmptyDir,
|
|
53
|
-
deleteFileInDir,
|
|
54
|
-
readJsonDir,
|
|
55
|
-
resolveProjectRelativeDir,
|
|
56
|
-
writeJsonFile,
|
|
57
|
-
} from "./internal/json-dir-store.js";
|
|
58
51
|
import type { SessionEvent } from "./types.js";
|
|
59
52
|
|
|
60
53
|
export type Expectation =
|
|
@@ -322,35 +315,6 @@ export interface EvalRunSnapshot {
|
|
|
322
315
|
};
|
|
323
316
|
}
|
|
324
317
|
|
|
325
|
-
/** Context passed to {@link EvalRunPersistence} hooks. */
|
|
326
|
-
export interface EvalRunPersistContext {
|
|
327
|
-
/** Absolute agent project root (directory that contains `evals/`). */
|
|
328
|
-
projectRoot: string;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
/**
|
|
332
|
-
* Durable store for playground eval batches. Without {@link EvalConfigFile.persistRuns},
|
|
333
|
-
* runs live only in the serve process memory — lost on restart, and capped by
|
|
334
|
-
* {@link EvalConfigFile.maxPlaygroundRuns} (default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}).
|
|
335
|
-
*
|
|
336
|
-
* Implement your own (S3, DB, …) or use {@link persistEvalRunsToDir}.
|
|
337
|
-
* `delete` is required so {@link EvalConfigFile.maxPlaygroundRuns} can prune
|
|
338
|
-
* durable history (otherwise pruned rows reload on the next hydrate).
|
|
339
|
-
*/
|
|
340
|
-
export interface EvalRunPersistence {
|
|
341
|
-
/** Load saved runs (any order). Called once when the serve process hydrates. */
|
|
342
|
-
load(
|
|
343
|
-
ctx: EvalRunPersistContext
|
|
344
|
-
): EvalRunSnapshot[] | Promise<EvalRunSnapshot[]>;
|
|
345
|
-
/**
|
|
346
|
-
* Persist a full snapshot. Called when a batch starts, after each case
|
|
347
|
-
* finishes, and when the batch completes or fails.
|
|
348
|
-
*/
|
|
349
|
-
save(run: EvalRunSnapshot, ctx: EvalRunPersistContext): void | Promise<void>;
|
|
350
|
-
/** Remove a run that falls outside {@link EvalConfigFile.maxPlaygroundRuns}. */
|
|
351
|
-
delete(runId: string, ctx: EvalRunPersistContext): void | Promise<void>;
|
|
352
|
-
}
|
|
353
|
-
|
|
354
318
|
export interface EvalConfigFile {
|
|
355
319
|
timeoutMs?: number;
|
|
356
320
|
/**
|
|
@@ -361,52 +325,30 @@ export interface EvalConfigFile {
|
|
|
361
325
|
maxConcurrency: number;
|
|
362
326
|
/**
|
|
363
327
|
* Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
|
|
364
|
-
* history window (in-memory, and pruned from
|
|
365
|
-
*
|
|
328
|
+
* history window (in-memory, and pruned from the `defineStorage` evals
|
|
329
|
+
* table when one is configured).
|
|
366
330
|
* Does not affect `agent-serve eval` CLI runs.
|
|
367
331
|
*
|
|
368
332
|
* Default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}; hard-capped at
|
|
369
333
|
* {@link EVAL_MAX_PLAYGROUND_RUNS}. Newest batches win.
|
|
370
334
|
*/
|
|
371
335
|
maxPlaygroundRuns?: number;
|
|
372
|
-
/**
|
|
373
|
-
* Optional durable store for playground eval batches. Omit this and runs
|
|
374
|
-
* are **not** surviving process restart (only the in-memory
|
|
375
|
-
* {@link maxPlaygroundRuns} window).
|
|
376
|
-
*
|
|
377
|
-
* ```ts
|
|
378
|
-
* import {
|
|
379
|
-
* defineEvalConfig,
|
|
380
|
-
* persistEvalRunsToDir,
|
|
381
|
-
* } from "@anysphere/agent-serve/evals";
|
|
382
|
-
*
|
|
383
|
-
* export default defineEvalConfig({
|
|
384
|
-
* maxConcurrency: 20,
|
|
385
|
-
* maxPlaygroundRuns: 50,
|
|
386
|
-
* // Survive serve restarts (relative to the agent project root):
|
|
387
|
-
* persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
|
|
388
|
-
* });
|
|
389
|
-
* ```
|
|
390
|
-
*/
|
|
391
|
-
persistRuns?: EvalRunPersistence;
|
|
392
336
|
}
|
|
393
337
|
|
|
394
338
|
/**
|
|
395
339
|
* Defaults for every eval under `evals/` (`evals/evals.config.ts`).
|
|
396
340
|
* `maxConcurrency` is required (1–{@link EVAL_MAX_CONCURRENCY_LIMIT}).
|
|
397
341
|
*
|
|
398
|
-
* Playground batches default to **in-memory only
|
|
399
|
-
*
|
|
342
|
+
* Playground batches default to **in-memory only**; author an `evals`
|
|
343
|
+
* table in `agent/storage.ts` (`defineStorage`) to survive serve restart.
|
|
400
344
|
*/
|
|
401
345
|
export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
|
|
402
|
-
const persistRuns = normalizePersistRuns(config.persistRuns);
|
|
403
346
|
return {
|
|
404
347
|
...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
|
|
405
348
|
maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency),
|
|
406
349
|
maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
|
|
407
350
|
config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS
|
|
408
351
|
),
|
|
409
|
-
...(persistRuns === undefined ? {} : { persistRuns }),
|
|
410
352
|
};
|
|
411
353
|
}
|
|
412
354
|
|
|
@@ -443,59 +385,3 @@ export function resolveEvalMaxPlaygroundRuns(value: unknown): number {
|
|
|
443
385
|
return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
|
|
444
386
|
}
|
|
445
387
|
}
|
|
446
|
-
|
|
447
|
-
function normalizePersistRuns(
|
|
448
|
-
value: EvalRunPersistence | undefined
|
|
449
|
-
): EvalRunPersistence | undefined {
|
|
450
|
-
if (value === undefined) {
|
|
451
|
-
return undefined;
|
|
452
|
-
}
|
|
453
|
-
if (
|
|
454
|
-
typeof value !== "object" ||
|
|
455
|
-
value === null ||
|
|
456
|
-
typeof value.load !== "function" ||
|
|
457
|
-
typeof value.save !== "function" ||
|
|
458
|
-
typeof value.delete !== "function"
|
|
459
|
-
) {
|
|
460
|
-
throw new Error(
|
|
461
|
-
"evals.config.ts: persistRuns must define load(ctx), save(run, ctx), and delete(runId, ctx)"
|
|
462
|
-
);
|
|
463
|
-
}
|
|
464
|
-
return value;
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
/**
|
|
468
|
-
* Filesystem {@link EvalRunPersistence}: one JSON file per run under `dir`
|
|
469
|
-
* (relative paths resolve against the agent project root).
|
|
470
|
-
*
|
|
471
|
-
* ```ts
|
|
472
|
-
* export default defineEvalConfig({
|
|
473
|
-
* maxConcurrency: 20,
|
|
474
|
-
* persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
|
|
475
|
-
* });
|
|
476
|
-
* ```
|
|
477
|
-
*/
|
|
478
|
-
export function persistEvalRunsToDir(dir: string): EvalRunPersistence {
|
|
479
|
-
const relativeDir = assertNonEmptyDir(dir, "persistEvalRunsToDir");
|
|
480
|
-
const resolveDir = (projectRoot: string): string =>
|
|
481
|
-
resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
|
|
482
|
-
|
|
483
|
-
return {
|
|
484
|
-
async load(ctx) {
|
|
485
|
-
return readJsonDir<EvalRunSnapshot>(resolveDir(ctx.projectRoot));
|
|
486
|
-
},
|
|
487
|
-
async save(run, ctx) {
|
|
488
|
-
await writeJsonFile({
|
|
489
|
-
root: resolveDir(ctx.projectRoot),
|
|
490
|
-
fileName: `${run.runId}.json`,
|
|
491
|
-
value: run,
|
|
492
|
-
});
|
|
493
|
-
},
|
|
494
|
-
async delete(runId, ctx) {
|
|
495
|
-
await deleteFileInDir({
|
|
496
|
-
root: resolveDir(ctx.projectRoot),
|
|
497
|
-
fileName: `${runId}.json`,
|
|
498
|
-
});
|
|
499
|
-
},
|
|
500
|
-
};
|
|
501
|
-
}
|
package/src/index.ts
CHANGED
|
@@ -105,10 +105,12 @@ export type {
|
|
|
105
105
|
export { defineSchedule } from "./schedules.js";
|
|
106
106
|
export { defineSkill } from "./skills.js";
|
|
107
107
|
export type {
|
|
108
|
+
StorageABTable,
|
|
108
109
|
StorageConfig,
|
|
109
110
|
StorageContext,
|
|
110
111
|
StorageDefinition,
|
|
111
112
|
StorageEntry,
|
|
113
|
+
StorageEvalsTable,
|
|
112
114
|
StoragePolicy,
|
|
113
115
|
StorageRestorePolicy,
|
|
114
116
|
} from "./storage.js";
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import {
|
|
18
18
|
type ABDefinition,
|
|
19
|
-
type
|
|
19
|
+
type ABMetricSample,
|
|
20
20
|
type ABSampleReason,
|
|
21
21
|
cloneABMetrics,
|
|
22
22
|
} from "../ab.js";
|
|
@@ -33,8 +33,11 @@ import { describeError } from "./describe-error.js";
|
|
|
33
33
|
export type { DiscoveredAB };
|
|
34
34
|
|
|
35
35
|
export interface ABCollectorOptions {
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Durable sample export (the `defineStorage` `abs` table, via the
|
|
38
|
+
* coordinator). Fire-and-forget; must never throw.
|
|
39
|
+
*/
|
|
40
|
+
persistSample?: (sample: ABMetricSample) => void;
|
|
38
41
|
}
|
|
39
42
|
|
|
40
43
|
/** Drop cache after these — next event rehydrates from the durable log. */
|
|
@@ -47,8 +50,9 @@ const EVICT_ON: ReadonlySet<SessionEvent["type"]> = new Set([
|
|
|
47
50
|
export class ABCollector {
|
|
48
51
|
private readonly state = new Map<string, FoldedABSession>();
|
|
49
52
|
private readonly hydrated = new Set<string>();
|
|
50
|
-
private readonly
|
|
51
|
-
|
|
53
|
+
private readonly persistSample:
|
|
54
|
+
| ((sample: ABMetricSample) => void)
|
|
55
|
+
| undefined;
|
|
52
56
|
|
|
53
57
|
constructor(
|
|
54
58
|
private readonly experiments: readonly DiscoveredAB[],
|
|
@@ -56,8 +60,7 @@ export class ABCollector {
|
|
|
56
60
|
private readonly loadEvents: (sessionId: string) => Promise<SessionEvent[]>,
|
|
57
61
|
options?: ABCollectorOptions
|
|
58
62
|
) {
|
|
59
|
-
this.
|
|
60
|
-
this.persistSamples = options?.persistSamples;
|
|
63
|
+
this.persistSample = options?.persistSample;
|
|
61
64
|
}
|
|
62
65
|
|
|
63
66
|
get enabled(): boolean {
|
|
@@ -155,7 +158,7 @@ export class ABCollector {
|
|
|
155
158
|
}
|
|
156
159
|
const reason = sampleReason(event);
|
|
157
160
|
if (reason !== undefined) {
|
|
158
|
-
const sample = {
|
|
161
|
+
const sample: ABMetricSample = {
|
|
159
162
|
experiment: experiment.name,
|
|
160
163
|
variant,
|
|
161
164
|
...(def.variants[variant]?.label !== undefined
|
|
@@ -174,20 +177,7 @@ export class ABCollector {
|
|
|
174
177
|
`[agentkit] ab "${experiment.name}" onSample threw: ${describeError(error)}`
|
|
175
178
|
);
|
|
176
179
|
}
|
|
177
|
-
|
|
178
|
-
this.persistSamples !== undefined &&
|
|
179
|
-
this.projectRoot !== undefined
|
|
180
|
-
) {
|
|
181
|
-
try {
|
|
182
|
-
await this.persistSamples.save(sample, {
|
|
183
|
-
projectRoot: this.projectRoot,
|
|
184
|
-
});
|
|
185
|
-
} catch (error) {
|
|
186
|
-
this.logger(
|
|
187
|
-
`[agentkit] ab persistSamples.save threw: ${describeError(error)}`
|
|
188
|
-
);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
180
|
+
this.persistSample?.(sample);
|
|
191
181
|
}
|
|
192
182
|
}
|
|
193
183
|
}
|
|
@@ -58,12 +58,10 @@ export interface ABSnapshotExperiment {
|
|
|
58
58
|
export interface ABSnapshotConfig {
|
|
59
59
|
/** Newest-first session fold window for this snapshot (`GET /v1/abs`). */
|
|
60
60
|
maxPlaygroundSessions?: number;
|
|
61
|
-
/** True when `
|
|
61
|
+
/** True when the `defineStorage` `abs` table is configured. */
|
|
62
62
|
durableSamples?: boolean;
|
|
63
|
-
/** True when `
|
|
63
|
+
/** True when the `abs` table also stores the aggregate snapshot. */
|
|
64
64
|
durableSnapshots?: boolean;
|
|
65
|
-
/** Set when writing via filesystem snapshot persistence helpers. */
|
|
66
|
-
snapshotId?: string;
|
|
67
65
|
}
|
|
68
66
|
|
|
69
67
|
export interface ABSnapshot {
|