@cursor/july 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/dist/ab.d.ts +8 -95
  2. package/dist/ab.d.ts.map +1 -1
  3. package/dist/ab.js +9 -150
  4. package/dist/bin/agent-serve.js +14 -6
  5. package/dist/docs/404.html +2 -2
  6. package/dist/docs/ab.html +4 -4
  7. package/dist/docs/assets/{app.DabPG-io.js → app.DMboGKUD.js} +1 -1
  8. package/dist/docs/assets/chunks/@localSearchIndexroot.O6eP7QZr.js +1 -0
  9. package/dist/docs/assets/chunks/{VPLocalSearchBox.jmyr0bU0.js → VPLocalSearchBox.DkEqEnJU.js} +1 -1
  10. package/dist/docs/assets/chunks/{theme.DysN9-VN.js → theme.ASugdMNm.js} +2 -2
  11. package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
  12. package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
  13. package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
  14. package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
  15. package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
  16. package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
  17. package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
  18. package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
  19. package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
  20. package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
  21. package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
  22. package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
  23. package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
  24. package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
  25. package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
  26. package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
  27. package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
  28. package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
  29. package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
  30. package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
  31. package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
  32. package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
  33. package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
  34. package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
  35. package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
  36. package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
  37. package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
  38. package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
  39. package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
  40. package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
  41. package/dist/docs/building-with-agents.html +4 -4
  42. package/dist/docs/concepts.html +4 -4
  43. package/dist/docs/deployment.html +58 -17
  44. package/dist/docs/evals.html +4 -4
  45. package/dist/docs/example-agents/approval-buddy.html +34 -0
  46. package/dist/docs/example-agents/benny.html +31 -0
  47. package/dist/docs/example-agents/bugbot.html +35 -0
  48. package/dist/docs/example-agents/codebase-wiki.html +32 -0
  49. package/dist/docs/example-agents/codeowners-review.html +32 -0
  50. package/dist/docs/example-agents/concierge.html +47 -0
  51. package/dist/docs/example-agents/fsd.html +39 -0
  52. package/dist/docs/example-agents/index.html +26 -0
  53. package/dist/docs/example-agents/knowledge-base.html +35 -0
  54. package/dist/docs/example-agents/security-reviewer.html +43 -0
  55. package/dist/docs/example-agents/slack-agent.html +29 -0
  56. package/dist/docs/example-agents/weather-agent.html +48 -0
  57. package/dist/docs/guides/agent-to-agent.html +4 -4
  58. package/dist/docs/guides/cloud-runtime.html +5 -5
  59. package/dist/docs/guides/github.html +4 -4
  60. package/dist/docs/guides/human-in-the-loop.html +4 -4
  61. package/dist/docs/guides/slack.html +4 -4
  62. package/dist/docs/guides/webhooks.html +4 -4
  63. package/dist/docs/hashmap.json +1 -1
  64. package/dist/docs/hillclimbing.html +4 -4
  65. package/dist/docs/index.html +7 -7
  66. package/dist/docs/quickstart.html +4 -4
  67. package/dist/docs/reference/agent-config.html +4 -4
  68. package/dist/docs/reference/channels.html +4 -4
  69. package/dist/docs/reference/cli.html +52 -30
  70. package/dist/docs/reference/connections.html +4 -4
  71. package/dist/docs/reference/hooks.html +4 -4
  72. package/dist/docs/reference/http-api.html +4 -4
  73. package/dist/docs/reference/instructions.html +4 -4
  74. package/dist/docs/reference/playground.html +4 -4
  75. package/dist/docs/reference/project-layout.html +4 -4
  76. package/dist/docs/reference/schedules.html +4 -4
  77. package/dist/docs/reference/sessions.html +4 -4
  78. package/dist/docs/reference/skills.html +4 -4
  79. package/dist/docs/reference/subagents.html +4 -4
  80. package/dist/docs/reference/tools.html +4 -4
  81. package/dist/docs/scaffolding-agents.html +4 -4
  82. package/dist/docs/storage.html +4 -4
  83. package/dist/docs/troubleshooting.html +4 -4
  84. package/dist/evals.d.ts +4 -61
  85. package/dist/evals.d.ts.map +1 -1
  86. package/dist/evals.js +3 -66
  87. package/dist/index.d.ts +1 -1
  88. package/dist/index.d.ts.map +1 -1
  89. package/dist/internal/ab-collector.d.ts +7 -5
  90. package/dist/internal/ab-collector.d.ts.map +1 -1
  91. package/dist/internal/ab-collector.js +3 -14
  92. package/dist/internal/ab-snapshot.d.ts +2 -4
  93. package/dist/internal/ab-snapshot.d.ts.map +1 -1
  94. package/dist/internal/cli-ax.d.ts +18 -3
  95. package/dist/internal/cli-ax.d.ts.map +1 -1
  96. package/dist/internal/cli-ax.js +80 -42
  97. package/dist/internal/cli-deploy.js +1 -1
  98. package/dist/internal/discovery.js +3 -3
  99. package/dist/internal/eval-run-store.d.ts +33 -30
  100. package/dist/internal/eval-run-store.d.ts.map +1 -1
  101. package/dist/internal/eval-run-store.js +84 -96
  102. package/dist/internal/init-project.d.ts.map +1 -1
  103. package/dist/internal/init-project.js +3 -2
  104. package/dist/internal/playground-proxy.d.ts +5 -5
  105. package/dist/internal/playground-proxy.js +3 -3
  106. package/dist/internal/resolve-prod-target.d.ts +30 -0
  107. package/dist/internal/resolve-prod-target.d.ts.map +1 -1
  108. package/dist/internal/resolve-prod-target.js +74 -2
  109. package/dist/internal/server.d.ts.map +1 -1
  110. package/dist/internal/server.js +5 -3
  111. package/dist/internal/session-engine.d.ts +1 -2
  112. package/dist/internal/session-engine.d.ts.map +1 -1
  113. package/dist/internal/session-engine.js +14 -31
  114. package/dist/internal/storage-coordinator.d.ts +16 -15
  115. package/dist/internal/storage-coordinator.d.ts.map +1 -1
  116. package/dist/internal/storage-coordinator.js +73 -80
  117. package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
  118. package/dist/playground/assets/index-7GJIUFQj.css +1 -0
  119. package/dist/playground/assets/index-mux9aj5P.js +319 -0
  120. package/dist/playground/index.html +2 -2
  121. package/dist/storage.d.ts +51 -10
  122. package/dist/storage.d.ts.map +1 -1
  123. package/dist/storage.js +27 -10
  124. package/docs/README.md +34 -5
  125. package/docs/deployment.md +352 -149
  126. package/docs/example-agents/approval-buddy.md +270 -0
  127. package/docs/example-agents/benny.md +186 -0
  128. package/docs/example-agents/bugbot.md +231 -0
  129. package/docs/example-agents/codebase-wiki.md +174 -0
  130. package/docs/example-agents/codeowners-review.md +195 -0
  131. package/docs/example-agents/concierge.md +205 -0
  132. package/docs/example-agents/fsd.md +330 -0
  133. package/docs/example-agents/index.md +102 -0
  134. package/docs/example-agents/knowledge-base.md +171 -0
  135. package/docs/example-agents/security-reviewer.md +296 -0
  136. package/docs/example-agents/slack-agent.md +146 -0
  137. package/docs/example-agents/weather-agent.md +302 -0
  138. package/docs/reference/cli.md +546 -147
  139. package/package.json +1 -1
  140. package/src/ab.ts +9 -261
  141. package/src/bin/agent-serve.ts +14 -6
  142. package/src/evals.ts +4 -118
  143. package/src/index.ts +2 -0
  144. package/src/internal/ab-collector.ts +12 -22
  145. package/src/internal/ab-snapshot.ts +2 -4
  146. package/src/internal/cli-ax.ts +101 -45
  147. package/src/internal/cli-deploy.ts +1 -1
  148. package/src/internal/discovery.ts +2 -2
  149. package/src/internal/eval-run-store.ts +86 -100
  150. package/src/internal/init-project.ts +3 -2
  151. package/src/internal/playground-proxy.ts +5 -5
  152. package/src/internal/resolve-prod-target.ts +101 -3
  153. package/src/internal/server.ts +5 -3
  154. package/src/internal/session-engine.ts +9 -29
  155. package/src/internal/storage-coordinator.ts +109 -101
  156. package/src/storage.ts +79 -14
  157. package/dist/docs/assets/chunks/@localSearchIndexroot.QwK5BtEH.js +0 -1
  158. package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
  159. package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
  160. package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
  161. package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
  162. package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
  163. package/dist/internal/json-dir-store.d.ts +0 -32
  164. package/dist/internal/json-dir-store.d.ts.map +0 -1
  165. package/dist/internal/json-dir-store.js +0 -100
  166. package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
  167. package/dist/playground/assets/index-BEauYlII.css +0 -1
  168. package/dist/playground/assets/index-BtM0wEGg.js +0 -319
  169. package/src/internal/json-dir-store.ts +0 -109
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cursor/july",
3
- "version": "0.1.5",
3
+ "version": "0.1.6",
4
4
  "description": "(early alpha) Filesystem-first framework for defining Cursor agents as markdown and TypeScript and serving them over channels with the Cursor SDK.",
5
5
  "license": "SEE LICENSE IN LICENSE.md",
6
6
  "repository": {
package/src/ab.ts CHANGED
@@ -10,9 +10,8 @@
10
10
  *
11
11
  * Assignments and foldable metrics live in each session's `events.ndjson`
12
12
  * (survive serve restart). Optional project defaults in `agent/ab.config.ts`
13
- * ({@link defineABConfig}) let authors retain the playground fold window and
14
- * persist samples / aggregate snapshots wherever they like same idea as
15
- * eval `persistRuns`.
13
+ * ({@link defineABConfig}) cap the playground fold window; durable
14
+ * sample/snapshot export lives on the `defineStorage` `abs` table.
16
15
  *
17
16
  * ```ts
18
17
  * import { defineAB, splitBySessionHash } from "@anysphere/agent-serve/ab";
@@ -31,20 +30,8 @@
31
30
  * ```
32
31
  */
33
32
 
34
- import { randomUUID } from "node:crypto";
35
- import { appendFile, mkdir } from "node:fs/promises";
36
- import { join } from "node:path";
37
- import type { ABSnapshot } from "./internal/ab-snapshot.js";
38
33
  import { normalizePositiveInt } from "./internal/bounded-int.js";
39
34
  import { brandDefinition } from "./internal/brand.js";
40
- import {
41
- assertNonEmptyDir,
42
- deleteFileInDir,
43
- listJsonBasenames,
44
- readJsonDir,
45
- resolveProjectRelativeDir,
46
- writeJsonFile,
47
- } from "./internal/json-dir-store.js";
48
35
  import type { SessionEvent, SessionInfo } from "./types.js";
49
36
 
50
37
  export type { ABSnapshot } from "./internal/ab-snapshot.js";
@@ -370,7 +357,7 @@ export function mergeABInstructions(
370
357
  }
371
358
 
372
359
  // ---------------------------------------------------------------------------
373
- // Project config (`agent/ab.config.ts`) — fold window + optional persistence
360
+ // Project config (`agent/ab.config.ts`) — playground fold window
374
361
  // ---------------------------------------------------------------------------
375
362
 
376
363
  /** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
@@ -379,128 +366,36 @@ export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
379
366
  /** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
380
367
  export const AB_MAX_PLAYGROUND_SESSIONS = 10_000;
381
368
 
382
- /** Default snapshot files kept when {@link ABConfigFile.persistSnapshots} is set. */
383
- export const AB_DEFAULT_RETAIN_SNAPSHOTS = 20;
384
-
385
- /** Hard ceiling for {@link ABConfigFile.retainSnapshots}. */
386
- export const AB_MAX_RETAIN_SNAPSHOTS = 500;
387
-
388
- /** Context passed to A/B persistence hooks. */
389
- export interface ABPersistContext {
390
- /** Absolute agent project root (directory that contains `agent/`). */
391
- projectRoot: string;
392
- }
393
-
394
- /**
395
- * Durable sink for {@link ABMetricSample} rows (same payload as
396
- * {@link ABConfig.onSample}). Session `events.ndjson` remains the source of
397
- * truth for assignment + fold; this is for analytics export.
398
- */
399
- export interface ABSamplePersistence {
400
- save(sample: ABMetricSample, ctx: ABPersistContext): void | Promise<void>;
401
- }
402
-
403
- /**
404
- * Durable store for playground-style aggregate snapshots (`GET /v1/abs`).
405
- * Without this, aggregates are always recomputed from session logs (nothing
406
- * is kept after sessions are pruned).
407
- *
408
- * **Runtime note:** `serve` currently only calls {@link save} (after each
409
- * `GET /v1/abs`). {@link load} / {@link delete} are for author tooling or a
410
- * custom store; they are not wired into the playground hydrate path.
411
- */
412
- export interface ABSnapshotPersistence {
413
- load(ctx: ABPersistContext): ABSnapshot[] | Promise<ABSnapshot[]>;
414
- save(snapshot: ABSnapshot, ctx: ABPersistContext): void | Promise<void>;
415
- delete?(snapshotId: string, ctx: ABPersistContext): void | Promise<void>;
416
- }
417
-
418
- /** Lets {@link defineABConfig} push `retainSnapshots` into {@link persistABSnapshotsToDir}. */
419
- const abSnapshotRetainState = new WeakMap<
420
- ABSnapshotPersistence,
421
- { retain: number }
422
- >();
423
-
424
369
  /**
425
370
  * Project-wide A/B defaults (`agent/ab.config.ts`).
426
371
  *
427
372
  * **Assignments and foldable metrics** already persist in each session's
428
- * `events.ndjson` under `--state-root`. Use this config to:
429
- * - cap how many sessions the playground / `GET /v1/abs` folds
430
- * - export samples and/or aggregate snapshots to author-chosen storage
373
+ * `events.ndjson` under `--state-root`; durable sample/snapshot export
374
+ * lives on the `defineStorage` `abs` table (`agent/storage.ts`). This
375
+ * config only caps how many sessions the playground / `GET /v1/abs` folds.
431
376
  */
432
377
  export interface ABConfigFile {
433
378
  /**
434
379
  * Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
435
- * tab** (and into {@link persistSnapshots} payloads). Does not prune
436
- * session logs, change assignment, or limit {@link persistSamples}.
380
+ * tab**. Does not prune session logs or change assignment.
437
381
  *
438
382
  * Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
439
383
  * {@link AB_MAX_PLAYGROUND_SESSIONS}.
440
384
  */
441
385
  maxPlaygroundSessions?: number;
442
- /**
443
- * Snapshot-file budget when {@link persistSnapshots} is set.
444
- * Default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}; hard-capped at
445
- * {@link AB_MAX_RETAIN_SNAPSHOTS}.
446
- *
447
- * When using {@link persistABSnapshotsToDir}, this value controls on-disk
448
- * pruning (overriding the helper's `{ retain }` default). Custom stores
449
- * should apply the same budget themselves.
450
- */
451
- retainSnapshots?: number;
452
- /**
453
- * Optional sink for cumulative metric samples (in addition to each
454
- * experiment's {@link ABConfig.onSample}).
455
- *
456
- * ```ts
457
- * persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
458
- * ```
459
- */
460
- persistSamples?: ABSamplePersistence;
461
- /**
462
- * Optional store for aggregate snapshots written when `GET /v1/abs` runs.
463
- *
464
- * ```ts
465
- * persistSnapshots: persistABSnapshotsToDir(".agent-serve/ab-snapshots"),
466
- * ```
467
- */
468
- persistSnapshots?: ABSnapshotPersistence;
469
386
  }
470
387
 
471
388
  /**
472
389
  * Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
473
390
  *
474
- * Session event logs remain the assignment/metrics source of truth; this
475
- * file controls the playground fold window and optional durable exports.
391
+ * Session event logs remain the assignment/metrics source of truth;
392
+ * durable exports go through the `defineStorage` `abs` table.
476
393
  */
477
394
  export function defineABConfig(config: ABConfigFile): ABConfigFile {
478
- const persistSamples = normalizeSamplePersistence(config.persistSamples);
479
- const persistSnapshots = normalizeSnapshotPersistence(
480
- config.persistSnapshots
481
- );
482
- const helperRetain =
483
- persistSnapshots !== undefined
484
- ? abSnapshotRetainState.get(persistSnapshots)
485
- : undefined;
486
- let retainSnapshots: number | undefined;
487
- if (config.retainSnapshots !== undefined) {
488
- retainSnapshots = normalizeABRetainSnapshots(config.retainSnapshots);
489
- if (helperRetain !== undefined) {
490
- helperRetain.retain = retainSnapshots;
491
- }
492
- } else if (persistSnapshots !== undefined) {
493
- retainSnapshots =
494
- helperRetain?.retain ??
495
- normalizeABRetainSnapshots(AB_DEFAULT_RETAIN_SNAPSHOTS);
496
- }
497
395
  return {
498
396
  maxPlaygroundSessions: normalizeABMaxPlaygroundSessions(
499
397
  config.maxPlaygroundSessions ?? AB_DEFAULT_MAX_PLAYGROUND_SESSIONS
500
398
  ),
501
- ...(retainSnapshots === undefined ? {} : { retainSnapshots }),
502
- ...(persistSamples === undefined ? {} : { persistSamples }),
503
- ...(persistSnapshots === undefined ? {} : { persistSnapshots }),
504
399
  };
505
400
  }
506
401
 
@@ -526,150 +421,3 @@ export function resolveABMaxPlaygroundSessions(value: unknown): number {
526
421
  return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
527
422
  }
528
423
  }
529
-
530
- /** Validate {@link ABConfigFile.retainSnapshots}. */
531
- export function normalizeABRetainSnapshots(value: unknown): number {
532
- return normalizePositiveInt(value, {
533
- label: "ab.config.ts: retainSnapshots",
534
- max: AB_MAX_RETAIN_SNAPSHOTS,
535
- });
536
- }
537
-
538
- function normalizeSamplePersistence(
539
- value: ABSamplePersistence | undefined
540
- ): ABSamplePersistence | undefined {
541
- if (value === undefined) {
542
- return undefined;
543
- }
544
- if (
545
- typeof value !== "object" ||
546
- value === null ||
547
- typeof value.save !== "function"
548
- ) {
549
- throw new Error(
550
- "ab.config.ts: persistSamples must define save(sample, ctx)"
551
- );
552
- }
553
- return value;
554
- }
555
-
556
- function normalizeSnapshotPersistence(
557
- value: ABSnapshotPersistence | undefined
558
- ): ABSnapshotPersistence | undefined {
559
- if (value === undefined) {
560
- return undefined;
561
- }
562
- if (
563
- typeof value !== "object" ||
564
- value === null ||
565
- typeof value.load !== "function" ||
566
- typeof value.save !== "function"
567
- ) {
568
- throw new Error(
569
- "ab.config.ts: persistSnapshots must define load(ctx) and save(snapshot, ctx)"
570
- );
571
- }
572
- if (value.delete !== undefined && typeof value.delete !== "function") {
573
- throw new Error("ab.config.ts: persistSnapshots.delete must be a function");
574
- }
575
- return value;
576
- }
577
-
578
- /**
579
- * Append-only NDJSON {@link ABSamplePersistence} under `dir` (relative to
580
- * the agent project root).
581
- *
582
- * ```ts
583
- * // agent/ab.config.ts
584
- * export default defineABConfig({
585
- * persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
586
- * });
587
- * ```
588
- */
589
- export function persistABSamplesToDir(dir: string): ABSamplePersistence {
590
- const relativeDir = assertNonEmptyDir(dir, "persistABSamplesToDir");
591
- const resolveDir = (projectRoot: string): string =>
592
- resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
593
-
594
- return {
595
- async save(sample, ctx) {
596
- const root = resolveDir(ctx.projectRoot);
597
- await mkdir(root, { recursive: true });
598
- await appendFile(
599
- join(root, "samples.ndjson"),
600
- `${JSON.stringify(sample)}\n`,
601
- "utf8"
602
- );
603
- },
604
- };
605
- }
606
-
607
- /**
608
- * Filesystem {@link ABSnapshotPersistence}: one JSON file per snapshot under
609
- * `dir`, pruned to `retain` files (default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}).
610
- *
611
- * Prefer setting {@link ABConfigFile.retainSnapshots} in `defineABConfig` —
612
- * that value is applied to this helper's prune budget. Pass `{ retain }` here
613
- * only when constructing the helper outside `defineABConfig`, or as a default
614
- * that `retainSnapshots` may override.
615
- */
616
- export function persistABSnapshotsToDir(
617
- dir: string,
618
- options?: { retain?: number }
619
- ): ABSnapshotPersistence {
620
- const relativeDir = assertNonEmptyDir(dir, "persistABSnapshotsToDir");
621
- const retainState = {
622
- retain: normalizeABRetainSnapshots(
623
- options?.retain ?? AB_DEFAULT_RETAIN_SNAPSHOTS
624
- ),
625
- };
626
- const resolveDir = (projectRoot: string): string =>
627
- resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
628
- let saveChain: Promise<void> = Promise.resolve();
629
-
630
- const persist: ABSnapshotPersistence = {
631
- async load(ctx) {
632
- return readJsonDir<ABSnapshot>(resolveDir(ctx.projectRoot), {
633
- sortBasenames: "desc",
634
- });
635
- },
636
- async save(snapshot, ctx) {
637
- const run = async (): Promise<void> => {
638
- const root = resolveDir(ctx.projectRoot);
639
- const snapshotId =
640
- snapshot.config?.snapshotId ??
641
- `absnap_${randomUUID().replace(/-/g, "")}`;
642
- const toWrite: ABSnapshot = {
643
- ...snapshot,
644
- config: {
645
- ...snapshot.config,
646
- snapshotId,
647
- },
648
- };
649
- await writeJsonFile({
650
- root,
651
- fileName: `${snapshotId}.json`,
652
- value: toWrite,
653
- });
654
- const names = (await listJsonBasenames(root)).sort().reverse();
655
- for (const stale of names.slice(retainState.retain)) {
656
- await deleteFileInDir({ root, fileName: stale });
657
- }
658
- };
659
- const next = saveChain.then(run, run);
660
- saveChain = next.then(
661
- () => undefined,
662
- () => undefined
663
- );
664
- await next;
665
- },
666
- async delete(snapshotId, ctx) {
667
- await deleteFileInDir({
668
- root: resolveDir(ctx.projectRoot),
669
- fileName: `${snapshotId}.json`,
670
- });
671
- },
672
- };
673
- abSnapshotRetainState.set(persist, retainState);
674
- return persist;
675
- }
@@ -90,7 +90,7 @@ Usage:
90
90
  ${CLI} init [dir] [--json]
91
91
  ${CLI} eval [--dir <path>] [evalId...] [--list] [--json]
92
92
  [--tag <tag>]... [--verbose] [--timeout-ms <n>]
93
- [--no-stream] [--url <baseUrl> | --prod] [--slug <slug>]
93
+ [--no-stream] [--local | --url <baseUrl>] [--slug <slug>]
94
94
  [--team <id>]
95
95
  ${CLI} info [--dir <path>] [--json]
96
96
  ${CLI} validate [--dir <path>]
@@ -155,9 +155,11 @@ Commands:
155
155
  machine-readable and never blocks on login. No directory:
156
156
  print the setup guide.
157
157
  eval Run filesystem evals under evals/**/*.eval.ts (streams on a TTY).
158
- Pass evalId args to select: one datapoint (weather/nyc), a whole
159
- file (weather all cases in that file), several of either, or
160
- omit to run all.
158
+ Runs against the team's hosted deployment by default (slug from
159
+ --slug or the directory name); pass --local for an ephemeral
160
+ local server, or --url for a running one. Pass evalId args to
161
+ select: one datapoint (weather/nyc), a whole file (weather →
162
+ all cases in that file), several of either, or omit to run all.
161
163
  info Print the discovered agent surface (tools, skills, channels, ...).
162
164
  validate Check the project(s) and exit non-zero on error diagnostics.
163
165
  login Sign the host in to Cursor: browser sign-in mints a named,
@@ -192,7 +194,7 @@ Agent loop:
192
194
  ${CLI} validate --dir .
193
195
  ${CLI} info --dir . --json
194
196
  ${CLI} run --dir . --message "..."
195
- ${CLI} eval --dir .
197
+ ${CLI} eval --local --dir .
196
198
  ${CLI} dev .
197
199
 
198
200
  Options:
@@ -231,7 +233,10 @@ Options:
231
233
  --prod chat/resume/run/call/eval/logs/sessions/session/playground:
232
234
  use the team's hosted deployment. Slug from --slug or the
233
235
  --dir basename. Mutually exclusive with --url. Requires
234
- \`${CLI} login\`.
236
+ \`${CLI} login\`. (eval already defaults to the
237
+ deployment; there --prod is just being explicit.)
238
+ --local eval: run against an ephemeral local server instead of
239
+ the hosted deployment. Mutually exclusive with --url.
235
240
  --once logs: print current progress / log buffer once and exit
236
241
  (default is to follow).
237
242
  --print playground: print the URL only; do not open a browser.
@@ -347,6 +352,7 @@ async function main(argv: string[]): Promise<number> {
347
352
  "no-docs": { type: "boolean", default: false },
348
353
  url: { type: "string" },
349
354
  prod: { type: "boolean", default: false },
355
+ local: { type: "boolean", default: false },
350
356
  message: { type: "string", multiple: true },
351
357
  input: { type: "string" },
352
358
  "messages-file": { type: "string" },
@@ -628,6 +634,7 @@ async function main(argv: string[]): Promise<number> {
628
634
  tags: values.tag,
629
635
  url: values.url,
630
636
  prod: values.prod,
637
+ local: values.local,
631
638
  team: values.team,
632
639
  apiKey: values["api-key"],
633
640
  slug: values.slug,
@@ -1045,6 +1052,7 @@ function axFromOptions(
1045
1052
  toolInput: partial.toolInput,
1046
1053
  url: partial.url,
1047
1054
  prod: partial.prod,
1055
+ local: partial.local,
1048
1056
  team: partial.team,
1049
1057
  apiKey: partial.apiKey,
1050
1058
  fetchImpl: partial.fetchImpl,
package/src/evals.ts CHANGED
@@ -48,13 +48,6 @@
48
48
 
49
49
  import { normalizePositiveInt } from "./internal/bounded-int.js";
50
50
  import { brandDefinition } from "./internal/brand.js";
51
- import {
52
- assertNonEmptyDir,
53
- deleteFileInDir,
54
- readJsonDir,
55
- resolveProjectRelativeDir,
56
- writeJsonFile,
57
- } from "./internal/json-dir-store.js";
58
51
  import type { SessionEvent } from "./types.js";
59
52
 
60
53
  export type Expectation =
@@ -322,35 +315,6 @@ export interface EvalRunSnapshot {
322
315
  };
323
316
  }
324
317
 
325
- /** Context passed to {@link EvalRunPersistence} hooks. */
326
- export interface EvalRunPersistContext {
327
- /** Absolute agent project root (directory that contains `evals/`). */
328
- projectRoot: string;
329
- }
330
-
331
- /**
332
- * Durable store for playground eval batches. Without {@link EvalConfigFile.persistRuns},
333
- * runs live only in the serve process memory — lost on restart, and capped by
334
- * {@link EvalConfigFile.maxPlaygroundRuns} (default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}).
335
- *
336
- * Implement your own (S3, DB, …) or use {@link persistEvalRunsToDir}.
337
- * `delete` is required so {@link EvalConfigFile.maxPlaygroundRuns} can prune
338
- * durable history (otherwise pruned rows reload on the next hydrate).
339
- */
340
- export interface EvalRunPersistence {
341
- /** Load saved runs (any order). Called once when the serve process hydrates. */
342
- load(
343
- ctx: EvalRunPersistContext
344
- ): EvalRunSnapshot[] | Promise<EvalRunSnapshot[]>;
345
- /**
346
- * Persist a full snapshot. Called when a batch starts, after each case
347
- * finishes, and when the batch completes or fails.
348
- */
349
- save(run: EvalRunSnapshot, ctx: EvalRunPersistContext): void | Promise<void>;
350
- /** Remove a run that falls outside {@link EvalConfigFile.maxPlaygroundRuns}. */
351
- delete(runId: string, ctx: EvalRunPersistContext): void | Promise<void>;
352
- }
353
-
354
318
  export interface EvalConfigFile {
355
319
  timeoutMs?: number;
356
320
  /**
@@ -361,52 +325,30 @@ export interface EvalConfigFile {
361
325
  maxConcurrency: number;
362
326
  /**
363
327
  * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
364
- * history window (in-memory, and pruned from {@link persistRuns} via
365
- * {@link EvalRunPersistence.delete}).
328
+ * history window (in-memory, and pruned from the `defineStorage` evals
329
+ * table when one is configured).
366
330
  * Does not affect `agent-serve eval` CLI runs.
367
331
  *
368
332
  * Default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}; hard-capped at
369
333
  * {@link EVAL_MAX_PLAYGROUND_RUNS}. Newest batches win.
370
334
  */
371
335
  maxPlaygroundRuns?: number;
372
- /**
373
- * Optional durable store for playground eval batches. Omit this and runs
374
- * are **not** surviving process restart (only the in-memory
375
- * {@link maxPlaygroundRuns} window).
376
- *
377
- * ```ts
378
- * import {
379
- * defineEvalConfig,
380
- * persistEvalRunsToDir,
381
- * } from "@anysphere/agent-serve/evals";
382
- *
383
- * export default defineEvalConfig({
384
- * maxConcurrency: 20,
385
- * maxPlaygroundRuns: 50,
386
- * // Survive serve restarts (relative to the agent project root):
387
- * persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
388
- * });
389
- * ```
390
- */
391
- persistRuns?: EvalRunPersistence;
392
336
  }
393
337
 
394
338
  /**
395
339
  * Defaults for every eval under `evals/` (`evals/evals.config.ts`).
396
340
  * `maxConcurrency` is required (1–{@link EVAL_MAX_CONCURRENCY_LIMIT}).
397
341
  *
398
- * Playground batches default to **in-memory only** (see {@link maxPlaygroundRuns} /
399
- * {@link persistRuns}).
342
+ * Playground batches default to **in-memory only**; author an `evals`
343
+ * table in `agent/storage.ts` (`defineStorage`) to survive serve restart.
400
344
  */
401
345
  export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
402
- const persistRuns = normalizePersistRuns(config.persistRuns);
403
346
  return {
404
347
  ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
405
348
  maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency),
406
349
  maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
407
350
  config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS
408
351
  ),
409
- ...(persistRuns === undefined ? {} : { persistRuns }),
410
352
  };
411
353
  }
412
354
 
@@ -443,59 +385,3 @@ export function resolveEvalMaxPlaygroundRuns(value: unknown): number {
443
385
  return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
444
386
  }
445
387
  }
446
-
447
- function normalizePersistRuns(
448
- value: EvalRunPersistence | undefined
449
- ): EvalRunPersistence | undefined {
450
- if (value === undefined) {
451
- return undefined;
452
- }
453
- if (
454
- typeof value !== "object" ||
455
- value === null ||
456
- typeof value.load !== "function" ||
457
- typeof value.save !== "function" ||
458
- typeof value.delete !== "function"
459
- ) {
460
- throw new Error(
461
- "evals.config.ts: persistRuns must define load(ctx), save(run, ctx), and delete(runId, ctx)"
462
- );
463
- }
464
- return value;
465
- }
466
-
467
- /**
468
- * Filesystem {@link EvalRunPersistence}: one JSON file per run under `dir`
469
- * (relative paths resolve against the agent project root).
470
- *
471
- * ```ts
472
- * export default defineEvalConfig({
473
- * maxConcurrency: 20,
474
- * persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
475
- * });
476
- * ```
477
- */
478
- export function persistEvalRunsToDir(dir: string): EvalRunPersistence {
479
- const relativeDir = assertNonEmptyDir(dir, "persistEvalRunsToDir");
480
- const resolveDir = (projectRoot: string): string =>
481
- resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
482
-
483
- return {
484
- async load(ctx) {
485
- return readJsonDir<EvalRunSnapshot>(resolveDir(ctx.projectRoot));
486
- },
487
- async save(run, ctx) {
488
- await writeJsonFile({
489
- root: resolveDir(ctx.projectRoot),
490
- fileName: `${run.runId}.json`,
491
- value: run,
492
- });
493
- },
494
- async delete(runId, ctx) {
495
- await deleteFileInDir({
496
- root: resolveDir(ctx.projectRoot),
497
- fileName: `${runId}.json`,
498
- });
499
- },
500
- };
501
- }
package/src/index.ts CHANGED
@@ -105,10 +105,12 @@ export type {
105
105
  export { defineSchedule } from "./schedules.js";
106
106
  export { defineSkill } from "./skills.js";
107
107
  export type {
108
+ StorageABTable,
108
109
  StorageConfig,
109
110
  StorageContext,
110
111
  StorageDefinition,
111
112
  StorageEntry,
113
+ StorageEvalsTable,
112
114
  StoragePolicy,
113
115
  StorageRestorePolicy,
114
116
  } from "./storage.js";
@@ -16,7 +16,7 @@
16
16
 
17
17
  import {
18
18
  type ABDefinition,
19
- type ABSamplePersistence,
19
+ type ABMetricSample,
20
20
  type ABSampleReason,
21
21
  cloneABMetrics,
22
22
  } from "../ab.js";
@@ -33,8 +33,11 @@ import { describeError } from "./describe-error.js";
33
33
  export type { DiscoveredAB };
34
34
 
35
35
  export interface ABCollectorOptions {
36
- projectRoot: string;
37
- persistSamples?: ABSamplePersistence;
36
+ /**
37
+ * Durable sample export (the `defineStorage` `abs` table, via the
38
+ * coordinator). Fire-and-forget; must never throw.
39
+ */
40
+ persistSample?: (sample: ABMetricSample) => void;
38
41
  }
39
42
 
40
43
  /** Drop cache after these — next event rehydrates from the durable log. */
@@ -47,8 +50,9 @@ const EVICT_ON: ReadonlySet<SessionEvent["type"]> = new Set([
47
50
  export class ABCollector {
48
51
  private readonly state = new Map<string, FoldedABSession>();
49
52
  private readonly hydrated = new Set<string>();
50
- private readonly projectRoot: string | undefined;
51
- private readonly persistSamples: ABSamplePersistence | undefined;
53
+ private readonly persistSample:
54
+ | ((sample: ABMetricSample) => void)
55
+ | undefined;
52
56
 
53
57
  constructor(
54
58
  private readonly experiments: readonly DiscoveredAB[],
@@ -56,8 +60,7 @@ export class ABCollector {
56
60
  private readonly loadEvents: (sessionId: string) => Promise<SessionEvent[]>,
57
61
  options?: ABCollectorOptions
58
62
  ) {
59
- this.projectRoot = options?.projectRoot;
60
- this.persistSamples = options?.persistSamples;
63
+ this.persistSample = options?.persistSample;
61
64
  }
62
65
 
63
66
  get enabled(): boolean {
@@ -155,7 +158,7 @@ export class ABCollector {
155
158
  }
156
159
  const reason = sampleReason(event);
157
160
  if (reason !== undefined) {
158
- const sample = {
161
+ const sample: ABMetricSample = {
159
162
  experiment: experiment.name,
160
163
  variant,
161
164
  ...(def.variants[variant]?.label !== undefined
@@ -174,20 +177,7 @@ export class ABCollector {
174
177
  `[agentkit] ab "${experiment.name}" onSample threw: ${describeError(error)}`
175
178
  );
176
179
  }
177
- if (
178
- this.persistSamples !== undefined &&
179
- this.projectRoot !== undefined
180
- ) {
181
- try {
182
- await this.persistSamples.save(sample, {
183
- projectRoot: this.projectRoot,
184
- });
185
- } catch (error) {
186
- this.logger(
187
- `[agentkit] ab persistSamples.save threw: ${describeError(error)}`
188
- );
189
- }
190
- }
180
+ this.persistSample?.(sample);
191
181
  }
192
182
  }
193
183
  }
@@ -58,12 +58,10 @@ export interface ABSnapshotExperiment {
58
58
  export interface ABSnapshotConfig {
59
59
  /** Newest-first session fold window for this snapshot (`GET /v1/abs`). */
60
60
  maxPlaygroundSessions?: number;
61
- /** True when `persistSamples` is configured. */
61
+ /** True when the `defineStorage` `abs` table is configured. */
62
62
  durableSamples?: boolean;
63
- /** True when `persistSnapshots` is configured. */
63
+ /** True when the `abs` table also stores the aggregate snapshot. */
64
64
  durableSnapshots?: boolean;
65
- /** Set when writing via filesystem snapshot persistence helpers. */
66
- snapshotId?: string;
67
65
  }
68
66
 
69
67
  export interface ABSnapshot {