@cursor/july 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/dist/ab.d.ts +8 -95
  2. package/dist/ab.d.ts.map +1 -1
  3. package/dist/ab.js +9 -150
  4. package/dist/bin/agent-serve.js +41 -8
  5. package/dist/channels/slack/post-update-delivery.d.ts +85 -0
  6. package/dist/channels/slack/post-update-delivery.d.ts.map +1 -0
  7. package/dist/docs/404.html +2 -2
  8. package/dist/docs/ab.html +4 -4
  9. package/dist/docs/assets/{app.DabPG-io.js → app.COTN7wgo.js} +1 -1
  10. package/dist/docs/assets/chunks/@localSearchIndexroot.B7UcKvIn.js +1 -0
  11. package/dist/docs/assets/chunks/{VPLocalSearchBox.jmyr0bU0.js → VPLocalSearchBox.BW3TBdT0.js} +1 -1
  12. package/dist/docs/assets/chunks/{theme.DysN9-VN.js → theme.BEJW0vE7.js} +2 -2
  13. package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
  14. package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
  15. package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
  16. package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
  17. package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
  18. package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
  19. package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
  20. package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
  21. package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
  22. package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
  23. package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
  24. package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
  25. package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
  26. package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
  27. package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
  28. package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
  29. package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
  30. package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
  31. package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
  32. package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
  33. package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
  34. package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
  35. package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
  36. package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
  37. package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
  38. package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
  39. package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
  40. package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
  41. package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
  42. package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
  43. package/dist/docs/building-with-agents.html +4 -4
  44. package/dist/docs/concepts.html +4 -4
  45. package/dist/docs/deployment.html +58 -17
  46. package/dist/docs/evals.html +4 -4
  47. package/dist/docs/example-agents/approval-buddy.html +34 -0
  48. package/dist/docs/example-agents/benny.html +31 -0
  49. package/dist/docs/example-agents/bugbot.html +35 -0
  50. package/dist/docs/example-agents/codebase-wiki.html +32 -0
  51. package/dist/docs/example-agents/codeowners-review.html +32 -0
  52. package/dist/docs/example-agents/concierge.html +47 -0
  53. package/dist/docs/example-agents/fsd.html +39 -0
  54. package/dist/docs/example-agents/index.html +26 -0
  55. package/dist/docs/example-agents/knowledge-base.html +35 -0
  56. package/dist/docs/example-agents/security-reviewer.html +43 -0
  57. package/dist/docs/example-agents/slack-agent.html +29 -0
  58. package/dist/docs/example-agents/weather-agent.html +48 -0
  59. package/dist/docs/guides/agent-to-agent.html +4 -4
  60. package/dist/docs/guides/cloud-runtime.html +5 -5
  61. package/dist/docs/guides/github.html +4 -4
  62. package/dist/docs/guides/human-in-the-loop.html +4 -4
  63. package/dist/docs/guides/slack.html +4 -4
  64. package/dist/docs/guides/webhooks.html +4 -4
  65. package/dist/docs/hashmap.json +1 -1
  66. package/dist/docs/hillclimbing.html +4 -4
  67. package/dist/docs/index.html +7 -7
  68. package/dist/docs/quickstart.html +4 -4
  69. package/dist/docs/reference/agent-config.html +4 -4
  70. package/dist/docs/reference/channels.html +4 -4
  71. package/dist/docs/reference/cli.html +52 -30
  72. package/dist/docs/reference/connections.html +4 -4
  73. package/dist/docs/reference/hooks.html +4 -4
  74. package/dist/docs/reference/http-api.html +4 -4
  75. package/dist/docs/reference/instructions.html +4 -4
  76. package/dist/docs/reference/playground.html +4 -4
  77. package/dist/docs/reference/project-layout.html +4 -4
  78. package/dist/docs/reference/schedules.html +4 -4
  79. package/dist/docs/reference/sessions.html +4 -4
  80. package/dist/docs/reference/skills.html +4 -4
  81. package/dist/docs/reference/subagents.html +4 -4
  82. package/dist/docs/reference/tools.html +4 -4
  83. package/dist/docs/scaffolding-agents.html +4 -4
  84. package/dist/docs/storage.html +4 -4
  85. package/dist/docs/troubleshooting.html +4 -4
  86. package/dist/evals.d.ts +5 -62
  87. package/dist/evals.d.ts.map +1 -1
  88. package/dist/evals.js +3 -66
  89. package/dist/index.d.ts +1 -1
  90. package/dist/index.d.ts.map +1 -1
  91. package/dist/internal/ab-collector.d.ts +7 -5
  92. package/dist/internal/ab-collector.d.ts.map +1 -1
  93. package/dist/internal/ab-collector.js +3 -14
  94. package/dist/internal/ab-snapshot.d.ts +2 -4
  95. package/dist/internal/ab-snapshot.d.ts.map +1 -1
  96. package/dist/internal/cli-ax.d.ts +33 -5
  97. package/dist/internal/cli-ax.d.ts.map +1 -1
  98. package/dist/internal/cli-ax.js +428 -87
  99. package/dist/internal/cli-deploy.js +1 -1
  100. package/dist/internal/discovery.js +3 -3
  101. package/dist/internal/eval-run-store.d.ts +35 -30
  102. package/dist/internal/eval-run-store.d.ts.map +1 -1
  103. package/dist/internal/eval-run-store.js +88 -100
  104. package/dist/internal/evals-client.d.ts +96 -0
  105. package/dist/internal/evals-client.d.ts.map +1 -0
  106. package/dist/internal/evals-client.js +262 -0
  107. package/dist/internal/init-project.d.ts.map +1 -1
  108. package/dist/internal/init-project.js +1 -0
  109. package/dist/internal/persistence-coordinator.d.ts +127 -0
  110. package/dist/internal/persistence-coordinator.d.ts.map +1 -0
  111. package/dist/internal/playground-proxy.d.ts +5 -5
  112. package/dist/internal/playground-proxy.js +3 -3
  113. package/dist/internal/resolve-prod-target.d.ts +30 -0
  114. package/dist/internal/resolve-prod-target.d.ts.map +1 -1
  115. package/dist/internal/resolve-prod-target.js +74 -2
  116. package/dist/internal/server.d.ts.map +1 -1
  117. package/dist/internal/server.js +16 -5
  118. package/dist/internal/session-engine.d.ts +1 -2
  119. package/dist/internal/session-engine.d.ts.map +1 -1
  120. package/dist/internal/session-engine.js +14 -31
  121. package/dist/internal/storage-coordinator.d.ts +16 -15
  122. package/dist/internal/storage-coordinator.d.ts.map +1 -1
  123. package/dist/internal/storage-coordinator.js +73 -80
  124. package/dist/persistence.d.ts +184 -0
  125. package/dist/persistence.d.ts.map +1 -0
  126. package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
  127. package/dist/playground/assets/index-72vCOBWO.js +86 -0
  128. package/dist/playground/assets/index-BjnMwYoR.css +1 -0
  129. package/dist/playground/index.html +2 -2
  130. package/dist/storage.d.ts +51 -10
  131. package/dist/storage.d.ts.map +1 -1
  132. package/dist/storage.js +27 -10
  133. package/docs/README.md +34 -5
  134. package/docs/deployment.md +352 -149
  135. package/docs/example-agents/approval-buddy.md +270 -0
  136. package/docs/example-agents/benny.md +186 -0
  137. package/docs/example-agents/bugbot.md +231 -0
  138. package/docs/example-agents/codebase-wiki.md +174 -0
  139. package/docs/example-agents/codeowners-review.md +195 -0
  140. package/docs/example-agents/concierge.md +205 -0
  141. package/docs/example-agents/fsd.md +330 -0
  142. package/docs/example-agents/index.md +102 -0
  143. package/docs/example-agents/knowledge-base.md +171 -0
  144. package/docs/example-agents/security-reviewer.md +296 -0
  145. package/docs/example-agents/slack-agent.md +146 -0
  146. package/docs/example-agents/weather-agent.md +302 -0
  147. package/docs/reference/cli.md +546 -147
  148. package/package.json +1 -1
  149. package/src/ab.ts +9 -261
  150. package/src/bin/agent-serve.ts +46 -7
  151. package/src/evals.ts +5 -119
  152. package/src/index.ts +2 -0
  153. package/src/internal/ab-collector.ts +12 -22
  154. package/src/internal/ab-snapshot.ts +2 -4
  155. package/src/internal/cli-ax.ts +551 -104
  156. package/src/internal/cli-deploy.ts +1 -1
  157. package/src/internal/discovery.ts +2 -2
  158. package/src/internal/eval-run-store.ts +91 -100
  159. package/src/internal/evals-client.ts +431 -0
  160. package/src/internal/init-project.ts +1 -0
  161. package/src/internal/playground-proxy.ts +5 -5
  162. package/src/internal/resolve-prod-target.ts +101 -3
  163. package/src/internal/server.ts +17 -3
  164. package/src/internal/session-engine.ts +9 -29
  165. package/src/internal/storage-coordinator.ts +109 -101
  166. package/src/storage.ts +79 -14
  167. package/dist/docs/assets/chunks/@localSearchIndexroot.QwK5BtEH.js +0 -1
  168. package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
  169. package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
  170. package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
  171. package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
  172. package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
  173. package/dist/internal/json-dir-store.js +0 -100
  174. package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
  175. package/dist/playground/assets/index-BEauYlII.css +0 -1
  176. package/dist/playground/assets/index-BtM0wEGg.js +0 -319
  177. package/src/internal/json-dir-store.ts +0 -109
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cursor/july",
3
- "version": "0.1.5",
3
+ "version": "0.1.7",
4
4
  "description": "(early alpha) Filesystem-first framework for defining Cursor agents as markdown and TypeScript and serving them over channels with the Cursor SDK.",
5
5
  "license": "SEE LICENSE IN LICENSE.md",
6
6
  "repository": {
package/src/ab.ts CHANGED
@@ -10,9 +10,8 @@
10
10
  *
11
11
  * Assignments and foldable metrics live in each session's `events.ndjson`
12
12
  * (survive serve restart). Optional project defaults in `agent/ab.config.ts`
13
- * ({@link defineABConfig}) let authors retain the playground fold window and
14
- * persist samples / aggregate snapshots wherever they like same idea as
15
- * eval `persistRuns`.
13
+ * ({@link defineABConfig}) cap the playground fold window; durable
14
+ * sample/snapshot export lives on the `defineStorage` `abs` table.
16
15
  *
17
16
  * ```ts
18
17
  * import { defineAB, splitBySessionHash } from "@anysphere/agent-serve/ab";
@@ -31,20 +30,8 @@
31
30
  * ```
32
31
  */
33
32
 
34
- import { randomUUID } from "node:crypto";
35
- import { appendFile, mkdir } from "node:fs/promises";
36
- import { join } from "node:path";
37
- import type { ABSnapshot } from "./internal/ab-snapshot.js";
38
33
  import { normalizePositiveInt } from "./internal/bounded-int.js";
39
34
  import { brandDefinition } from "./internal/brand.js";
40
- import {
41
- assertNonEmptyDir,
42
- deleteFileInDir,
43
- listJsonBasenames,
44
- readJsonDir,
45
- resolveProjectRelativeDir,
46
- writeJsonFile,
47
- } from "./internal/json-dir-store.js";
48
35
  import type { SessionEvent, SessionInfo } from "./types.js";
49
36
 
50
37
  export type { ABSnapshot } from "./internal/ab-snapshot.js";
@@ -370,7 +357,7 @@ export function mergeABInstructions(
370
357
  }
371
358
 
372
359
  // ---------------------------------------------------------------------------
373
- // Project config (`agent/ab.config.ts`) — fold window + optional persistence
360
+ // Project config (`agent/ab.config.ts`) — playground fold window
374
361
  // ---------------------------------------------------------------------------
375
362
 
376
363
  /** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
@@ -379,128 +366,36 @@ export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
379
366
  /** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
380
367
  export const AB_MAX_PLAYGROUND_SESSIONS = 10_000;
381
368
 
382
- /** Default snapshot files kept when {@link ABConfigFile.persistSnapshots} is set. */
383
- export const AB_DEFAULT_RETAIN_SNAPSHOTS = 20;
384
-
385
- /** Hard ceiling for {@link ABConfigFile.retainSnapshots}. */
386
- export const AB_MAX_RETAIN_SNAPSHOTS = 500;
387
-
388
- /** Context passed to A/B persistence hooks. */
389
- export interface ABPersistContext {
390
- /** Absolute agent project root (directory that contains `agent/`). */
391
- projectRoot: string;
392
- }
393
-
394
- /**
395
- * Durable sink for {@link ABMetricSample} rows (same payload as
396
- * {@link ABConfig.onSample}). Session `events.ndjson` remains the source of
397
- * truth for assignment + fold; this is for analytics export.
398
- */
399
- export interface ABSamplePersistence {
400
- save(sample: ABMetricSample, ctx: ABPersistContext): void | Promise<void>;
401
- }
402
-
403
- /**
404
- * Durable store for playground-style aggregate snapshots (`GET /v1/abs`).
405
- * Without this, aggregates are always recomputed from session logs (nothing
406
- * is kept after sessions are pruned).
407
- *
408
- * **Runtime note:** `serve` currently only calls {@link save} (after each
409
- * `GET /v1/abs`). {@link load} / {@link delete} are for author tooling or a
410
- * custom store; they are not wired into the playground hydrate path.
411
- */
412
- export interface ABSnapshotPersistence {
413
- load(ctx: ABPersistContext): ABSnapshot[] | Promise<ABSnapshot[]>;
414
- save(snapshot: ABSnapshot, ctx: ABPersistContext): void | Promise<void>;
415
- delete?(snapshotId: string, ctx: ABPersistContext): void | Promise<void>;
416
- }
417
-
418
- /** Lets {@link defineABConfig} push `retainSnapshots` into {@link persistABSnapshotsToDir}. */
419
- const abSnapshotRetainState = new WeakMap<
420
- ABSnapshotPersistence,
421
- { retain: number }
422
- >();
423
-
424
369
  /**
425
370
  * Project-wide A/B defaults (`agent/ab.config.ts`).
426
371
  *
427
372
  * **Assignments and foldable metrics** already persist in each session's
428
- * `events.ndjson` under `--state-root`. Use this config to:
429
- * - cap how many sessions the playground / `GET /v1/abs` folds
430
- * - export samples and/or aggregate snapshots to author-chosen storage
373
+ * `events.ndjson` under `--state-root`; durable sample/snapshot export
374
+ * lives on the `defineStorage` `abs` table (`agent/storage.ts`). This
375
+ * config only caps how many sessions the playground / `GET /v1/abs` folds.
431
376
  */
432
377
  export interface ABConfigFile {
433
378
  /**
434
379
  * Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
435
- * tab** (and into {@link persistSnapshots} payloads). Does not prune
436
- * session logs, change assignment, or limit {@link persistSamples}.
380
+ * tab**. Does not prune session logs or change assignment.
437
381
  *
438
382
  * Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
439
383
  * {@link AB_MAX_PLAYGROUND_SESSIONS}.
440
384
  */
441
385
  maxPlaygroundSessions?: number;
442
- /**
443
- * Snapshot-file budget when {@link persistSnapshots} is set.
444
- * Default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}; hard-capped at
445
- * {@link AB_MAX_RETAIN_SNAPSHOTS}.
446
- *
447
- * When using {@link persistABSnapshotsToDir}, this value controls on-disk
448
- * pruning (overriding the helper's `{ retain }` default). Custom stores
449
- * should apply the same budget themselves.
450
- */
451
- retainSnapshots?: number;
452
- /**
453
- * Optional sink for cumulative metric samples (in addition to each
454
- * experiment's {@link ABConfig.onSample}).
455
- *
456
- * ```ts
457
- * persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
458
- * ```
459
- */
460
- persistSamples?: ABSamplePersistence;
461
- /**
462
- * Optional store for aggregate snapshots written when `GET /v1/abs` runs.
463
- *
464
- * ```ts
465
- * persistSnapshots: persistABSnapshotsToDir(".agent-serve/ab-snapshots"),
466
- * ```
467
- */
468
- persistSnapshots?: ABSnapshotPersistence;
469
386
  }
470
387
 
471
388
  /**
472
389
  * Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
473
390
  *
474
- * Session event logs remain the assignment/metrics source of truth; this
475
- * file controls the playground fold window and optional durable exports.
391
+ * Session event logs remain the assignment/metrics source of truth;
392
+ * durable exports go through the `defineStorage` `abs` table.
476
393
  */
477
394
  export function defineABConfig(config: ABConfigFile): ABConfigFile {
478
- const persistSamples = normalizeSamplePersistence(config.persistSamples);
479
- const persistSnapshots = normalizeSnapshotPersistence(
480
- config.persistSnapshots
481
- );
482
- const helperRetain =
483
- persistSnapshots !== undefined
484
- ? abSnapshotRetainState.get(persistSnapshots)
485
- : undefined;
486
- let retainSnapshots: number | undefined;
487
- if (config.retainSnapshots !== undefined) {
488
- retainSnapshots = normalizeABRetainSnapshots(config.retainSnapshots);
489
- if (helperRetain !== undefined) {
490
- helperRetain.retain = retainSnapshots;
491
- }
492
- } else if (persistSnapshots !== undefined) {
493
- retainSnapshots =
494
- helperRetain?.retain ??
495
- normalizeABRetainSnapshots(AB_DEFAULT_RETAIN_SNAPSHOTS);
496
- }
497
395
  return {
498
396
  maxPlaygroundSessions: normalizeABMaxPlaygroundSessions(
499
397
  config.maxPlaygroundSessions ?? AB_DEFAULT_MAX_PLAYGROUND_SESSIONS
500
398
  ),
501
- ...(retainSnapshots === undefined ? {} : { retainSnapshots }),
502
- ...(persistSamples === undefined ? {} : { persistSamples }),
503
- ...(persistSnapshots === undefined ? {} : { persistSnapshots }),
504
399
  };
505
400
  }
506
401
 
@@ -526,150 +421,3 @@ export function resolveABMaxPlaygroundSessions(value: unknown): number {
526
421
  return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
527
422
  }
528
423
  }
529
-
530
- /** Validate {@link ABConfigFile.retainSnapshots}. */
531
- export function normalizeABRetainSnapshots(value: unknown): number {
532
- return normalizePositiveInt(value, {
533
- label: "ab.config.ts: retainSnapshots",
534
- max: AB_MAX_RETAIN_SNAPSHOTS,
535
- });
536
- }
537
-
538
- function normalizeSamplePersistence(
539
- value: ABSamplePersistence | undefined
540
- ): ABSamplePersistence | undefined {
541
- if (value === undefined) {
542
- return undefined;
543
- }
544
- if (
545
- typeof value !== "object" ||
546
- value === null ||
547
- typeof value.save !== "function"
548
- ) {
549
- throw new Error(
550
- "ab.config.ts: persistSamples must define save(sample, ctx)"
551
- );
552
- }
553
- return value;
554
- }
555
-
556
- function normalizeSnapshotPersistence(
557
- value: ABSnapshotPersistence | undefined
558
- ): ABSnapshotPersistence | undefined {
559
- if (value === undefined) {
560
- return undefined;
561
- }
562
- if (
563
- typeof value !== "object" ||
564
- value === null ||
565
- typeof value.load !== "function" ||
566
- typeof value.save !== "function"
567
- ) {
568
- throw new Error(
569
- "ab.config.ts: persistSnapshots must define load(ctx) and save(snapshot, ctx)"
570
- );
571
- }
572
- if (value.delete !== undefined && typeof value.delete !== "function") {
573
- throw new Error("ab.config.ts: persistSnapshots.delete must be a function");
574
- }
575
- return value;
576
- }
577
-
578
- /**
579
- * Append-only NDJSON {@link ABSamplePersistence} under `dir` (relative to
580
- * the agent project root).
581
- *
582
- * ```ts
583
- * // agent/ab.config.ts
584
- * export default defineABConfig({
585
- * persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
586
- * });
587
- * ```
588
- */
589
- export function persistABSamplesToDir(dir: string): ABSamplePersistence {
590
- const relativeDir = assertNonEmptyDir(dir, "persistABSamplesToDir");
591
- const resolveDir = (projectRoot: string): string =>
592
- resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
593
-
594
- return {
595
- async save(sample, ctx) {
596
- const root = resolveDir(ctx.projectRoot);
597
- await mkdir(root, { recursive: true });
598
- await appendFile(
599
- join(root, "samples.ndjson"),
600
- `${JSON.stringify(sample)}\n`,
601
- "utf8"
602
- );
603
- },
604
- };
605
- }
606
-
607
- /**
608
- * Filesystem {@link ABSnapshotPersistence}: one JSON file per snapshot under
609
- * `dir`, pruned to `retain` files (default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}).
610
- *
611
- * Prefer setting {@link ABConfigFile.retainSnapshots} in `defineABConfig` —
612
- * that value is applied to this helper's prune budget. Pass `{ retain }` here
613
- * only when constructing the helper outside `defineABConfig`, or as a default
614
- * that `retainSnapshots` may override.
615
- */
616
- export function persistABSnapshotsToDir(
617
- dir: string,
618
- options?: { retain?: number }
619
- ): ABSnapshotPersistence {
620
- const relativeDir = assertNonEmptyDir(dir, "persistABSnapshotsToDir");
621
- const retainState = {
622
- retain: normalizeABRetainSnapshots(
623
- options?.retain ?? AB_DEFAULT_RETAIN_SNAPSHOTS
624
- ),
625
- };
626
- const resolveDir = (projectRoot: string): string =>
627
- resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
628
- let saveChain: Promise<void> = Promise.resolve();
629
-
630
- const persist: ABSnapshotPersistence = {
631
- async load(ctx) {
632
- return readJsonDir<ABSnapshot>(resolveDir(ctx.projectRoot), {
633
- sortBasenames: "desc",
634
- });
635
- },
636
- async save(snapshot, ctx) {
637
- const run = async (): Promise<void> => {
638
- const root = resolveDir(ctx.projectRoot);
639
- const snapshotId =
640
- snapshot.config?.snapshotId ??
641
- `absnap_${randomUUID().replace(/-/g, "")}`;
642
- const toWrite: ABSnapshot = {
643
- ...snapshot,
644
- config: {
645
- ...snapshot.config,
646
- snapshotId,
647
- },
648
- };
649
- await writeJsonFile({
650
- root,
651
- fileName: `${snapshotId}.json`,
652
- value: toWrite,
653
- });
654
- const names = (await listJsonBasenames(root)).sort().reverse();
655
- for (const stale of names.slice(retainState.retain)) {
656
- await deleteFileInDir({ root, fileName: stale });
657
- }
658
- };
659
- const next = saveChain.then(run, run);
660
- saveChain = next.then(
661
- () => undefined,
662
- () => undefined
663
- );
664
- await next;
665
- },
666
- async delete(snapshotId, ctx) {
667
- await deleteFileInDir({
668
- root: resolveDir(ctx.projectRoot),
669
- fileName: `${snapshotId}.json`,
670
- });
671
- },
672
- };
673
- abSnapshotRetainState.set(persist, retainState);
674
- return persist;
675
- }
@@ -29,6 +29,7 @@ import {
29
29
  cmdCall,
30
30
  cmdChat,
31
31
  cmdEval,
32
+ cmdEvalStatus,
32
33
  cmdInit,
33
34
  cmdLogs,
34
35
  cmdPlayground,
@@ -90,8 +91,11 @@ Usage:
90
91
  ${CLI} init [dir] [--json]
91
92
  ${CLI} eval [--dir <path>] [evalId...] [--list] [--json]
92
93
  [--tag <tag>]... [--verbose] [--timeout-ms <n>]
93
- [--no-stream] [--url <baseUrl> | --prod] [--slug <slug>]
94
+ [--no-stream] [--no-wait]
95
+ [--prod | --url <baseUrl>] [--slug <slug>]
94
96
  [--team <id>]
97
+ ${CLI} eval status [runId] --prod|--url <baseUrl> [--dir <path>] [--json]
98
+ [--slug <slug>] [--team <id>]
95
99
  ${CLI} info [--dir <path>] [--json]
96
100
  ${CLI} validate [--dir <path>]
97
101
  ${CLI} login [--api-key <key>] [--key-name <name>]
@@ -155,9 +159,19 @@ Commands:
155
159
  machine-readable and never blocks on login. No directory:
156
160
  print the setup guide.
157
161
  eval Run filesystem evals under evals/**/*.eval.ts (streams on a TTY).
158
- Pass evalId args to select: one datapoint (weather/nyc), a whole
159
- file (weather all cases in that file), several of either, or
160
- omit to run all.
162
+ Default: ephemeral local harness against evals on disk. Pass
163
+ --prod to kick off the batch on the team's hosted deployment
164
+ (slug from --slug or the directory name) via /v1/dev/evals/runs
165
+ so results appear in the playground and persist when the agent
166
+ declares an evals storage table; or --url for a running server
167
+ (same server-side path). With --prod/--url, pass --no-wait to
168
+ return the runId immediately without polling; check later with
169
+ \`${CLI} eval status <runId> --prod\` (omit runId to list recent
170
+ runs; exit 3 while still running). The status subcommand only
171
+ binds when --prod/--url is set, so an eval id named status still
172
+ runs locally. Pass evalId args to select: one datapoint
173
+ (weather/nyc), a whole file (weather → all cases in that file),
174
+ several of either, or omit to run all.
161
175
  info Print the discovered agent surface (tools, skills, channels, ...).
162
176
  validate Check the project(s) and exit non-zero on error diagnostics.
163
177
  login Sign the host in to Cursor: browser sign-in mints a named,
@@ -193,6 +207,7 @@ Agent loop:
193
207
  ${CLI} info --dir . --json
194
208
  ${CLI} run --dir . --message "..."
195
209
  ${CLI} eval --dir .
210
+ ${CLI} eval --prod --dir .
196
211
  ${CLI} dev .
197
212
 
198
213
  Options:
@@ -231,12 +246,14 @@ Options:
231
246
  --prod chat/resume/run/call/eval/logs/sessions/session/playground:
232
247
  use the team's hosted deployment. Slug from --slug or the
233
248
  --dir basename. Mutually exclusive with --url. Requires
234
- \`${CLI} login\`.
249
+ \`${CLI} login\`. For eval, this is how you opt into the
250
+ persisted server-side batch (default is local).
235
251
  --once logs: print current progress / log buffer once and exit
236
252
  (default is to follow).
237
253
  --print playground: print the URL only; do not open a browser.
238
- --no-wait deploy/stop: return right after the request is accepted
239
- instead of polling until a terminal status.
254
+ --no-wait deploy/stop/eval: return right after the request is
255
+ accepted instead of polling until a terminal status.
256
+ For eval, prints the runId (deployment / --url only).
240
257
  --no-playground Disable the bundled web playground at /playground (also skips
241
258
  auto-build / HMR).
242
259
  --no-docs Disable the documentation site at /docs (also skips its
@@ -618,6 +635,26 @@ async function main(argv: string[]): Promise<number> {
618
635
  }
619
636
 
620
637
  case "eval": {
638
+ // Remote status subcommand only when targeting --prod/--url. Otherwise
639
+ // the bare token "status" is a normal eval id (evals/status.eval.ts).
640
+ if (
641
+ positionals[0] === "status" &&
642
+ (values.prod === true || values.url !== undefined)
643
+ ) {
644
+ return cmdEvalStatus(
645
+ positionals[1],
646
+ axFromOptions({
647
+ dir,
648
+ json: values.json,
649
+ url: values.url,
650
+ prod: values.prod,
651
+ team: values.team,
652
+ apiKey: values["api-key"],
653
+ slug: values.slug,
654
+ bearerToken: values["bearer-token"],
655
+ })
656
+ );
657
+ }
621
658
  return cmdEval(
622
659
  axFromOptions({
623
660
  dir,
@@ -628,6 +665,7 @@ async function main(argv: string[]): Promise<number> {
628
665
  tags: values.tag,
629
666
  url: values.url,
630
667
  prod: values.prod,
668
+ noWait: values["no-wait"],
631
669
  team: values.team,
632
670
  apiKey: values["api-key"],
633
671
  slug: values.slug,
@@ -1045,6 +1083,7 @@ function axFromOptions(
1045
1083
  toolInput: partial.toolInput,
1046
1084
  url: partial.url,
1047
1085
  prod: partial.prod,
1086
+ noWait: partial.noWait,
1048
1087
  team: partial.team,
1049
1088
  apiKey: partial.apiKey,
1050
1089
  fetchImpl: partial.fetchImpl,
package/src/evals.ts CHANGED
@@ -48,13 +48,6 @@
48
48
 
49
49
  import { normalizePositiveInt } from "./internal/bounded-int.js";
50
50
  import { brandDefinition } from "./internal/brand.js";
51
- import {
52
- assertNonEmptyDir,
53
- deleteFileInDir,
54
- readJsonDir,
55
- resolveProjectRelativeDir,
56
- writeJsonFile,
57
- } from "./internal/json-dir-store.js";
58
51
  import type { SessionEvent } from "./types.js";
59
52
 
60
53
  export type Expectation =
@@ -322,35 +315,6 @@ export interface EvalRunSnapshot {
322
315
  };
323
316
  }
324
317
 
325
- /** Context passed to {@link EvalRunPersistence} hooks. */
326
- export interface EvalRunPersistContext {
327
- /** Absolute agent project root (directory that contains `evals/`). */
328
- projectRoot: string;
329
- }
330
-
331
- /**
332
- * Durable store for playground eval batches. Without {@link EvalConfigFile.persistRuns},
333
- * runs live only in the serve process memory — lost on restart, and capped by
334
- * {@link EvalConfigFile.maxPlaygroundRuns} (default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}).
335
- *
336
- * Implement your own (S3, DB, …) or use {@link persistEvalRunsToDir}.
337
- * `delete` is required so {@link EvalConfigFile.maxPlaygroundRuns} can prune
338
- * durable history (otherwise pruned rows reload on the next hydrate).
339
- */
340
- export interface EvalRunPersistence {
341
- /** Load saved runs (any order). Called once when the serve process hydrates. */
342
- load(
343
- ctx: EvalRunPersistContext
344
- ): EvalRunSnapshot[] | Promise<EvalRunSnapshot[]>;
345
- /**
346
- * Persist a full snapshot. Called when a batch starts, after each case
347
- * finishes, and when the batch completes or fails.
348
- */
349
- save(run: EvalRunSnapshot, ctx: EvalRunPersistContext): void | Promise<void>;
350
- /** Remove a run that falls outside {@link EvalConfigFile.maxPlaygroundRuns}. */
351
- delete(runId: string, ctx: EvalRunPersistContext): void | Promise<void>;
352
- }
353
-
354
318
  export interface EvalConfigFile {
355
319
  timeoutMs?: number;
356
320
  /**
@@ -361,52 +325,30 @@ export interface EvalConfigFile {
361
325
  maxConcurrency: number;
362
326
  /**
363
327
  * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
364
- * history window (in-memory, and pruned from {@link persistRuns} via
365
- * {@link EvalRunPersistence.delete}).
366
- * Does not affect `agent-serve eval` CLI runs.
328
+ * history window (in-memory, and pruned from the `defineStorage` evals
329
+ * table when one is configured). Includes server-side batches started by
330
+ * `agentkit eval --prod` or `--url` (not the default local harness).
367
331
  *
368
332
  * Default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}; hard-capped at
369
333
  * {@link EVAL_MAX_PLAYGROUND_RUNS}. Newest batches win.
370
334
  */
371
335
  maxPlaygroundRuns?: number;
372
- /**
373
- * Optional durable store for playground eval batches. Omit this and runs
374
- * are **not** surviving process restart (only the in-memory
375
- * {@link maxPlaygroundRuns} window).
376
- *
377
- * ```ts
378
- * import {
379
- * defineEvalConfig,
380
- * persistEvalRunsToDir,
381
- * } from "@anysphere/agent-serve/evals";
382
- *
383
- * export default defineEvalConfig({
384
- * maxConcurrency: 20,
385
- * maxPlaygroundRuns: 50,
386
- * // Survive serve restarts (relative to the agent project root):
387
- * persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
388
- * });
389
- * ```
390
- */
391
- persistRuns?: EvalRunPersistence;
392
336
  }
393
337
 
394
338
  /**
395
339
  * Defaults for every eval under `evals/` (`evals/evals.config.ts`).
396
340
  * `maxConcurrency` is required (1–{@link EVAL_MAX_CONCURRENCY_LIMIT}).
397
341
  *
398
- * Playground batches default to **in-memory only** (see {@link maxPlaygroundRuns} /
399
- * {@link persistRuns}).
342
+ * Playground batches default to **in-memory only**; author an `evals`
343
+ * table in `agent/storage.ts` (`defineStorage`) to survive serve restart.
400
344
  */
401
345
  export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
402
- const persistRuns = normalizePersistRuns(config.persistRuns);
403
346
  return {
404
347
  ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
405
348
  maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency),
406
349
  maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
407
350
  config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS
408
351
  ),
409
- ...(persistRuns === undefined ? {} : { persistRuns }),
410
352
  };
411
353
  }
412
354
 
@@ -443,59 +385,3 @@ export function resolveEvalMaxPlaygroundRuns(value: unknown): number {
443
385
  return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
444
386
  }
445
387
  }
446
-
447
- function normalizePersistRuns(
448
- value: EvalRunPersistence | undefined
449
- ): EvalRunPersistence | undefined {
450
- if (value === undefined) {
451
- return undefined;
452
- }
453
- if (
454
- typeof value !== "object" ||
455
- value === null ||
456
- typeof value.load !== "function" ||
457
- typeof value.save !== "function" ||
458
- typeof value.delete !== "function"
459
- ) {
460
- throw new Error(
461
- "evals.config.ts: persistRuns must define load(ctx), save(run, ctx), and delete(runId, ctx)"
462
- );
463
- }
464
- return value;
465
- }
466
-
467
- /**
468
- * Filesystem {@link EvalRunPersistence}: one JSON file per run under `dir`
469
- * (relative paths resolve against the agent project root).
470
- *
471
- * ```ts
472
- * export default defineEvalConfig({
473
- * maxConcurrency: 20,
474
- * persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
475
- * });
476
- * ```
477
- */
478
- export function persistEvalRunsToDir(dir: string): EvalRunPersistence {
479
- const relativeDir = assertNonEmptyDir(dir, "persistEvalRunsToDir");
480
- const resolveDir = (projectRoot: string): string =>
481
- resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
482
-
483
- return {
484
- async load(ctx) {
485
- return readJsonDir<EvalRunSnapshot>(resolveDir(ctx.projectRoot));
486
- },
487
- async save(run, ctx) {
488
- await writeJsonFile({
489
- root: resolveDir(ctx.projectRoot),
490
- fileName: `${run.runId}.json`,
491
- value: run,
492
- });
493
- },
494
- async delete(runId, ctx) {
495
- await deleteFileInDir({
496
- root: resolveDir(ctx.projectRoot),
497
- fileName: `${runId}.json`,
498
- });
499
- },
500
- };
501
- }
package/src/index.ts CHANGED
@@ -105,10 +105,12 @@ export type {
105
105
  export { defineSchedule } from "./schedules.js";
106
106
  export { defineSkill } from "./skills.js";
107
107
  export type {
108
+ StorageABTable,
108
109
  StorageConfig,
109
110
  StorageContext,
110
111
  StorageDefinition,
111
112
  StorageEntry,
113
+ StorageEvalsTable,
112
114
  StoragePolicy,
113
115
  StorageRestorePolicy,
114
116
  } from "./storage.js";