@cursor/july 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ab.d.ts +8 -95
- package/dist/ab.d.ts.map +1 -1
- package/dist/ab.js +9 -150
- package/dist/bin/agent-serve.js +41 -8
- package/dist/channels/slack/post-update-delivery.d.ts +85 -0
- package/dist/channels/slack/post-update-delivery.d.ts.map +1 -0
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +4 -4
- package/dist/docs/assets/{app.DabPG-io.js → app.COTN7wgo.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.B7UcKvIn.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.jmyr0bU0.js → VPLocalSearchBox.BW3TBdT0.js} +1 -1
- package/dist/docs/assets/chunks/{theme.DysN9-VN.js → theme.BEJW0vE7.js} +2 -2
- package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
- package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
- package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
- package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
- package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
- package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
- package/dist/docs/building-with-agents.html +4 -4
- package/dist/docs/concepts.html +4 -4
- package/dist/docs/deployment.html +58 -17
- package/dist/docs/evals.html +4 -4
- package/dist/docs/example-agents/approval-buddy.html +34 -0
- package/dist/docs/example-agents/benny.html +31 -0
- package/dist/docs/example-agents/bugbot.html +35 -0
- package/dist/docs/example-agents/codebase-wiki.html +32 -0
- package/dist/docs/example-agents/codeowners-review.html +32 -0
- package/dist/docs/example-agents/concierge.html +47 -0
- package/dist/docs/example-agents/fsd.html +39 -0
- package/dist/docs/example-agents/index.html +26 -0
- package/dist/docs/example-agents/knowledge-base.html +35 -0
- package/dist/docs/example-agents/security-reviewer.html +43 -0
- package/dist/docs/example-agents/slack-agent.html +29 -0
- package/dist/docs/example-agents/weather-agent.html +48 -0
- package/dist/docs/guides/agent-to-agent.html +4 -4
- package/dist/docs/guides/cloud-runtime.html +5 -5
- package/dist/docs/guides/github.html +4 -4
- package/dist/docs/guides/human-in-the-loop.html +4 -4
- package/dist/docs/guides/slack.html +4 -4
- package/dist/docs/guides/webhooks.html +4 -4
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +4 -4
- package/dist/docs/index.html +7 -7
- package/dist/docs/quickstart.html +4 -4
- package/dist/docs/reference/agent-config.html +4 -4
- package/dist/docs/reference/channels.html +4 -4
- package/dist/docs/reference/cli.html +52 -30
- package/dist/docs/reference/connections.html +4 -4
- package/dist/docs/reference/hooks.html +4 -4
- package/dist/docs/reference/http-api.html +4 -4
- package/dist/docs/reference/instructions.html +4 -4
- package/dist/docs/reference/playground.html +4 -4
- package/dist/docs/reference/project-layout.html +4 -4
- package/dist/docs/reference/schedules.html +4 -4
- package/dist/docs/reference/sessions.html +4 -4
- package/dist/docs/reference/skills.html +4 -4
- package/dist/docs/reference/subagents.html +4 -4
- package/dist/docs/reference/tools.html +4 -4
- package/dist/docs/scaffolding-agents.html +4 -4
- package/dist/docs/storage.html +4 -4
- package/dist/docs/troubleshooting.html +4 -4
- package/dist/evals.d.ts +5 -62
- package/dist/evals.d.ts.map +1 -1
- package/dist/evals.js +3 -66
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/internal/ab-collector.d.ts +7 -5
- package/dist/internal/ab-collector.d.ts.map +1 -1
- package/dist/internal/ab-collector.js +3 -14
- package/dist/internal/ab-snapshot.d.ts +2 -4
- package/dist/internal/ab-snapshot.d.ts.map +1 -1
- package/dist/internal/cli-ax.d.ts +33 -5
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +428 -87
- package/dist/internal/cli-deploy.js +1 -1
- package/dist/internal/discovery.js +3 -3
- package/dist/internal/eval-run-store.d.ts +35 -30
- package/dist/internal/eval-run-store.d.ts.map +1 -1
- package/dist/internal/eval-run-store.js +88 -100
- package/dist/internal/evals-client.d.ts +96 -0
- package/dist/internal/evals-client.d.ts.map +1 -0
- package/dist/internal/evals-client.js +262 -0
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +1 -0
- package/dist/internal/persistence-coordinator.d.ts +127 -0
- package/dist/internal/persistence-coordinator.d.ts.map +1 -0
- package/dist/internal/playground-proxy.d.ts +5 -5
- package/dist/internal/playground-proxy.js +3 -3
- package/dist/internal/resolve-prod-target.d.ts +30 -0
- package/dist/internal/resolve-prod-target.d.ts.map +1 -1
- package/dist/internal/resolve-prod-target.js +74 -2
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +16 -5
- package/dist/internal/session-engine.d.ts +1 -2
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +14 -31
- package/dist/internal/storage-coordinator.d.ts +16 -15
- package/dist/internal/storage-coordinator.d.ts.map +1 -1
- package/dist/internal/storage-coordinator.js +73 -80
- package/dist/persistence.d.ts +184 -0
- package/dist/persistence.d.ts.map +1 -0
- package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
- package/dist/playground/assets/index-72vCOBWO.js +86 -0
- package/dist/playground/assets/index-BjnMwYoR.css +1 -0
- package/dist/playground/index.html +2 -2
- package/dist/storage.d.ts +51 -10
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +27 -10
- package/docs/README.md +34 -5
- package/docs/deployment.md +352 -149
- package/docs/example-agents/approval-buddy.md +270 -0
- package/docs/example-agents/benny.md +186 -0
- package/docs/example-agents/bugbot.md +231 -0
- package/docs/example-agents/codebase-wiki.md +174 -0
- package/docs/example-agents/codeowners-review.md +195 -0
- package/docs/example-agents/concierge.md +205 -0
- package/docs/example-agents/fsd.md +330 -0
- package/docs/example-agents/index.md +102 -0
- package/docs/example-agents/knowledge-base.md +171 -0
- package/docs/example-agents/security-reviewer.md +296 -0
- package/docs/example-agents/slack-agent.md +146 -0
- package/docs/example-agents/weather-agent.md +302 -0
- package/docs/reference/cli.md +546 -147
- package/package.json +1 -1
- package/src/ab.ts +9 -261
- package/src/bin/agent-serve.ts +46 -7
- package/src/evals.ts +5 -119
- package/src/index.ts +2 -0
- package/src/internal/ab-collector.ts +12 -22
- package/src/internal/ab-snapshot.ts +2 -4
- package/src/internal/cli-ax.ts +551 -104
- package/src/internal/cli-deploy.ts +1 -1
- package/src/internal/discovery.ts +2 -2
- package/src/internal/eval-run-store.ts +91 -100
- package/src/internal/evals-client.ts +431 -0
- package/src/internal/init-project.ts +1 -0
- package/src/internal/playground-proxy.ts +5 -5
- package/src/internal/resolve-prod-target.ts +101 -3
- package/src/internal/server.ts +17 -3
- package/src/internal/session-engine.ts +9 -29
- package/src/internal/storage-coordinator.ts +109 -101
- package/src/storage.ts +79 -14
- package/dist/docs/assets/chunks/@localSearchIndexroot.QwK5BtEH.js +0 -1
- package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
- package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
- package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
- package/dist/internal/json-dir-store.js +0 -100
- package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
- package/dist/playground/assets/index-BEauYlII.css +0 -1
- package/dist/playground/assets/index-BtM0wEGg.js +0 -319
- package/src/internal/json-dir-store.ts +0 -109
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cursor/july",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "(early alpha) Filesystem-first framework for defining Cursor agents as markdown and TypeScript and serving them over channels with the Cursor SDK.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE.md",
|
|
6
6
|
"repository": {
|
package/src/ab.ts
CHANGED
|
@@ -10,9 +10,8 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Assignments and foldable metrics live in each session's `events.ndjson`
|
|
12
12
|
* (survive serve restart). Optional project defaults in `agent/ab.config.ts`
|
|
13
|
-
* ({@link defineABConfig})
|
|
14
|
-
*
|
|
15
|
-
* eval `persistRuns`.
|
|
13
|
+
* ({@link defineABConfig}) cap the playground fold window; durable
|
|
14
|
+
* sample/snapshot export lives on the `defineStorage` `abs` table.
|
|
16
15
|
*
|
|
17
16
|
* ```ts
|
|
18
17
|
* import { defineAB, splitBySessionHash } from "@anysphere/agent-serve/ab";
|
|
@@ -31,20 +30,8 @@
|
|
|
31
30
|
* ```
|
|
32
31
|
*/
|
|
33
32
|
|
|
34
|
-
import { randomUUID } from "node:crypto";
|
|
35
|
-
import { appendFile, mkdir } from "node:fs/promises";
|
|
36
|
-
import { join } from "node:path";
|
|
37
|
-
import type { ABSnapshot } from "./internal/ab-snapshot.js";
|
|
38
33
|
import { normalizePositiveInt } from "./internal/bounded-int.js";
|
|
39
34
|
import { brandDefinition } from "./internal/brand.js";
|
|
40
|
-
import {
|
|
41
|
-
assertNonEmptyDir,
|
|
42
|
-
deleteFileInDir,
|
|
43
|
-
listJsonBasenames,
|
|
44
|
-
readJsonDir,
|
|
45
|
-
resolveProjectRelativeDir,
|
|
46
|
-
writeJsonFile,
|
|
47
|
-
} from "./internal/json-dir-store.js";
|
|
48
35
|
import type { SessionEvent, SessionInfo } from "./types.js";
|
|
49
36
|
|
|
50
37
|
export type { ABSnapshot } from "./internal/ab-snapshot.js";
|
|
@@ -370,7 +357,7 @@ export function mergeABInstructions(
|
|
|
370
357
|
}
|
|
371
358
|
|
|
372
359
|
// ---------------------------------------------------------------------------
|
|
373
|
-
// Project config (`agent/ab.config.ts`) — fold window
|
|
360
|
+
// Project config (`agent/ab.config.ts`) — playground fold window
|
|
374
361
|
// ---------------------------------------------------------------------------
|
|
375
362
|
|
|
376
363
|
/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
|
|
@@ -379,128 +366,36 @@ export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
|
|
|
379
366
|
/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
|
|
380
367
|
export const AB_MAX_PLAYGROUND_SESSIONS = 10_000;
|
|
381
368
|
|
|
382
|
-
/** Default snapshot files kept when {@link ABConfigFile.persistSnapshots} is set. */
|
|
383
|
-
export const AB_DEFAULT_RETAIN_SNAPSHOTS = 20;
|
|
384
|
-
|
|
385
|
-
/** Hard ceiling for {@link ABConfigFile.retainSnapshots}. */
|
|
386
|
-
export const AB_MAX_RETAIN_SNAPSHOTS = 500;
|
|
387
|
-
|
|
388
|
-
/** Context passed to A/B persistence hooks. */
|
|
389
|
-
export interface ABPersistContext {
|
|
390
|
-
/** Absolute agent project root (directory that contains `agent/`). */
|
|
391
|
-
projectRoot: string;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
/**
|
|
395
|
-
* Durable sink for {@link ABMetricSample} rows (same payload as
|
|
396
|
-
* {@link ABConfig.onSample}). Session `events.ndjson` remains the source of
|
|
397
|
-
* truth for assignment + fold; this is for analytics export.
|
|
398
|
-
*/
|
|
399
|
-
export interface ABSamplePersistence {
|
|
400
|
-
save(sample: ABMetricSample, ctx: ABPersistContext): void | Promise<void>;
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
/**
|
|
404
|
-
* Durable store for playground-style aggregate snapshots (`GET /v1/abs`).
|
|
405
|
-
* Without this, aggregates are always recomputed from session logs (nothing
|
|
406
|
-
* is kept after sessions are pruned).
|
|
407
|
-
*
|
|
408
|
-
* **Runtime note:** `serve` currently only calls {@link save} (after each
|
|
409
|
-
* `GET /v1/abs`). {@link load} / {@link delete} are for author tooling or a
|
|
410
|
-
* custom store; they are not wired into the playground hydrate path.
|
|
411
|
-
*/
|
|
412
|
-
export interface ABSnapshotPersistence {
|
|
413
|
-
load(ctx: ABPersistContext): ABSnapshot[] | Promise<ABSnapshot[]>;
|
|
414
|
-
save(snapshot: ABSnapshot, ctx: ABPersistContext): void | Promise<void>;
|
|
415
|
-
delete?(snapshotId: string, ctx: ABPersistContext): void | Promise<void>;
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/** Lets {@link defineABConfig} push `retainSnapshots` into {@link persistABSnapshotsToDir}. */
|
|
419
|
-
const abSnapshotRetainState = new WeakMap<
|
|
420
|
-
ABSnapshotPersistence,
|
|
421
|
-
{ retain: number }
|
|
422
|
-
>();
|
|
423
|
-
|
|
424
369
|
/**
|
|
425
370
|
* Project-wide A/B defaults (`agent/ab.config.ts`).
|
|
426
371
|
*
|
|
427
372
|
* **Assignments and foldable metrics** already persist in each session's
|
|
428
|
-
* `events.ndjson` under `--state-root
|
|
429
|
-
*
|
|
430
|
-
*
|
|
373
|
+
* `events.ndjson` under `--state-root`; durable sample/snapshot export
|
|
374
|
+
* lives on the `defineStorage` `abs` table (`agent/storage.ts`). This
|
|
375
|
+
* config only caps how many sessions the playground / `GET /v1/abs` folds.
|
|
431
376
|
*/
|
|
432
377
|
export interface ABConfigFile {
|
|
433
378
|
/**
|
|
434
379
|
* Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
|
|
435
|
-
* tab
|
|
436
|
-
* session logs, change assignment, or limit {@link persistSamples}.
|
|
380
|
+
* tab**. Does not prune session logs or change assignment.
|
|
437
381
|
*
|
|
438
382
|
* Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
|
|
439
383
|
* {@link AB_MAX_PLAYGROUND_SESSIONS}.
|
|
440
384
|
*/
|
|
441
385
|
maxPlaygroundSessions?: number;
|
|
442
|
-
/**
|
|
443
|
-
* Snapshot-file budget when {@link persistSnapshots} is set.
|
|
444
|
-
* Default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}; hard-capped at
|
|
445
|
-
* {@link AB_MAX_RETAIN_SNAPSHOTS}.
|
|
446
|
-
*
|
|
447
|
-
* When using {@link persistABSnapshotsToDir}, this value controls on-disk
|
|
448
|
-
* pruning (overriding the helper's `{ retain }` default). Custom stores
|
|
449
|
-
* should apply the same budget themselves.
|
|
450
|
-
*/
|
|
451
|
-
retainSnapshots?: number;
|
|
452
|
-
/**
|
|
453
|
-
* Optional sink for cumulative metric samples (in addition to each
|
|
454
|
-
* experiment's {@link ABConfig.onSample}).
|
|
455
|
-
*
|
|
456
|
-
* ```ts
|
|
457
|
-
* persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
|
|
458
|
-
* ```
|
|
459
|
-
*/
|
|
460
|
-
persistSamples?: ABSamplePersistence;
|
|
461
|
-
/**
|
|
462
|
-
* Optional store for aggregate snapshots written when `GET /v1/abs` runs.
|
|
463
|
-
*
|
|
464
|
-
* ```ts
|
|
465
|
-
* persistSnapshots: persistABSnapshotsToDir(".agent-serve/ab-snapshots"),
|
|
466
|
-
* ```
|
|
467
|
-
*/
|
|
468
|
-
persistSnapshots?: ABSnapshotPersistence;
|
|
469
386
|
}
|
|
470
387
|
|
|
471
388
|
/**
|
|
472
389
|
* Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
|
|
473
390
|
*
|
|
474
|
-
* Session event logs remain the assignment/metrics source of truth;
|
|
475
|
-
*
|
|
391
|
+
* Session event logs remain the assignment/metrics source of truth;
|
|
392
|
+
* durable exports go through the `defineStorage` `abs` table.
|
|
476
393
|
*/
|
|
477
394
|
export function defineABConfig(config: ABConfigFile): ABConfigFile {
|
|
478
|
-
const persistSamples = normalizeSamplePersistence(config.persistSamples);
|
|
479
|
-
const persistSnapshots = normalizeSnapshotPersistence(
|
|
480
|
-
config.persistSnapshots
|
|
481
|
-
);
|
|
482
|
-
const helperRetain =
|
|
483
|
-
persistSnapshots !== undefined
|
|
484
|
-
? abSnapshotRetainState.get(persistSnapshots)
|
|
485
|
-
: undefined;
|
|
486
|
-
let retainSnapshots: number | undefined;
|
|
487
|
-
if (config.retainSnapshots !== undefined) {
|
|
488
|
-
retainSnapshots = normalizeABRetainSnapshots(config.retainSnapshots);
|
|
489
|
-
if (helperRetain !== undefined) {
|
|
490
|
-
helperRetain.retain = retainSnapshots;
|
|
491
|
-
}
|
|
492
|
-
} else if (persistSnapshots !== undefined) {
|
|
493
|
-
retainSnapshots =
|
|
494
|
-
helperRetain?.retain ??
|
|
495
|
-
normalizeABRetainSnapshots(AB_DEFAULT_RETAIN_SNAPSHOTS);
|
|
496
|
-
}
|
|
497
395
|
return {
|
|
498
396
|
maxPlaygroundSessions: normalizeABMaxPlaygroundSessions(
|
|
499
397
|
config.maxPlaygroundSessions ?? AB_DEFAULT_MAX_PLAYGROUND_SESSIONS
|
|
500
398
|
),
|
|
501
|
-
...(retainSnapshots === undefined ? {} : { retainSnapshots }),
|
|
502
|
-
...(persistSamples === undefined ? {} : { persistSamples }),
|
|
503
|
-
...(persistSnapshots === undefined ? {} : { persistSnapshots }),
|
|
504
399
|
};
|
|
505
400
|
}
|
|
506
401
|
|
|
@@ -526,150 +421,3 @@ export function resolveABMaxPlaygroundSessions(value: unknown): number {
|
|
|
526
421
|
return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
|
|
527
422
|
}
|
|
528
423
|
}
|
|
529
|
-
|
|
530
|
-
/** Validate {@link ABConfigFile.retainSnapshots}. */
|
|
531
|
-
export function normalizeABRetainSnapshots(value: unknown): number {
|
|
532
|
-
return normalizePositiveInt(value, {
|
|
533
|
-
label: "ab.config.ts: retainSnapshots",
|
|
534
|
-
max: AB_MAX_RETAIN_SNAPSHOTS,
|
|
535
|
-
});
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
function normalizeSamplePersistence(
|
|
539
|
-
value: ABSamplePersistence | undefined
|
|
540
|
-
): ABSamplePersistence | undefined {
|
|
541
|
-
if (value === undefined) {
|
|
542
|
-
return undefined;
|
|
543
|
-
}
|
|
544
|
-
if (
|
|
545
|
-
typeof value !== "object" ||
|
|
546
|
-
value === null ||
|
|
547
|
-
typeof value.save !== "function"
|
|
548
|
-
) {
|
|
549
|
-
throw new Error(
|
|
550
|
-
"ab.config.ts: persistSamples must define save(sample, ctx)"
|
|
551
|
-
);
|
|
552
|
-
}
|
|
553
|
-
return value;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
function normalizeSnapshotPersistence(
|
|
557
|
-
value: ABSnapshotPersistence | undefined
|
|
558
|
-
): ABSnapshotPersistence | undefined {
|
|
559
|
-
if (value === undefined) {
|
|
560
|
-
return undefined;
|
|
561
|
-
}
|
|
562
|
-
if (
|
|
563
|
-
typeof value !== "object" ||
|
|
564
|
-
value === null ||
|
|
565
|
-
typeof value.load !== "function" ||
|
|
566
|
-
typeof value.save !== "function"
|
|
567
|
-
) {
|
|
568
|
-
throw new Error(
|
|
569
|
-
"ab.config.ts: persistSnapshots must define load(ctx) and save(snapshot, ctx)"
|
|
570
|
-
);
|
|
571
|
-
}
|
|
572
|
-
if (value.delete !== undefined && typeof value.delete !== "function") {
|
|
573
|
-
throw new Error("ab.config.ts: persistSnapshots.delete must be a function");
|
|
574
|
-
}
|
|
575
|
-
return value;
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
/**
|
|
579
|
-
* Append-only NDJSON {@link ABSamplePersistence} under `dir` (relative to
|
|
580
|
-
* the agent project root).
|
|
581
|
-
*
|
|
582
|
-
* ```ts
|
|
583
|
-
* // agent/ab.config.ts
|
|
584
|
-
* export default defineABConfig({
|
|
585
|
-
* persistSamples: persistABSamplesToDir(".agent-serve/ab-samples"),
|
|
586
|
-
* });
|
|
587
|
-
* ```
|
|
588
|
-
*/
|
|
589
|
-
export function persistABSamplesToDir(dir: string): ABSamplePersistence {
|
|
590
|
-
const relativeDir = assertNonEmptyDir(dir, "persistABSamplesToDir");
|
|
591
|
-
const resolveDir = (projectRoot: string): string =>
|
|
592
|
-
resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
|
|
593
|
-
|
|
594
|
-
return {
|
|
595
|
-
async save(sample, ctx) {
|
|
596
|
-
const root = resolveDir(ctx.projectRoot);
|
|
597
|
-
await mkdir(root, { recursive: true });
|
|
598
|
-
await appendFile(
|
|
599
|
-
join(root, "samples.ndjson"),
|
|
600
|
-
`${JSON.stringify(sample)}\n`,
|
|
601
|
-
"utf8"
|
|
602
|
-
);
|
|
603
|
-
},
|
|
604
|
-
};
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
/**
|
|
608
|
-
* Filesystem {@link ABSnapshotPersistence}: one JSON file per snapshot under
|
|
609
|
-
* `dir`, pruned to `retain` files (default {@link AB_DEFAULT_RETAIN_SNAPSHOTS}).
|
|
610
|
-
*
|
|
611
|
-
* Prefer setting {@link ABConfigFile.retainSnapshots} in `defineABConfig` —
|
|
612
|
-
* that value is applied to this helper's prune budget. Pass `{ retain }` here
|
|
613
|
-
* only when constructing the helper outside `defineABConfig`, or as a default
|
|
614
|
-
* that `retainSnapshots` may override.
|
|
615
|
-
*/
|
|
616
|
-
export function persistABSnapshotsToDir(
|
|
617
|
-
dir: string,
|
|
618
|
-
options?: { retain?: number }
|
|
619
|
-
): ABSnapshotPersistence {
|
|
620
|
-
const relativeDir = assertNonEmptyDir(dir, "persistABSnapshotsToDir");
|
|
621
|
-
const retainState = {
|
|
622
|
-
retain: normalizeABRetainSnapshots(
|
|
623
|
-
options?.retain ?? AB_DEFAULT_RETAIN_SNAPSHOTS
|
|
624
|
-
),
|
|
625
|
-
};
|
|
626
|
-
const resolveDir = (projectRoot: string): string =>
|
|
627
|
-
resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
|
|
628
|
-
let saveChain: Promise<void> = Promise.resolve();
|
|
629
|
-
|
|
630
|
-
const persist: ABSnapshotPersistence = {
|
|
631
|
-
async load(ctx) {
|
|
632
|
-
return readJsonDir<ABSnapshot>(resolveDir(ctx.projectRoot), {
|
|
633
|
-
sortBasenames: "desc",
|
|
634
|
-
});
|
|
635
|
-
},
|
|
636
|
-
async save(snapshot, ctx) {
|
|
637
|
-
const run = async (): Promise<void> => {
|
|
638
|
-
const root = resolveDir(ctx.projectRoot);
|
|
639
|
-
const snapshotId =
|
|
640
|
-
snapshot.config?.snapshotId ??
|
|
641
|
-
`absnap_${randomUUID().replace(/-/g, "")}`;
|
|
642
|
-
const toWrite: ABSnapshot = {
|
|
643
|
-
...snapshot,
|
|
644
|
-
config: {
|
|
645
|
-
...snapshot.config,
|
|
646
|
-
snapshotId,
|
|
647
|
-
},
|
|
648
|
-
};
|
|
649
|
-
await writeJsonFile({
|
|
650
|
-
root,
|
|
651
|
-
fileName: `${snapshotId}.json`,
|
|
652
|
-
value: toWrite,
|
|
653
|
-
});
|
|
654
|
-
const names = (await listJsonBasenames(root)).sort().reverse();
|
|
655
|
-
for (const stale of names.slice(retainState.retain)) {
|
|
656
|
-
await deleteFileInDir({ root, fileName: stale });
|
|
657
|
-
}
|
|
658
|
-
};
|
|
659
|
-
const next = saveChain.then(run, run);
|
|
660
|
-
saveChain = next.then(
|
|
661
|
-
() => undefined,
|
|
662
|
-
() => undefined
|
|
663
|
-
);
|
|
664
|
-
await next;
|
|
665
|
-
},
|
|
666
|
-
async delete(snapshotId, ctx) {
|
|
667
|
-
await deleteFileInDir({
|
|
668
|
-
root: resolveDir(ctx.projectRoot),
|
|
669
|
-
fileName: `${snapshotId}.json`,
|
|
670
|
-
});
|
|
671
|
-
},
|
|
672
|
-
};
|
|
673
|
-
abSnapshotRetainState.set(persist, retainState);
|
|
674
|
-
return persist;
|
|
675
|
-
}
|
package/src/bin/agent-serve.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
cmdCall,
|
|
30
30
|
cmdChat,
|
|
31
31
|
cmdEval,
|
|
32
|
+
cmdEvalStatus,
|
|
32
33
|
cmdInit,
|
|
33
34
|
cmdLogs,
|
|
34
35
|
cmdPlayground,
|
|
@@ -90,8 +91,11 @@ Usage:
|
|
|
90
91
|
${CLI} init [dir] [--json]
|
|
91
92
|
${CLI} eval [--dir <path>] [evalId...] [--list] [--json]
|
|
92
93
|
[--tag <tag>]... [--verbose] [--timeout-ms <n>]
|
|
93
|
-
[--no-stream] [--
|
|
94
|
+
[--no-stream] [--no-wait]
|
|
95
|
+
[--prod | --url <baseUrl>] [--slug <slug>]
|
|
94
96
|
[--team <id>]
|
|
97
|
+
${CLI} eval status [runId] --prod|--url <baseUrl> [--dir <path>] [--json]
|
|
98
|
+
[--slug <slug>] [--team <id>]
|
|
95
99
|
${CLI} info [--dir <path>] [--json]
|
|
96
100
|
${CLI} validate [--dir <path>]
|
|
97
101
|
${CLI} login [--api-key <key>] [--key-name <name>]
|
|
@@ -155,9 +159,19 @@ Commands:
|
|
|
155
159
|
machine-readable and never blocks on login. No directory:
|
|
156
160
|
print the setup guide.
|
|
157
161
|
eval Run filesystem evals under evals/**/*.eval.ts (streams on a TTY).
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
162
|
+
Default: ephemeral local harness against evals on disk. Pass
|
|
163
|
+
--prod to kick off the batch on the team's hosted deployment
|
|
164
|
+
(slug from --slug or the directory name) via /v1/dev/evals/runs
|
|
165
|
+
so results appear in the playground and persist when the agent
|
|
166
|
+
declares an evals storage table; or --url for a running server
|
|
167
|
+
(same server-side path). With --prod/--url, pass --no-wait to
|
|
168
|
+
return the runId immediately without polling; check later with
|
|
169
|
+
\`${CLI} eval status <runId> --prod\` (omit runId to list recent
|
|
170
|
+
runs; exit 3 while still running). The status subcommand only
|
|
171
|
+
binds when --prod/--url is set, so an eval id named status still
|
|
172
|
+
runs locally. Pass evalId args to select: one datapoint
|
|
173
|
+
(weather/nyc), a whole file (weather → all cases in that file),
|
|
174
|
+
several of either, or omit to run all.
|
|
161
175
|
info Print the discovered agent surface (tools, skills, channels, ...).
|
|
162
176
|
validate Check the project(s) and exit non-zero on error diagnostics.
|
|
163
177
|
login Sign the host in to Cursor: browser sign-in mints a named,
|
|
@@ -193,6 +207,7 @@ Agent loop:
|
|
|
193
207
|
${CLI} info --dir . --json
|
|
194
208
|
${CLI} run --dir . --message "..."
|
|
195
209
|
${CLI} eval --dir .
|
|
210
|
+
${CLI} eval --prod --dir .
|
|
196
211
|
${CLI} dev .
|
|
197
212
|
|
|
198
213
|
Options:
|
|
@@ -231,12 +246,14 @@ Options:
|
|
|
231
246
|
--prod chat/resume/run/call/eval/logs/sessions/session/playground:
|
|
232
247
|
use the team's hosted deployment. Slug from --slug or the
|
|
233
248
|
--dir basename. Mutually exclusive with --url. Requires
|
|
234
|
-
\`${CLI} login\`.
|
|
249
|
+
\`${CLI} login\`. For eval, this is how you opt into the
|
|
250
|
+
persisted server-side batch (default is local).
|
|
235
251
|
--once logs: print current progress / log buffer once and exit
|
|
236
252
|
(default is to follow).
|
|
237
253
|
--print playground: print the URL only; do not open a browser.
|
|
238
|
-
--no-wait deploy/stop: return right after the request is
|
|
239
|
-
instead of polling until a terminal status.
|
|
254
|
+
--no-wait deploy/stop/eval: return right after the request is
|
|
255
|
+
accepted instead of polling until a terminal status.
|
|
256
|
+
For eval, prints the runId (deployment / --url only).
|
|
240
257
|
--no-playground Disable the bundled web playground at /playground (also skips
|
|
241
258
|
auto-build / HMR).
|
|
242
259
|
--no-docs Disable the documentation site at /docs (also skips its
|
|
@@ -618,6 +635,26 @@ async function main(argv: string[]): Promise<number> {
|
|
|
618
635
|
}
|
|
619
636
|
|
|
620
637
|
case "eval": {
|
|
638
|
+
// Remote status subcommand only when targeting --prod/--url. Otherwise
|
|
639
|
+
// the bare token "status" is a normal eval id (evals/status.eval.ts).
|
|
640
|
+
if (
|
|
641
|
+
positionals[0] === "status" &&
|
|
642
|
+
(values.prod === true || values.url !== undefined)
|
|
643
|
+
) {
|
|
644
|
+
return cmdEvalStatus(
|
|
645
|
+
positionals[1],
|
|
646
|
+
axFromOptions({
|
|
647
|
+
dir,
|
|
648
|
+
json: values.json,
|
|
649
|
+
url: values.url,
|
|
650
|
+
prod: values.prod,
|
|
651
|
+
team: values.team,
|
|
652
|
+
apiKey: values["api-key"],
|
|
653
|
+
slug: values.slug,
|
|
654
|
+
bearerToken: values["bearer-token"],
|
|
655
|
+
})
|
|
656
|
+
);
|
|
657
|
+
}
|
|
621
658
|
return cmdEval(
|
|
622
659
|
axFromOptions({
|
|
623
660
|
dir,
|
|
@@ -628,6 +665,7 @@ async function main(argv: string[]): Promise<number> {
|
|
|
628
665
|
tags: values.tag,
|
|
629
666
|
url: values.url,
|
|
630
667
|
prod: values.prod,
|
|
668
|
+
noWait: values["no-wait"],
|
|
631
669
|
team: values.team,
|
|
632
670
|
apiKey: values["api-key"],
|
|
633
671
|
slug: values.slug,
|
|
@@ -1045,6 +1083,7 @@ function axFromOptions(
|
|
|
1045
1083
|
toolInput: partial.toolInput,
|
|
1046
1084
|
url: partial.url,
|
|
1047
1085
|
prod: partial.prod,
|
|
1086
|
+
noWait: partial.noWait,
|
|
1048
1087
|
team: partial.team,
|
|
1049
1088
|
apiKey: partial.apiKey,
|
|
1050
1089
|
fetchImpl: partial.fetchImpl,
|
package/src/evals.ts
CHANGED
|
@@ -48,13 +48,6 @@
|
|
|
48
48
|
|
|
49
49
|
import { normalizePositiveInt } from "./internal/bounded-int.js";
|
|
50
50
|
import { brandDefinition } from "./internal/brand.js";
|
|
51
|
-
import {
|
|
52
|
-
assertNonEmptyDir,
|
|
53
|
-
deleteFileInDir,
|
|
54
|
-
readJsonDir,
|
|
55
|
-
resolveProjectRelativeDir,
|
|
56
|
-
writeJsonFile,
|
|
57
|
-
} from "./internal/json-dir-store.js";
|
|
58
51
|
import type { SessionEvent } from "./types.js";
|
|
59
52
|
|
|
60
53
|
export type Expectation =
|
|
@@ -322,35 +315,6 @@ export interface EvalRunSnapshot {
|
|
|
322
315
|
};
|
|
323
316
|
}
|
|
324
317
|
|
|
325
|
-
/** Context passed to {@link EvalRunPersistence} hooks. */
|
|
326
|
-
export interface EvalRunPersistContext {
|
|
327
|
-
/** Absolute agent project root (directory that contains `evals/`). */
|
|
328
|
-
projectRoot: string;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
/**
|
|
332
|
-
* Durable store for playground eval batches. Without {@link EvalConfigFile.persistRuns},
|
|
333
|
-
* runs live only in the serve process memory — lost on restart, and capped by
|
|
334
|
-
* {@link EvalConfigFile.maxPlaygroundRuns} (default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}).
|
|
335
|
-
*
|
|
336
|
-
* Implement your own (S3, DB, …) or use {@link persistEvalRunsToDir}.
|
|
337
|
-
* `delete` is required so {@link EvalConfigFile.maxPlaygroundRuns} can prune
|
|
338
|
-
* durable history (otherwise pruned rows reload on the next hydrate).
|
|
339
|
-
*/
|
|
340
|
-
export interface EvalRunPersistence {
|
|
341
|
-
/** Load saved runs (any order). Called once when the serve process hydrates. */
|
|
342
|
-
load(
|
|
343
|
-
ctx: EvalRunPersistContext
|
|
344
|
-
): EvalRunSnapshot[] | Promise<EvalRunSnapshot[]>;
|
|
345
|
-
/**
|
|
346
|
-
* Persist a full snapshot. Called when a batch starts, after each case
|
|
347
|
-
* finishes, and when the batch completes or fails.
|
|
348
|
-
*/
|
|
349
|
-
save(run: EvalRunSnapshot, ctx: EvalRunPersistContext): void | Promise<void>;
|
|
350
|
-
/** Remove a run that falls outside {@link EvalConfigFile.maxPlaygroundRuns}. */
|
|
351
|
-
delete(runId: string, ctx: EvalRunPersistContext): void | Promise<void>;
|
|
352
|
-
}
|
|
353
|
-
|
|
354
318
|
export interface EvalConfigFile {
|
|
355
319
|
timeoutMs?: number;
|
|
356
320
|
/**
|
|
@@ -361,52 +325,30 @@ export interface EvalConfigFile {
|
|
|
361
325
|
maxConcurrency: number;
|
|
362
326
|
/**
|
|
363
327
|
* Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
|
|
364
|
-
* history window (in-memory, and pruned from
|
|
365
|
-
*
|
|
366
|
-
*
|
|
328
|
+
* history window (in-memory, and pruned from the `defineStorage` evals
|
|
329
|
+
* table when one is configured). Includes server-side batches started by
|
|
330
|
+
* `agentkit eval --prod` or `--url` (not the default local harness).
|
|
367
331
|
*
|
|
368
332
|
* Default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}; hard-capped at
|
|
369
333
|
* {@link EVAL_MAX_PLAYGROUND_RUNS}. Newest batches win.
|
|
370
334
|
*/
|
|
371
335
|
maxPlaygroundRuns?: number;
|
|
372
|
-
/**
|
|
373
|
-
* Optional durable store for playground eval batches. Omit this and runs
|
|
374
|
-
* are **not** surviving process restart (only the in-memory
|
|
375
|
-
* {@link maxPlaygroundRuns} window).
|
|
376
|
-
*
|
|
377
|
-
* ```ts
|
|
378
|
-
* import {
|
|
379
|
-
* defineEvalConfig,
|
|
380
|
-
* persistEvalRunsToDir,
|
|
381
|
-
* } from "@anysphere/agent-serve/evals";
|
|
382
|
-
*
|
|
383
|
-
* export default defineEvalConfig({
|
|
384
|
-
* maxConcurrency: 20,
|
|
385
|
-
* maxPlaygroundRuns: 50,
|
|
386
|
-
* // Survive serve restarts (relative to the agent project root):
|
|
387
|
-
* persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
|
|
388
|
-
* });
|
|
389
|
-
* ```
|
|
390
|
-
*/
|
|
391
|
-
persistRuns?: EvalRunPersistence;
|
|
392
336
|
}
|
|
393
337
|
|
|
394
338
|
/**
|
|
395
339
|
* Defaults for every eval under `evals/` (`evals/evals.config.ts`).
|
|
396
340
|
* `maxConcurrency` is required (1–{@link EVAL_MAX_CONCURRENCY_LIMIT}).
|
|
397
341
|
*
|
|
398
|
-
* Playground batches default to **in-memory only
|
|
399
|
-
*
|
|
342
|
+
* Playground batches default to **in-memory only**; author an `evals`
|
|
343
|
+
* table in `agent/storage.ts` (`defineStorage`) to survive serve restart.
|
|
400
344
|
*/
|
|
401
345
|
export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
|
|
402
|
-
const persistRuns = normalizePersistRuns(config.persistRuns);
|
|
403
346
|
return {
|
|
404
347
|
...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
|
|
405
348
|
maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency),
|
|
406
349
|
maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
|
|
407
350
|
config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS
|
|
408
351
|
),
|
|
409
|
-
...(persistRuns === undefined ? {} : { persistRuns }),
|
|
410
352
|
};
|
|
411
353
|
}
|
|
412
354
|
|
|
@@ -443,59 +385,3 @@ export function resolveEvalMaxPlaygroundRuns(value: unknown): number {
|
|
|
443
385
|
return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
|
|
444
386
|
}
|
|
445
387
|
}
|
|
446
|
-
|
|
447
|
-
function normalizePersistRuns(
|
|
448
|
-
value: EvalRunPersistence | undefined
|
|
449
|
-
): EvalRunPersistence | undefined {
|
|
450
|
-
if (value === undefined) {
|
|
451
|
-
return undefined;
|
|
452
|
-
}
|
|
453
|
-
if (
|
|
454
|
-
typeof value !== "object" ||
|
|
455
|
-
value === null ||
|
|
456
|
-
typeof value.load !== "function" ||
|
|
457
|
-
typeof value.save !== "function" ||
|
|
458
|
-
typeof value.delete !== "function"
|
|
459
|
-
) {
|
|
460
|
-
throw new Error(
|
|
461
|
-
"evals.config.ts: persistRuns must define load(ctx), save(run, ctx), and delete(runId, ctx)"
|
|
462
|
-
);
|
|
463
|
-
}
|
|
464
|
-
return value;
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
/**
|
|
468
|
-
* Filesystem {@link EvalRunPersistence}: one JSON file per run under `dir`
|
|
469
|
-
* (relative paths resolve against the agent project root).
|
|
470
|
-
*
|
|
471
|
-
* ```ts
|
|
472
|
-
* export default defineEvalConfig({
|
|
473
|
-
* maxConcurrency: 20,
|
|
474
|
-
* persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
|
|
475
|
-
* });
|
|
476
|
-
* ```
|
|
477
|
-
*/
|
|
478
|
-
export function persistEvalRunsToDir(dir: string): EvalRunPersistence {
|
|
479
|
-
const relativeDir = assertNonEmptyDir(dir, "persistEvalRunsToDir");
|
|
480
|
-
const resolveDir = (projectRoot: string): string =>
|
|
481
|
-
resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
|
|
482
|
-
|
|
483
|
-
return {
|
|
484
|
-
async load(ctx) {
|
|
485
|
-
return readJsonDir<EvalRunSnapshot>(resolveDir(ctx.projectRoot));
|
|
486
|
-
},
|
|
487
|
-
async save(run, ctx) {
|
|
488
|
-
await writeJsonFile({
|
|
489
|
-
root: resolveDir(ctx.projectRoot),
|
|
490
|
-
fileName: `${run.runId}.json`,
|
|
491
|
-
value: run,
|
|
492
|
-
});
|
|
493
|
-
},
|
|
494
|
-
async delete(runId, ctx) {
|
|
495
|
-
await deleteFileInDir({
|
|
496
|
-
root: resolveDir(ctx.projectRoot),
|
|
497
|
-
fileName: `${runId}.json`,
|
|
498
|
-
});
|
|
499
|
-
},
|
|
500
|
-
};
|
|
501
|
-
}
|
package/src/index.ts
CHANGED
|
@@ -105,10 +105,12 @@ export type {
|
|
|
105
105
|
export { defineSchedule } from "./schedules.js";
|
|
106
106
|
export { defineSkill } from "./skills.js";
|
|
107
107
|
export type {
|
|
108
|
+
StorageABTable,
|
|
108
109
|
StorageConfig,
|
|
109
110
|
StorageContext,
|
|
110
111
|
StorageDefinition,
|
|
111
112
|
StorageEntry,
|
|
113
|
+
StorageEvalsTable,
|
|
112
114
|
StoragePolicy,
|
|
113
115
|
StorageRestorePolicy,
|
|
114
116
|
} from "./storage.js";
|