@cursor/july 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/channels/slack/post-update-delivery.d.ts +85 -0
  2. package/dist/channels/slack/post-update-delivery.d.ts.map +1 -0
  3. package/dist/docs/404.html +2 -2
  4. package/dist/docs/ab.html +3 -3
  5. package/dist/docs/assets/{app.CKFFLL4Z.js → app.DjHC-Ggf.js} +1 -1
  6. package/dist/docs/assets/chunks/@localSearchIndexroot.DZtu0ZIc.js +1 -0
  7. package/dist/docs/assets/chunks/{VPLocalSearchBox.BXk19JBN.js → VPLocalSearchBox.DwviwmIq.js} +1 -1
  8. package/dist/docs/assets/chunks/{theme.BXSPBMyn.js → theme.BIpeYSon.js} +2 -2
  9. package/dist/docs/assets/{example-agents_index.md.BPXjUtRB.js → example-agents_index.md.D2PEVSXl.js} +2 -2
  10. package/dist/docs/assets/{example-agents_index.md.BPXjUtRB.lean.js → example-agents_index.md.D2PEVSXl.lean.js} +1 -1
  11. package/dist/docs/assets/example-agents_security-reviewer.md.BZSNycMf.js +19 -0
  12. package/dist/docs/assets/{example-agents_security-reviewer.md.CE9v-aKm.lean.js → example-agents_security-reviewer.md.BZSNycMf.lean.js} +1 -1
  13. package/dist/docs/building-with-agents.html +3 -3
  14. package/dist/docs/concepts.html +3 -3
  15. package/dist/docs/deployment.html +3 -3
  16. package/dist/docs/evals.html +3 -3
  17. package/dist/docs/example-agents/approval-buddy.html +3 -3
  18. package/dist/docs/example-agents/benny.html +3 -3
  19. package/dist/docs/example-agents/bugbot.html +3 -3
  20. package/dist/docs/example-agents/codebase-wiki.html +3 -3
  21. package/dist/docs/example-agents/codeowners-review.html +3 -3
  22. package/dist/docs/example-agents/concierge.html +3 -3
  23. package/dist/docs/example-agents/fsd.html +3 -3
  24. package/dist/docs/example-agents/index.html +5 -5
  25. package/dist/docs/example-agents/knowledge-base.html +3 -3
  26. package/dist/docs/example-agents/oncall.html +3 -3
  27. package/dist/docs/example-agents/security-reviewer.html +14 -14
  28. package/dist/docs/example-agents/slack-agent.html +3 -3
  29. package/dist/docs/example-agents/weather-agent.html +3 -3
  30. package/dist/docs/guides/agent-to-agent.html +3 -3
  31. package/dist/docs/guides/cloud-runtime.html +3 -3
  32. package/dist/docs/guides/github.html +3 -3
  33. package/dist/docs/guides/human-in-the-loop.html +3 -3
  34. package/dist/docs/guides/mcp-oauth.html +3 -3
  35. package/dist/docs/guides/slack.html +3 -3
  36. package/dist/docs/guides/webhooks.html +3 -3
  37. package/dist/docs/hashmap.json +1 -1
  38. package/dist/docs/hillclimbing.html +3 -3
  39. package/dist/docs/index.html +3 -3
  40. package/dist/docs/quickstart.html +3 -3
  41. package/dist/docs/reference/agent-config.html +3 -3
  42. package/dist/docs/reference/channels.html +3 -3
  43. package/dist/docs/reference/cli.html +3 -3
  44. package/dist/docs/reference/connections.html +3 -3
  45. package/dist/docs/reference/hooks.html +3 -3
  46. package/dist/docs/reference/http-api.html +3 -3
  47. package/dist/docs/reference/instructions.html +3 -3
  48. package/dist/docs/reference/playground.html +3 -3
  49. package/dist/docs/reference/project-layout.html +3 -3
  50. package/dist/docs/reference/prompt.html +3 -3
  51. package/dist/docs/reference/schedules.html +3 -3
  52. package/dist/docs/reference/sessions.html +3 -3
  53. package/dist/docs/reference/skills.html +3 -3
  54. package/dist/docs/reference/subagents.html +3 -3
  55. package/dist/docs/reference/tools.html +3 -3
  56. package/dist/docs/scaffolding-agents.html +3 -3
  57. package/dist/docs/storage.html +3 -3
  58. package/dist/docs/troubleshooting.html +3 -3
  59. package/dist/evals.d.ts +26 -0
  60. package/dist/evals.d.ts.map +1 -1
  61. package/dist/evals.js +30 -0
  62. package/dist/internal/cli-ax.d.ts.map +1 -1
  63. package/dist/internal/cli-ax.js +5 -2
  64. package/dist/internal/eval-run-store.d.ts.map +1 -1
  65. package/dist/internal/eval-run-store.js +7 -0
  66. package/dist/internal/eval-runner.d.ts +2 -0
  67. package/dist/internal/eval-runner.d.ts.map +1 -1
  68. package/dist/internal/eval-runner.js +17 -2
  69. package/dist/internal/evals-client.d.ts.map +1 -1
  70. package/dist/internal/evals-client.js +4 -3
  71. package/dist/internal/persistence-coordinator.d.ts +127 -0
  72. package/dist/internal/persistence-coordinator.d.ts.map +1 -0
  73. package/dist/persistence.d.ts +184 -0
  74. package/dist/persistence.d.ts.map +1 -0
  75. package/dist/playground/assets/index-Bwl4Onx4.js +85 -0
  76. package/dist/playground/assets/index-ByV4nfch.css +1 -0
  77. package/dist/playground/index.html +2 -2
  78. package/docs/example-agents/index.md +1 -1
  79. package/docs/example-agents/security-reviewer.md +23 -21
  80. package/package.json +1 -1
  81. package/skills/evals/SKILL.md +10 -0
  82. package/src/evals.ts +52 -0
  83. package/src/internal/cli-ax.ts +3 -0
  84. package/src/internal/eval-run-store.ts +8 -0
  85. package/src/internal/eval-runner.ts +22 -1
  86. package/src/internal/evals-client.ts +10 -2
  87. package/dist/channels/github/instrument.d.ts +0 -20
  88. package/dist/channels/github/instrument.d.ts.map +0 -1
  89. package/dist/docs/assets/chunks/@localSearchIndexroot.DGWkvXJc.js +0 -1
  90. package/dist/docs/assets/example-agents_security-reviewer.md.CE9v-aKm.js +0 -19
  91. package/dist/internal/json-dir-store.js +0 -100
  92. package/dist/playground/assets/index-CuXb4j4V.css +0 -1
  93. package/dist/playground/assets/index-DeyYpOit.js +0 -86
package/dist/playground/index.html CHANGED
@@ -8,8 +8,8 @@
8
8
  />
9
9
  <meta name="viewport" content="width=device-width, initial-scale=1" />
10
10
  <title>agent-serve playground</title>
11
- <script type="module" crossorigin src="./assets/index-DeyYpOit.js"></script>
12
- <link rel="stylesheet" crossorigin href="./assets/index-CuXb4j4V.css">
11
+ <script type="module" crossorigin src="./assets/index-Bwl4Onx4.js"></script>
12
+ <link rel="stylesheet" crossorigin href="./assets/index-ByV4nfch.css">
13
13
  </head>
14
14
  <body>
15
15
  <div id="root"></div>
package/docs/example-agents/index.md CHANGED
@@ -26,7 +26,7 @@ The source projects live under
26
26
  | [Alert investigator](./oncall.md) | Local | Watched Slack alerts channel | Bot-post channel watching, per-thread debounce, reminder tools, and host Slack calls | Every alert gets a thread-pinned investigation that schedules its own re-checks. |
27
27
  | [PR evidence reviewer](./bugbot.md) | Local | Custom HTTP and Slack | Host tool, skill, seeded workspaces, and an eval | The model receives a prepared diff-first evidence tree instead of a checkout. |
28
28
  | [Approval Buddy](./approval-buddy.md) | Local | GitHub and Slack | Policy tools, two subagents, durable storage, and evals | Code decides whether a PR may be approved. Reviews stay informational. |
29
- | [Security Reviewer](./security-reviewer.md) | Local host pipeline | GitHub and chat | Staged tools, parallel SDK agents, progress UI, durable storage, A/B, and evals | Reviewers and triage overlap while the playground shows every stage. |
29
+ | [Security Reviewer](./security-reviewer.md) | Local host pipeline | GitHub and chat | Staged tools, parallel SDK agents, progress UI, durable storage, A/B, and evals | Lives in `factory/security-reviewer/`. Reviewers and triage overlap while the playground shows every stage. |
30
30
  | [Remote PR coordinator](./fsd.md) | Local coordinator and remote PR sessions | HTTP, GitHub, and Slack | Remote handoff, hooks, affinity, buffering, reminders, and workflow MCP | One remote conversation follows a PR across chat, webhooks, and timed wakes. |
31
31
  | [Knowledge base](./knowledge-base.md) | Local | Built-in HTTP chat | Durable host-side state, a conventions skill, a schedule, unit tests, and evals | People curate shared facts in chat, and fresh sessions retrieve them from markdown. |
32
32
  | [Codebase wiki](./codebase-wiki.md) | Local | GitHub and chat | Task-dispatch webhooks, seeded digests, a mapping skill, a schedule, and evals | Merged PRs accumulate into per-feature wiki pages with a daily digest. |
package/docs/example-agents/security-reviewer.md CHANGED
@@ -13,7 +13,9 @@ confirmed findings, writes artifacts, and may publish a GitHub review.
13
13
  Use this example when the workflow needs several model workers, but the host
14
14
  must own orchestration, progress, artifacts, and the final write.
15
15
 
16
- [Browse the Security Reviewer source.](../../examples/security-reviewer/)
16
+ Source lives under [`factory/security-reviewer/`](../../../../factory/security-reviewer/) (factory agent, not under `examples/`).
17
+
18
+ [Browse the Security Reviewer source.](../../../../factory/security-reviewer/)
17
19
 
18
20
  ## Run a three-stage host pipeline
19
21
 
@@ -65,15 +67,15 @@ status from the finding count even when the review posting result says
65
67
 
66
68
  | Capability | Source | Role |
67
69
  | --- | --- | --- |
68
- | Root agent | [`agent/agent.ts`](../../examples/security-reviewer/agent/agent.ts), [`agent/instructions.md`](../../examples/security-reviewer/agent/instructions.md) | Configure local chat and explain the three-stage contract. |
69
- | Server tools | [`agent/tools/`](../../examples/security-reviewer/agent/tools/) | Expose each review stage to chat and host orchestration. |
70
- | GitHub channel | [`agent/channels/github.ts`](../../examples/security-reviewer/agent/channels/github.ts) | Filter wakes, run background tasks, and publish status. |
71
- | Progress channel | [`agent/channels/asr-progress.ts`](../../examples/security-reviewer/agent/channels/asr-progress.ts) | Serve live reviewer and triage state by `runId`. |
72
- | Playground renderer | [`agent/playground/tools/run_reviewers.tsx`](../../examples/security-reviewer/agent/playground/tools/run_reviewers.tsx) | Replace the generic tool chip with live module rows. |
73
- | SDK review pipeline | [`agent/lib/asr/`](../../examples/security-reviewer/agent/lib/asr/), [`review-stages.ts`](../../examples/security-reviewer/agent/lib/review-stages.ts) | Select modules, call model workers, triage, deduplicate, and write artifacts. |
74
- | Storage | [`agent/storage.ts`](../../examples/security-reviewer/agent/storage.ts) | Persist framework sessions with `cursorHostedStorage` (lazy restore). |
75
- | A/B | [`agent/ab.ts`](../../examples/security-reviewer/agent/ab.ts), [`agent/ab.config.ts`](../../examples/security-reviewer/agent/ab.config.ts) | Compare all-severity versus high-only GitHub comments. |
76
- | Eval | [`evals/`](../../examples/security-reviewer/evals/) | Check stage-tool presence against a pinned sample. |
70
+ | Root agent | [`agent/agent.ts`](../../../../factory/security-reviewer/agent/agent.ts), [`agent/instructions.md`](../../../../factory/security-reviewer/agent/instructions.md) | Configure local chat and explain the three-stage contract. |
71
+ | Server tools | [`agent/tools/`](../../../../factory/security-reviewer/agent/tools/) | Expose each review stage to chat and host orchestration. |
72
+ | GitHub channel | [`agent/channels/github.ts`](../../../../factory/security-reviewer/agent/channels/github.ts) | Filter wakes, run background tasks, and publish status. |
73
+ | Progress channel | [`agent/channels/asr-progress.ts`](../../../../factory/security-reviewer/agent/channels/asr-progress.ts) | Serve live reviewer and triage state by `runId`. |
74
+ | Playground renderer | [`agent/playground/tools/run_reviewers.tsx`](../../../../factory/security-reviewer/agent/playground/tools/run_reviewers.tsx) | Replace the generic tool chip with live module rows. |
75
+ | SDK review pipeline | [`agent/lib/asr/`](../../../../factory/security-reviewer/agent/lib/asr/), [`review-stages.ts`](../../../../factory/security-reviewer/agent/lib/review-stages.ts) | Select modules, call model workers, triage, deduplicate, and write artifacts. |
76
+ | Storage | [`agent/storage.ts`](../../../../factory/security-reviewer/agent/storage.ts) | Persist framework sessions with `cursorHostedStorage` (lazy restore). |
77
+ | A/B | [`agent/ab.ts`](../../../../factory/security-reviewer/agent/ab.ts) | Compare all-severity versus high-only GitHub comments. |
78
+ | Eval | [`evals/`](../../../../factory/security-reviewer/evals/) | Check stage-tool presence against a pinned sample. |
77
79
 
78
80
  There is no Slack channel, authored skill, discovered subagent, MCP
79
81
  connection, schedule, reminder, hook, tool approval, or cloud runtime.
@@ -95,14 +97,14 @@ The pipeline exposes settings for:
95
97
  - prior-comment loading.
96
98
 
97
99
  The active names live beside the orchestration in
98
- [`review-stages.ts`](../../examples/security-reviewer/agent/lib/review-stages.ts).
100
+ [`review-stages.ts`](../../../../factory/security-reviewer/agent/lib/review-stages.ts).
99
101
 
100
102
  ## Validate the discovered agent
101
103
 
102
104
  ```bash
103
- agentkit validate --dir examples/security-reviewer
104
- agentkit info --dir examples/security-reviewer --json
105
- agentkit eval --dir examples/security-reviewer --list
105
+ agentkit validate --dir ../../factory/security-reviewer
106
+ agentkit info --dir ../../factory/security-reviewer --json
107
+ agentkit eval --dir ../../factory/security-reviewer --list
106
108
  ```
107
109
 
108
110
  The manifest should show four server tools, two authored channels, one
@@ -128,15 +130,15 @@ Call each stage and pass `postComment: false` yourself:
128
130
 
129
131
  ```bash
130
132
  agentkit call prepare_review \
131
- --dir examples/security-reviewer \
133
+ --dir ../../factory/security-reviewer \
132
134
  --input '{"prUrl":"https://github.com/owner/repo/pull/123"}'
133
135
 
134
136
  agentkit call run_reviewers \
135
- --dir examples/security-reviewer \
137
+ --dir ../../factory/security-reviewer \
136
138
  --input '{"runId":"<run-id>"}'
137
139
 
138
140
  agentkit call finalize_review \
139
- --dir examples/security-reviewer \
141
+ --dir ../../factory/security-reviewer \
140
142
  --input '{"runId":"<run-id>","postComment":false}'
141
143
  ```
142
144
 
@@ -152,7 +154,7 @@ stages can open the prepared `runId`.
152
154
  Run the dev server:
153
155
 
154
156
  ```bash
155
- agentkit dev examples/security-reviewer
157
+ agentkit dev ../../factory/security-reviewer
156
158
  ```
157
159
 
158
160
  Open the printed playground and start a review. The custom
@@ -233,7 +235,7 @@ Inspect its event surface:
233
235
 
234
236
  ```bash
235
237
  agentkit github events \
236
- --dir examples/security-reviewer \
238
+ --dir ../../factory/security-reviewer \
237
239
  --json
238
240
  ```
239
241
 
@@ -243,7 +245,7 @@ Replay reaches the full publishing path:
243
245
  TEST_PR_URL=https://github.com/your-org/allowlisted-test-repo/pull/123
244
246
  agentkit github replay \
245
247
  "$TEST_PR_URL" \
246
- --dir examples/security-reviewer \
248
+ --dir ../../factory/security-reviewer \
247
249
  --action opened
248
250
  ```
249
251
 
@@ -254,7 +256,7 @@ status and may post findings.
254
256
  ## Inspect the eval before running it
255
257
 
256
258
  ```bash
257
- agentkit eval --dir examples/security-reviewer --list
259
+ agentkit eval --dir ../../factory/security-reviewer --list
258
260
  ```
259
261
 
260
262
  The case reads pinned metadata from the committed fixture and checks for all
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cursor/july",
3
- "version": "0.1.14",
3
+ "version": "0.1.16",
4
4
  "description": "(early alpha) Filesystem-first framework for defining Cursor agents as markdown and TypeScript and serving them over channels with the Cursor SDK.",
5
5
  "license": "SEE LICENSE IN LICENSE.md",
6
6
  "repository": {
package/skills/evals/SKILL.md CHANGED
@@ -168,9 +168,19 @@ export default defineEvalConfig({
168
168
  Options: `workspaceDir`, `workspaceFiles` (`{ path: contents }` seeded
169
169
  before the first turn). Multiple `t.send` calls share the session.
170
170
  - `t.reply` / `t.events` / `t.sessionId` for assertions and correlation.
171
+ - `t.metric(name, value)` — structured scores on the case snapshot (playground
172
+ badges + CLI). Prefer this over encoding scores only in `t.log`.
171
173
  - Builders: `includes(string | RegExp)`, `equals(value)`,
172
174
  `satisfies(predicate, label)`.
173
175
 
176
+ ### Playground customization (optional)
177
+
178
+ Drop `agent/playground/evals.tsx` exporting `agentName` (must match
179
+ `defineAgent({ name })`) and
180
+ `default: { RunSummary?, CaseMetrics?, CaseExtras? }`. The playground
181
+ loads these via Vite glob (`examples/*` and `factory/*`). Without a
182
+ module, cases still show default `t.metric` badges.
183
+
174
184
  ## What good cases assert
175
185
 
176
186
  Gate **decisions and shape, not prose**. Model wording varies run to run;
package/src/evals.ts CHANGED
@@ -129,6 +129,12 @@ export interface EvalTestContext {
129
129
  check(value: unknown, assertion: Expectation): void;
130
130
  /** Debug line (surfaced under `agent-serve eval --verbose`). */
131
131
  log(message: string): void;
132
+ /**
133
+ * Structured score for the playground (and CLI). Values appear on the
134
+ * case card; agents can further customize rendering via
135
+ * `agent/playground/evals.tsx`.
136
+ */
137
+ metric(name: string, value: string | number): void;
132
138
  }
133
139
 
134
140
  /** Hard ceiling for {@link EvalConfig.iterations} / case `iterations`. */
@@ -334,6 +340,50 @@ export function matchesEvalIdFilter(caseId: string, filter: string): boolean {
334
340
  return caseId === filter || caseId.startsWith(`${filter}/`);
335
341
  }
336
342
 
343
+ /**
344
+ * Tag for suite-aggregator datapoints that wait on sibling cases from the
345
+ * same eval file (e.g. `deepsec-recall/overall`). Selecting a summary case
346
+ * alone expands the batch to the whole file via
347
+ * {@link expandSummarySuiteSelection}.
348
+ */
349
+ export const EVAL_SUMMARY_TAG = "summary";
350
+
351
+ /** True when `tags` marks a suite-aggregator case. */
352
+ export function isEvalSummaryCase(
353
+ tags: readonly string[] | undefined
354
+ ): boolean {
355
+ return tags?.includes(EVAL_SUMMARY_TAG) ?? false;
356
+ }
357
+
358
+ /**
359
+ * If `selected` includes any summary-tagged suite aggregator, pull in every
360
+ * other datapoint from the same `fileId` so the aggregator is not run alone.
361
+ * Preserves discovery order from `all`.
362
+ */
363
+ export function expandSummarySuiteSelection<
364
+ T extends { id: string; fileId: string },
365
+ >(
366
+ all: readonly T[],
367
+ selected: readonly T[],
368
+ isSummary: (item: T) => boolean
369
+ ): T[] {
370
+ const fileIdsNeedingSuite = new Set(
371
+ selected.filter((item) => isSummary(item)).map((item) => item.fileId)
372
+ );
373
+ if (fileIdsNeedingSuite.size === 0) {
374
+ return [...selected];
375
+ }
376
+ const selectedIds = new Set(selected.map((item) => item.id));
377
+ const extras = all.filter(
378
+ (item) => fileIdsNeedingSuite.has(item.fileId) && !selectedIds.has(item.id)
379
+ );
380
+ if (extras.length === 0) {
381
+ return [...selected];
382
+ }
383
+ const want = new Set([...selectedIds, ...extras.map((item) => item.id)]);
384
+ return all.filter((item) => want.has(item.id));
385
+ }
386
+
337
387
  /**
338
388
  * Hard ceiling for concurrent eval cases.
339
389
  *
@@ -364,6 +414,8 @@ export interface EvalCaseSnapshot {
364
414
  durationMs?: number;
365
415
  error?: string;
366
416
  logs?: string[];
417
+ /** Structured scores from {@link EvalTestContext.metric}. */
418
+ metrics?: Record<string, string | number>;
367
419
  inputs?: string[];
368
420
  finalText?: string;
369
421
  tools?: string[];
package/src/internal/cli-ax.ts CHANGED
@@ -1412,6 +1412,9 @@ function printEvalResults(
1412
1412
  }\n`
1413
1413
  );
1414
1414
  }
1415
+ for (const [name, value] of Object.entries(r.metrics ?? {})) {
1416
+ process.stdout.write(` metric ${name}=${String(value)}\n`);
1417
+ }
1415
1418
  for (const line of r.logs ?? []) {
1416
1419
  process.stdout.write(` ${line}\n`);
1417
1420
  }
package/src/internal/eval-run-store.ts CHANGED
@@ -383,6 +383,10 @@ export class EvalRunStore {
383
383
  c.inputs = [];
384
384
  c.toolCalls = [];
385
385
  c.finalText = undefined;
386
+ c.metrics = undefined;
387
+ c.logs = undefined;
388
+ c.assertions = undefined;
389
+ c.error = undefined;
386
390
  }
387
391
  },
388
392
  onCaseEvent: (id, event) => {
@@ -473,6 +477,10 @@ export class EvalRunStore {
473
477
  c.assertions = result.assertions;
474
478
  c.durationMs = result.durationMs;
475
479
  c.logs = result.logs;
480
+ c.metrics =
481
+ Object.keys(result.metrics).length > 0
482
+ ? { ...result.metrics }
483
+ : undefined;
476
484
  c.inputs = result.inputs;
477
485
  if (result.toolCalls !== undefined) {
478
486
  c.toolCalls = result.toolCalls;
package/src/internal/eval-runner.ts CHANGED
@@ -25,6 +25,8 @@ import type {
25
25
  import {
26
26
  EVAL_MAX_CONCURRENCY_LIMIT,
27
27
  expandEvalDefinition,
28
+ expandSummarySuiteSelection,
29
+ isEvalSummaryCase,
28
30
  matchesEvalIdFilter,
29
31
  normalizeEvalMaxConcurrency,
30
32
  } from "../evals.js";
@@ -63,6 +65,8 @@ export interface EvalRunResult {
63
65
  }>;
64
66
  error?: string;
65
67
  logs: string[];
68
+ /** Structured scores from {@link import("../evals.js").EvalTestContext.metric}. */
69
+ metrics: Record<string, string | number>;
66
70
  durationMs: number;
67
71
  }
68
72
 
@@ -206,7 +210,10 @@ export function filterDiscoveredEvals(
206
210
  (e.definition.tags ?? []).some((t) => options.tags!.includes(t))
207
211
  );
208
212
  }
209
- return selected;
213
+ // Suite aggregators (`summary` tag) need sibling cases from the same file.
214
+ return expandSummarySuiteSelection(evals, selected, (e) =>
215
+ isEvalSummaryCase(e.definition.tags)
216
+ );
210
217
  }
211
218
 
212
219
  export async function runDiscoveredEvals(
@@ -342,6 +349,7 @@ async function runOneEval(
342
349
  }
343
350
  ): Promise<EvalRunResult> {
344
351
  const logs: string[] = [];
352
+ const metrics: Record<string, string | number> = {};
345
353
  const assertions: EvalAssertionResult[] = [];
346
354
  const def = discovered.definition;
347
355
  const timeoutMs = def.timeoutMs ?? opts.defaultTimeoutMs;
@@ -364,6 +372,7 @@ async function runOneEval(
364
372
  assertions,
365
373
  error: "cancelled",
366
374
  logs,
375
+ metrics,
367
376
  inputs,
368
377
  toolCalls: [],
369
378
  tools: [],
@@ -491,6 +500,16 @@ async function runOneEval(
491
500
  process.stderr.write(` ${message}\n`);
492
501
  }
493
502
  },
503
+ metric(name: string, value: string | number) {
504
+ const key = name.trim();
505
+ if (key === "") {
506
+ return;
507
+ }
508
+ metrics[key] = value;
509
+ if (opts.verbose) {
510
+ process.stderr.write(` metric ${key}=${String(value)}\n`);
511
+ }
512
+ },
494
513
  };
495
514
 
496
515
  try {
@@ -510,6 +529,7 @@ async function runOneEval(
510
529
  assertions,
511
530
  error: message,
512
531
  logs,
532
+ metrics,
513
533
  inputs,
514
534
  toolCalls: toolCallIo(),
515
535
  ...(sessionId === undefined ? {} : { sessionId }),
@@ -532,6 +552,7 @@ async function runOneEval(
532
552
  ok,
533
553
  assertions,
534
554
  logs,
555
+ metrics,
535
556
  inputs,
536
557
  toolCalls: toolCallIo(),
537
558
  ...(sessionId === undefined ? {} : { sessionId }),
package/src/internal/evals-client.ts CHANGED
@@ -8,7 +8,11 @@
8
8
  */
9
9
 
10
10
  import type { EvalRunSnapshot } from "../evals.js";
11
- import { matchesEvalIdFilter } from "../evals.js";
11
+ import {
12
+ expandSummarySuiteSelection,
13
+ isEvalSummaryCase,
14
+ matchesEvalIdFilter,
15
+ } from "../evals.js";
12
16
  import type { EvalRunResult } from "./eval-runner.js";
13
17
  import { mergeRequestHeaders, type RequestAuth } from "./request-headers.js";
14
18
 
@@ -170,7 +174,10 @@ export async function listRemoteEvals(args: {
170
174
  (e.tags ?? []).some((t) => args.tags!.includes(t))
171
175
  );
172
176
  }
173
- return selected;
177
+ // Match server-side filterDiscoveredEvals: summary → whole file.
178
+ return expandSummarySuiteSelection(listed, selected, (e) =>
179
+ isEvalSummaryCase(e.tags)
180
+ );
174
181
  }
175
182
 
176
183
  export async function startRemoteEvalRun(args: {
@@ -489,6 +496,7 @@ export function snapshotToEvalRunResults(
489
496
  ...(c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls }),
490
497
  ...(c.error === undefined ? {} : { error: c.error }),
491
498
  logs: c.logs ?? [],
499
+ metrics: c.metrics ?? {},
492
500
  durationMs: c.durationMs ?? 0,
493
501
  }));
494
502
  }
package/dist/channels/github/instrument.d.ts DELETED
@@ -1,20 +0,0 @@
1
- /**
2
- * Per-request observability for GitHub channel Octokit clients.
3
- *
4
- * Every Octokit built by `createGitHubOctokit` / `callGitHubApi` gets one
5
- * structured `[agentkit/github]` line per GitHub API request (REST and
6
- * GraphQL), including rate-limit headers so installation-quota pressure can
7
- * be attributed to agentkit deployments from hosted-runtime logs.
8
- */
9
- import type { Octokit } from "@octokit/rest";
10
- export type GitHubRequestLogWrite = (line: string) => void;
11
- /** Best-effort GraphQL operation name (`query Foo` / `mutation Bar`). */
12
- export declare function parseGraphQLOperationName(query: unknown): string | undefined;
13
- /**
14
- * Wrap an Octokit so every request emits one structured log line. Returns
15
- * the same instance for call-through convenience. Idempotent per instance.
16
- */
17
- export declare function instrumentGitHubOctokit(octokit: Octokit, input?: {
18
- write?: GitHubRequestLogWrite;
19
- }): Octokit;
20
- //# sourceMappingURL=instrument.d.ts.map
package/dist/channels/github/instrument.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"instrument.d.ts","sourceRoot":"","sources":["../../../src/channels/github/instrument.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAI7C,MAAM,MAAM,qBAAqB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;AA4C3D,yEAAyE;AACzE,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAQ5E;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,OAAO,EAChB,KAAK,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,qBAAqB,CAAA;CAAE,GACxC,OAAO,CAmDT"}