@rulvar/cli 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,9 +6,9 @@ import { createHash } from "node:crypto";
6
6
  * suite runners. A separate quality-measurement package built strictly on
7
7
  * the public APIs (L6).
8
8
  *
9
- * Owning spec: docs/09-observability-testing-spec.md, section
10
- * "@rulvar/evals"; determinism rules in docs/11-testing-strategy.md,
11
- * section "Eval CI": judge graders run THROUGH the engine, so judge calls
9
+ * Determinism rules (https://docs.rulvar.com/guide/evals and
10
+ * https://docs.rulvar.com/guide/testing): judge graders run THROUGH the
11
+ * engine, so judge calls
12
12
  * are journaled, budgeted, and VCR-recordable, and an eval suite replays
13
13
  * deterministically from cassettes with zero live calls.
14
14
  */
@@ -244,14 +244,14 @@ function judgeGrader(options) {
244
244
  };
245
245
  }
246
246
  /**
247
- * The eval-committer identity (M11-T01; docs/05, sections "Data model"
248
- * and "Commit discipline"). The pipeline-side commit path: builds
247
+ * The eval-committer identity (M11-T01; https://docs.rulvar.com/guide/model-knowledge).
248
+ * The pipeline-side commit path: builds
249
249
  * eval-committer-gated ops (the coherence square: class eval-measured,
250
250
  * author eval-pipeline, metrics present) and commits them with the
251
251
  * documented CAS-rebase recipe. Humans never call this; their path is
252
252
  * the human gate and it structurally cannot carry metrics.
253
253
  */
254
- /** One measured claim, TTL applied per the docs/05 decay table. */
254
+ /** One measured claim; claimExpiry applies the TTL from the decay table. */
255
255
  function evalMeasuredClaim(input, committerId) {
256
256
  return {
257
257
  id: input.id,
@@ -306,8 +306,7 @@ async function commitEvalMeasured(store, claims, options) {
306
306
  throw lastCas ?? /* @__PURE__ */ new Error("commitEvalMeasured: unreachable");
307
307
  }
308
308
  /**
309
- * The canary fingerprint (M11-T04; docs/05, section "Grounding and
310
- * decay"; OQ-06). The optional compensation for silent alias
309
+ * The canary fingerprint (M11-T04; OQ-06). The optional compensation for silent alias
311
310
  * re-pointing that modelEpoch honestly cannot catch: a FIXED probe set
312
311
  * at temperature 0, run through the ordinary engine (journaled,
313
312
  * budgeted, VCR-recordable), hashed over normalized outputs. A
@@ -341,9 +340,8 @@ async function canaryFingerprint(engine, probes) {
341
340
  }
342
341
  /**
343
342
  * Flips the model's ACTIVE eval-measured claims to stale when their
344
- * recorded canary fingerprint differs from the fresh one (docs/05:
345
- * "a fingerprint change immediately flips the model's eval claims to
346
- * stale"). Claims without a recorded fingerprint have no baseline and
343
+ * recorded canary fingerprint differs from the fresh one. Claims
344
+ * without a recorded fingerprint have no baseline and
347
345
  * stay untouched (the documented no-probe posture); a second run is
348
346
  * an idempotent noop. CAS-rebased like every maintenance commit.
349
347
  */
@@ -382,9 +380,8 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
382
380
  throw lastCas ?? /* @__PURE__ */ new Error("flipStaleOnCanaryDrift: unreachable");
383
381
  }
384
382
  /**
385
- * The phases 1-2 measured-value checkpoint (M12-T01; docs/05, section
386
- * "Phases and placement"; the quantitative criteria of OQ-09,
387
- * 14-open-questions.md, closed at M11-T06). The M12 gate: kb_propose
383
+ * The phases 1-2 measured-value checkpoint (M12-T01; the quantitative
384
+ * criteria of OQ-09, closed at M11-T06). The M12 gate: kb_propose
388
385
  * and the proposal loop ship ONLY if the knowledge card demonstrably
389
386
  * improves tier and agentType selection on eval cases.
390
387
  *
@@ -404,7 +401,11 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
404
401
  * with and without the knowledge store configured (the card docks
405
402
  * into the spawn tool description when configured). Criterion 2
406
403
  * holds when the card-informed arm matches or beats the baseline
407
- * pass rate at no more than 105 percent of its cost.
404
+ * pass rate at no more than 105 percent of its cost, OR beats it
405
+ * by at least 15 points at no more than 115 percent of its cost
406
+ * (the quality branch; OQ-09 as amended 2026-07-12: the baseline
407
+ * fails CHEAPLY, so the flat cost bar tightened exactly when the
408
+ * card was winning on quality).
408
409
  *
409
410
  * The checkpoint PASSES only when both criteria hold. Methodology
410
411
  * guard: the claims the treatment consumes MUST come from a seeding
@@ -413,6 +414,18 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
413
414
  */
414
415
  /** IEEE754 guard for the rule boundaries (0.8 + 0.05 exceeds 0.85). */
415
416
  const EPSILON = 1e-9;
417
+ /**
418
+ * The OQ-09 criterion 2 rule (as amended 2026-07-12): match-or-beat at
419
+ * 105 percent of baseline cost, OR at least 15 points better at 115
420
+ * percent (the quality branch: the baseline fails cheaply, so the flat
421
+ * bar tightened exactly when the card won on quality). The vacuous-pass
422
+ * guard stays with the caller.
423
+ */
424
+ function agentTypeRuleHolds(baseline, informed) {
425
+ const matchesCheaply = informed.passRate >= baseline.passRate - EPSILON && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON;
426
+ const clearlyBetterNearCost = informed.passRate >= baseline.passRate + .15 - EPSILON && informed.totalCostUsd <= 1.15 * baseline.totalCostUsd + EPSILON;
427
+ return matchesCheaply || clearlyBetterNearCost;
428
+ }
416
429
  /** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
417
430
  function rungRuleHolds(baseline, treatment) {
418
431
  const equalOrBetterCheaper = treatment.passRate >= baseline.passRate - EPSILON && treatment.totalCostUsd <= .9 * baseline.totalCostUsd + EPSILON;
@@ -493,7 +506,7 @@ async function runValueCheckpoint(checkpointPool, options) {
493
506
  criterion2 = {
494
507
  baseline,
495
508
  informed,
496
- passed: informed.n > 0 && informed.passRate > 0 && informed.passRate >= baseline.passRate && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON
509
+ passed: informed.n > 0 && informed.passRate > 0 && agentTypeRuleHolds(baseline, informed)
497
510
  };
498
511
  }
499
512
  return {
package/dist/index.d.ts CHANGED
@@ -37,7 +37,7 @@ declare function inspectCommand(argv: string[], context: CommandContext): Promis
37
37
  interface CliConfig {
38
38
  engineOptions?: Partial<CreateEngineOptions>;
39
39
  workflows?: WorkflowRegistry;
40
- /** rulvar kb sweep configuration (M11-T05; docs/05, "Grounding and decay"). */
40
+ /** rulvar kb sweep configuration (M11-T05). */
41
41
  kbSweep?: KbSweepCliConfig;
42
42
  }
43
43
  /**
@@ -95,7 +95,7 @@ interface AssembledCli {
95
95
  engine: Engine;
96
96
  store: JournalStore;
97
97
  workflows: WorkflowRegistry;
98
- /** The journal-fold price function (table wins over caps; docs/04, section 10). */
98
+ /** The journal-fold price function (table wins over caps). */
99
99
  priceUsd: (servedBy: ModelRef, usage: Usage) => number | undefined;
100
100
  }
101
101
  declare function assembleEngine(options: {
@@ -115,7 +115,7 @@ declare function driveRun(options: {
115
115
  engine: Engine;
116
116
  workflow: Workflow<never, unknown>;
117
117
  first: RunHandle<unknown>;
118
- io: CliIo; /** Original run arguments: not journaled in v1, the host re-supplies them (docs/14). */
118
+ io: CliIo; /** Original run arguments: not journaled in v1, the host re-supplies them. */
119
119
  args?: unknown;
120
120
  }): Promise<RunOutcome<unknown>>;
121
121
  /** Renders the settled outcome; returns the process exit code. */
@@ -124,17 +124,17 @@ declare function reportOutcome(outcome: RunOutcome<unknown>, io: CliIo): number;
124
124
  //#region src/server.d.ts
125
125
  interface CreateServerOptions {
126
126
  engine: Engine;
127
- /** The explicit, first-class registry (docs/06, section 10.4). */
127
+ /** The explicit, first-class registry. */
128
128
  workflows: WorkflowRegistry;
129
129
  /**
130
130
  * Prices the journal fold behind GET /runs/:id/cost for runs without a
131
131
  * settled in-process outcome (the host assembles pricing exactly as it
132
132
  * does for the CLI); absent means those usages surface as `unpriced`,
133
- * never a silent zero (docs/04, section 10).
133
+ * never a silent zero.
134
134
  */
135
135
  priceUsd?: (servedBy: ModelRef, usage: Usage) => number | undefined;
136
136
  /**
137
- * Opt-in retention (docs/02, 8.2; OQ-20 executed at M8-T04): evaluated
137
+ * Opt-in retention (OQ-20 executed at M8-T04): evaluated
138
138
  * when a tracked run settles terminally; a true verdict applies
139
139
  * engine.deleteRun (transcript cascade, then the journal) and
140
140
  * untracks the run. Absent means everything persists indefinitely.
@@ -147,7 +147,7 @@ interface RulvarServer {
147
147
  declare function createServer(options: CreateServerOptions): RulvarServer;
148
148
  //#endregion
149
149
  //#region src/worker.d.ts
150
- /** Appendix A: the committed reference lease ttl (docs/06). */
150
+ /** Appendix A: the committed reference lease ttl. */
151
151
  declare const DEFAULT_WORKER_TTL_MS = 6e4;
152
152
  interface CreateWorkerOptions {
153
153
  /**
@@ -162,7 +162,7 @@ interface CreateWorkerOptions {
162
162
  owner?: string;
163
163
  /**
164
164
  * The store's lease ttl; the worker renews at ttl/3 (the normative
165
- * bound, docs/03 12.3). Default: the Appendix A reference 60000 ms.
165
+ * bound). Default: the Appendix A reference 60000 ms.
166
166
  * MUST match the store's configured ttl.
167
167
  */
168
168
  ttlMs?: number;
@@ -179,7 +179,7 @@ interface CreateWorkerOptions {
179
179
  /** Observability hook for per-run failures; never throws into the loop. */
180
180
  onError?: (runId: string, error: unknown) => void;
181
181
  /**
182
- * Opt-in retention (docs/02, 8.3; OQ-20 executed at M8-T04): evaluated
182
+ * Opt-in retention (OQ-20 executed at M8-T04): evaluated
183
183
  * during sweeps over SETTLED runs (terminal meta); a true verdict
184
184
  * applies engine.deleteRun under a briefly held lease. Absent means
185
185
  * everything persists indefinitely.
package/dist/index.js CHANGED
@@ -1,12 +1,12 @@
1
- import { a as resumeCommand, c as driveRun, d as renderEventLine, f as DEFAULT_STORE_DIR, g as looksLikeFile, h as loadWorkflowModule, i as inspectCommand, l as reportOutcome, m as loadCliConfig, n as HELP, o as runCommand, p as assembleEngine, r as runCli, s as runsLsCommand, t as processIo, u as attachProgress } from "./io-TjsY28AR.js";
1
+ import { a as resumeCommand, c as driveRun, d as renderEventLine, f as DEFAULT_STORE_DIR, g as looksLikeFile, h as loadWorkflowModule, i as inspectCommand, l as reportOutcome, m as loadCliConfig, n as HELP, o as runCommand, p as assembleEngine, r as runCli, s as runsLsCommand, t as processIo, u as attachProgress } from "./io-DSpNsEKg.js";
2
2
  import { ConfigError, InvalidResolutionError, JournalCompatibilityError, LeaseHeldError, Replayer, RulvarError, buildDeriverRegistry, costReportFromJournal, maskSecrets, normalizeEntry, scanJournalCompatibility, validateSchemaSpec } from "@rulvar/core";
3
3
  //#region src/server.ts
4
4
  /**
5
5
  * createServer (M8-T01): the HTTP shell over the public engine API
6
- * (docs/02, section 8.2; FR-702). Canonical signature
6
+ * (FR-702). Canonical signature
7
7
  * `createServer({ engine, workflows })` returning
8
8
  * `{ fetch(req: Request): Promise<Response> }`; the journal store comes
9
- * from the engine (Engine.stores, docs/06 10.2, M8 entry amendment).
9
+ * from the engine (Engine.stores, M8 entry amendment).
10
10
  *
11
11
  * Routes:
12
12
  * POST /runs start a run of a registered workflow
@@ -16,20 +16,19 @@ import { ConfigError, InvalidResolutionError, JournalCompatibilityError, LeaseHe
16
16
  * GET /runs/:id/cost CostReport
17
17
  *
18
18
  * Authentication is explicitly out of scope: the server is host-embedded
19
- * and auth belongs to host middleware (docs/14, OQ-16). SSE reconnection
20
- * maps Last-Event-ID to the event seq (the per-run telemetry counter,
21
- * docs/09, section 1.1); replay is at-least-once by design, matching the
22
- * journal-backed re-emission contract (docs/09, section 1.5: consumers
19
+ * and auth belongs to host middleware (OQ-16). SSE reconnection
20
+ * maps Last-Event-ID to the event seq (the per-run telemetry counter);
21
+ * replay is at-least-once by design, matching the
22
+ * journal-backed re-emission contract (consumers
23
23
  * deduplicate on `replayed`).
24
24
  *
25
25
  * The server is a single-process shell: it tracks the runs it started
26
26
  * (or resumed) in memory and serves everything else from the engine's
27
27
  * stores. A resolution posted for a run that is NOT live in this process
28
- * is the documented offline append (docs/03, section 8: load, compute
28
+ * is the documented offline append (load, compute
29
29
  * next seq, append, under a lease where the store is leasable); such a
30
30
  * run resumes on a queue worker (createWorker, M8-T02), not here,
31
- * because original run arguments are not journaled in v1 (docs/14,
32
- * OQ-21).
31
+ * because original run arguments are not journaled in v1 (OQ-21).
33
32
  */
34
33
  const JSON_HEADERS = { "content-type": "application/json; charset=utf-8" };
35
34
  const wallClock = Date.now.bind(globalThis);
@@ -64,7 +63,7 @@ function isLeasable(store) {
64
63
  const candidate = store;
65
64
  return typeof candidate.acquire === "function" && typeof candidate.renew === "function" && typeof candidate.release === "function";
66
65
  }
67
- /** The approval-suspension resolution key (docs/08, section 3.6). */
66
+ /** The approval-suspension resolution key. */
68
67
  const APPROVAL_KEY_PREFIX = "approval:";
69
68
  function suspensionKeyOf(entry) {
70
69
  if (entry.status !== "suspended") return;
@@ -299,7 +298,7 @@ function createServer(options) {
299
298
  return section;
300
299
  }
301
300
  /**
302
- * The offline path (docs/03, section 8): the run is not live in this
301
+ * The offline path: the run is not live in this
303
302
  * process; append the resolution under a lease where the store is
304
303
  * leasable and leave the resume to a queue worker.
305
304
  */
@@ -433,7 +432,7 @@ function createServer(options) {
433
432
  //#region src/worker.ts
434
433
  /**
435
434
  * createWorker (M8-T02): the queue shell over the public engine API
436
- * (docs/02, section 8.3; FR-703). Canonical signature
435
+ * (FR-703). Canonical signature
437
436
  * `createWorker(engine, { store: LeasableStore, concurrency? })`.
438
437
  *
439
438
  * The worker leases resumable ('running' meta: a crashed or currently
@@ -441,7 +440,7 @@ function createServer(options) {
441
440
  * fencing epoch; acquire on a held lease rejects with LeaseHeldError and
442
441
  * the worker simply moves on. Stateless workers call engine.resume,
443
442
  * passing the lease via ResumeOptions.lease so EVERY engine append of
444
- * the resumed run is fenced (docs/03, section 12.3, M8 entry amendment):
443
+ * the resumed run is fenced (M8 entry amendment):
445
444
  * lease theft is impossible because a stale writer's appends are
446
445
  * rejected by the store and never become visible, whether or not the
447
446
  * stale worker noticed it lost the lease.
@@ -452,23 +451,23 @@ function createServer(options) {
452
451
  * this worker (an older library never writes into a newer journal).
453
452
  *
454
453
  * Queue semantics are honestly at-least-once with deduplication by the
455
- * journal (docs/03, section 13.1): re-leasing a settled or unchanged
454
+ * journal: re-leasing a settled or unchanged
456
455
  * run replays to the same outcome with zero live calls. Workflows
457
456
  * resolve through the engine's defaults.workflows registry plus the
458
- * persisted CompiledWorkflow sources, never through a worker parameter
459
- * (docs/06, section 10.4); original in-process run arguments are not
457
+ * persisted CompiledWorkflow sources, never through a worker parameter;
458
+ * original in-process run arguments are not
460
459
  * journaled in v1, so the host MAY re-supply them per run via `argsFor`
461
- * (docs/14, OQ-21).
460
+ * (OQ-21).
462
461
  *
463
462
  * Appendix A (committed at M8 entry): concurrency defaults to 1 (one
464
463
  * leased run per worker process; hosts scale out by adding workers,
465
464
  * which the fencing epoch makes safe by construction); the renew
466
465
  * cadence is ttl/3 with the reference ttl of 60000 ms. There is no
467
- * distributed cross-process rate limiter in v1 (EXC-14; docs/14,
468
- * OQ-17): divide provider quota per worker or front an external
466
+ * distributed cross-process rate limiter in v1 (EXC-14; OQ-17):
467
+ * divide provider quota per worker or front an external
469
468
  * gateway.
470
469
  */
471
- /** Appendix A: the committed reference lease ttl (docs/06). */
470
+ /** Appendix A: the committed reference lease ttl. */
472
471
  const DEFAULT_WORKER_TTL_MS = 6e4;
473
472
  const CANDIDATE_STATUSES = /* @__PURE__ */ new Set(["running", "suspended"]);
474
473
  let workerOrdinal = 0;
@@ -544,7 +543,7 @@ function createWorker(engine, options) {
544
543
  });
545
544
  await settled;
546
545
  }
547
- /** Opt-in retention over settled runs (docs/02, 8.3; M8-T04). */
546
+ /** Opt-in retention over settled runs (M8-T04). */
548
547
  async function applyRetention(meta) {
549
548
  if (options.retention?.(meta) !== true) return;
550
549
  let lease;
@@ -630,9 +629,9 @@ function createWorker(engine, options) {
630
629
  //#endregion
631
630
  //#region src/otel.ts
632
631
  /**
633
- * OpenTelemetry exporter (M5-T08; docs/09, section 3). `toOtel(run,
632
+ * OpenTelemetry exporter (M5-T08; https://docs.rulvar.com/guide/observability). `toOtel(run,
634
633
  * tracer)` maps the spanId tree of a run 1:1 onto OTel spans: one span
635
- * per rulvar span, parented per the docs/09 1.2 hierarchy (run > phase >
634
+ * per rulvar span, parented per the span hierarchy (run > phase >
636
635
  * agent > tool > child), with start/end timestamps from the lifecycle
637
636
  * events. Events without an own span (log, budget:update) attach as span
638
637
  * events on their enclosing span.