@rulvar/cli 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1 -1
- package/dist/{dist-DtuSmcav.js → dist-Cdj5YBRG.js} +17 -18
- package/dist/dist-Clz0iBp5.js +6902 -0
- package/dist/{dist-CU6iYxCW.js → dist-M-vPRtBM.js} +29 -16
- package/dist/index.d.ts +9 -9
- package/dist/index.js +23 -24
- package/dist/{io-TjsY28AR.js → io-DSpNsEKg.js} +263 -23
- package/dist/{src-DVFj9U50.js → src-BA7JP7vn.js} +1 -1
- package/package.json +7 -7
|
@@ -6,9 +6,9 @@ import { createHash } from "node:crypto";
|
|
|
6
6
|
* suite runners. A separate quality-measurement package built strictly on
|
|
7
7
|
* the public APIs (L6).
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* Determinism rules (https://docs.rulvar.com/guide/evals and
|
|
10
|
+
* https://docs.rulvar.com/guide/testing): judge graders run THROUGH the
|
|
11
|
+
* engine, so judge calls
|
|
12
12
|
* are journaled, budgeted, and VCR-recordable, and an eval suite replays
|
|
13
13
|
* deterministically from cassettes with zero live calls.
|
|
14
14
|
*/
|
|
@@ -244,14 +244,14 @@ function judgeGrader(options) {
|
|
|
244
244
|
};
|
|
245
245
|
}
|
|
246
246
|
/**
|
|
247
|
-
* The eval-committer identity (M11-T01; docs/
|
|
248
|
-
*
|
|
247
|
+
* The eval-committer identity (M11-T01; https://docs.rulvar.com/guide/model-knowledge).
|
|
248
|
+
* The pipeline-side commit path: builds
|
|
249
249
|
* eval-committer-gated ops (the coherence square: class eval-measured,
|
|
250
250
|
* author eval-pipeline, metrics present) and commits them with the
|
|
251
251
|
* documented CAS-rebase recipe. Humans never call this; their path is
|
|
252
252
|
* the human gate and it structurally cannot carry metrics.
|
|
253
253
|
*/
|
|
254
|
-
/** One measured claim
|
|
254
|
+
/** One measured claim; claimExpiry applies the TTL from the decay table. */
|
|
255
255
|
function evalMeasuredClaim(input, committerId) {
|
|
256
256
|
return {
|
|
257
257
|
id: input.id,
|
|
@@ -306,8 +306,7 @@ async function commitEvalMeasured(store, claims, options) {
|
|
|
306
306
|
throw lastCas ?? /* @__PURE__ */ new Error("commitEvalMeasured: unreachable");
|
|
307
307
|
}
|
|
308
308
|
/**
|
|
309
|
-
* The canary fingerprint (M11-T04;
|
|
310
|
-
* decay"; OQ-06). The optional compensation for silent alias
|
|
309
|
+
* The canary fingerprint (M11-T04; OQ-06). The optional compensation for silent alias
|
|
311
310
|
* re-pointing that modelEpoch honestly cannot catch: a FIXED probe set
|
|
312
311
|
* at temperature 0, run through the ordinary engine (journaled,
|
|
313
312
|
* budgeted, VCR-recordable), hashed over normalized outputs. A
|
|
@@ -341,9 +340,8 @@ async function canaryFingerprint(engine, probes) {
|
|
|
341
340
|
}
|
|
342
341
|
/**
|
|
343
342
|
* Flips the model's ACTIVE eval-measured claims to stale when their
|
|
344
|
-
* recorded canary fingerprint differs from the fresh one
|
|
345
|
-
*
|
|
346
|
-
* stale"). Claims without a recorded fingerprint have no baseline and
|
|
343
|
+
* recorded canary fingerprint differs from the fresh one. Claims
|
|
344
|
+
* without a recorded fingerprint have no baseline and
|
|
347
345
|
* stay untouched (the documented no-probe posture); a second run is
|
|
348
346
|
* an idempotent noop. CAS-rebased like every maintenance commit.
|
|
349
347
|
*/
|
|
@@ -382,9 +380,8 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
|
|
|
382
380
|
throw lastCas ?? /* @__PURE__ */ new Error("flipStaleOnCanaryDrift: unreachable");
|
|
383
381
|
}
|
|
384
382
|
/**
|
|
385
|
-
* The phases 1-2 measured-value checkpoint (M12-T01;
|
|
386
|
-
*
|
|
387
|
-
* 14-open-questions.md, closed at M11-T06). The M12 gate: kb_propose
|
|
383
|
+
* The phases 1-2 measured-value checkpoint (M12-T01; the quantitative
|
|
384
|
+
* criteria of OQ-09, closed at M11-T06). The M12 gate: kb_propose
|
|
388
385
|
* and the proposal loop ship ONLY if the knowledge card demonstrably
|
|
389
386
|
* improves tier and agentType selection on eval cases.
|
|
390
387
|
*
|
|
@@ -404,7 +401,11 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
|
|
|
404
401
|
* with and without the knowledge store configured (the card docks
|
|
405
402
|
* into the spawn tool description when configured). Criterion 2
|
|
406
403
|
* holds when the card-informed arm matches or beats the baseline
|
|
407
|
-
* pass rate at no more than 105 percent of its cost
|
|
404
|
+
* pass rate at no more than 105 percent of its cost, OR beats it
|
|
405
|
+
* by at least 15 points at no more than 115 percent of its cost
|
|
406
|
+
* (the quality branch; OQ-09 as amended 2026-07-12: the baseline
|
|
407
|
+
* fails CHEAPLY, so the flat cost bar tightened exactly when the
|
|
408
|
+
* card was winning on quality).
|
|
408
409
|
*
|
|
409
410
|
* The checkpoint PASSES only when both criteria hold. Methodology
|
|
410
411
|
* guard: the claims the treatment consumes MUST come from a seeding
|
|
@@ -413,6 +414,18 @@ async function flipStaleOnCanaryDrift(store, model, freshFingerprint, options) {
|
|
|
413
414
|
*/
|
|
414
415
|
/** IEEE754 guard for the rule boundaries (0.8 + 0.05 exceeds 0.85). */
|
|
415
416
|
const EPSILON = 1e-9;
|
|
417
|
+
/**
|
|
418
|
+
* The OQ-09 criterion 2 rule (as amended 2026-07-12): match-or-beat at
|
|
419
|
+
* 105 percent of baseline cost, OR at least 15 points better at 115
|
|
420
|
+
* percent (the quality branch: the baseline fails cheaply, so the flat
|
|
421
|
+
* bar tightened exactly when the card won on quality). The vacuous-pass
|
|
422
|
+
* guard stays with the caller.
|
|
423
|
+
*/
|
|
424
|
+
function agentTypeRuleHolds(baseline, informed) {
|
|
425
|
+
const matchesCheaply = informed.passRate >= baseline.passRate - EPSILON && informed.totalCostUsd <= 1.05 * baseline.totalCostUsd + EPSILON;
|
|
426
|
+
const clearlyBetterNearCost = informed.passRate >= baseline.passRate + .15 - EPSILON && informed.totalCostUsd <= 1.15 * baseline.totalCostUsd + EPSILON;
|
|
427
|
+
return matchesCheaply || clearlyBetterNearCost;
|
|
428
|
+
}
|
|
416
429
|
/** The OQ-09 cell rule (shared by the per-cell and pooled verdicts). */
|
|
417
430
|
function rungRuleHolds(baseline, treatment) {
|
|
418
431
|
const equalOrBetterCheaper = treatment.passRate >= baseline.passRate - EPSILON && treatment.totalCostUsd <= .9 * baseline.totalCostUsd + EPSILON;
|
|
@@ -493,7 +506,7 @@ async function runValueCheckpoint(checkpointPool, options) {
|
|
|
493
506
|
criterion2 = {
|
|
494
507
|
baseline,
|
|
495
508
|
informed,
|
|
496
|
-
passed: informed.n > 0 && informed.passRate > 0 &&
|
|
509
|
+
passed: informed.n > 0 && informed.passRate > 0 && agentTypeRuleHolds(baseline, informed)
|
|
497
510
|
};
|
|
498
511
|
}
|
|
499
512
|
return {
|
package/dist/index.d.ts
CHANGED
|
@@ -37,7 +37,7 @@ declare function inspectCommand(argv: string[], context: CommandContext): Promis
|
|
|
37
37
|
interface CliConfig {
|
|
38
38
|
engineOptions?: Partial<CreateEngineOptions>;
|
|
39
39
|
workflows?: WorkflowRegistry;
|
|
40
|
-
/** rulvar kb sweep configuration (M11-T05
|
|
40
|
+
/** rulvar kb sweep configuration (M11-T05). */
|
|
41
41
|
kbSweep?: KbSweepCliConfig;
|
|
42
42
|
}
|
|
43
43
|
/**
|
|
@@ -95,7 +95,7 @@ interface AssembledCli {
|
|
|
95
95
|
engine: Engine;
|
|
96
96
|
store: JournalStore;
|
|
97
97
|
workflows: WorkflowRegistry;
|
|
98
|
-
/** The journal-fold price function (table wins over caps
|
|
98
|
+
/** The journal-fold price function (table wins over caps). */
|
|
99
99
|
priceUsd: (servedBy: ModelRef, usage: Usage) => number | undefined;
|
|
100
100
|
}
|
|
101
101
|
declare function assembleEngine(options: {
|
|
@@ -115,7 +115,7 @@ declare function driveRun(options: {
|
|
|
115
115
|
engine: Engine;
|
|
116
116
|
workflow: Workflow<never, unknown>;
|
|
117
117
|
first: RunHandle<unknown>;
|
|
118
|
-
io: CliIo; /** Original run arguments: not journaled in v1, the host re-supplies them
|
|
118
|
+
io: CliIo; /** Original run arguments: not journaled in v1, the host re-supplies them. */
|
|
119
119
|
args?: unknown;
|
|
120
120
|
}): Promise<RunOutcome<unknown>>;
|
|
121
121
|
/** Renders the settled outcome; returns the process exit code. */
|
|
@@ -124,17 +124,17 @@ declare function reportOutcome(outcome: RunOutcome<unknown>, io: CliIo): number;
|
|
|
124
124
|
//#region src/server.d.ts
|
|
125
125
|
interface CreateServerOptions {
|
|
126
126
|
engine: Engine;
|
|
127
|
-
/** The explicit, first-class registry
|
|
127
|
+
/** The explicit, first-class registry. */
|
|
128
128
|
workflows: WorkflowRegistry;
|
|
129
129
|
/**
|
|
130
130
|
* Prices the journal fold behind GET /runs/:id/cost for runs without a
|
|
131
131
|
* settled in-process outcome (the host assembles pricing exactly as it
|
|
132
132
|
* does for the CLI); absent means those usages surface as `unpriced`,
|
|
133
|
-
* never a silent zero
|
|
133
|
+
* never a silent zero.
|
|
134
134
|
*/
|
|
135
135
|
priceUsd?: (servedBy: ModelRef, usage: Usage) => number | undefined;
|
|
136
136
|
/**
|
|
137
|
-
* Opt-in retention (
|
|
137
|
+
* Opt-in retention (OQ-20 executed at M8-T04): evaluated
|
|
138
138
|
* when a tracked run settles terminally; a true verdict applies
|
|
139
139
|
* engine.deleteRun (transcript cascade, then the journal) and
|
|
140
140
|
* untracks the run. Absent means everything persists indefinitely.
|
|
@@ -147,7 +147,7 @@ interface RulvarServer {
|
|
|
147
147
|
declare function createServer(options: CreateServerOptions): RulvarServer;
|
|
148
148
|
//#endregion
|
|
149
149
|
//#region src/worker.d.ts
|
|
150
|
-
/** Appendix A: the committed reference lease ttl
|
|
150
|
+
/** Appendix A: the committed reference lease ttl. */
|
|
151
151
|
declare const DEFAULT_WORKER_TTL_MS = 6e4;
|
|
152
152
|
interface CreateWorkerOptions {
|
|
153
153
|
/**
|
|
@@ -162,7 +162,7 @@ interface CreateWorkerOptions {
|
|
|
162
162
|
owner?: string;
|
|
163
163
|
/**
|
|
164
164
|
* The store's lease ttl; the worker renews at ttl/3 (the normative
|
|
165
|
-
* bound
|
|
165
|
+
* bound). Default: the Appendix A reference 60000 ms.
|
|
166
166
|
* MUST match the store's configured ttl.
|
|
167
167
|
*/
|
|
168
168
|
ttlMs?: number;
|
|
@@ -179,7 +179,7 @@ interface CreateWorkerOptions {
|
|
|
179
179
|
/** Observability hook for per-run failures; never throws into the loop. */
|
|
180
180
|
onError?: (runId: string, error: unknown) => void;
|
|
181
181
|
/**
|
|
182
|
-
* Opt-in retention (
|
|
182
|
+
* Opt-in retention (OQ-20 executed at M8-T04): evaluated
|
|
183
183
|
* during sweeps over SETTLED runs (terminal meta); a true verdict
|
|
184
184
|
* applies engine.deleteRun under a briefly held lease. Absent means
|
|
185
185
|
* everything persists indefinitely.
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { a as resumeCommand, c as driveRun, d as renderEventLine, f as DEFAULT_STORE_DIR, g as looksLikeFile, h as loadWorkflowModule, i as inspectCommand, l as reportOutcome, m as loadCliConfig, n as HELP, o as runCommand, p as assembleEngine, r as runCli, s as runsLsCommand, t as processIo, u as attachProgress } from "./io-
|
|
1
|
+
import { a as resumeCommand, c as driveRun, d as renderEventLine, f as DEFAULT_STORE_DIR, g as looksLikeFile, h as loadWorkflowModule, i as inspectCommand, l as reportOutcome, m as loadCliConfig, n as HELP, o as runCommand, p as assembleEngine, r as runCli, s as runsLsCommand, t as processIo, u as attachProgress } from "./io-DSpNsEKg.js";
|
|
2
2
|
import { ConfigError, InvalidResolutionError, JournalCompatibilityError, LeaseHeldError, Replayer, RulvarError, buildDeriverRegistry, costReportFromJournal, maskSecrets, normalizeEntry, scanJournalCompatibility, validateSchemaSpec } from "@rulvar/core";
|
|
3
3
|
//#region src/server.ts
|
|
4
4
|
/**
|
|
5
5
|
* createServer (M8-T01): the HTTP shell over the public engine API
|
|
6
|
-
* (
|
|
6
|
+
* (FR-702). Canonical signature
|
|
7
7
|
* `createServer({ engine, workflows })` returning
|
|
8
8
|
* `{ fetch(req: Request): Promise<Response> }`; the journal store comes
|
|
9
|
-
* from the engine (Engine.stores,
|
|
9
|
+
* from the engine (Engine.stores, M8 entry amendment).
|
|
10
10
|
*
|
|
11
11
|
* Routes:
|
|
12
12
|
* POST /runs start a run of a registered workflow
|
|
@@ -16,20 +16,19 @@ import { ConfigError, InvalidResolutionError, JournalCompatibilityError, LeaseHe
|
|
|
16
16
|
* GET /runs/:id/cost CostReport
|
|
17
17
|
*
|
|
18
18
|
* Authentication is explicitly out of scope: the server is host-embedded
|
|
19
|
-
* and auth belongs to host middleware (
|
|
20
|
-
* maps Last-Event-ID to the event seq (the per-run telemetry counter
|
|
21
|
-
*
|
|
22
|
-
* journal-backed re-emission contract (
|
|
19
|
+
* and auth belongs to host middleware (OQ-16). SSE reconnection
|
|
20
|
+
* maps Last-Event-ID to the event seq (the per-run telemetry counter);
|
|
21
|
+
* replay is at-least-once by design, matching the
|
|
22
|
+
* journal-backed re-emission contract (consumers
|
|
23
23
|
* deduplicate on `replayed`).
|
|
24
24
|
*
|
|
25
25
|
* The server is a single-process shell: it tracks the runs it started
|
|
26
26
|
* (or resumed) in memory and serves everything else from the engine's
|
|
27
27
|
* stores. A resolution posted for a run that is NOT live in this process
|
|
28
|
-
* is the documented offline append (
|
|
28
|
+
* is the documented offline append (load, compute
|
|
29
29
|
* next seq, append, under a lease where the store is leasable); such a
|
|
30
30
|
* run resumes on a queue worker (createWorker, M8-T02), not here,
|
|
31
|
-
* because original run arguments are not journaled in v1 (
|
|
32
|
-
* OQ-21).
|
|
31
|
+
* because original run arguments are not journaled in v1 (OQ-21).
|
|
33
32
|
*/
|
|
34
33
|
const JSON_HEADERS = { "content-type": "application/json; charset=utf-8" };
|
|
35
34
|
const wallClock = Date.now.bind(globalThis);
|
|
@@ -64,7 +63,7 @@ function isLeasable(store) {
|
|
|
64
63
|
const candidate = store;
|
|
65
64
|
return typeof candidate.acquire === "function" && typeof candidate.renew === "function" && typeof candidate.release === "function";
|
|
66
65
|
}
|
|
67
|
-
/** The approval-suspension resolution key
|
|
66
|
+
/** The approval-suspension resolution key. */
|
|
68
67
|
const APPROVAL_KEY_PREFIX = "approval:";
|
|
69
68
|
function suspensionKeyOf(entry) {
|
|
70
69
|
if (entry.status !== "suspended") return;
|
|
@@ -299,7 +298,7 @@ function createServer(options) {
|
|
|
299
298
|
return section;
|
|
300
299
|
}
|
|
301
300
|
/**
|
|
302
|
-
* The offline path
|
|
301
|
+
* The offline path: the run is not live in this
|
|
303
302
|
* process; append the resolution under a lease where the store is
|
|
304
303
|
* leasable and leave the resume to a queue worker.
|
|
305
304
|
*/
|
|
@@ -433,7 +432,7 @@ function createServer(options) {
|
|
|
433
432
|
//#region src/worker.ts
|
|
434
433
|
/**
|
|
435
434
|
* createWorker (M8-T02): the queue shell over the public engine API
|
|
436
|
-
* (
|
|
435
|
+
* (FR-703). Canonical signature
|
|
437
436
|
* `createWorker(engine, { store: LeasableStore, concurrency? })`.
|
|
438
437
|
*
|
|
439
438
|
* The worker leases resumable ('running' meta: a crashed or currently
|
|
@@ -441,7 +440,7 @@ function createServer(options) {
|
|
|
441
440
|
* fencing epoch; acquire on a held lease rejects with LeaseHeldError and
|
|
442
441
|
* the worker simply moves on. Stateless workers call engine.resume,
|
|
443
442
|
* passing the lease via ResumeOptions.lease so EVERY engine append of
|
|
444
|
-
* the resumed run is fenced (
|
|
443
|
+
* the resumed run is fenced (M8 entry amendment):
|
|
445
444
|
* lease theft is impossible because a stale writer's appends are
|
|
446
445
|
* rejected by the store and never become visible, whether or not the
|
|
447
446
|
* stale worker noticed it lost the lease.
|
|
@@ -452,23 +451,23 @@ function createServer(options) {
|
|
|
452
451
|
* this worker (an older library never writes into a newer journal).
|
|
453
452
|
*
|
|
454
453
|
* Queue semantics are honestly at-least-once with deduplication by the
|
|
455
|
-
* journal
|
|
454
|
+
* journal: re-leasing a settled or unchanged
|
|
456
455
|
* run replays to the same outcome with zero live calls. Workflows
|
|
457
456
|
* resolve through the engine's defaults.workflows registry plus the
|
|
458
|
-
* persisted CompiledWorkflow sources, never through a worker parameter
|
|
459
|
-
*
|
|
457
|
+
* persisted CompiledWorkflow sources, never through a worker parameter;
|
|
458
|
+
* original in-process run arguments are not
|
|
460
459
|
* journaled in v1, so the host MAY re-supply them per run via `argsFor`
|
|
461
|
-
* (
|
|
460
|
+
* (OQ-21).
|
|
462
461
|
*
|
|
463
462
|
* Appendix A (committed at M8 entry): concurrency defaults to 1 (one
|
|
464
463
|
* leased run per worker process; hosts scale out by adding workers,
|
|
465
464
|
* which the fencing epoch makes safe by construction); the renew
|
|
466
465
|
* cadence is ttl/3 with the reference ttl of 60000 ms. There is no
|
|
467
|
-
* distributed cross-process rate limiter in v1 (EXC-14;
|
|
468
|
-
*
|
|
466
|
+
* distributed cross-process rate limiter in v1 (EXC-14; OQ-17):
|
|
467
|
+
* divide provider quota per worker or front an external
|
|
469
468
|
* gateway.
|
|
470
469
|
*/
|
|
471
|
-
/** Appendix A: the committed reference lease ttl
|
|
470
|
+
/** Appendix A: the committed reference lease ttl. */
|
|
472
471
|
const DEFAULT_WORKER_TTL_MS = 6e4;
|
|
473
472
|
const CANDIDATE_STATUSES = /* @__PURE__ */ new Set(["running", "suspended"]);
|
|
474
473
|
let workerOrdinal = 0;
|
|
@@ -544,7 +543,7 @@ function createWorker(engine, options) {
|
|
|
544
543
|
});
|
|
545
544
|
await settled;
|
|
546
545
|
}
|
|
547
|
-
/** Opt-in retention over settled runs (
|
|
546
|
+
/** Opt-in retention over settled runs (M8-T04). */
|
|
548
547
|
async function applyRetention(meta) {
|
|
549
548
|
if (options.retention?.(meta) !== true) return;
|
|
550
549
|
let lease;
|
|
@@ -630,9 +629,9 @@ function createWorker(engine, options) {
|
|
|
630
629
|
//#endregion
|
|
631
630
|
//#region src/otel.ts
|
|
632
631
|
/**
|
|
633
|
-
* OpenTelemetry exporter (M5-T08; docs/
|
|
632
|
+
* OpenTelemetry exporter (M5-T08; https://docs.rulvar.com/guide/observability). `toOtel(run,
|
|
634
633
|
* tracer)` maps the spanId tree of a run 1:1 onto OTel spans: one span
|
|
635
|
-
* per rulvar span, parented per the
|
|
634
|
+
* per rulvar span, parented per the span hierarchy (run > phase >
|
|
636
635
|
* agent > tool > child), with start/end timestamps from the lifecycle
|
|
637
636
|
* events. Events without an own span (log, budget:update) attach as span
|
|
638
637
|
* events on their enclosing span.
|