@crewhaus/watchme-store 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -7
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/join.d.ts +42 -3
- package/dist/join.js +30 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -56,13 +56,30 @@ rewritten by `register`/`deregister`, so reads never race writers.
|
|
|
56
56
|
## Quality join
|
|
57
57
|
|
|
58
58
|
`joinQualityToArms(decisions, quality)` joins durable route decisions to
|
|
59
|
-
delayed quality scores per `(sessionId, turnNumber)`; the decision's model
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
59
|
+
delayed quality scores per `(sessionId, turnNumber)`; the decision's `model`
|
|
60
|
+
is the ARM id — the `models:` profile name when the candidate is a profile,
|
|
61
|
+
else its spec model string, which is what the live scoreboard keys on. Emitted
|
|
62
|
+
rows carry SHADOW routeKeys — `"q:" + routeKey`, a namespace the runtime router
|
|
63
|
+
never mints or reads — so recording them observes routing quality without
|
|
64
|
+
steering it, until `crewhaus route promote` folds the lane. Scores clamp to
|
|
65
|
+
`[0, 1]` and multiple scores for one turn average. Rewards are computed by the
|
|
66
|
+
caller via routing-store's `computeReward` with `obs.quality` set; this module
|
|
67
|
+
stays reward-free and fs-free.
|
|
68
|
+
|
|
69
|
+
**One row per decision (0.6.0 §7.9).** A hybrid turn routes more than once — a
|
|
70
|
+
cascade drafts on the cheap arm, then escalates to the strong one — and each
|
|
71
|
+
decision carries its own `stage`. `watchme report --feed-routing` feeds every
|
|
72
|
+
stage's decision, so the turn's one delayed quality folds onto all of them
|
|
73
|
+
(the same one-quality-to-N-decisions fan-out the in-loop path performs at the
|
|
74
|
+
strategy-turn boundary). Keeping only the first decision credited the whole
|
|
75
|
+
turn to the drafting arm and made the escalation invisible. Per-turn latency
|
|
76
|
+
and cost are turn totals, so the caller attaches them to the turn's first
|
|
77
|
+
stage only — and a later stage's row therefore carries **no** `latencyMs` at
|
|
78
|
+
all. Do not coerce that gap to `0`: `computeReward` reads `0` as a perfect
|
|
79
|
+
latency term, which would hand the expensive escalation rung a free score it
|
|
80
|
+
did not earn. `RouteObservation.latencyMs` is optional for exactly this
|
|
81
|
+
reason; the reward drops the term and redistributes its weight, as it already
|
|
82
|
+
does for an unknown `costUsd`.
|
|
66
83
|
|
|
67
84
|
## Exports
|
|
68
85
|
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* an injected callback at every CLI append site — this package never imports
|
|
9
9
|
* PII detectors.
|
|
10
10
|
*/
|
|
11
|
-
export { joinQualityToArms, type QualityArmRow, type RouteDecision, type TurnQuality, } from "./join.js";
|
|
11
|
+
export { joinQualityToArms, QUALITY_LANE_PREFIX, type QualityArmRow, type RouteDecision, type TurnQuality, } from "./join.js";
|
|
12
12
|
export { openHarnessRegistry, type HarnessRegistry, type HarnessRegistryOptions, } from "./registry.js";
|
|
13
13
|
export { openWatchmeStore, type WatchmeStore, type WatchmeStoreOptions } from "./store.js";
|
|
14
14
|
export type { HarnessEntry, WatchmeAggregate, WatchmeJudgment, WatchmeObservation, WatchmeState, } from "./types.js";
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,6 @@
|
|
|
8
8
|
* an injected callback at every CLI append site — this package never imports
|
|
9
9
|
* PII detectors.
|
|
10
10
|
*/
|
|
11
|
-
export { joinQualityToArms, } from "./join.js";
|
|
11
|
+
export { joinQualityToArms, QUALITY_LANE_PREFIX, } from "./join.js";
|
|
12
12
|
export { openHarnessRegistry, } from "./registry.js";
|
|
13
13
|
export { openWatchmeStore } from "./store.js";
|
package/dist/join.d.ts
CHANGED
|
@@ -3,18 +3,44 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Route decisions (durable `model_route` lines) and delayed quality scores
|
|
5
5
|
* (phase-2 judgments or normalized ratings) meet per `(sessionId, turnNumber)`;
|
|
6
|
-
* the
|
|
6
|
+
* the PROFILE on the decision names the arm when the candidate is a `models:`
|
|
7
|
+
* profile, else its spec model string — the same arm id the live scoreboard
|
|
8
|
+
* keys on (§7.9), so a profiled roster's arms are reachable from the join.
|
|
9
|
+
*
|
|
10
|
+
* 0.6.0 §7.9 — the join is PER STAGE, not per turn. A hybrid turn makes
|
|
11
|
+
* several routed decisions (a cascade drafts on the cheap arm, then escalates
|
|
12
|
+
* to the strong one), and each carries a `stage` on its durable line. Folding
|
|
13
|
+
* the turn's one delayed quality onto every stage is the same one-quality-to-
|
|
14
|
+
* N-decisions fan-out the in-loop path performs at the strategy-turn boundary;
|
|
15
|
+
* keeping only the FIRST decision, as this join used to, credited the whole
|
|
16
|
+
* turn to the drafting arm and made the escalation invisible. Emitted rows
|
|
17
|
+
* carry SHADOW routeKeys
|
|
7
18
|
* — `"q:" + originalRouteKey` — a namespace the runtime router never mints or
|
|
8
19
|
* reads, so recording them observes routing quality without steering it.
|
|
9
20
|
* Rewards are computed by the CALLER via routing-store's `computeReward` with
|
|
10
21
|
* `obs.quality` set; this module stays reward-free.
|
|
22
|
+
*
|
|
23
|
+
* 0.6.0 (design §7.8) — this OFFLINE lane has an ONLINE twin: `strategy.shadow`
|
|
24
|
+
* re-runs a sampled share of turns on an audition candidate after the primary
|
|
25
|
+
* has answered, grades it blind against the primary with order-swapped
|
|
26
|
+
* pairwise judging, and records both arms under `shadow:<scope>/<band>`
|
|
27
|
+
* (`@crewhaus/routing-store`'s `shadowRouteKey`). Both lanes are observe-only:
|
|
28
|
+
* the runtime router mints neither prefix and reads neither back, so nothing
|
|
29
|
+
* recorded here steers a live decision until `route promote` folds it.
|
|
11
30
|
*/
|
|
12
31
|
/** One durable route decision (a `model_route` line, turn-attributed). */
|
|
13
32
|
export type RouteDecision = {
|
|
14
33
|
sessionId: string;
|
|
15
34
|
turnNumber: number;
|
|
16
35
|
routeKey: string;
|
|
36
|
+
/** The scoreboard ARM id: the `models:` profile name, else the spec model string. */
|
|
17
37
|
model: string;
|
|
38
|
+
/**
|
|
39
|
+
* 0.6.0 §7.9 — the hybrid stage this decision served (`"draft"`,
|
|
40
|
+
* `"escalate"`, …). Absent on an unstaged turn (every pre-0.6.0 line and
|
|
41
|
+
* every plain pooled turn), which is exactly one decision per turn.
|
|
42
|
+
*/
|
|
43
|
+
stage?: string;
|
|
18
44
|
latencyMs?: number;
|
|
19
45
|
costUsd?: number;
|
|
20
46
|
success: boolean;
|
|
@@ -31,15 +57,28 @@ export type QualityArmRow = {
|
|
|
31
57
|
model: string;
|
|
32
58
|
obs: {
|
|
33
59
|
success: boolean;
|
|
34
|
-
|
|
60
|
+
/**
|
|
61
|
+
* 0.6.0 §7.9 — ABSENT when the decision carried no latency. Per-turn
|
|
62
|
+
* latency is a TURN total, so only a turn's first stage carries one; a
|
|
63
|
+
* later stage records the quality alone. Coercing the gap to `0` would
|
|
64
|
+
* hand the escalation arm a perfect latency term in `computeReward`,
|
|
65
|
+
* which is the flattery §6.3 item 1 exists to prevent.
|
|
66
|
+
*/
|
|
67
|
+
latencyMs?: number;
|
|
35
68
|
costUsd?: number;
|
|
36
69
|
quality: number;
|
|
70
|
+
/** Carried onto the `v:2` line so a promoted observation says which stage produced it. */
|
|
71
|
+
stage?: string;
|
|
37
72
|
};
|
|
38
73
|
};
|
|
74
|
+
/** The offline quality lane prefix (its runtime twin is `shadow:`, see above). */
|
|
75
|
+
export declare const QUALITY_LANE_PREFIX = "q:";
|
|
39
76
|
/**
|
|
40
77
|
* Join route decisions to quality scores per `(sessionId, turnNumber)`.
|
|
41
78
|
* Decisions without a quality score are dropped (nothing to learn from);
|
|
42
79
|
* multiple scores for the same turn (a rating AND a judgment) average.
|
|
43
|
-
* Row order follows `decisions` order
|
|
80
|
+
* Row order follows `decisions` order — ONE ROW PER DECISION, so a hybrid
|
|
81
|
+
* turn's stages each get their own row (the caller decides which decisions
|
|
82
|
+
* to feed; this function never collapses them).
|
|
44
83
|
*/
|
|
45
84
|
export declare function joinQualityToArms(decisions: ReadonlyArray<RouteDecision>, quality: ReadonlyArray<TurnQuality>): ReadonlyArray<QualityArmRow>;
|
package/dist/join.js
CHANGED
|
@@ -3,19 +3,42 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Route decisions (durable `model_route` lines) and delayed quality scores
|
|
5
5
|
* (phase-2 judgments or normalized ratings) meet per `(sessionId, turnNumber)`;
|
|
6
|
-
* the
|
|
6
|
+
* the PROFILE on the decision names the arm when the candidate is a `models:`
|
|
7
|
+
* profile, else its spec model string — the same arm id the live scoreboard
|
|
8
|
+
* keys on (§7.9), so a profiled roster's arms are reachable from the join.
|
|
9
|
+
*
|
|
10
|
+
* 0.6.0 §7.9 — the join is PER STAGE, not per turn. A hybrid turn makes
|
|
11
|
+
* several routed decisions (a cascade drafts on the cheap arm, then escalates
|
|
12
|
+
* to the strong one), and each carries a `stage` on its durable line. Folding
|
|
13
|
+
* the turn's one delayed quality onto every stage is the same one-quality-to-
|
|
14
|
+
* N-decisions fan-out the in-loop path performs at the strategy-turn boundary;
|
|
15
|
+
* keeping only the FIRST decision, as this join used to, credited the whole
|
|
16
|
+
* turn to the drafting arm and made the escalation invisible. Emitted rows
|
|
17
|
+
* carry SHADOW routeKeys
|
|
7
18
|
* — `"q:" + originalRouteKey` — a namespace the runtime router never mints or
|
|
8
19
|
* reads, so recording them observes routing quality without steering it.
|
|
9
20
|
* Rewards are computed by the CALLER via routing-store's `computeReward` with
|
|
10
21
|
* `obs.quality` set; this module stays reward-free.
|
|
22
|
+
*
|
|
23
|
+
* 0.6.0 (design §7.8) — this OFFLINE lane has an ONLINE twin: `strategy.shadow`
|
|
24
|
+
* re-runs a sampled share of turns on an audition candidate after the primary
|
|
25
|
+
* has answered, grades it blind against the primary with order-swapped
|
|
26
|
+
* pairwise judging, and records both arms under `shadow:<scope>/<band>`
|
|
27
|
+
* (`@crewhaus/routing-store`'s `shadowRouteKey`). Both lanes are observe-only:
|
|
28
|
+
* the runtime router mints neither prefix and reads neither back, so nothing
|
|
29
|
+
* recorded here steers a live decision until `route promote` folds it.
|
|
11
30
|
*/
|
|
12
|
-
|
|
31
|
+
/** The offline quality lane prefix (its runtime twin is `shadow:`, see above). */
|
|
32
|
+
export const QUALITY_LANE_PREFIX = "q:";
|
|
33
|
+
const SHADOW_PREFIX = QUALITY_LANE_PREFIX;
|
|
13
34
|
const clamp01 = (x) => Math.min(1, Math.max(0, x));
|
|
14
35
|
/**
|
|
15
36
|
* Join route decisions to quality scores per `(sessionId, turnNumber)`.
|
|
16
37
|
* Decisions without a quality score are dropped (nothing to learn from);
|
|
17
38
|
* multiple scores for the same turn (a rating AND a judgment) average.
|
|
18
|
-
* Row order follows `decisions` order
|
|
39
|
+
* Row order follows `decisions` order — ONE ROW PER DECISION, so a hybrid
|
|
40
|
+
* turn's stages each get their own row (the caller decides which decisions
|
|
41
|
+
* to feed; this function never collapses them).
|
|
19
42
|
*/
|
|
20
43
|
export function joinQualityToArms(decisions, quality) {
|
|
21
44
|
const scores = new Map();
|
|
@@ -33,11 +56,14 @@ export function joinQualityToArms(decisions, quality) {
|
|
|
33
56
|
continue;
|
|
34
57
|
const obs = {
|
|
35
58
|
success: d.success,
|
|
36
|
-
latencyMs: Math.max(0, d.latencyMs ?? 0),
|
|
37
59
|
quality: clamp01(acc.sum / acc.n),
|
|
38
60
|
};
|
|
61
|
+
if (d.latencyMs !== undefined)
|
|
62
|
+
obs.latencyMs = Math.max(0, d.latencyMs);
|
|
39
63
|
if (d.costUsd !== undefined)
|
|
40
64
|
obs.costUsd = d.costUsd;
|
|
65
|
+
if (d.stage !== undefined)
|
|
66
|
+
obs.stage = d.stage;
|
|
41
67
|
rows.push({ routeKey: `${SHADOW_PREFIX}${d.routeKey}`, model: d.model, obs });
|
|
42
68
|
}
|
|
43
69
|
return rows;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/watchme-store",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Durable \"watch me\" observation store — per-harness session digests, judge verdicts, and the cross-harness registry behind crewhaus watchme",
|
|
6
6
|
"main": "dist/index.js",
|