@lanefoundry/source-prod-slice 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit-export.d.ts +58 -0
- package/dist/audit-export.d.ts.map +1 -0
- package/dist/audit-export.js +341 -0
- package/dist/audit-export.js.map +1 -0
- package/dist/canary-orchestrator.d.ts +150 -0
- package/dist/canary-orchestrator.d.ts.map +1 -0
- package/dist/canary-orchestrator.js +274 -0
- package/dist/canary-orchestrator.js.map +1 -0
- package/dist/compare-scores.d.ts +55 -0
- package/dist/compare-scores.d.ts.map +1 -0
- package/dist/compare-scores.js +131 -0
- package/dist/compare-scores.js.map +1 -0
- package/dist/freeze-slice.d.ts +63 -0
- package/dist/freeze-slice.d.ts.map +1 -0
- package/dist/freeze-slice.js +85 -0
- package/dist/freeze-slice.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/promotion-decision.d.ts +64 -0
- package/dist/promotion-decision.d.ts.map +1 -0
- package/dist/promotion-decision.js +105 -0
- package/dist/promotion-decision.js.map +1 -0
- package/dist/replay-batch.d.ts +71 -0
- package/dist/replay-batch.d.ts.map +1 -0
- package/dist/replay-batch.js +82 -0
- package/dist/replay-batch.js.map +1 -0
- package/dist/signed-report.d.ts +82 -0
- package/dist/signed-report.d.ts.map +1 -0
- package/dist/signed-report.js +104 -0
- package/dist/signed-report.js.map +1 -0
- package/package.json +65 -4
- package/README.md +0 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @lanefoundry/source-prod-slice — Production slice tooling.
|
|
3
|
+
*
|
|
4
|
+
* Freeze, replay, compare, promote, and audit production traffic slices.
|
|
5
|
+
*
|
|
6
|
+
* @see docs/prd.md §5.2 — Three dataset sources (prod slice)
|
|
7
|
+
*/
|
|
8
|
+
export * from './freeze-slice.js';
|
|
9
|
+
export * from './replay-batch.js';
|
|
10
|
+
export * from './compare-scores.js';
|
|
11
|
+
export * from './promotion-decision.js';
|
|
12
|
+
export * from './signed-report.js';
|
|
13
|
+
export * from './canary-orchestrator.js';
|
|
14
|
+
export * from './audit-export.js';
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* promotion-decision.ts — Apply PromotionPolicy rules to compare output.
|
|
3
|
+
*
|
|
4
|
+
* Reuses engine's evaluate() to produce a PromotionDecision.
|
|
5
|
+
* Adds policy validation and detailed rule-result reporting.
|
|
6
|
+
*
|
|
7
|
+
* @see docs/prd.md §5.1 — The gate
|
|
8
|
+
*/
|
|
9
|
+
import type { PromotionPolicy, PromotionDecision, JudgeStabilityMatrix } from '@lanefoundry/gatelane-sdk';
|
|
10
|
+
import type { CandidateMetric } from '@lanefoundry/gatelane-engine';
|
|
11
|
+
import { type EvaluateResult } from '@lanefoundry/gatelane-engine';
|
|
12
|
+
/** Input for promotion decision. */
|
|
13
|
+
export type PromotionDecisionArgs = {
|
|
14
|
+
/** Candidate references in evaluation order. */
|
|
15
|
+
candidates: ReadonlyArray<string>;
|
|
16
|
+
/** Per-candidate metrics from compareScores(). */
|
|
17
|
+
perCandidate: Record<string, CandidateMetric>;
|
|
18
|
+
/** Judge stability matrix from compareScores(). */
|
|
19
|
+
judgeMatrix: JudgeStabilityMatrix;
|
|
20
|
+
/** Promotion policy to apply. Defaults to DEFAULT_POLICY. */
|
|
21
|
+
policy?: PromotionPolicy;
|
|
22
|
+
/** Optional approver (required when policy.approval_required = true). */
|
|
23
|
+
approver?: string;
|
|
24
|
+
};
|
|
25
|
+
/** Result of promotion decision with extended detail. */
|
|
26
|
+
export type PromotionDecisionResult = {
|
|
27
|
+
/** The promotion decision. */
|
|
28
|
+
decision: PromotionDecision;
|
|
29
|
+
/** Per-rule pass/fail detail. */
|
|
30
|
+
ruleResults: EvaluateResult['rule_results'];
|
|
31
|
+
/** Policy that was applied. */
|
|
32
|
+
policy: PromotionPolicy;
|
|
33
|
+
/** Whether approval is required. */
|
|
34
|
+
approvalRequired: boolean;
|
|
35
|
+
/** Approver if provided. */
|
|
36
|
+
approver?: string;
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Evaluate candidates against promotion policy.
|
|
40
|
+
*
|
|
41
|
+
* This wraps the engine's evaluate() and adds:
|
|
42
|
+
* - Input validation
|
|
43
|
+
* - Policy merge with defaults
|
|
44
|
+
* - Extended result with policy and approval metadata
|
|
45
|
+
*/
|
|
46
|
+
export declare function makePromotionDecision(args: PromotionDecisionArgs): PromotionDecisionResult;
|
|
47
|
+
/**
|
|
48
|
+
* Check if a candidate passes all policy rules without making a final decision.
|
|
49
|
+
* Returns per-rule results for inspection.
|
|
50
|
+
*/
|
|
51
|
+
export declare function checkCandidateAgainstPolicy(candidateRef: string, metric: CandidateMetric, judgeMatrix: JudgeStabilityMatrix, policy?: PromotionPolicy): {
|
|
52
|
+
passes: boolean;
|
|
53
|
+
rules: Record<string, {
|
|
54
|
+
pass: boolean;
|
|
55
|
+
value: number;
|
|
56
|
+
threshold: number;
|
|
57
|
+
}>;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* Validate a PromotionPolicy for correctness.
|
|
61
|
+
* Throws if policy has invalid values.
|
|
62
|
+
*/
|
|
63
|
+
export declare function validatePolicy(policy: PromotionPolicy): void;
|
|
64
|
+
//# sourceMappingURL=promotion-decision.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promotion-decision.d.ts","sourceRoot":"","sources":["../src/promotion-decision.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAC1G,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAA+B,KAAK,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAGhG,oCAAoC;AACpC,MAAM,MAAM,qBAAqB,GAAG;IAClC,gDAAgD;IAChD,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAClC,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,mDAAmD;IACnD,WAAW,EAAE,oBAAoB,CAAC;IAClC,6DAA6D;IAC7D,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,yDAAyD;AACzD,MAAM,MAAM,uBAAuB,GAAG;IACpC,8BAA8B;IAC9B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,iCAAiC;IACjC,WAAW,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;IAC5C,+BAA+B;IAC/B,MAAM,EAAE,eAAe,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,qBAAqB,GAAG,uBAAuB,CA+B1F;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,eAAe,EACvB,WAAW,EAAE,oBAAoB,EACjC,MAAM,GAAE,eAAgC,GACvC;IACD,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC5E,CA8BA;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAqB5D"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* promotion-decision.ts — Apply PromotionPolicy rules to compare output.
|
|
3
|
+
*
|
|
4
|
+
* Reuses engine's evaluate() to produce a PromotionDecision.
|
|
5
|
+
* Adds policy validation and detailed rule-result reporting.
|
|
6
|
+
*
|
|
7
|
+
* @see docs/prd.md §5.1 — The gate
|
|
8
|
+
*/
|
|
9
|
+
import { evaluate } from '@lanefoundry/gatelane-engine';
|
|
10
|
+
import { DEFAULT_POLICY } from '@lanefoundry/gatelane-sdk';
|
|
11
|
+
/**
|
|
12
|
+
* Evaluate candidates against promotion policy.
|
|
13
|
+
*
|
|
14
|
+
* This wraps the engine's evaluate() and adds:
|
|
15
|
+
* - Input validation
|
|
16
|
+
* - Policy merge with defaults
|
|
17
|
+
* - Extended result with policy and approval metadata
|
|
18
|
+
*/
|
|
19
|
+
export function makePromotionDecision(args) {
|
|
20
|
+
const { candidates, perCandidate, judgeMatrix, policy = DEFAULT_POLICY, approver } = args;
|
|
21
|
+
// Validate inputs
|
|
22
|
+
if (candidates.length === 0) {
|
|
23
|
+
throw new Error('promotion-decision: at least one candidate is required');
|
|
24
|
+
}
|
|
25
|
+
for (const ref of candidates) {
|
|
26
|
+
if (!perCandidate[ref]) {
|
|
27
|
+
throw new Error(`promotion-decision: missing metrics for candidate "${ref}"`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
// Evaluate using engine
|
|
31
|
+
const evaluateArgs = {
|
|
32
|
+
candidates,
|
|
33
|
+
perCandidate,
|
|
34
|
+
judgeMatrix,
|
|
35
|
+
policy,
|
|
36
|
+
approver,
|
|
37
|
+
};
|
|
38
|
+
const result = evaluate(evaluateArgs);
|
|
39
|
+
return {
|
|
40
|
+
decision: result.decision,
|
|
41
|
+
ruleResults: result.rule_results,
|
|
42
|
+
policy,
|
|
43
|
+
approvalRequired: policy.approval_required,
|
|
44
|
+
approver,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Check if a candidate passes all policy rules without making a final decision.
|
|
49
|
+
* Returns per-rule results for inspection.
|
|
50
|
+
*/
|
|
51
|
+
export function checkCandidateAgainstPolicy(candidateRef, metric, judgeMatrix, policy = DEFAULT_POLICY) {
|
|
52
|
+
const consensus = new Set(judgeMatrix.consensus_winners);
|
|
53
|
+
const isConsensus = consensus.has(candidateRef);
|
|
54
|
+
const rules = {
|
|
55
|
+
min_delta: {
|
|
56
|
+
pass: metric.aggregate_delta >= policy.min_delta,
|
|
57
|
+
value: metric.aggregate_delta,
|
|
58
|
+
threshold: policy.min_delta,
|
|
59
|
+
},
|
|
60
|
+
judge_stability: {
|
|
61
|
+
pass: isConsensus,
|
|
62
|
+
value: isConsensus ? 1 : 0,
|
|
63
|
+
threshold: policy.judge_stability_threshold,
|
|
64
|
+
},
|
|
65
|
+
cost_ceiling: {
|
|
66
|
+
pass: metric.cost_delta <= policy.cost_ceiling,
|
|
67
|
+
value: metric.cost_delta,
|
|
68
|
+
threshold: policy.cost_ceiling,
|
|
69
|
+
},
|
|
70
|
+
latency_ceiling: {
|
|
71
|
+
pass: metric.latency_delta <= policy.latency_ceiling,
|
|
72
|
+
value: metric.latency_delta,
|
|
73
|
+
threshold: policy.latency_ceiling,
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
const passes = Object.values(rules).every((r) => r.pass);
|
|
77
|
+
return { passes, rules };
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Validate a PromotionPolicy for correctness.
|
|
81
|
+
* Throws if policy has invalid values.
|
|
82
|
+
*/
|
|
83
|
+
export function validatePolicy(policy) {
|
|
84
|
+
if (policy.min_delta < 0 || policy.min_delta > 1) {
|
|
85
|
+
throw new Error('policy.min_delta must be in [0, 1]');
|
|
86
|
+
}
|
|
87
|
+
if (policy.judge_stability_threshold < 0 || policy.judge_stability_threshold > 1) {
|
|
88
|
+
throw new Error('policy.judge_stability_threshold must be in [0, 1]');
|
|
89
|
+
}
|
|
90
|
+
if (policy.cost_ceiling < 0) {
|
|
91
|
+
throw new Error('policy.cost_ceiling must be >= 0');
|
|
92
|
+
}
|
|
93
|
+
if (policy.latency_ceiling < 0) {
|
|
94
|
+
throw new Error('policy.latency_ceiling must be >= 0');
|
|
95
|
+
}
|
|
96
|
+
if (policy.auto_block_rule !== undefined) {
|
|
97
|
+
if (policy.auto_block_rule.metric_drop < 0 || policy.auto_block_rule.metric_drop > 1) {
|
|
98
|
+
throw new Error('policy.auto_block_rule.metric_drop must be in [0, 1]');
|
|
99
|
+
}
|
|
100
|
+
if (!policy.auto_block_rule.window || typeof policy.auto_block_rule.window !== 'string') {
|
|
101
|
+
throw new Error('policy.auto_block_rule.window must be a non-empty string (e.g., "24h")');
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=promotion-decision.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promotion-decision.js","sourceRoot":"","sources":["../src/promotion-decision.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EAAE,QAAQ,EAA0C,MAAM,8BAA8B,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AA8B3D;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAA2B;IAC/D,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,cAAc,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAE1F,kBAAkB;IAClB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,GAAG,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,YAAY,GAAiB;QACjC,UAAU;QACV,YAAY;QACZ,WAAW;QACX,MAAM;QACN,QAAQ;KACT,CAAC;IAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IAEtC,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,WAAW,EAAE,MAAM,CAAC,YAAY;QAChC,MAAM;QACN,gBAAgB,EAAE,MAAM,CAAC,iBAAiB;QAC1C,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,2BAA2B,CACzC,YAAoB,EACpB,MAAuB,EACvB,WAAiC,EACjC,SAA0B,cAAc;IAKxC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAEhD,MAAM,KAAK,GAAG;QACZ,SAAS,EAAE;YACT,IAAI,EAAE,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,SAAS;YAChD,KAAK,EAAE,MAAM,CAAC,eAAe;YAC7B,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B;QACD,eAAe,EAAE;YACf,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,SAAS,EAAE,MAAM,CAAC,yBAAyB;SAC5C;QACD,YAAY,EAAE;YACZ,IAAI,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY;YAC9C,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,SAAS,EAAE,MAAM,CAAC,YAAY;SAC/B;QACD,eAAe,EAAE;YACf,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,eAAe;YACpD,KAAK,EAAE,MAAM,CAAC,aAAa;YAC3B,SAAS,EAAE,MAAM,CAAC,eAAe;SAClC;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAEzD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAuB;IACpD,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,MAAM,CAAC,yBAAyB,GAAG,CAAC,IAAI,MAAM,CAAC,yBAAyB,GAAG,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,MAAM,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QACzC,IAAI,MAAM,CAAC,eAAe,CAAC,WAAW,GAAG,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxF,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* replay-batch.ts — Replay a FrozenDataset against candidate models.
|
|
3
|
+
*
|
|
4
|
+
* Reuses the engine's replay() function with a deterministic seed for idempotency.
|
|
5
|
+
* Returns ReplayResult with per-item responses, costs, latency, and determinism_score.
|
|
6
|
+
*
|
|
7
|
+
* @see docs/prd.md §5.1 — The gate
|
|
8
|
+
*/
|
|
9
|
+
import type { FrozenDataset, DatasetItem } from '@lanefoundry/gatelane-sdk';
|
|
10
|
+
import type { LLMCaller } from '@lanefoundry/gatelane-engine';
|
|
11
|
+
import { type ReplayResult } from '@lanefoundry/gatelane-engine';
|
|
12
|
+
/** Candidate configuration for replay. */
|
|
13
|
+
export type ReplayCandidate = {
|
|
14
|
+
/** Unique candidate reference (e.g., "gpt-4o", "model:looplane-v3"). */
|
|
15
|
+
ref: string;
|
|
16
|
+
/** Model identifier for the LLM caller. */
|
|
17
|
+
model: string;
|
|
18
|
+
/** Optional system prompt override. */
|
|
19
|
+
systemPrompt?: string;
|
|
20
|
+
/** Sampling temperature. Default: 0.0 for determinism. */
|
|
21
|
+
temperature?: number;
|
|
22
|
+
/** Max tokens. */
|
|
23
|
+
maxTokens?: number;
|
|
24
|
+
};
|
|
25
|
+
/** Configuration for batch replay. */
|
|
26
|
+
export type ReplayBatchArgs = {
|
|
27
|
+
/** The frozen dataset to replay against. */
|
|
28
|
+
dataset: FrozenDataset;
|
|
29
|
+
/** Candidates to evaluate. */
|
|
30
|
+
candidates: ReadonlyArray<ReplayCandidate>;
|
|
31
|
+
/** LLM caller implementation (MockLLMCaller, OpenAIChatCaller, etc.). */
|
|
32
|
+
caller: LLMCaller;
|
|
33
|
+
/** Master seed for deterministic replay. Same seed = same results. */
|
|
34
|
+
seed?: number;
|
|
35
|
+
/** If true, replay each (candidate, item) twice to measure determinism. */
|
|
36
|
+
measureDeterminism?: boolean;
|
|
37
|
+
/** Optional custom message extractor from dataset items. */
|
|
38
|
+
messagesFromItem?: (item: DatasetItem) => {
|
|
39
|
+
role: 'system' | 'user' | 'assistant';
|
|
40
|
+
content: string;
|
|
41
|
+
}[];
|
|
42
|
+
};
|
|
43
|
+
/** Result of a batch replay operation. */
|
|
44
|
+
export type ReplayBatchResult = {
|
|
45
|
+
/** Per-candidate replay results. */
|
|
46
|
+
perCandidate: Record<string, ReplayResult>;
|
|
47
|
+
/** Aggregated determinism score across all candidates. */
|
|
48
|
+
overallDeterminismScore: number;
|
|
49
|
+
/** Total cost across all candidates (USD). */
|
|
50
|
+
totalCostUsd: number;
|
|
51
|
+
/** Total latency across all candidates (ms). */
|
|
52
|
+
totalLatencyMs: number;
|
|
53
|
+
/** Timestamp of replay execution. */
|
|
54
|
+
timestamp: string;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Replay a frozen dataset against multiple candidates.
|
|
58
|
+
*
|
|
59
|
+
* This is a thin wrapper around the engine's replay() that:
|
|
60
|
+
* 1. Converts ReplayCandidate[] to the engine's expected format.
|
|
61
|
+
* 2. Runs replay for each candidate (can be parallelized in future).
|
|
62
|
+
* 3. Aggregates results with overall metrics.
|
|
63
|
+
* 4. Uses a stable seed per (candidate, item) for idempotency.
|
|
64
|
+
*/
|
|
65
|
+
export declare function replayBatch(args: ReplayBatchArgs): Promise<ReplayBatchResult>;
|
|
66
|
+
/**
|
|
67
|
+
* Replay a single candidate for incremental evaluation.
|
|
68
|
+
* Useful when adding a new candidate to an existing gate run.
|
|
69
|
+
*/
|
|
70
|
+
export declare function replaySingle(dataset: FrozenDataset, candidate: ReplayCandidate, caller: LLMCaller, seed?: number, measureDeterminism?: boolean, messagesFromItem?: ReplayBatchArgs['messagesFromItem']): Promise<ReplayResult>;
|
|
71
|
+
//# sourceMappingURL=replay-batch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay-batch.d.ts","sourceRoot":"","sources":["../src/replay-batch.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAA2B,KAAK,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC1F,0CAA0C;AAC1C,MAAM,MAAM,eAAe,GAAG;IAC5B,wEAAwE;IACxE,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,sCAAsC;AACtC,MAAM,MAAM,eAAe,GAAG;IAC5B,4CAA4C;IAC5C,OAAO,EAAE,aAAa,CAAC;IACvB,8BAA8B;IAC9B,UAAU,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC3C,yEAAyE;IACzE,MAAM,EAAE,SAAS,CAAC;IAClB,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,4DAA4D;IAC5D,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE,WAAW,KAAK;QAAE,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACxG,CAAC;AACF,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG;IAC9B,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC3C,0DAA0D;IAC1D,uBAAuB,EAAE,MAAM,CAAC;IAChC,8CAA8C;IAC9C,YAAY,EAAE,MAAM,CAAC;IACrB,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkDnF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,eAAe,EAC1B,MAAM,EAAE,SAAS,EACjB,IAAI,CAAC,EAAE,MAAM,EACb,kBAAkB,CAAC,EAAE,OAAO,EAC5B,gBAAgB,CAAC,EAAE,eAAe,CAAC,kBAAkB,CAAC,GACrD,OAAO,CAAC,YAAY,CAAC,CAcvB"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* replay-batch.ts — Replay a FrozenDataset against candidate models.
|
|
3
|
+
*
|
|
4
|
+
* Reuses the engine's replay() function with a deterministic seed for idempotency.
|
|
5
|
+
* Returns ReplayResult with per-item responses, costs, latency, and determinism_score.
|
|
6
|
+
*
|
|
7
|
+
* @see docs/prd.md §5.1 — The gate
|
|
8
|
+
*/
|
|
9
|
+
import { replay } from '@lanefoundry/gatelane-engine';
|
|
10
|
+
/**
|
|
11
|
+
* Replay a frozen dataset against multiple candidates.
|
|
12
|
+
*
|
|
13
|
+
* This is a thin wrapper around the engine's replay() that:
|
|
14
|
+
* 1. Converts ReplayCandidate[] to the engine's expected format.
|
|
15
|
+
* 2. Runs replay for each candidate (can be parallelized in future).
|
|
16
|
+
* 3. Aggregates results with overall metrics.
|
|
17
|
+
* 4. Uses a stable seed per (candidate, item) for idempotency.
|
|
18
|
+
*/
|
|
19
|
+
export async function replayBatch(args) {
|
|
20
|
+
const { dataset, candidates, caller, seed = 0xC0FFEE, measureDeterminism = false, messagesFromItem } = args;
|
|
21
|
+
const perCandidate = {};
|
|
22
|
+
let overallDeterminismScore = 0;
|
|
23
|
+
let totalCostUsd = 0;
|
|
24
|
+
let totalLatencyMs = 0;
|
|
25
|
+
let totalRows = 0;
|
|
26
|
+
for (const candidate of candidates) {
|
|
27
|
+
const messagesFromItemFn = candidate.systemPrompt
|
|
28
|
+
? (item) => {
|
|
29
|
+
const messages = [
|
|
30
|
+
{ role: 'system', content: candidate.systemPrompt },
|
|
31
|
+
{ role: 'user', content: String(item.input) },
|
|
32
|
+
];
|
|
33
|
+
return messages;
|
|
34
|
+
}
|
|
35
|
+
: messagesFromItem;
|
|
36
|
+
const candidateConfig = {
|
|
37
|
+
ref: candidate.ref,
|
|
38
|
+
model: candidate.model,
|
|
39
|
+
messages_from_item: messagesFromItemFn,
|
|
40
|
+
};
|
|
41
|
+
const replayArgs = {
|
|
42
|
+
dataset,
|
|
43
|
+
candidates: [candidateConfig],
|
|
44
|
+
caller,
|
|
45
|
+
seed,
|
|
46
|
+
measureDeterminism,
|
|
47
|
+
};
|
|
48
|
+
const result = await replay(replayArgs);
|
|
49
|
+
perCandidate[candidate.ref] = result;
|
|
50
|
+
overallDeterminismScore += result.determinism_score * result.rows.length;
|
|
51
|
+
totalCostUsd += result.total_cost_usd;
|
|
52
|
+
totalLatencyMs += result.total_latency_ms;
|
|
53
|
+
totalRows += result.rows.length;
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
perCandidate,
|
|
57
|
+
overallDeterminismScore: totalRows > 0 ? overallDeterminismScore / totalRows : 0,
|
|
58
|
+
totalCostUsd,
|
|
59
|
+
totalLatencyMs,
|
|
60
|
+
timestamp: new Date().toISOString(),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Replay a single candidate for incremental evaluation.
|
|
65
|
+
* Useful when adding a new candidate to an existing gate run.
|
|
66
|
+
*/
|
|
67
|
+
export async function replaySingle(dataset, candidate, caller, seed, measureDeterminism, messagesFromItem) {
|
|
68
|
+
const result = await replayBatch({
|
|
69
|
+
dataset,
|
|
70
|
+
candidates: [candidate],
|
|
71
|
+
caller,
|
|
72
|
+
seed,
|
|
73
|
+
measureDeterminism,
|
|
74
|
+
messagesFromItem,
|
|
75
|
+
});
|
|
76
|
+
const replayResult = result.perCandidate[candidate.ref];
|
|
77
|
+
if (!replayResult) {
|
|
78
|
+
throw new Error(`Replay result not found for candidate: ${candidate.ref}`);
|
|
79
|
+
}
|
|
80
|
+
return replayResult;
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=replay-batch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay-batch.js","sourceRoot":"","sources":["../src/replay-batch.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EAAE,MAAM,EAAsC,MAAM,8BAA8B,CAAC;AA4C1F;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAqB;IACrD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,kBAAkB,GAAG,KAAK,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC;IAE5G,MAAM,YAAY,GAAiC,EAAE,CAAC;IACtD,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,kBAAkB,GAAG,SAAS,CAAC,YAAY;YAC/C,CAAC,CAAC,CAAC,IAAiB,EAAqE,EAAE;gBACvF,MAAM,QAAQ,GAAsE;oBAClF,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,YAAa,EAAE;oBACpD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;iBAC9C,CAAC;gBACF,OAAO,QAAQ,CAAC;YAClB,CAAC;YACH,CAAC,CAAC,gBAAgB,CAAC;QAErB,MAAM,eAAe,GAAG;YACtB,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,kBAAkB,EAAE,kBAAkB;SACvC,CAAC;QAEF,MAAM,UAAU,GAAe;YAC7B,OAAO;YACP,UAAU,EAAE,CAAC,eAAe,CAAC;YAC7B,MAAM;YACN,IAAI;YACJ,kBAAkB;SACnB,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;QAExC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QACrC,uBAAuB,IAAI,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzE,YAAY,IAAI,MAAM,CAAC,cAAc,CAAC;QACtC,cAAc,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAC1C,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;IAClC,CAAC;IAED,OAAO;QACL,YAAY;QACZ,uBAAuB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAChF,YAAY;QACZ,cAAc;QACd,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAsB,EACtB,SAA0B,EAC1B,MAAiB,EACjB,IAAa,EACb,kBAA4B,EAC5B,gBAAsD;IAEtD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,OAAO;QACP,UAAU,EAAE,CAAC,SAAS,CAAC;QACvB,MAAM;QACN,IAAI;QACJ,kBAAkB;QAClB,gBAAgB;KACjB,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACxD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,0CAA0C,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* signed-report.ts — Generate signed PromotionReport with trace IDs and candidate SHAs.
|
|
3
|
+
*
|
|
4
|
+
* Builds a complete PromotionReport from gate run artifacts and signs it
|
|
5
|
+
* using HMAC-SHA256 via the engine's signReport().
|
|
6
|
+
*
|
|
7
|
+
* @see docs/prd.md §5.1 — The gate
|
|
8
|
+
*/
|
|
9
|
+
import type { PromotionReport, PromotionPolicy, PromotionDecision, JudgeStabilityMatrix } from '@lanefoundry/gatelane-sdk';
|
|
10
|
+
import type { FrozenDataset } from '@lanefoundry/gatelane-sdk';
|
|
11
|
+
import type { CandidateMetric } from '@lanefoundry/gatelane-engine';
|
|
12
|
+
/** Input for building a signed PromotionReport. */
|
|
13
|
+
export type SignedReportArgs = {
|
|
14
|
+
/** Unique gate run ID (e.g., UUID). */
|
|
15
|
+
gateRunId: string;
|
|
16
|
+
/** The frozen dataset used for this gate run. */
|
|
17
|
+
dataset: FrozenDataset;
|
|
18
|
+
/** Candidate SHAs (ref -> SHA). From candidate build / source control. */
|
|
19
|
+
candidateShas: Record<string, string>;
|
|
20
|
+
/** Judge SHAs (ref -> SHA). From judge model versions. */
|
|
21
|
+
judgeShas: Record<string, string>;
|
|
22
|
+
/** SHA of the scorer/evaluator code. */
|
|
23
|
+
scorerCodeSha: string;
|
|
24
|
+
/** Baseline metrics (per-metric name -> value). */
|
|
25
|
+
baselineMetrics: Record<string, number>;
|
|
26
|
+
/** Per-candidate metrics from compareScores(). */
|
|
27
|
+
candidateMetrics: Record<string, CandidateMetric>;
|
|
28
|
+
/** Judge stability matrix from compareScores(). */
|
|
29
|
+
judgeMatrix: JudgeStabilityMatrix;
|
|
30
|
+
/** Policy applied to this gate run. */
|
|
31
|
+
policy: PromotionPolicy;
|
|
32
|
+
/** The promotion decision. */
|
|
33
|
+
decision: PromotionDecision;
|
|
34
|
+
/** Optional approver (when policy.approval_required). */
|
|
35
|
+
approver?: string;
|
|
36
|
+
/** Trace IDs for observability (from capture SDK / OTel). */
|
|
37
|
+
traceIds?: ReadonlyArray<string>;
|
|
38
|
+
/** HMAC signing key (from env GATELANE_REPORT_SIGNING_KEY). */
|
|
39
|
+
signingKey: string;
|
|
40
|
+
};
|
|
41
|
+
/** Result of signed report generation. */
|
|
42
|
+
export type SignedReportResult = {
|
|
43
|
+
/** The signed PromotionReport. */
|
|
44
|
+
report: PromotionReport;
|
|
45
|
+
/** Canonical JSON used for signing (without signature field). */
|
|
46
|
+
canonicalJson: string;
|
|
47
|
+
/** The signature (base64url-encoded HMAC-SHA256). */
|
|
48
|
+
signature: string;
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Build and sign a PromotionReport from gate run artifacts.
|
|
52
|
+
*
|
|
53
|
+
* This assembles all the pieces required by the PromotionReport type,
|
|
54
|
+
* computes the signature, and returns the complete signed report.
|
|
55
|
+
*/
|
|
56
|
+
export declare function buildSignedReport(args: SignedReportArgs): Promise<SignedReportResult>;
|
|
57
|
+
/**
|
|
58
|
+
* Verify a PromotionReport's signature.
|
|
59
|
+
* Returns true if the signature matches the canonical JSON.
|
|
60
|
+
*/
|
|
61
|
+
export declare function verifySignedReport(report: PromotionReport, signingKey: string): Promise<boolean>;
|
|
62
|
+
/**
|
|
63
|
+
* Generate candidate SHAs from source references.
|
|
64
|
+
* In production, these would come from git commit SHAs or container image digests.
|
|
65
|
+
*/
|
|
66
|
+
export declare function generateCandidateShas(candidates: ReadonlyArray<{
|
|
67
|
+
ref: string;
|
|
68
|
+
sourceRef: string;
|
|
69
|
+
}>): Promise<Record<string, string>>;
|
|
70
|
+
/**
|
|
71
|
+
* Generate judge SHAs from judge references.
|
|
72
|
+
*/
|
|
73
|
+
export declare function generateJudgeShas(judges: ReadonlyArray<{
|
|
74
|
+
ref: string;
|
|
75
|
+
model: string;
|
|
76
|
+
version?: string;
|
|
77
|
+
}>): Promise<Record<string, string>>;
|
|
78
|
+
/**
|
|
79
|
+
* Export a PromotionReport as canonical JSON for audit logs.
|
|
80
|
+
*/
|
|
81
|
+
export declare function exportReportForAudit(report: PromotionReport): string;
|
|
82
|
+
//# sourceMappingURL=signed-report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"signed-report.d.ts","sourceRoot":"","sources":["../src/signed-report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,oBAAoB,EACrB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAIpE,mDAAmD;AACnD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,OAAO,EAAE,aAAa,CAAC;IACvB,0EAA0E;IAC1E,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAClD,mDAAmD;IACnD,WAAW,EAAE,oBAAoB,CAAC;IAClC,uCAAuC;IACvC,MAAM,EAAE,eAAe,CAAC;IACxB,8BAA8B;IAC9B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IACjC,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,0CAA0C;AAC1C,MAAM,MAAM,kBAAkB,GAAG;IAC/B,kCAAkC;IAClC,MAAM,EAAE,eAAe,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAoE3F;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGtG;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,aAAa,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,GAC5D,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAMjC;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,aAAa,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GACtE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAOjC;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAEpE"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* signed-report.ts — Generate signed PromotionReport with trace IDs and candidate SHAs.
|
|
3
|
+
*
|
|
4
|
+
* Builds a complete PromotionReport from gate run artifacts and signs it
|
|
5
|
+
* using HMAC-SHA256 via the engine's signReport().
|
|
6
|
+
*
|
|
7
|
+
* @see docs/prd.md §5.1 — The gate
|
|
8
|
+
*/
|
|
9
|
+
import { signReport, canonicalizeReport } from '@lanefoundry/gatelane-engine';
|
|
10
|
+
import { shaOfCandidateRef } from '@lanefoundry/gatelane-sdk';
|
|
11
|
+
/**
|
|
12
|
+
* Build and sign a PromotionReport from gate run artifacts.
|
|
13
|
+
*
|
|
14
|
+
* This assembles all the pieces required by the PromotionReport type,
|
|
15
|
+
* computes the signature, and returns the complete signed report.
|
|
16
|
+
*/
|
|
17
|
+
export async function buildSignedReport(args) {
|
|
18
|
+
const { gateRunId, dataset, candidateShas, judgeShas, scorerCodeSha, baselineMetrics, candidateMetrics, judgeMatrix, policy, approver, signingKey, } = args;
|
|
19
|
+
// Validate required fields
|
|
20
|
+
if (!signingKey || signingKey.length < 16) {
|
|
21
|
+
throw new Error('signingKey must be >= 16 characters');
|
|
22
|
+
}
|
|
23
|
+
if (!gateRunId) {
|
|
24
|
+
throw new Error('gateRunId is required');
|
|
25
|
+
}
|
|
26
|
+
if (!dataset.content_hash) {
|
|
27
|
+
throw new Error('dataset must have content_hash');
|
|
28
|
+
}
|
|
29
|
+
// Build candidate_metrics in the shape expected by PromotionReport
|
|
30
|
+
const candidateMetricsForReport = {};
|
|
31
|
+
for (const [ref, metric] of Object.entries(candidateMetrics)) {
|
|
32
|
+
candidateMetricsForReport[ref] = {
|
|
33
|
+
aggregate_delta: metric.aggregate_delta,
|
|
34
|
+
cost_delta: metric.cost_delta,
|
|
35
|
+
latency_delta: metric.latency_delta,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
// Build the report (without signature initially)
|
|
39
|
+
const report = {
|
|
40
|
+
id: `report-${gateRunId}`,
|
|
41
|
+
gate_run_id: gateRunId,
|
|
42
|
+
dataset_content_hash: dataset.content_hash,
|
|
43
|
+
dataset_version: dataset.version,
|
|
44
|
+
candidate_shas: candidateShas,
|
|
45
|
+
judge_shas: judgeShas,
|
|
46
|
+
scorer_code_sha: scorerCodeSha,
|
|
47
|
+
baseline_metrics: baselineMetrics,
|
|
48
|
+
candidate_metrics: candidateMetricsForReport,
|
|
49
|
+
judge_matrix: judgeMatrix,
|
|
50
|
+
policy,
|
|
51
|
+
approver,
|
|
52
|
+
timestamp: new Date().toISOString(),
|
|
53
|
+
signature: '', // placeholder; will be filled after signing
|
|
54
|
+
};
|
|
55
|
+
// Sign the report
|
|
56
|
+
const signature = await signReport(report, signingKey);
|
|
57
|
+
// Return signed report
|
|
58
|
+
const signedReport = {
|
|
59
|
+
...report,
|
|
60
|
+
signature,
|
|
61
|
+
};
|
|
62
|
+
return {
|
|
63
|
+
report: signedReport,
|
|
64
|
+
canonicalJson: canonicalizeReport(signedReport),
|
|
65
|
+
signature,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Verify a PromotionReport's signature.
|
|
70
|
+
* Returns true if the signature matches the canonical JSON.
|
|
71
|
+
*/
|
|
72
|
+
export async function verifySignedReport(report, signingKey) {
|
|
73
|
+
const { verifyReport } = await import('@lanefoundry/gatelane-engine/sign');
|
|
74
|
+
return verifyReport(report, signingKey);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Generate candidate SHAs from source references.
|
|
78
|
+
* In production, these would come from git commit SHAs or container image digests.
|
|
79
|
+
*/
|
|
80
|
+
export async function generateCandidateShas(candidates) {
|
|
81
|
+
const shas = {};
|
|
82
|
+
for (const { ref, sourceRef } of candidates) {
|
|
83
|
+
shas[ref] = await shaOfCandidateRef(sourceRef);
|
|
84
|
+
}
|
|
85
|
+
return shas;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Generate judge SHAs from judge references.
|
|
89
|
+
*/
|
|
90
|
+
export async function generateJudgeShas(judges) {
|
|
91
|
+
const shas = {};
|
|
92
|
+
for (const { ref, model, version } of judges) {
|
|
93
|
+
const versioned = version ? `${model}@${version}` : model;
|
|
94
|
+
shas[ref] = await shaOfCandidateRef(versioned);
|
|
95
|
+
}
|
|
96
|
+
return shas;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Export a PromotionReport as canonical JSON for audit logs.
|
|
100
|
+
*/
|
|
101
|
+
export function exportReportForAudit(report) {
|
|
102
|
+
return canonicalizeReport(report);
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=signed-report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"signed-report.js","sourceRoot":"","sources":["../src/signed-report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAUH,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AA0C9D;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAsB;IAC5D,MAAM,EACJ,SAAS,EACT,OAAO,EACP,aAAa,EACb,SAAS,EACT,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,MAAM,EACN,QAAQ,EACR,UAAU,GACX,GAAG,IAAI,CAAC;IAET,2BAA2B;IAC3B,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,mEAAmE;IACnE,MAAM,yBAAyB,GAAyC,EAAE,CAAC;IAC3E,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAC7D,yBAAyB,CAAC,GAAG,CAAC,GAAG;YAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC;IACJ,CAAC;IAED,iDAAiD;IACjD,MAAM,MAAM,GAAoB;QAC9B,EAAE,EAAE,UAAU,SAAS,EAAE;QACzB,WAAW,EAAE,SAAS;QACtB,oBAAoB,EAAE,OAAO,CAAC,YAAY;QAC1C,eAAe,EAAE,OAAO,CAAC,OAAO;QAChC,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,SAAS;QACrB,eAAe,EAAE,aAAa;QAC9B,gBAAgB,EAAE,eAAe;QACjC,iBAAiB,EAAE,yBAAyB;QAC5C,YAAY,EAAE,WAAW;QACzB,MAAM;QACN,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,SAAS,EAAE,EAAE,EAAE,4CAA4C;KAC5D,CAAC;IAEF,kBAAkB;IAClB,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAEvD,uBAAuB;IACvB,MAAM,YAAY,GAAoB;QACpC,GAAG,MAAM;QACT,SAAS;KACV,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,YAAY;QACpB,aAAa,EAAE,kBAAkB,CAAC,YAAY,CAAC;QAC/C,SAAS;KACV,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAuB,EAAE,UAAkB;IAClF,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAC;IAC3E,OAAO,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,UAA6D;IAE7D,MAAM,IAAI,GAA2B,EAAE,CAAC;IACxC,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAuE;IAEvE,MAAM,IAAI,GAA2B,EAAE,CAAC;IACxC,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,MAAM,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC1D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAuB;IAC1D,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,67 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lanefoundry/source-prod-slice",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Production slice tooling — freeze, replay, compare, promote, audit.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/lanefoundry/gatelane.git",
|
|
9
|
+
"directory": "packages/source-prod-slice"
|
|
10
|
+
},
|
|
11
|
+
"homepage": "https://github.com/lanefoundry/gatelane/tree/main/packages/source-prod-slice",
|
|
12
|
+
"type": "module",
|
|
13
|
+
"main": "./dist/index.js",
|
|
14
|
+
"types": "./dist/index.d.ts",
|
|
15
|
+
"exports": {
|
|
16
|
+
".": {
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"import": "./dist/index.js"
|
|
19
|
+
},
|
|
20
|
+
"./freeze-slice": {
|
|
21
|
+
"types": "./dist/freeze-slice.d.ts",
|
|
22
|
+
"import": "./dist/freeze-slice.js"
|
|
23
|
+
},
|
|
24
|
+
"./replay-batch": {
|
|
25
|
+
"types": "./dist/replay-batch.d.ts",
|
|
26
|
+
"import": "./dist/replay-batch.js"
|
|
27
|
+
},
|
|
28
|
+
"./compare-scores": {
|
|
29
|
+
"types": "./dist/compare-scores.d.ts",
|
|
30
|
+
"import": "./dist/compare-scores.js"
|
|
31
|
+
},
|
|
32
|
+
"./promotion-decision": {
|
|
33
|
+
"types": "./dist/promotion-decision.d.ts",
|
|
34
|
+
"import": "./dist/promotion-decision.js"
|
|
35
|
+
},
|
|
36
|
+
"./signed-report": {
|
|
37
|
+
"types": "./dist/signed-report.d.ts",
|
|
38
|
+
"import": "./dist/signed-report.js"
|
|
39
|
+
},
|
|
40
|
+
"./canary-orchestrator": {
|
|
41
|
+
"types": "./dist/canary-orchestrator.d.ts",
|
|
42
|
+
"import": "./dist/canary-orchestrator.js"
|
|
43
|
+
},
|
|
44
|
+
"./audit-export": {
|
|
45
|
+
"types": "./dist/audit-export.d.ts",
|
|
46
|
+
"import": "./dist/audit-export.js"
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
"files": [
|
|
50
|
+
"dist"
|
|
51
|
+
],
|
|
52
|
+
"publishConfig": {
|
|
53
|
+
"access": "public"
|
|
54
|
+
},
|
|
55
|
+
"scripts": {
|
|
56
|
+
"build": "tsc -p tsconfig.json",
|
|
57
|
+
"test": "vitest run"
|
|
58
|
+
},
|
|
59
|
+
"dependencies": {
|
|
60
|
+
"@lanefoundry/gatelane-sdk": "0.1.0",
|
|
61
|
+
"@lanefoundry/gatelane-engine": "0.1.0"
|
|
62
|
+
},
|
|
63
|
+
"devDependencies": {
|
|
64
|
+
"typescript": "^5.5.4",
|
|
65
|
+
"vitest": "^2.0.5"
|
|
66
|
+
}
|
|
67
|
+
}
|
package/README.md
DELETED