@trazum/core 1.50.7 → 1.50.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/quality-gate.d.ts +134 -0
- package/dist/quality-gate.d.ts.map +1 -0
- package/dist/quality-gate.js +185 -0
- package/dist/quality-gate.js.map +1 -0
- package/dist/semantic.d.ts +148 -0
- package/dist/semantic.d.ts.map +1 -0
- package/dist/semantic.js +163 -0
- package/dist/semantic.js.map +1 -0
- package/package.json +1 -1
- package/src/index.ts +21 -0
- package/src/quality-gate.ts +253 -0
- package/src/semantic.ts +252 -0
package/dist/index.d.ts
CHANGED
|
@@ -19,7 +19,11 @@ export type { PlanAction, PlanActionKind, PlanAssumption, PlanDocument } from '.
|
|
|
19
19
|
export { verifyPlan } from './verify.js';
|
|
20
20
|
export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
|
|
21
21
|
export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
|
|
22
|
+
export { verifySemanticProposals, semanticPassCost, SEMANTIC_SYSTEM_PROMPT, ALREADY_DETECTED_SIMILARITY, } from './semantic.js';
|
|
23
|
+
export type { SemanticFinding, SemanticKind, SemanticProposal, SemanticRejection, SemanticResult, } from './semantic.js';
|
|
22
24
|
export { runExperiment } from './experiment.js';
|
|
25
|
+
export { qualityGate, MIN_OUTCOMES_EACH_SIDE, MAX_MODEL_MIX_DRIFT, MAX_VOLUME_RATIO, MAX_COVERAGE_DRIFT, } from './quality-gate.js';
|
|
26
|
+
export type { Confounder, GateSide, GateUnknown, GateVerdict, QualityGate } from './quality-gate.js';
|
|
23
27
|
export type { ArmResult, ExperimentArm, ExperimentDeclaration, ExperimentResult, Marginal, NotSeparableReason, Separation, } from './experiment.js';
|
|
24
28
|
export { ladderArithmetic, ladderPosition, validateLadder, MIN_CALLS_FOR_LADDER, BREAK_EVEN_BAND, } from './ladder.js';
|
|
25
29
|
export type { LadderArithmetic, LadderPolicy, LadderPosition, LadderProblem, LadderUnknown, LadderVerdict, } from './ladder.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAGlF,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAQ7F,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC3F,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,YAAY,EACV,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,gBAAgB,EAChB,QAAQ,EACR,kBAAkB,EAClB,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,aAAa,EACb,aAAa,EACb,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACpF,YAAY,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC7E,YAAY,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7E,YAAY,EACV,WAAW,EACX,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,aAAa,GACd,MAAM,WAAW,CAAC;AACnB,YAAY,EACV,aAAa,EACb,aAAa,EACb,cAAc,EACd,YAAY,EACZ,sBAAsB,EACtB,UAAU,GACX,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAC7E,YAAY,EACV,YAAY,EACZ,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,WAAW,EACX,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC1F,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,OAAO,EACP,kBAAkB,EAClB,WAAW,GACZ,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAClG,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrG,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC3D,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKxE,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACxE,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGzF,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGnE,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACxF,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC7G,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAIlF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAExF,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAChF,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAClG,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACtG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC1F,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpG,YAAY,EACV,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AACjF,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAG3E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAMpD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrF,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAKhD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,YAAY,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAKtD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,eAAe,EACf,YAAY,EACZ,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAItF,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGhG,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAGpE,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAG3E,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAK3F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACzF,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAGjF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGxF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,aAAa,GACd,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKzD,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAGlF,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAQ7F,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC3F,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,GACf,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrG,YAAY,EACV,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,gBAAgB,EAChB,QAAQ,EACR,kBAAkB,EAClB,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,aAAa,EACb,aAAa,EACb,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACpF,YAAY,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC7E,YAAY,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7E,YAAY,EACV,WAAW,EACX,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,aAAa,GACd,MAAM,WAAW,CAAC;AACnB,YAAY,EACV,aAAa,EACb,aAAa,EACb,cAAc,EACd,YAAY,EACZ,sBAAsB,EACtB,UAAU,GACX,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAC7E,YAAY,EACV,YAAY,EACZ,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,WAAW,EACX,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC1F,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,OAAO,EACP,kBAAkB,EAClB,WAAW,GACZ,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAClG,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrG,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC3D,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKxE,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACxE,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGzF,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGnE,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACxF,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC7G,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAIlF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAExF,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAChF,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAClG,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACtG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC1F,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpG,YAAY,EACV,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AACjF,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAG3E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAMpD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrF,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAKhD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,YAAY,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAKtD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,eAAe,EACf,YAAY,EACZ,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAItF,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGhG,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAGpE,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAG3E,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAK3F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACzF,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAGjF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGxF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,aAAa,GACd,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKzD,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -22,7 +22,9 @@ export { buildPlan, planLabelName, parsePlanDocument, PLAN_ACTION_KINDS } from '
|
|
|
22
22
|
export { verifyPlan } from './verify.js';
|
|
23
23
|
export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
|
|
24
24
|
export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
|
|
25
|
+
export { verifySemanticProposals, semanticPassCost, SEMANTIC_SYSTEM_PROMPT, ALREADY_DETECTED_SIMILARITY, } from './semantic.js';
|
|
25
26
|
export { runExperiment } from './experiment.js';
|
|
27
|
+
export { qualityGate, MIN_OUTCOMES_EACH_SIDE, MAX_MODEL_MIX_DRIFT, MAX_VOLUME_RATIO, MAX_COVERAGE_DRIFT, } from './quality-gate.js';
|
|
26
28
|
export { ladderArithmetic, ladderPosition, validateLadder, MIN_CALLS_FOR_LADDER, BREAK_EVEN_BAND, } from './ladder.js';
|
|
27
29
|
export { perOutcome, rankPerOutcome, MIN_OUTCOMES_FOR_RATE, MIN_COVERAGE_FOR_RATE, } from './per-outcome.js';
|
|
28
30
|
export { gatewayDecision, usageFromResponse, FAILURE_POLICIES } from './gateway.js';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AAWpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAClF,gFAAgF;AAChF,4EAA4E;AAC5E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtF,2EAA2E;AAC3E,+EAA+E;AAC/E,8EAA8E;AAC9E,0EAA0E;AAC1E,yCAAyC;AACzC,4EAA4E;AAC5E,qDAAqD;AACrD,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG3F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AAWpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAClF,gFAAgF;AAChF,4EAA4E;AAC5E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtF,2EAA2E;AAC3E,+EAA+E;AAC/E,8EAA8E;AAC9E,0EAA0E;AAC1E,yCAAyC;AACzC,4EAA4E;AAC5E,qDAAqD;AACrD,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG3F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,eAAe,CAAC;AAQvB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAW3B,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AASrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAiB1B,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAYpF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAW7E,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAqB7E,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAU7E,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AAiBxB,4EAA4E;AAC5E,8EAA8E;AAC9E,8EAA8E;AAC9E,sBAAsB;AACtB,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAExE,2EAA2E;AAC3E,8DAA8D;AAC9D,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnE,2EAA2E;AAC3E,+DAA+D;AAC/D,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGxF,yEAAyE;AACzE,mEAAmE;AACnE,2BAA2B;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE7D,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACxF,4EAA4E;AAC5E,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAOhF,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAI1C,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQpG,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AAItB,uEAAuE;AACvE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,8EAA8E;AAC9E,+EAA+E;AAC/E,6EAA6E;AAC7E,UAAU;AACV,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAErF,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AAQvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,+EAA+E;AAC/E,+EAA+E;AAC/E,6DAA6D;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/C,gFAAgF;AAChF,gFAAgF;AAChF,iEAAiE;AACjE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAOtE,4EAA4E;AAC5E,6EAA6E;AAC7E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAGtF,8EAA8E;AAC9E,2EAA2E;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAWlB,uBAAuB;AACvB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AAGzB,sBAAsB;AACtB,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEpE,kDAAkD;AAClD,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAGlB,6BAA6B;AAC7B,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGpE,qEAAqE;AACrE,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAG3E,kFAAkF;AAClF,kFAAkF;AAClF,wCAAwC;AACxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGzF,mEAAmE;AACnE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGjE,gCAAgC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQ9C,0EAA0E;AAC1E,+EAA+E;AAC/E,8EAA8E;AAC9E,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzD,8EAA8E;AAC9E,6EAA6E;AAC7E,2DAA2D;AAC3D,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The failure that actually matters, and the four ways of getting it wrong.
|
|
3
|
+
*
|
|
4
|
+
* CI has been able to fail a build for tokens since 1.4 and for dollars since
|
|
5
|
+
* 1.21. A prompt edit that quietly made the product worse has never been
|
|
6
|
+
* gateable — which means every cost saving this tool has ever recommended went
|
|
7
|
+
* into a repository with its most important consequence unmeasured.
|
|
8
|
+
*
|
|
9
|
+
* ## This is a before-and-after, not an experiment
|
|
10
|
+
*
|
|
11
|
+
* Worth saying in the first paragraph, because the arithmetic is the same as
|
|
12
|
+
* `experiment`'s and the epistemics are not. An experiment splits traffic
|
|
13
|
+
* randomly, so the two arms differ only in the thing under test. A
|
|
14
|
+
* before-and-after splits it by **time**, and everything else that changed at
|
|
15
|
+
* the same time is in the difference too.
|
|
16
|
+
*
|
|
17
|
+
* So this module spends most of its code looking for reasons *not* to blame the
|
|
18
|
+
* prompt, and it reports `cannot-tell` far more readily than a randomised
|
|
19
|
+
* comparison would. That is not timidity. A gate that blames the prompt because
|
|
20
|
+
* the prompt is the thing it can see will be switched off within a month, and a
|
|
21
|
+
* switched-off gate catches nothing at all.
|
|
22
|
+
*
|
|
23
|
+
* ## The three confounders it can actually see
|
|
24
|
+
*
|
|
25
|
+
* **The model moved.** If the mix of models shifted across the boundary, the
|
|
26
|
+
* prompt is not the only variable, and the drop may be entirely somebody else's
|
|
27
|
+
* migration.
|
|
28
|
+
*
|
|
29
|
+
* **The volume moved.** A workload whose traffic tripled is usually a workload
|
|
30
|
+
* whose *population* changed — a new surface, a new customer, a marketing
|
|
31
|
+
* campaign — and the questions being asked are not the questions from before.
|
|
32
|
+
*
|
|
33
|
+
* **The coverage moved.** The subtle one, and the one nobody thinks of: if the
|
|
34
|
+
* share of calls recording an outcome changed, the two rates describe different
|
|
35
|
+
* populations. A team that started instrumenting its hard cases will see its
|
|
36
|
+
* measured rate fall without anything getting worse.
|
|
37
|
+
*
|
|
38
|
+
* Any of them present, and the verdict is `cannot-tell` with the confounder
|
|
39
|
+
* **named**. Not a hedge attached to a blame — a refusal to blame.
|
|
40
|
+
*
|
|
41
|
+
* ## What it cannot see, and says so
|
|
42
|
+
*
|
|
43
|
+
* Everything else deployed that day. This module has one label's numbers and no
|
|
44
|
+
* knowledge of the world, and it never pretends otherwise: a `dropped` verdict
|
|
45
|
+
* is a statement that the rate fell and the three things it can check did not
|
|
46
|
+
* move, which is a smaller claim than "the prompt did it" and is the largest
|
|
47
|
+
* claim the evidence supports.
|
|
48
|
+
*/
|
|
49
|
+
import type { ArmResult, ExperimentArm } from './experiment.js';
|
|
50
|
+
import type { OutcomeVocabulary } from './outcome.js';
|
|
51
|
+
/**
|
|
52
|
+
* Outcomes each side of the change needs before a verdict is offered.
|
|
53
|
+
*
|
|
54
|
+
* A hundred rather than the ten used for a rate elsewhere, because this one
|
|
55
|
+
* fails builds. The cost of a wrong `dropped` is somebody reverting a good
|
|
56
|
+
* change and losing the saving; the cost of a wrong `cannot-tell` is waiting a
|
|
57
|
+
* day. Those are not symmetric and the threshold is not either.
|
|
58
|
+
*/
|
|
59
|
+
export declare const MIN_OUTCOMES_EACH_SIDE = 100;
|
|
60
|
+
/** How far the model mix may move before the comparison stops being about the prompt. */
|
|
61
|
+
export declare const MAX_MODEL_MIX_DRIFT = 0.1;
|
|
62
|
+
/**
|
|
63
|
+
* How far the call volume may move, as a ratio.
|
|
64
|
+
*
|
|
65
|
+
* Half again, either way. Ordinary week-to-week traffic moves less than that;
|
|
66
|
+
* a workload that doubled is usually a workload whose population changed.
|
|
67
|
+
*/
|
|
68
|
+
export declare const MAX_VOLUME_RATIO = 1.5;
|
|
69
|
+
/** How far outcome coverage may move before the two rates describe different populations. */
|
|
70
|
+
export declare const MAX_COVERAGE_DRIFT = 0.1;
|
|
71
|
+
export type Confounder = {
|
|
72
|
+
kind: 'model-mix-moved';
|
|
73
|
+
drift: number;
|
|
74
|
+
model: string;
|
|
75
|
+
} | {
|
|
76
|
+
kind: 'volume-moved';
|
|
77
|
+
beforeCalls: number;
|
|
78
|
+
afterCalls: number;
|
|
79
|
+
ratio: number;
|
|
80
|
+
} | {
|
|
81
|
+
kind: 'coverage-moved';
|
|
82
|
+
before: number;
|
|
83
|
+
after: number;
|
|
84
|
+
};
|
|
85
|
+
export type GateVerdict = 'held' | 'dropped' | 'cannot-tell';
|
|
86
|
+
export type GateUnknown = 'too-few-before' | 'too-few-after' | 'confounded' | 'no-vocabulary' | 'not-separable';
|
|
87
|
+
/** One side of the boundary, as the caller measured it. */
|
|
88
|
+
export interface GateSide {
|
|
89
|
+
arm: ExperimentArm;
|
|
90
|
+
/** Every call in the window, outcome or not — the coverage denominator. */
|
|
91
|
+
calls: number;
|
|
92
|
+
/** Spend per model, for the mix comparison. */
|
|
93
|
+
usdByModel: Array<{
|
|
94
|
+
model: string;
|
|
95
|
+
usd: number;
|
|
96
|
+
}>;
|
|
97
|
+
}
|
|
98
|
+
export interface QualityGate {
|
|
99
|
+
verdict: GateVerdict;
|
|
100
|
+
/** Why, when the verdict is `cannot-tell`. A refusal never arrives bare. */
|
|
101
|
+
unknown: GateUnknown | null;
|
|
102
|
+
before: ArmResult;
|
|
103
|
+
after: ArmResult;
|
|
104
|
+
/** The 95% interval on (after − before), or null. */
|
|
105
|
+
difference: {
|
|
106
|
+
point: number;
|
|
107
|
+
low: number;
|
|
108
|
+
high: number;
|
|
109
|
+
} | null;
|
|
110
|
+
/**
|
|
111
|
+
* Everything found that means the prompt is not the only variable.
|
|
112
|
+
*
|
|
113
|
+
* Non-empty forces `cannot-tell`. Reported even on a `held` verdict, because
|
|
114
|
+
* a rate that held while the model changed underneath is not evidence that
|
|
115
|
+
* the prompt is fine either.
|
|
116
|
+
*/
|
|
117
|
+
confounders: Confounder[];
|
|
118
|
+
/**
|
|
119
|
+
* The cost half — measured, and kept beside the quality half rather than
|
|
120
|
+
* merged with it. The sentence teams argue about needs both.
|
|
121
|
+
*/
|
|
122
|
+
cost: {
|
|
123
|
+
beforeUsdPerCall: number;
|
|
124
|
+
afterUsdPerCall: number;
|
|
125
|
+
deltaUsdPerCall: number;
|
|
126
|
+
} | null;
|
|
127
|
+
/** Outcomes behind the comparison, so the claim carries its own sample size. */
|
|
128
|
+
outcomes: {
|
|
129
|
+
before: number;
|
|
130
|
+
after: number;
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
export declare function qualityGate(before: GateSide, after: GateSide, vocabulary: OutcomeVocabulary | null): QualityGate;
|
|
134
|
+
//# sourceMappingURL=quality-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quality-gate.d.ts","sourceRoot":"","sources":["../src/quality-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAEtD;;;;;;;GAOG;AACH,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,yFAAyF;AACzF,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC,6FAA6F;AAC7F,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChF;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9D,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC;AAE7D,MAAM,MAAM,WAAW,GACnB,gBAAgB,GAChB,eAAe,GACf,YAAY,GACZ,eAAe,GACf,eAAe,CAAC;AAEpB,2DAA2D;AAC3D,MAAM,WAAW,QAAQ;IACvB,GAAG,EAAE,aAAa,CAAC;IACnB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,UAAU,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,WAAW,CAAC;IACrB,4EAA4E;IAC5E,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,SAAS,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,qDAAqD;IACrD,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAChE;;;;;;OAMG;IACH,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B;;;OAGG;IACH,IAAI,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC5F,gFAAgF;IAChF,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7C;AAwBD,wBAAgB,WAAW,CACzB,MAAM,EAAE,QAAQ,EAChB,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,iBAAiB,GAAG,IAAI,GACnC,WAAW,CAqGb"}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The failure that actually matters, and the four ways of getting it wrong.
|
|
3
|
+
*
|
|
4
|
+
* CI has been able to fail a build for tokens since 1.4 and for dollars since
|
|
5
|
+
* 1.21. A prompt edit that quietly made the product worse has never been
|
|
6
|
+
* gateable — which means every cost saving this tool has ever recommended went
|
|
7
|
+
* into a repository with its most important consequence unmeasured.
|
|
8
|
+
*
|
|
9
|
+
* ## This is a before-and-after, not an experiment
|
|
10
|
+
*
|
|
11
|
+
* Worth saying in the first paragraph, because the arithmetic is the same as
|
|
12
|
+
* `experiment`'s and the epistemics are not. An experiment splits traffic
|
|
13
|
+
* randomly, so the two arms differ only in the thing under test. A
|
|
14
|
+
* before-and-after splits it by **time**, and everything else that changed at
|
|
15
|
+
* the same time is in the difference too.
|
|
16
|
+
*
|
|
17
|
+
* So this module spends most of its code looking for reasons *not* to blame the
|
|
18
|
+
* prompt, and it reports `cannot-tell` far more readily than a randomised
|
|
19
|
+
* comparison would. That is not timidity. A gate that blames the prompt because
|
|
20
|
+
* the prompt is the thing it can see will be switched off within a month, and a
|
|
21
|
+
* switched-off gate catches nothing at all.
|
|
22
|
+
*
|
|
23
|
+
* ## The three confounders it can actually see
|
|
24
|
+
*
|
|
25
|
+
* **The model moved.** If the mix of models shifted across the boundary, the
|
|
26
|
+
* prompt is not the only variable, and the drop may be entirely somebody else's
|
|
27
|
+
* migration.
|
|
28
|
+
*
|
|
29
|
+
* **The volume moved.** A workload whose traffic tripled is usually a workload
|
|
30
|
+
* whose *population* changed — a new surface, a new customer, a marketing
|
|
31
|
+
* campaign — and the questions being asked are not the questions from before.
|
|
32
|
+
*
|
|
33
|
+
* **The coverage moved.** The subtle one, and the one nobody thinks of: if the
|
|
34
|
+
* share of calls recording an outcome changed, the two rates describe different
|
|
35
|
+
* populations. A team that started instrumenting its hard cases will see its
|
|
36
|
+
* measured rate fall without anything getting worse.
|
|
37
|
+
*
|
|
38
|
+
* Any of them present, and the verdict is `cannot-tell` with the confounder
|
|
39
|
+
* **named**. Not a hedge attached to a blame — a refusal to blame.
|
|
40
|
+
*
|
|
41
|
+
* ## What it cannot see, and says so
|
|
42
|
+
*
|
|
43
|
+
* Everything else deployed that day. This module has one label's numbers and no
|
|
44
|
+
* knowledge of the world, and it never pretends otherwise: a `dropped` verdict
|
|
45
|
+
* is a statement that the rate fell and the three things it can check did not
|
|
46
|
+
* move, which is a smaller claim than "the prompt did it" and is the largest
|
|
47
|
+
* claim the evidence supports.
|
|
48
|
+
*/
|
|
49
|
+
import { runExperiment } from './experiment.js';
|
|
50
|
+
/**
|
|
51
|
+
* Outcomes each side of the change needs before a verdict is offered.
|
|
52
|
+
*
|
|
53
|
+
* A hundred rather than the ten used for a rate elsewhere, because this one
|
|
54
|
+
* fails builds. The cost of a wrong `dropped` is somebody reverting a good
|
|
55
|
+
* change and losing the saving; the cost of a wrong `cannot-tell` is waiting a
|
|
56
|
+
* day. Those are not symmetric and the threshold is not either.
|
|
57
|
+
*/
|
|
58
|
+
export const MIN_OUTCOMES_EACH_SIDE = 100;
|
|
59
|
+
/** How far the model mix may move before the comparison stops being about the prompt. */
|
|
60
|
+
export const MAX_MODEL_MIX_DRIFT = 0.1;
|
|
61
|
+
/**
|
|
62
|
+
* How far the call volume may move, as a ratio.
|
|
63
|
+
*
|
|
64
|
+
* Half again, either way. Ordinary week-to-week traffic moves less than that;
|
|
65
|
+
* a workload that doubled is usually a workload whose population changed.
|
|
66
|
+
*/
|
|
67
|
+
export const MAX_VOLUME_RATIO = 1.5;
|
|
68
|
+
/** How far outcome coverage may move before the two rates describe different populations. */
|
|
69
|
+
export const MAX_COVERAGE_DRIFT = 0.1;
|
|
70
|
+
function mixDrift(before, after) {
|
|
71
|
+
const share = (rows) => {
|
|
72
|
+
const total = rows.reduce((sum, row) => sum + row.usd, 0);
|
|
73
|
+
const out = new Map();
|
|
74
|
+
if (total <= 0)
|
|
75
|
+
return out;
|
|
76
|
+
for (const row of rows)
|
|
77
|
+
out.set(row.model, (out.get(row.model) ?? 0) + row.usd / total);
|
|
78
|
+
return out;
|
|
79
|
+
};
|
|
80
|
+
const a = share(before);
|
|
81
|
+
const b = share(after);
|
|
82
|
+
let worst = null;
|
|
83
|
+
for (const model of new Set([...a.keys(), ...b.keys()])) {
|
|
84
|
+
const drift = Math.abs((b.get(model) ?? 0) - (a.get(model) ?? 0));
|
|
85
|
+
if (worst === null || drift > worst.drift)
|
|
86
|
+
worst = { model, drift };
|
|
87
|
+
}
|
|
88
|
+
if (worst === null || worst.drift <= MAX_MODEL_MIX_DRIFT)
|
|
89
|
+
return null;
|
|
90
|
+
return { kind: 'model-mix-moved', drift: worst.drift, model: worst.model };
|
|
91
|
+
}
|
|
92
|
+
export function qualityGate(before, after, vocabulary) {
|
|
93
|
+
/**
|
|
94
|
+
* The statistics are `experiment`'s, deliberately.
|
|
95
|
+
*
|
|
96
|
+
* One implementation of a two-proportion comparison, so a gate and a
|
|
97
|
+
* deliberate experiment can never disagree about the same two numbers — the
|
|
98
|
+
* failure mode where a team runs both and gets two answers, then trusts
|
|
99
|
+
* whichever one they liked.
|
|
100
|
+
*/
|
|
101
|
+
const stats = runExperiment({ arms: ['before', 'after'], minOutcomesPerArm: MIN_OUTCOMES_EACH_SIDE }, { a: { ...after.arm, name: 'after' }, b: { ...before.arm, name: 'before' } }, vocabulary);
|
|
102
|
+
const outcomes = { before: stats.b.recorded, after: stats.a.recorded };
|
|
103
|
+
const cost = before.calls > 0 && after.calls > 0
|
|
104
|
+
? {
|
|
105
|
+
beforeUsdPerCall: before.arm.totalUsd / before.calls,
|
|
106
|
+
afterUsdPerCall: after.arm.totalUsd / after.calls,
|
|
107
|
+
deltaUsdPerCall: after.arm.totalUsd / after.calls - before.arm.totalUsd / before.calls,
|
|
108
|
+
}
|
|
109
|
+
: null;
|
|
110
|
+
const confounders = [];
|
|
111
|
+
const mix = mixDrift(before.usdByModel, after.usdByModel);
|
|
112
|
+
if (mix !== null)
|
|
113
|
+
confounders.push(mix);
|
|
114
|
+
if (before.calls > 0 && after.calls > 0) {
|
|
115
|
+
const ratio = after.calls / before.calls;
|
|
116
|
+
if (ratio > MAX_VOLUME_RATIO || ratio < 1 / MAX_VOLUME_RATIO) {
|
|
117
|
+
confounders.push({
|
|
118
|
+
kind: 'volume-moved',
|
|
119
|
+
beforeCalls: before.calls,
|
|
120
|
+
afterCalls: after.calls,
|
|
121
|
+
ratio,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Coverage, the one nobody thinks of.
|
|
126
|
+
*
|
|
127
|
+
* A team that starts instrumenting its hard cases sees its measured rate
|
|
128
|
+
* fall without anything having got worse. Comparing two rates over
|
|
129
|
+
* differently-selected populations is the most convincing wrong answer
|
|
130
|
+
* this module could produce.
|
|
131
|
+
*/
|
|
132
|
+
const coverageBefore = outcomes.before / before.calls;
|
|
133
|
+
const coverageAfter = outcomes.after / after.calls;
|
|
134
|
+
if (Math.abs(coverageAfter - coverageBefore) > MAX_COVERAGE_DRIFT) {
|
|
135
|
+
confounders.push({ kind: 'coverage-moved', before: coverageBefore, after: coverageAfter });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const shell = {
|
|
139
|
+
before: stats.b,
|
|
140
|
+
after: stats.a,
|
|
141
|
+
difference: stats.difference,
|
|
142
|
+
confounders,
|
|
143
|
+
cost,
|
|
144
|
+
outcomes,
|
|
145
|
+
};
|
|
146
|
+
if (vocabulary === null || vocabulary.success.length === 0) {
|
|
147
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'no-vocabulary' };
|
|
148
|
+
}
|
|
149
|
+
if (outcomes.before < MIN_OUTCOMES_EACH_SIDE) {
|
|
150
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-before' };
|
|
151
|
+
}
|
|
152
|
+
if (outcomes.after < MIN_OUTCOMES_EACH_SIDE) {
|
|
153
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-after' };
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* A confounder outranks the statistics.
|
|
157
|
+
*
|
|
158
|
+
* Checked after the sample sizes and before the verdict, so a build is never
|
|
159
|
+
* failed on a difference that something else could equally explain. A gate
|
|
160
|
+
* that blames the prompt because the prompt is the thing it can see gets
|
|
161
|
+
* switched off within a month, and a switched-off gate catches nothing.
|
|
162
|
+
*/
|
|
163
|
+
if (confounders.length > 0) {
|
|
164
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'confounded' };
|
|
165
|
+
}
|
|
166
|
+
if (stats.separation === 'not-separable') {
|
|
167
|
+
/**
|
|
168
|
+
* Not separable is `cannot-tell`, never `held`.
|
|
169
|
+
*
|
|
170
|
+
* "The rate did not measurably fall" and "the rate held" are different
|
|
171
|
+
* claims, and a gate that spells them the same way passes a real
|
|
172
|
+
* regression it merely lacked the power to see. `verify` has held this
|
|
173
|
+
* line since 1.39 and it matters more here, because this one is wired to
|
|
174
|
+
* an exit code.
|
|
175
|
+
*/
|
|
176
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'not-separable' };
|
|
177
|
+
}
|
|
178
|
+
// `a` is the *after* arm, so `a-wins` means the rate went up.
|
|
179
|
+
return {
|
|
180
|
+
...shell,
|
|
181
|
+
verdict: stats.separation === 'a-wins' ? 'held' : 'dropped',
|
|
182
|
+
unknown: null,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
//# sourceMappingURL=quality-gate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quality-gate.js","sourceRoot":"","sources":["../src/quality-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,yFAAyF;AACzF,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEvC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,6FAA6F;AAC7F,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAkDtC,SAAS,QAAQ,CACf,MAA6C,EAC7C,KAA4C;IAE5C,MAAM,KAAK,GAAG,CAAC,IAA2C,EAAuB,EAAE;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;QACxF,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACvB,IAAI,KAAK,GAA4C,IAAI,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK;YAAE,KAAK,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,mBAAmB;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,MAAgB,EAChB,KAAe,EACf,UAAoC;IAEpC;;;;;;;OAOG;IACH,MAAM,KAAK,GAAG,aAAa,CACzB,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,EACxE,EAAE,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAC5E,UAAU,CACX,CAAC;IAEF,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvE,MAAM,IAAI,GACR,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;QACjC,CAAC,CAAC;YACE,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK;YACpD,eAAe,EAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;YACjD,eAAe,EAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK;SACvF;QACH,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC1D,IAAI,GAAG,KAAK,IAAI;QAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACzC,IAAI,KAAK,GAAG,gBAAgB,IAAI,KAAK,GAAG,CAAC,GAAG,gBAAgB,EAAE,CAAC;YAC7D,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,cAAc;gBACpB,WAAW,EAAE,MAAM,CAAC,KAAK;gBACzB,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QACD;;;;;;;WAOG;QACH,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;QACtD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACnD,IAAI,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,cAAc,CAAC,GAAG,kBAAkB,EAAE,CAAC;YAClE,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE,KAAK,CAAC,CAAC;QACf,KAAK,EAAE,KAAK,CAAC,CAAC;QACd,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,IAAI;QACJ,QAAQ;KACT,CAAC;IAEF,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC7C,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IACzE,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,GAAG,sBAAsB,EAAE,CAAC;QAC5C,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IACxE,CAAC;IACD;;;;;;;OAOG;IACH,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;IACrE,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,eAAe,EAAE,CAAC;QACzC;;;;;;;;WAQG;QACH,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IACxE,CAAC;IACD,8DAA8D;IAC9D,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC3D,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The findings the rules engine has deferred since 0.1.0, and the layer that
|
|
3
|
+
* throws most of them away.
|
|
4
|
+
*
|
|
5
|
+
* A dictionary cannot see meaning. Two few-shot examples teaching the same
|
|
6
|
+
* boundary in different words, an instruction restated four paragraphs later, a
|
|
7
|
+
* policy contradicted by a clarification — none of them are findable by
|
|
8
|
+
* pattern, and the near-copy detector deliberately does not flag them because
|
|
9
|
+
* flagging them would mean flagging everything.
|
|
10
|
+
*
|
|
11
|
+
* A model can see them. A model can also **invent** them, fluently, and a
|
|
12
|
+
* hallucinated finding is worse than a missed one: a missed finding costs
|
|
13
|
+
* somebody nothing, and an invented one costs them an afternoon and the next
|
|
14
|
+
* finding's credibility.
|
|
15
|
+
*
|
|
16
|
+
* ## The model proposes; this disposes
|
|
17
|
+
*
|
|
18
|
+
* The discipline `--suggest` has had since 1.6, applied to a harder claim. A
|
|
19
|
+
* rewrite suggestion is checkable by construction — the replacement is either
|
|
20
|
+
* shorter or it is not. A *semantic* finding is a claim about meaning, and the
|
|
21
|
+
* only part of it this layer can check is the **evidence**.
|
|
22
|
+
*
|
|
23
|
+
* So it checks the evidence, ruthlessly:
|
|
24
|
+
*
|
|
25
|
+
* 1. **Every quoted span must appear in the prompt character for character.**
|
|
26
|
+
* The single strongest signal available. A model reporting on a prompt
|
|
27
|
+
* while paraphrasing what it quotes has stopped reading and started
|
|
28
|
+
* writing, and everything else it said in that response is suspect.
|
|
29
|
+
* 2. **The spans must be distinct and must not overlap.** A "pair" that is one
|
|
30
|
+
* span quoted twice is not a finding about redundancy; it is a finding
|
|
31
|
+
* about nothing.
|
|
32
|
+
* 3. **The near-copy detector must not already catch it.** A model paid to
|
|
33
|
+
* re-report a finding the deterministic layer produces for free is a model
|
|
34
|
+
* being paid for nothing, and the reader sees the same thing twice with two
|
|
35
|
+
* different confidences attached.
|
|
36
|
+
* 4. **Nothing the model says about size is believed.** Tokens are counted
|
|
37
|
+
* here, from the spans, with the same counter everything else uses.
|
|
38
|
+
*
|
|
39
|
+
* ## It never becomes a prerequisite
|
|
40
|
+
*
|
|
41
|
+
* The deterministic core keeps working with no key, no network and no model.
|
|
42
|
+
* This is a pass *on top*, and the rule from 0.1.0 stands unchanged — which is
|
|
43
|
+
* why the verification lives here, in the package that has no network, and the
|
|
44
|
+
* call lives in the CLI.
|
|
45
|
+
*/
|
|
46
|
+
import type { TokenCounter } from './types.js';
|
|
47
|
+
export type SemanticKind =
|
|
48
|
+
/** Two spans teaching the same boundary in different words. */
|
|
49
|
+
'paraphrase-pair'
|
|
50
|
+
/** An instruction repeated in a distant part of the prompt. */
|
|
51
|
+
| 'restated-instruction'
|
|
52
|
+
/** A policy and a later clarification that disagree. */
|
|
53
|
+
| 'contradiction';
|
|
54
|
+
/** What the model returns, before anything has been checked. */
|
|
55
|
+
export interface SemanticProposal {
|
|
56
|
+
kind: SemanticKind;
|
|
57
|
+
/** Two spans, quoted from the prompt. */
|
|
58
|
+
spans: [string, string];
|
|
59
|
+
/** One sentence saying what the relation is. Never used to decide anything. */
|
|
60
|
+
because: string;
|
|
61
|
+
}
|
|
62
|
+
export type SemanticRejection =
|
|
63
|
+
/** A span is not in the prompt. The model paraphrased its own evidence. */
|
|
64
|
+
'span-not-found'
|
|
65
|
+
/** The two spans are the same text. */
|
|
66
|
+
| 'spans-identical'
|
|
67
|
+
/** The two spans overlap in the prompt. */
|
|
68
|
+
| 'spans-overlap'
|
|
69
|
+
/** Near-identical: the deterministic detector already reports this for free. */
|
|
70
|
+
| 'already-detected'
|
|
71
|
+
/**
|
|
72
|
+
* A contradiction whose spans are near-identical.
|
|
73
|
+
*
|
|
74
|
+
* Two spans that say the same thing cannot contradict each other, and a
|
|
75
|
+
* model that labels a near-copy a contradiction has mislabelled it — which
|
|
76
|
+
* makes every other label in the same response worth less.
|
|
77
|
+
*/
|
|
78
|
+
| 'contradiction-of-a-copy'
|
|
79
|
+
/** A duplicate of a finding already accepted. */
|
|
80
|
+
| 'duplicate';
|
|
81
|
+
export interface SemanticFinding {
|
|
82
|
+
kind: SemanticKind;
|
|
83
|
+
spans: [string, string];
|
|
84
|
+
because: string;
|
|
85
|
+
/** Where each span sits, so a reader can go and look. */
|
|
86
|
+
offsets: [number, number];
|
|
87
|
+
/**
|
|
88
|
+
* Tokens the shorter span holds — a **ceiling** on what resolving this could
|
|
89
|
+
* save, and named as one.
|
|
90
|
+
*
|
|
91
|
+
* Not a saving. Resolving a paraphrase pair means rewriting both spans into
|
|
92
|
+
* one, and how much that recovers depends on what the merged version says,
|
|
93
|
+
* which nobody knows yet. The ceiling is the honest figure: it cannot save
|
|
94
|
+
* more than deleting the smaller half.
|
|
95
|
+
*/
|
|
96
|
+
ceilingTokens: number;
|
|
97
|
+
}
|
|
98
|
+
export interface SemanticResult {
|
|
99
|
+
findings: SemanticFinding[];
|
|
100
|
+
rejected: Array<{
|
|
101
|
+
proposal: SemanticProposal;
|
|
102
|
+
reason: SemanticRejection;
|
|
103
|
+
}>;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* How similar two spans may be before the deterministic detector owns them.
|
|
107
|
+
*
|
|
108
|
+
* **0.92, read off `rules.ts` rather than chosen.** The duplicate-example rule
|
|
109
|
+
* drops a few-shot example when its Jaccard similarity to a kept one is at
|
|
110
|
+
* least 0.92, so that is exactly the line above which a model is being paid to
|
|
111
|
+
* re-report something the rules engine produces for free.
|
|
112
|
+
*
|
|
113
|
+
* The first version of this file said 0.8 and claimed in a comment that it
|
|
114
|
+
* matched the deterministic pass. It did not, and the error ran the dangerous
|
|
115
|
+
* way: everything between 0.8 and 0.92 is a pair the rules engine does **not**
|
|
116
|
+
* catch, and this layer was silently throwing those away — discarding exactly
|
|
117
|
+
* the findings the chapter exists to surface, while a comment asserted the
|
|
118
|
+
* opposite.
|
|
119
|
+
*/
|
|
120
|
+
export declare const ALREADY_DETECTED_SIMILARITY = 0.92;
|
|
121
|
+
export interface VerifyOptions {
|
|
122
|
+
tokenCounter?: TokenCounter;
|
|
123
|
+
/** Cap on findings, largest ceiling first. Defaults to 20. */
|
|
124
|
+
max?: number;
|
|
125
|
+
}
|
|
126
|
+
export declare function verifySemanticProposals(prompt: string, proposals: readonly SemanticProposal[], options?: VerifyOptions): SemanticResult;
|
|
127
|
+
/**
|
|
128
|
+
* What running the pass costs, computed before it runs.
|
|
129
|
+
*
|
|
130
|
+
* A tool that spends somebody's money to tell them how to spend less must be
|
|
131
|
+
* the first thing audited by its own arithmetic. The figure is an estimate of
|
|
132
|
+
* one call — the prompt in, a small structured answer out — and it says which
|
|
133
|
+
* half is which by having only one half.
|
|
134
|
+
*/
|
|
135
|
+
export declare function semanticPassCost(prompt: string, rates: {
|
|
136
|
+
inputPerMTok: number;
|
|
137
|
+
outputPerMTok: number;
|
|
138
|
+
}, options?: {
|
|
139
|
+
tokenCounter?: TokenCounter;
|
|
140
|
+
expectedOutputTokens?: number;
|
|
141
|
+
}): {
|
|
142
|
+
inputTokens: number;
|
|
143
|
+
outputTokens: number;
|
|
144
|
+
usd: number;
|
|
145
|
+
provenance: 'estimated';
|
|
146
|
+
};
|
|
147
|
+
export declare const SEMANTIC_SYSTEM_PROMPT = "You find places where a prompt says the same thing twice in different words, or contradicts itself.\n\nReturn ONLY a JSON array. Each element is {\"kind\": \"...\", \"spans\": [\"...\", \"...\"], \"because\": \"...\"}.\n\n\"kind\" is one of:\n- \"paraphrase-pair\": two passages teaching the same rule or boundary in different words.\n- \"restated-instruction\": an instruction given once and then given again elsewhere.\n- \"contradiction\": two passages that cannot both be followed.\n\nRules:\n- Both strings in \"spans\" MUST be copied character for character from the prompt. Do not paraphrase them, do not fix their punctuation, do not trim them differently. If you cannot copy a passage exactly, leave the finding out.\n- The two spans must be different passages in different places. Never quote the same passage twice.\n- Do not report passages that are near-identical wordings of each other \u2014 those are already found without you. Report only pairs whose WORDS differ while their MEANING is the same.\n- \"because\" is one sentence naming the relation, for a human to read.\n- Prefer a few certain findings to many possible ones. Return [] if there is nothing you are sure of.\n\nNo explanation, no code fences, no commentary. The array alone.";
|
|
148
|
+
//# sourceMappingURL=semantic.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../src/semantic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,MAAM,MAAM,YAAY;AACtB,+DAA+D;AAC7D,iBAAiB;AACnB,+DAA+D;GAC7D,sBAAsB;AACxB,wDAAwD;GACtD,eAAe,CAAC;AAEpB,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,YAAY,CAAC;IACnB,yCAAyC;IACzC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,iBAAiB;AAC3B,2EAA2E;AACzE,gBAAgB;AAClB,uCAAuC;GACrC,iBAAiB;AACnB,2CAA2C;GACzC,eAAe;AACjB,gFAAgF;GAC9E,kBAAkB;AACpB;;;;;;GAMG;GACD,yBAAyB;AAC3B,iDAAiD;GAC/C,WAAW,CAAC;AAEhB,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,YAAY,CAAC;IACnB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1B;;;;;;;;OAQG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,gBAAgB,CAAC;QAAC,MAAM,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;CAC5E;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,8DAA8D;IAC9D,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,SAAS,gBAAgB,EAAE,EACtC,OAAO,GAAE,aAAkB,GAC1B,cAAc,CAuEhB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,EACtD,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,YAAY,CAAC;IAAC,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3E;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,WAAW,CAAA;CAAE,CAWrF;AAED,eAAO,MAAM,sBAAsB,6uCAgB6B,CAAC"}
|
package/dist/semantic.js
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The findings the rules engine has deferred since 0.1.0, and the layer that
|
|
3
|
+
* throws most of them away.
|
|
4
|
+
*
|
|
5
|
+
* A dictionary cannot see meaning. Two few-shot examples teaching the same
|
|
6
|
+
* boundary in different words, an instruction restated four paragraphs later, a
|
|
7
|
+
* policy contradicted by a clarification — none of them are findable by
|
|
8
|
+
* pattern, and the near-copy detector deliberately does not flag them because
|
|
9
|
+
* flagging them would mean flagging everything.
|
|
10
|
+
*
|
|
11
|
+
* A model can see them. A model can also **invent** them, fluently, and a
|
|
12
|
+
* hallucinated finding is worse than a missed one: a missed finding costs
|
|
13
|
+
* somebody nothing, and an invented one costs them an afternoon and the next
|
|
14
|
+
* finding's credibility.
|
|
15
|
+
*
|
|
16
|
+
* ## The model proposes; this disposes
|
|
17
|
+
*
|
|
18
|
+
* The discipline `--suggest` has had since 1.6, applied to a harder claim. A
|
|
19
|
+
* rewrite suggestion is checkable by construction — the replacement is either
|
|
20
|
+
* shorter or it is not. A *semantic* finding is a claim about meaning, and the
|
|
21
|
+
* only part of it this layer can check is the **evidence**.
|
|
22
|
+
*
|
|
23
|
+
* So it checks the evidence, ruthlessly:
|
|
24
|
+
*
|
|
25
|
+
* 1. **Every quoted span must appear in the prompt character for character.**
|
|
26
|
+
* The single strongest signal available. A model reporting on a prompt
|
|
27
|
+
* while paraphrasing what it quotes has stopped reading and started
|
|
28
|
+
* writing, and everything else it said in that response is suspect.
|
|
29
|
+
* 2. **The spans must be distinct and must not overlap.** A "pair" that is one
|
|
30
|
+
* span quoted twice is not a finding about redundancy; it is a finding
|
|
31
|
+
* about nothing.
|
|
32
|
+
* 3. **The near-copy detector must not already catch it.** A model paid to
|
|
33
|
+
* re-report a finding the deterministic layer produces for free is a model
|
|
34
|
+
* being paid for nothing, and the reader sees the same thing twice with two
|
|
35
|
+
* different confidences attached.
|
|
36
|
+
* 4. **Nothing the model says about size is believed.** Tokens are counted
|
|
37
|
+
* here, from the spans, with the same counter everything else uses.
|
|
38
|
+
*
|
|
39
|
+
* ## It never becomes a prerequisite
|
|
40
|
+
*
|
|
41
|
+
* The deterministic core keeps working with no key, no network and no model.
|
|
42
|
+
* This is a pass *on top*, and the rule from 0.1.0 stands unchanged — which is
|
|
43
|
+
* why the verification lives here, in the package that has no network, and the
|
|
44
|
+
* call lives in the CLI.
|
|
45
|
+
*/
|
|
46
|
+
import { normalizeForCompare, jaccard } from './similarity.js';
|
|
47
|
+
import { estimateTokens } from './tokenizer.js';
|
|
48
|
+
/**
|
|
49
|
+
* How similar two spans may be before the deterministic detector owns them.
|
|
50
|
+
*
|
|
51
|
+
* **0.92, read off `rules.ts` rather than chosen.** The duplicate-example rule
|
|
52
|
+
* drops a few-shot example when its Jaccard similarity to a kept one is at
|
|
53
|
+
* least 0.92, so that is exactly the line above which a model is being paid to
|
|
54
|
+
* re-report something the rules engine produces for free.
|
|
55
|
+
*
|
|
56
|
+
* The first version of this file said 0.8 and claimed in a comment that it
|
|
57
|
+
* matched the deterministic pass. It did not, and the error ran the dangerous
|
|
58
|
+
* way: everything between 0.8 and 0.92 is a pair the rules engine does **not**
|
|
59
|
+
* catch, and this layer was silently throwing those away — discarding exactly
|
|
60
|
+
* the findings the chapter exists to surface, while a comment asserted the
|
|
61
|
+
* opposite.
|
|
62
|
+
*/
|
|
63
|
+
export const ALREADY_DETECTED_SIMILARITY = 0.92;
|
|
64
|
+
export function verifySemanticProposals(prompt, proposals, options = {}) {
|
|
65
|
+
const { tokenCounter = estimateTokens, max = 20 } = options;
|
|
66
|
+
const findings = [];
|
|
67
|
+
const rejected = [];
|
|
68
|
+
const claimed = [];
|
|
69
|
+
for (const proposal of proposals) {
|
|
70
|
+
const [first, second] = proposal.spans;
|
|
71
|
+
const a = prompt.indexOf(first);
|
|
72
|
+
const b = prompt.indexOf(second);
|
|
73
|
+
if (a === -1 || b === -1) {
|
|
74
|
+
// The strongest check there is. A model that paraphrases what it quotes
|
|
75
|
+
// has stopped reading the prompt and started writing about it.
|
|
76
|
+
rejected.push({ proposal, reason: 'span-not-found' });
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
if (first === second) {
|
|
80
|
+
rejected.push({ proposal, reason: 'spans-identical' });
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const aEnd = a + first.length;
|
|
84
|
+
const bEnd = b + second.length;
|
|
85
|
+
if (a < bEnd && b < aEnd) {
|
|
86
|
+
rejected.push({ proposal, reason: 'spans-overlap' });
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
const similarity = jaccard(normalizeForCompare(first), normalizeForCompare(second));
|
|
90
|
+
if (proposal.kind === 'contradiction' && similarity >= ALREADY_DETECTED_SIMILARITY) {
|
|
91
|
+
rejected.push({ proposal, reason: 'contradiction-of-a-copy' });
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (proposal.kind !== 'contradiction' && similarity >= ALREADY_DETECTED_SIMILARITY) {
|
|
95
|
+
rejected.push({ proposal, reason: 'already-detected' });
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* A finding covering ground an accepted one already covers.
|
|
100
|
+
*
|
|
101
|
+
* Overlap against what has been *accepted*, not against every proposal —
|
|
102
|
+
* so a rejected finding never blocks a good one that touches the same
|
|
103
|
+
* paragraph.
|
|
104
|
+
*/
|
|
105
|
+
const overlapsAccepted = claimed.some(([start, end]) => (a < end && start < aEnd) || (b < end && start < bEnd));
|
|
106
|
+
if (overlapsAccepted) {
|
|
107
|
+
rejected.push({ proposal, reason: 'duplicate' });
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
claimed.push([a, aEnd], [b, bEnd]);
|
|
111
|
+
findings.push({
|
|
112
|
+
kind: proposal.kind,
|
|
113
|
+
spans: proposal.spans,
|
|
114
|
+
because: proposal.because,
|
|
115
|
+
offsets: [a, b],
|
|
116
|
+
// Counted here, from the spans. Nothing the model said about size is
|
|
117
|
+
// believed, and a contradiction saves nothing by being resolved — the
|
|
118
|
+
// point of finding one is that the prompt is wrong, not that it is long.
|
|
119
|
+
ceilingTokens: proposal.kind === 'contradiction'
|
|
120
|
+
? 0
|
|
121
|
+
: Math.min(tokenCounter(first), tokenCounter(second)),
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
findings.sort((x, y) => y.ceilingTokens - x.ceilingTokens);
|
|
125
|
+
return { findings: findings.slice(0, max), rejected };
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* What running the pass costs, computed before it runs.
|
|
129
|
+
*
|
|
130
|
+
* A tool that spends somebody's money to tell them how to spend less must be
|
|
131
|
+
* the first thing audited by its own arithmetic. The figure is an estimate of
|
|
132
|
+
* one call — the prompt in, a small structured answer out — and it says which
|
|
133
|
+
* half is which by having only one half.
|
|
134
|
+
*/
|
|
135
|
+
export function semanticPassCost(prompt, rates, options = {}) {
|
|
136
|
+
const { tokenCounter = estimateTokens, expectedOutputTokens = 800 } = options;
|
|
137
|
+
const inputTokens = tokenCounter(prompt) + tokenCounter(SEMANTIC_SYSTEM_PROMPT);
|
|
138
|
+
return {
|
|
139
|
+
inputTokens,
|
|
140
|
+
outputTokens: expectedOutputTokens,
|
|
141
|
+
usd: (inputTokens / 1_000_000) * rates.inputPerMTok +
|
|
142
|
+
(expectedOutputTokens / 1_000_000) * rates.outputPerMTok,
|
|
143
|
+
provenance: 'estimated',
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
export const SEMANTIC_SYSTEM_PROMPT = `You find places where a prompt says the same thing twice in different words, or contradicts itself.
|
|
147
|
+
|
|
148
|
+
Return ONLY a JSON array. Each element is {"kind": "...", "spans": ["...", "..."], "because": "..."}.
|
|
149
|
+
|
|
150
|
+
"kind" is one of:
|
|
151
|
+
- "paraphrase-pair": two passages teaching the same rule or boundary in different words.
|
|
152
|
+
- "restated-instruction": an instruction given once and then given again elsewhere.
|
|
153
|
+
- "contradiction": two passages that cannot both be followed.
|
|
154
|
+
|
|
155
|
+
Rules:
|
|
156
|
+
- Both strings in "spans" MUST be copied character for character from the prompt. Do not paraphrase them, do not fix their punctuation, do not trim them differently. If you cannot copy a passage exactly, leave the finding out.
|
|
157
|
+
- The two spans must be different passages in different places. Never quote the same passage twice.
|
|
158
|
+
- Do not report passages that are near-identical wordings of each other — those are already found without you. Report only pairs whose WORDS differ while their MEANING is the same.
|
|
159
|
+
- "because" is one sentence naming the relation, for a human to read.
|
|
160
|
+
- Prefer a few certain findings to many possible ones. Return [] if there is nothing you are sure of.
|
|
161
|
+
|
|
162
|
+
No explanation, no code fences, no commentary. The array alone.`;
|
|
163
|
+
//# sourceMappingURL=semantic.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semantic.js","sourceRoot":"","sources":["../src/semantic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA+DhD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAQhD,MAAM,UAAU,uBAAuB,CACrC,MAAc,EACd,SAAsC,EACtC,OAAO,GAAkB,EAAE;IAE3B,MAAM,EAAE,YAAY,GAAG,cAAc,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAC5D,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAA+B,EAAE,CAAC;IAChD,MAAM,OAAO,GAA4B,EAAE,CAAC;IAE5C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QAEvC,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACzB,wEAAwE;YACxE,+DAA+D;YAC/D,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YACtD,SAAS;QACX,CAAC;QACD,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC/B,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,IAAI,QAAQ,CAAC,IAAI,KAAK,eAAe,IAAI,UAAU,IAAI,2BAA2B,EAAE,CAAC;YACnF,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,eAAe,IAAI,UAAU,IAAI,2BAA2B,EAAE,CAAC;YACnF,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QAED;;;;;;WAMG;QACH,MAAM,gBAAgB,GAAG,OAAO,CAAC,IAAI,CACnC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,GAAG,IAAI,CAAC,CACzE,CAAC;QACF,IAAI,gBAAgB,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACjD,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACf,qEAAqE;YACrE,sEAAsE;YACtE,yEAAyE;YACzE,aAAa,EACX,QAAQ,CAAC,IAAI,KAAK,eAAe;gBAC/B,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;SAC1D,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;IAC3D,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC;AACxD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAc,EACd,KAAsD,EACtD,OAAO,GAAmE,EAAE;IAE5E,MAAM,EAAE,YAAY,GAAG,cAAc,EAAE,oBAAoB,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;IAC9E,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,YAAY,CAAC,sBAAsB,CAAC,CAAC;IAChF,OAAO;QACL,WAAW;QACX,YAAY,EAAE,oBAAoB;QAClC,GAAG,EACD,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,YAAY;YAC9C,CAAC,oBAAoB,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,aAAa;QAC1D,UAAU,EAAE,WAAW;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;gEAgB0B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@trazum/core",
|
|
3
|
-
"version": "1.50.
|
|
3
|
+
"version": "1.50.9",
|
|
4
4
|
"description": "Trazum core: priced advisories for LLM prompts (caching, model tier, batching, schemas), plus deterministic trimming, token counting and pricing.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "David Mu\u00f1oz Rey",
|
package/src/index.ts
CHANGED
|
@@ -50,7 +50,28 @@ export type { PlanAction, PlanActionKind, PlanAssumption, PlanDocument } from '.
|
|
|
50
50
|
export { verifyPlan } from './verify.js';
|
|
51
51
|
export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
|
|
52
52
|
export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
|
|
53
|
+
export {
|
|
54
|
+
verifySemanticProposals,
|
|
55
|
+
semanticPassCost,
|
|
56
|
+
SEMANTIC_SYSTEM_PROMPT,
|
|
57
|
+
ALREADY_DETECTED_SIMILARITY,
|
|
58
|
+
} from './semantic.js';
|
|
59
|
+
export type {
|
|
60
|
+
SemanticFinding,
|
|
61
|
+
SemanticKind,
|
|
62
|
+
SemanticProposal,
|
|
63
|
+
SemanticRejection,
|
|
64
|
+
SemanticResult,
|
|
65
|
+
} from './semantic.js';
|
|
53
66
|
export { runExperiment } from './experiment.js';
|
|
67
|
+
export {
|
|
68
|
+
qualityGate,
|
|
69
|
+
MIN_OUTCOMES_EACH_SIDE,
|
|
70
|
+
MAX_MODEL_MIX_DRIFT,
|
|
71
|
+
MAX_VOLUME_RATIO,
|
|
72
|
+
MAX_COVERAGE_DRIFT,
|
|
73
|
+
} from './quality-gate.js';
|
|
74
|
+
export type { Confounder, GateSide, GateUnknown, GateVerdict, QualityGate } from './quality-gate.js';
|
|
54
75
|
export type {
|
|
55
76
|
ArmResult,
|
|
56
77
|
ExperimentArm,
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The failure that actually matters, and the four ways of getting it wrong.
|
|
3
|
+
*
|
|
4
|
+
* CI has been able to fail a build for tokens since 1.4 and for dollars since
|
|
5
|
+
* 1.21. A prompt edit that quietly made the product worse has never been
|
|
6
|
+
* gateable — which means every cost saving this tool has ever recommended went
|
|
7
|
+
* into a repository with its most important consequence unmeasured.
|
|
8
|
+
*
|
|
9
|
+
* ## This is a before-and-after, not an experiment
|
|
10
|
+
*
|
|
11
|
+
* Worth saying in the first paragraph, because the arithmetic is the same as
|
|
12
|
+
* `experiment`'s and the epistemics are not. An experiment splits traffic
|
|
13
|
+
* randomly, so the two arms differ only in the thing under test. A
|
|
14
|
+
* before-and-after splits it by **time**, and everything else that changed at
|
|
15
|
+
* the same time is in the difference too.
|
|
16
|
+
*
|
|
17
|
+
* So this module spends most of its code looking for reasons *not* to blame the
|
|
18
|
+
* prompt, and it reports `cannot-tell` far more readily than a randomised
|
|
19
|
+
* comparison would. That is not timidity. A gate that blames the prompt because
|
|
20
|
+
* the prompt is the thing it can see will be switched off within a month, and a
|
|
21
|
+
* switched-off gate catches nothing at all.
|
|
22
|
+
*
|
|
23
|
+
* ## The three confounders it can actually see
|
|
24
|
+
*
|
|
25
|
+
* **The model moved.** If the mix of models shifted across the boundary, the
|
|
26
|
+
* prompt is not the only variable, and the drop may be entirely somebody else's
|
|
27
|
+
* migration.
|
|
28
|
+
*
|
|
29
|
+
* **The volume moved.** A workload whose traffic tripled is usually a workload
|
|
30
|
+
* whose *population* changed — a new surface, a new customer, a marketing
|
|
31
|
+
* campaign — and the questions being asked are not the questions from before.
|
|
32
|
+
*
|
|
33
|
+
* **The coverage moved.** The subtle one, and the one nobody thinks of: if the
|
|
34
|
+
* share of calls recording an outcome changed, the two rates describe different
|
|
35
|
+
* populations. A team that started instrumenting its hard cases will see its
|
|
36
|
+
* measured rate fall without anything getting worse.
|
|
37
|
+
*
|
|
38
|
+
* Any of them present, and the verdict is `cannot-tell` with the confounder
|
|
39
|
+
* **named**. Not a hedge attached to a blame — a refusal to blame.
|
|
40
|
+
*
|
|
41
|
+
* ## What it cannot see, and says so
|
|
42
|
+
*
|
|
43
|
+
* Everything else deployed that day. This module has one label's numbers and no
|
|
44
|
+
* knowledge of the world, and it never pretends otherwise: a `dropped` verdict
|
|
45
|
+
* is a statement that the rate fell and the three things it can check did not
|
|
46
|
+
* move, which is a smaller claim than "the prompt did it" and is the largest
|
|
47
|
+
* claim the evidence supports.
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
import { runExperiment } from './experiment.js';
|
|
51
|
+
import type { ArmResult, ExperimentArm } from './experiment.js';
|
|
52
|
+
import type { OutcomeVocabulary } from './outcome.js';
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Outcomes each side of the change needs before a verdict is offered.
|
|
56
|
+
*
|
|
57
|
+
* A hundred rather than the ten used for a rate elsewhere, because this one
|
|
58
|
+
* fails builds. The cost of a wrong `dropped` is somebody reverting a good
|
|
59
|
+
* change and losing the saving; the cost of a wrong `cannot-tell` is waiting a
|
|
60
|
+
* day. Those are not symmetric and the threshold is not either.
|
|
61
|
+
*/
|
|
62
|
+
export const MIN_OUTCOMES_EACH_SIDE = 100;
|
|
63
|
+
|
|
64
|
+
/** How far the model mix may move before the comparison stops being about the prompt. */
|
|
65
|
+
export const MAX_MODEL_MIX_DRIFT = 0.1;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* How far the call volume may move, as a ratio.
|
|
69
|
+
*
|
|
70
|
+
* Half again, either way. Ordinary week-to-week traffic moves less than that;
|
|
71
|
+
* a workload that doubled is usually a workload whose population changed.
|
|
72
|
+
*/
|
|
73
|
+
export const MAX_VOLUME_RATIO = 1.5;
|
|
74
|
+
|
|
75
|
+
/** How far outcome coverage may move before the two rates describe different populations. */
|
|
76
|
+
export const MAX_COVERAGE_DRIFT = 0.1;
|
|
77
|
+
|
|
78
|
+
export type Confounder =
|
|
79
|
+
| { kind: 'model-mix-moved'; drift: number; model: string }
|
|
80
|
+
| { kind: 'volume-moved'; beforeCalls: number; afterCalls: number; ratio: number }
|
|
81
|
+
| { kind: 'coverage-moved'; before: number; after: number };
|
|
82
|
+
|
|
83
|
+
export type GateVerdict = 'held' | 'dropped' | 'cannot-tell';
|
|
84
|
+
|
|
85
|
+
export type GateUnknown =
|
|
86
|
+
| 'too-few-before'
|
|
87
|
+
| 'too-few-after'
|
|
88
|
+
| 'confounded'
|
|
89
|
+
| 'no-vocabulary'
|
|
90
|
+
| 'not-separable';
|
|
91
|
+
|
|
92
|
+
/** One side of the boundary, as the caller measured it. */
|
|
93
|
+
export interface GateSide {
|
|
94
|
+
arm: ExperimentArm;
|
|
95
|
+
/** Every call in the window, outcome or not — the coverage denominator. */
|
|
96
|
+
calls: number;
|
|
97
|
+
/** Spend per model, for the mix comparison. */
|
|
98
|
+
usdByModel: Array<{ model: string; usd: number }>;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface QualityGate {
|
|
102
|
+
verdict: GateVerdict;
|
|
103
|
+
/** Why, when the verdict is `cannot-tell`. A refusal never arrives bare. */
|
|
104
|
+
unknown: GateUnknown | null;
|
|
105
|
+
before: ArmResult;
|
|
106
|
+
after: ArmResult;
|
|
107
|
+
/** The 95% interval on (after − before), or null. */
|
|
108
|
+
difference: { point: number; low: number; high: number } | null;
|
|
109
|
+
/**
|
|
110
|
+
* Everything found that means the prompt is not the only variable.
|
|
111
|
+
*
|
|
112
|
+
* Non-empty forces `cannot-tell`. Reported even on a `held` verdict, because
|
|
113
|
+
* a rate that held while the model changed underneath is not evidence that
|
|
114
|
+
* the prompt is fine either.
|
|
115
|
+
*/
|
|
116
|
+
confounders: Confounder[];
|
|
117
|
+
/**
|
|
118
|
+
* The cost half — measured, and kept beside the quality half rather than
|
|
119
|
+
* merged with it. The sentence teams argue about needs both.
|
|
120
|
+
*/
|
|
121
|
+
cost: { beforeUsdPerCall: number; afterUsdPerCall: number; deltaUsdPerCall: number } | null;
|
|
122
|
+
/** Outcomes behind the comparison, so the claim carries its own sample size. */
|
|
123
|
+
outcomes: { before: number; after: number };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function mixDrift(
|
|
127
|
+
before: Array<{ model: string; usd: number }>,
|
|
128
|
+
after: Array<{ model: string; usd: number }>,
|
|
129
|
+
): Confounder | null {
|
|
130
|
+
const share = (rows: Array<{ model: string; usd: number }>): Map<string, number> => {
|
|
131
|
+
const total = rows.reduce((sum, row) => sum + row.usd, 0);
|
|
132
|
+
const out = new Map<string, number>();
|
|
133
|
+
if (total <= 0) return out;
|
|
134
|
+
for (const row of rows) out.set(row.model, (out.get(row.model) ?? 0) + row.usd / total);
|
|
135
|
+
return out;
|
|
136
|
+
};
|
|
137
|
+
const a = share(before);
|
|
138
|
+
const b = share(after);
|
|
139
|
+
let worst: { model: string; drift: number } | null = null;
|
|
140
|
+
for (const model of new Set([...a.keys(), ...b.keys()])) {
|
|
141
|
+
const drift = Math.abs((b.get(model) ?? 0) - (a.get(model) ?? 0));
|
|
142
|
+
if (worst === null || drift > worst.drift) worst = { model, drift };
|
|
143
|
+
}
|
|
144
|
+
if (worst === null || worst.drift <= MAX_MODEL_MIX_DRIFT) return null;
|
|
145
|
+
return { kind: 'model-mix-moved', drift: worst.drift, model: worst.model };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function qualityGate(
|
|
149
|
+
before: GateSide,
|
|
150
|
+
after: GateSide,
|
|
151
|
+
vocabulary: OutcomeVocabulary | null,
|
|
152
|
+
): QualityGate {
|
|
153
|
+
/**
|
|
154
|
+
* The statistics are `experiment`'s, deliberately.
|
|
155
|
+
*
|
|
156
|
+
* One implementation of a two-proportion comparison, so a gate and a
|
|
157
|
+
* deliberate experiment can never disagree about the same two numbers — the
|
|
158
|
+
* failure mode where a team runs both and gets two answers, then trusts
|
|
159
|
+
* whichever one they liked.
|
|
160
|
+
*/
|
|
161
|
+
const stats = runExperiment(
|
|
162
|
+
{ arms: ['before', 'after'], minOutcomesPerArm: MIN_OUTCOMES_EACH_SIDE },
|
|
163
|
+
{ a: { ...after.arm, name: 'after' }, b: { ...before.arm, name: 'before' } },
|
|
164
|
+
vocabulary,
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
const outcomes = { before: stats.b.recorded, after: stats.a.recorded };
|
|
168
|
+
const cost =
|
|
169
|
+
before.calls > 0 && after.calls > 0
|
|
170
|
+
? {
|
|
171
|
+
beforeUsdPerCall: before.arm.totalUsd / before.calls,
|
|
172
|
+
afterUsdPerCall: after.arm.totalUsd / after.calls,
|
|
173
|
+
deltaUsdPerCall: after.arm.totalUsd / after.calls - before.arm.totalUsd / before.calls,
|
|
174
|
+
}
|
|
175
|
+
: null;
|
|
176
|
+
|
|
177
|
+
const confounders: Confounder[] = [];
|
|
178
|
+
const mix = mixDrift(before.usdByModel, after.usdByModel);
|
|
179
|
+
if (mix !== null) confounders.push(mix);
|
|
180
|
+
|
|
181
|
+
if (before.calls > 0 && after.calls > 0) {
|
|
182
|
+
const ratio = after.calls / before.calls;
|
|
183
|
+
if (ratio > MAX_VOLUME_RATIO || ratio < 1 / MAX_VOLUME_RATIO) {
|
|
184
|
+
confounders.push({
|
|
185
|
+
kind: 'volume-moved',
|
|
186
|
+
beforeCalls: before.calls,
|
|
187
|
+
afterCalls: after.calls,
|
|
188
|
+
ratio,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Coverage, the one nobody thinks of.
|
|
193
|
+
*
|
|
194
|
+
* A team that starts instrumenting its hard cases sees its measured rate
|
|
195
|
+
* fall without anything having got worse. Comparing two rates over
|
|
196
|
+
* differently-selected populations is the most convincing wrong answer
|
|
197
|
+
* this module could produce.
|
|
198
|
+
*/
|
|
199
|
+
const coverageBefore = outcomes.before / before.calls;
|
|
200
|
+
const coverageAfter = outcomes.after / after.calls;
|
|
201
|
+
if (Math.abs(coverageAfter - coverageBefore) > MAX_COVERAGE_DRIFT) {
|
|
202
|
+
confounders.push({ kind: 'coverage-moved', before: coverageBefore, after: coverageAfter });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const shell = {
|
|
207
|
+
before: stats.b,
|
|
208
|
+
after: stats.a,
|
|
209
|
+
difference: stats.difference,
|
|
210
|
+
confounders,
|
|
211
|
+
cost,
|
|
212
|
+
outcomes,
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
if (vocabulary === null || vocabulary.success.length === 0) {
|
|
216
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'no-vocabulary' };
|
|
217
|
+
}
|
|
218
|
+
if (outcomes.before < MIN_OUTCOMES_EACH_SIDE) {
|
|
219
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-before' };
|
|
220
|
+
}
|
|
221
|
+
if (outcomes.after < MIN_OUTCOMES_EACH_SIDE) {
|
|
222
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-after' };
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* A confounder outranks the statistics.
|
|
226
|
+
*
|
|
227
|
+
* Checked after the sample sizes and before the verdict, so a build is never
|
|
228
|
+
* failed on a difference that something else could equally explain. A gate
|
|
229
|
+
* that blames the prompt because the prompt is the thing it can see gets
|
|
230
|
+
* switched off within a month, and a switched-off gate catches nothing.
|
|
231
|
+
*/
|
|
232
|
+
if (confounders.length > 0) {
|
|
233
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'confounded' };
|
|
234
|
+
}
|
|
235
|
+
if (stats.separation === 'not-separable') {
|
|
236
|
+
/**
|
|
237
|
+
* Not separable is `cannot-tell`, never `held`.
|
|
238
|
+
*
|
|
239
|
+
* "The rate did not measurably fall" and "the rate held" are different
|
|
240
|
+
* claims, and a gate that spells them the same way passes a real
|
|
241
|
+
* regression it merely lacked the power to see. `verify` has held this
|
|
242
|
+
* line since 1.39 and it matters more here, because this one is wired to
|
|
243
|
+
* an exit code.
|
|
244
|
+
*/
|
|
245
|
+
return { ...shell, verdict: 'cannot-tell', unknown: 'not-separable' };
|
|
246
|
+
}
|
|
247
|
+
// `a` is the *after* arm, so `a-wins` means the rate went up.
|
|
248
|
+
return {
|
|
249
|
+
...shell,
|
|
250
|
+
verdict: stats.separation === 'a-wins' ? 'held' : 'dropped',
|
|
251
|
+
unknown: null,
|
|
252
|
+
};
|
|
253
|
+
}
|
package/src/semantic.ts
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The findings the rules engine has deferred since 0.1.0, and the layer that
|
|
3
|
+
* throws most of them away.
|
|
4
|
+
*
|
|
5
|
+
* A dictionary cannot see meaning. Two few-shot examples teaching the same
|
|
6
|
+
* boundary in different words, an instruction restated four paragraphs later, a
|
|
7
|
+
* policy contradicted by a clarification — none of them are findable by
|
|
8
|
+
* pattern, and the near-copy detector deliberately does not flag them because
|
|
9
|
+
* flagging them would mean flagging everything.
|
|
10
|
+
*
|
|
11
|
+
* A model can see them. A model can also **invent** them, fluently, and a
|
|
12
|
+
* hallucinated finding is worse than a missed one: a missed finding costs
|
|
13
|
+
* somebody nothing, and an invented one costs them an afternoon and the next
|
|
14
|
+
* finding's credibility.
|
|
15
|
+
*
|
|
16
|
+
* ## The model proposes; this disposes
|
|
17
|
+
*
|
|
18
|
+
* The discipline `--suggest` has had since 1.6, applied to a harder claim. A
|
|
19
|
+
* rewrite suggestion is checkable by construction — the replacement is either
|
|
20
|
+
* shorter or it is not. A *semantic* finding is a claim about meaning, and the
|
|
21
|
+
* only part of it this layer can check is the **evidence**.
|
|
22
|
+
*
|
|
23
|
+
* So it checks the evidence, ruthlessly:
|
|
24
|
+
*
|
|
25
|
+
* 1. **Every quoted span must appear in the prompt character for character.**
|
|
26
|
+
* The single strongest signal available. A model reporting on a prompt
|
|
27
|
+
* while paraphrasing what it quotes has stopped reading and started
|
|
28
|
+
* writing, and everything else it said in that response is suspect.
|
|
29
|
+
* 2. **The spans must be distinct and must not overlap.** A "pair" that is one
|
|
30
|
+
* span quoted twice is not a finding about redundancy; it is a finding
|
|
31
|
+
* about nothing.
|
|
32
|
+
* 3. **The near-copy detector must not already catch it.** A model paid to
|
|
33
|
+
* re-report a finding the deterministic layer produces for free is a model
|
|
34
|
+
* being paid for nothing, and the reader sees the same thing twice with two
|
|
35
|
+
* different confidences attached.
|
|
36
|
+
* 4. **Nothing the model says about size is believed.** Tokens are counted
|
|
37
|
+
* here, from the spans, with the same counter everything else uses.
|
|
38
|
+
*
|
|
39
|
+
* ## It never becomes a prerequisite
|
|
40
|
+
*
|
|
41
|
+
* The deterministic core keeps working with no key, no network and no model.
|
|
42
|
+
* This is a pass *on top*, and the rule from 0.1.0 stands unchanged — which is
|
|
43
|
+
* why the verification lives here, in the package that has no network, and the
|
|
44
|
+
* call lives in the CLI.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import { normalizeForCompare, jaccard } from './similarity.js';
|
|
48
|
+
import { estimateTokens } from './tokenizer.js';
|
|
49
|
+
import type { TokenCounter } from './types.js';
|
|
50
|
+
|
|
51
|
+
export type SemanticKind =
|
|
52
|
+
/** Two spans teaching the same boundary in different words. */
|
|
53
|
+
| 'paraphrase-pair'
|
|
54
|
+
/** An instruction repeated in a distant part of the prompt. */
|
|
55
|
+
| 'restated-instruction'
|
|
56
|
+
/** A policy and a later clarification that disagree. */
|
|
57
|
+
| 'contradiction';
|
|
58
|
+
|
|
59
|
+
/** What the model returns, before anything has been checked. */
|
|
60
|
+
export interface SemanticProposal {
|
|
61
|
+
kind: SemanticKind;
|
|
62
|
+
/** Two spans, quoted from the prompt. */
|
|
63
|
+
spans: [string, string];
|
|
64
|
+
/** One sentence saying what the relation is. Never used to decide anything. */
|
|
65
|
+
because: string;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export type SemanticRejection =
|
|
69
|
+
/** A span is not in the prompt. The model paraphrased its own evidence. */
|
|
70
|
+
| 'span-not-found'
|
|
71
|
+
/** The two spans are the same text. */
|
|
72
|
+
| 'spans-identical'
|
|
73
|
+
/** The two spans overlap in the prompt. */
|
|
74
|
+
| 'spans-overlap'
|
|
75
|
+
/** Near-identical: the deterministic detector already reports this for free. */
|
|
76
|
+
| 'already-detected'
|
|
77
|
+
/**
|
|
78
|
+
* A contradiction whose spans are near-identical.
|
|
79
|
+
*
|
|
80
|
+
* Two spans that say the same thing cannot contradict each other, and a
|
|
81
|
+
* model that labels a near-copy a contradiction has mislabelled it — which
|
|
82
|
+
* makes every other label in the same response worth less.
|
|
83
|
+
*/
|
|
84
|
+
| 'contradiction-of-a-copy'
|
|
85
|
+
/** A duplicate of a finding already accepted. */
|
|
86
|
+
| 'duplicate';
|
|
87
|
+
|
|
88
|
+
export interface SemanticFinding {
|
|
89
|
+
kind: SemanticKind;
|
|
90
|
+
spans: [string, string];
|
|
91
|
+
because: string;
|
|
92
|
+
/** Where each span sits, so a reader can go and look. */
|
|
93
|
+
offsets: [number, number];
|
|
94
|
+
/**
|
|
95
|
+
* Tokens the shorter span holds — a **ceiling** on what resolving this could
|
|
96
|
+
* save, and named as one.
|
|
97
|
+
*
|
|
98
|
+
* Not a saving. Resolving a paraphrase pair means rewriting both spans into
|
|
99
|
+
* one, and how much that recovers depends on what the merged version says,
|
|
100
|
+
* which nobody knows yet. The ceiling is the honest figure: it cannot save
|
|
101
|
+
* more than deleting the smaller half.
|
|
102
|
+
*/
|
|
103
|
+
ceilingTokens: number;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface SemanticResult {
|
|
107
|
+
findings: SemanticFinding[];
|
|
108
|
+
rejected: Array<{ proposal: SemanticProposal; reason: SemanticRejection }>;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* How similar two spans may be before the deterministic detector owns them.
|
|
113
|
+
*
|
|
114
|
+
* **0.92, read off `rules.ts` rather than chosen.** The duplicate-example rule
|
|
115
|
+
* drops a few-shot example when its Jaccard similarity to a kept one is at
|
|
116
|
+
* least 0.92, so that is exactly the line above which a model is being paid to
|
|
117
|
+
* re-report something the rules engine produces for free.
|
|
118
|
+
*
|
|
119
|
+
* The first version of this file said 0.8 and claimed in a comment that it
|
|
120
|
+
* matched the deterministic pass. It did not, and the error ran the dangerous
|
|
121
|
+
* way: everything between 0.8 and 0.92 is a pair the rules engine does **not**
|
|
122
|
+
* catch, and this layer was silently throwing those away — discarding exactly
|
|
123
|
+
* the findings the chapter exists to surface, while a comment asserted the
|
|
124
|
+
* opposite.
|
|
125
|
+
*/
|
|
126
|
+
export const ALREADY_DETECTED_SIMILARITY = 0.92;
|
|
127
|
+
|
|
128
|
+
export interface VerifyOptions {
|
|
129
|
+
tokenCounter?: TokenCounter;
|
|
130
|
+
/** Cap on findings, largest ceiling first. Defaults to 20. */
|
|
131
|
+
max?: number;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function verifySemanticProposals(
|
|
135
|
+
prompt: string,
|
|
136
|
+
proposals: readonly SemanticProposal[],
|
|
137
|
+
options: VerifyOptions = {},
|
|
138
|
+
): SemanticResult {
|
|
139
|
+
const { tokenCounter = estimateTokens, max = 20 } = options;
|
|
140
|
+
const findings: SemanticFinding[] = [];
|
|
141
|
+
const rejected: SemanticResult['rejected'] = [];
|
|
142
|
+
const claimed: Array<[number, number]> = [];
|
|
143
|
+
|
|
144
|
+
for (const proposal of proposals) {
|
|
145
|
+
const [first, second] = proposal.spans;
|
|
146
|
+
|
|
147
|
+
const a = prompt.indexOf(first);
|
|
148
|
+
const b = prompt.indexOf(second);
|
|
149
|
+
if (a === -1 || b === -1) {
|
|
150
|
+
// The strongest check there is. A model that paraphrases what it quotes
|
|
151
|
+
// has stopped reading the prompt and started writing about it.
|
|
152
|
+
rejected.push({ proposal, reason: 'span-not-found' });
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
if (first === second) {
|
|
156
|
+
rejected.push({ proposal, reason: 'spans-identical' });
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const aEnd = a + first.length;
|
|
160
|
+
const bEnd = b + second.length;
|
|
161
|
+
if (a < bEnd && b < aEnd) {
|
|
162
|
+
rejected.push({ proposal, reason: 'spans-overlap' });
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const similarity = jaccard(normalizeForCompare(first), normalizeForCompare(second));
|
|
167
|
+
if (proposal.kind === 'contradiction' && similarity >= ALREADY_DETECTED_SIMILARITY) {
|
|
168
|
+
rejected.push({ proposal, reason: 'contradiction-of-a-copy' });
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (proposal.kind !== 'contradiction' && similarity >= ALREADY_DETECTED_SIMILARITY) {
|
|
172
|
+
rejected.push({ proposal, reason: 'already-detected' });
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* A finding covering ground an accepted one already covers.
|
|
178
|
+
*
|
|
179
|
+
* Overlap against what has been *accepted*, not against every proposal —
|
|
180
|
+
* so a rejected finding never blocks a good one that touches the same
|
|
181
|
+
* paragraph.
|
|
182
|
+
*/
|
|
183
|
+
const overlapsAccepted = claimed.some(
|
|
184
|
+
([start, end]) => (a < end && start < aEnd) || (b < end && start < bEnd),
|
|
185
|
+
);
|
|
186
|
+
if (overlapsAccepted) {
|
|
187
|
+
rejected.push({ proposal, reason: 'duplicate' });
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
claimed.push([a, aEnd], [b, bEnd]);
|
|
192
|
+
findings.push({
|
|
193
|
+
kind: proposal.kind,
|
|
194
|
+
spans: proposal.spans,
|
|
195
|
+
because: proposal.because,
|
|
196
|
+
offsets: [a, b],
|
|
197
|
+
// Counted here, from the spans. Nothing the model said about size is
|
|
198
|
+
// believed, and a contradiction saves nothing by being resolved — the
|
|
199
|
+
// point of finding one is that the prompt is wrong, not that it is long.
|
|
200
|
+
ceilingTokens:
|
|
201
|
+
proposal.kind === 'contradiction'
|
|
202
|
+
? 0
|
|
203
|
+
: Math.min(tokenCounter(first), tokenCounter(second)),
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
findings.sort((x, y) => y.ceilingTokens - x.ceilingTokens);
|
|
208
|
+
return { findings: findings.slice(0, max), rejected };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* What running the pass costs, computed before it runs.
|
|
213
|
+
*
|
|
214
|
+
* A tool that spends somebody's money to tell them how to spend less must be
|
|
215
|
+
* the first thing audited by its own arithmetic. The figure is an estimate of
|
|
216
|
+
* one call — the prompt in, a small structured answer out — and it says which
|
|
217
|
+
* half is which by having only one half.
|
|
218
|
+
*/
|
|
219
|
+
export function semanticPassCost(
|
|
220
|
+
prompt: string,
|
|
221
|
+
rates: { inputPerMTok: number; outputPerMTok: number },
|
|
222
|
+
options: { tokenCounter?: TokenCounter; expectedOutputTokens?: number } = {},
|
|
223
|
+
): { inputTokens: number; outputTokens: number; usd: number; provenance: 'estimated' } {
|
|
224
|
+
const { tokenCounter = estimateTokens, expectedOutputTokens = 800 } = options;
|
|
225
|
+
const inputTokens = tokenCounter(prompt) + tokenCounter(SEMANTIC_SYSTEM_PROMPT);
|
|
226
|
+
return {
|
|
227
|
+
inputTokens,
|
|
228
|
+
outputTokens: expectedOutputTokens,
|
|
229
|
+
usd:
|
|
230
|
+
(inputTokens / 1_000_000) * rates.inputPerMTok +
|
|
231
|
+
(expectedOutputTokens / 1_000_000) * rates.outputPerMTok,
|
|
232
|
+
provenance: 'estimated',
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export const SEMANTIC_SYSTEM_PROMPT = `You find places where a prompt says the same thing twice in different words, or contradicts itself.
|
|
237
|
+
|
|
238
|
+
Return ONLY a JSON array. Each element is {"kind": "...", "spans": ["...", "..."], "because": "..."}.
|
|
239
|
+
|
|
240
|
+
"kind" is one of:
|
|
241
|
+
- "paraphrase-pair": two passages teaching the same rule or boundary in different words.
|
|
242
|
+
- "restated-instruction": an instruction given once and then given again elsewhere.
|
|
243
|
+
- "contradiction": two passages that cannot both be followed.
|
|
244
|
+
|
|
245
|
+
Rules:
|
|
246
|
+
- Both strings in "spans" MUST be copied character for character from the prompt. Do not paraphrase them, do not fix their punctuation, do not trim them differently. If you cannot copy a passage exactly, leave the finding out.
|
|
247
|
+
- The two spans must be different passages in different places. Never quote the same passage twice.
|
|
248
|
+
- Do not report passages that are near-identical wordings of each other — those are already found without you. Report only pairs whose WORDS differ while their MEANING is the same.
|
|
249
|
+
- "because" is one sentence naming the relation, for a human to read.
|
|
250
|
+
- Prefer a few certain findings to many possible ones. Return [] if there is nothing you are sure of.
|
|
251
|
+
|
|
252
|
+
No explanation, no code fences, no commentary. The array alone.`;
|