@holdyourvoice/hyv 3.4.0 → 3.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Readme.md +70 -127
- package/dist/cli.js +365 -327
- package/dist/fact-linter.js +107 -98
- package/dist/mcp.js +43 -192
- package/dist/rebuild-task.test.js +1 -1
- package/dist/stage1-evaluation.js +74 -48
- package/dist/text.js +8 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -432,77 +432,103 @@ export function sealRatings(packetInput, mappingInput, ratingsInput) {
|
|
|
432
432
|
const base = { kind: 'hyv-stage1-ratings-seal', version: '1', protocolDigest: packet.protocolDigest, candidateDigest: packet.candidateDigest, packetDigest: packet.packetDigest, mappingDigest: packet.mappingDigest, ratingsDigest: sha256Canonical(ratings), recordCount: ratings.length };
|
|
433
433
|
return { ...base, sealDigest: sha256Canonical(base) };
|
|
434
434
|
}
|
|
435
|
-
|
|
436
|
-
const protocol = assertCommittedProtocol(protocolInput);
|
|
437
|
-
const summary = validateRuns(protocol, runsInput);
|
|
438
|
-
const packet = parsePacket(packetInput);
|
|
435
|
+
function validateEvaluationPacket(protocol, runs, summary, packet) {
|
|
439
436
|
if (packet.protocolDigest !== protocol.protocolDigest || packet.candidateDigest !== candidateDigest(protocol) || packet.runsDigest !== summary.runsDigest)
|
|
440
437
|
fail('packet_digest_chain_mismatch');
|
|
441
|
-
const
|
|
442
|
-
|
|
443
|
-
if (packet.reviewableCount !== expectedReviewableCount || packet.nonReviewableCount !== protocol.cases.length - expectedReviewableCount)
|
|
438
|
+
const reviewableCount = protocol.cases.filter((entry) => runs.filter((run) => run.caseId === entry.caseId && effectiveRunOutcome(run) === 'completed').length === 2).length;
|
|
439
|
+
if (packet.reviewableCount !== reviewableCount || packet.nonReviewableCount !== protocol.cases.length - reviewableCount)
|
|
444
440
|
fail('blind_packet_count_mismatch');
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
const
|
|
448
|
-
const seal = sealRatings(packet, mapping, ratings);
|
|
449
|
-
if (sha256Canonical(seal) !== sha256Canonical(sealInput))
|
|
450
|
-
fail('ratings_seal_mismatch');
|
|
451
|
-
const audit = record(releaseAuditInput);
|
|
441
|
+
}
|
|
442
|
+
function parseReleaseAudit(protocol, input) {
|
|
443
|
+
const audit = record(input);
|
|
452
444
|
exactKeys(audit, ['kind', 'version', 'candidateCommit', 'protocolDigest', 'contractDigest', 'passed', 'digest']);
|
|
453
445
|
const { digest: auditDigest, ...auditCore } = audit;
|
|
454
446
|
if (auditDigest !== sha256Canonical(auditCore))
|
|
455
447
|
fail('release_audit_digest_mismatch');
|
|
456
448
|
if (audit.kind !== 'hyv-release-audit' || audit.version !== '1' || audit.candidateCommit !== protocol.stage1.sourceCommit || audit.protocolDigest !== protocol.protocolDigest || audit.contractDigest !== protocol.releaseAuditContractDigest || audit.passed !== true)
|
|
457
449
|
fail('release_audit_binding_mismatch');
|
|
450
|
+
return audit;
|
|
451
|
+
}
|
|
452
|
+
function correctionCount(ratings, mapping, arm, result) {
|
|
453
|
+
const label = mapping.labels.A === arm ? 'A' : 'B';
|
|
454
|
+
return ratings.filter((rating) => rating.correctionVersusConfirm[label] === result).length;
|
|
455
|
+
}
|
|
456
|
+
function workflowForArm(runs, arm) {
|
|
457
|
+
const armRuns = runs.filter((run) => run.arm === arm);
|
|
458
|
+
return { completed: armRuns.filter((run) => effectiveRunOutcome(run) === 'completed').length, denominator: armRuns.length };
|
|
459
|
+
}
|
|
460
|
+
function calculateEvaluation(protocol, runs, ratings, mapping, summary) {
|
|
461
|
+
const completedRatings = ratings.filter((rating) => rating.workflow === 'completed');
|
|
462
|
+
const preferenceDenominator = protocol.intentToTreat.expectedReviewers * protocol.cases.length;
|
|
463
|
+
const preferred = completedRatings.reduce((total, rating) => total + (rating.preferredLabel === 'tie' ? 0.5 : mapping.labels[rating.preferredLabel] === 'stage1' ? 1 : 0), 0);
|
|
464
|
+
const preferenceRate = preferenceDenominator ? preferred / preferenceDenominator : 0;
|
|
465
|
+
const stage1Confirm = correctionCount(completedRatings, mapping, 'stage1', 'confirm');
|
|
466
|
+
const baselineConfirm = correctionCount(completedRatings, mapping, 'baseline', 'confirm');
|
|
467
|
+
const stage1Correction = correctionCount(completedRatings, mapping, 'stage1', 'correction');
|
|
468
|
+
const baselineCorrection = correctionCount(completedRatings, mapping, 'baseline', 'correction');
|
|
469
|
+
const baselineWorkflow = workflowForArm(runs, 'baseline');
|
|
470
|
+
const stage1Workflow = workflowForArm(runs, 'stage1');
|
|
471
|
+
const correctionMarginMet = (stage1Confirm - baselineConfirm) / preferenceDenominator >= protocol.analysis.margin;
|
|
472
|
+
const workflowRegressed = stage1Workflow.denominator === 0 || baselineWorkflow.denominator === 0 || stage1Workflow.completed / stage1Workflow.denominator < baselineWorkflow.completed / baselineWorkflow.denominator;
|
|
473
|
+
const metrics = {
|
|
474
|
+
preference: { numerator: preferred, denominator: preferenceDenominator, rate: preferenceRate, uncertainty95: wilson95(preferred, preferenceDenominator) },
|
|
475
|
+
correctionVersusConfirm: { stage1: { corrections: stage1Correction, confirms: stage1Confirm, denominator: preferenceDenominator, correctionRate: stage1Correction / preferenceDenominator, uncertainty95: wilson95(stage1Correction, preferenceDenominator) }, baseline: { corrections: baselineCorrection, confirms: baselineConfirm, denominator: preferenceDenominator, correctionRate: baselineCorrection / preferenceDenominator, uncertainty95: wilson95(baselineCorrection, preferenceDenominator) } },
|
|
476
|
+
completion: { numerator: completedRatings.length, denominator: preferenceDenominator, rate: completedRatings.length / preferenceDenominator, uncertainty95: wilson95(completedRatings.length, preferenceDenominator) },
|
|
477
|
+
abandonment: { numerator: ratings.length - completedRatings.length, denominator: preferenceDenominator, rate: (ratings.length - completedRatings.length) / preferenceDenominator, uncertainty95: wilson95(ratings.length - completedRatings.length, preferenceDenominator) },
|
|
478
|
+
providerRuns: { completed: summary.completed, abandoned: summary.abandonments, hardFailures: summary.hardFailures, timeouts: summary.timeouts, denominator: summary.denominator },
|
|
479
|
+
};
|
|
480
|
+
return { correctionMarginMet, metrics, preferenceDenominator, preferenceRate, workflowRegressed };
|
|
481
|
+
}
|
|
482
|
+
function mappingCustodyIsValid(protocol, packet, mapping) {
|
|
483
|
+
const { attestation, ...openingCore } = mapping;
|
|
484
|
+
try {
|
|
485
|
+
validateAttestation(attestation, sha256Canonical(openingCore), protocol.protocolDigest, packet.packetDigest);
|
|
486
|
+
return attestation.purpose === 'stage1-mapping-custody';
|
|
487
|
+
}
|
|
488
|
+
catch {
|
|
489
|
+
return false;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
function evaluationBlockers(protocol, runs, packet, ratings, mapping, calculation) {
|
|
458
493
|
const blockers = ['human_writer_evidence_deferred'];
|
|
459
|
-
if (protocol.benchmark.synthetic || ratings.some((
|
|
494
|
+
if (protocol.benchmark.synthetic || ratings.some((rating) => rating.evidenceClass === 'synthetic-dry-run'))
|
|
460
495
|
blockers.push('synthetic_fixture_evidence');
|
|
461
496
|
if (protocol.mode !== 'locked-human')
|
|
462
497
|
blockers.push('locked_human_evidence_required');
|
|
463
498
|
if (protocol.mode === 'locked-human') {
|
|
464
499
|
blockers.push('reviewer_roster_verification_deferred');
|
|
465
|
-
|
|
466
|
-
try {
|
|
467
|
-
validateAttestation(attestation, sha256Canonical(openingCore), protocol.protocolDigest, packet.packetDigest);
|
|
468
|
-
if (attestation.purpose !== 'stage1-mapping-custody')
|
|
469
|
-
throw new Error('purpose');
|
|
470
|
-
}
|
|
471
|
-
catch {
|
|
500
|
+
if (!mappingCustodyIsValid(protocol, packet, mapping))
|
|
472
501
|
blockers.push('mapping_custody_attestation_required');
|
|
473
|
-
}
|
|
474
502
|
}
|
|
475
|
-
if (ratings.length !==
|
|
503
|
+
if (ratings.length !== calculation.preferenceDenominator)
|
|
476
504
|
blockers.push('reviewer_denominator_mismatch');
|
|
477
505
|
if (protocol.cases.length < protocol.analysis.minimumCases || ratings.length < protocol.analysis.minimumRatings)
|
|
478
506
|
blockers.push('preregistered_minimum_not_met');
|
|
479
|
-
if (
|
|
507
|
+
if (runs.some((run) => run.hardGates && Object.values(run.hardGates).some((passed) => !passed)))
|
|
480
508
|
blockers.push('hard_gate_regression');
|
|
481
|
-
|
|
482
|
-
const preferred = completedRatings.reduce((total, rating) => total + (rating.preferredLabel === 'tie' ? 0.5 : mapping.labels[rating.preferredLabel] === 'stage1' ? 1 : 0), 0);
|
|
483
|
-
const preferenceDenominator = protocol.intentToTreat.expectedReviewers * protocol.cases.length;
|
|
484
|
-
const preferenceRate = preferenceDenominator ? preferred / preferenceDenominator : 0;
|
|
485
|
-
const stage1Confirm = completedRatings.filter((rating) => rating.correctionVersusConfirm[mapping.labels.A === 'stage1' ? 'A' : 'B'] === 'confirm').length;
|
|
486
|
-
const baselineConfirm = completedRatings.filter((rating) => rating.correctionVersusConfirm[mapping.labels.A === 'baseline' ? 'A' : 'B'] === 'confirm').length;
|
|
487
|
-
const correctionMarginMet = (stage1Confirm - baselineConfirm) / preferenceDenominator >= protocol.analysis.margin;
|
|
488
|
-
if (preferenceRate < 0.5 + protocol.analysis.margin && !correctionMarginMet)
|
|
509
|
+
if (calculation.preferenceRate < 0.5 + protocol.analysis.margin && !calculation.correctionMarginMet)
|
|
489
510
|
blockers.push('checkpoint_threshold_not_met');
|
|
490
|
-
|
|
491
|
-
const baselineWorkflow = workflowByArm('baseline');
|
|
492
|
-
const stage1Workflow = workflowByArm('stage1');
|
|
493
|
-
if (stage1Workflow.denominator === 0 || baselineWorkflow.denominator === 0 || stage1Workflow.completed / stage1Workflow.denominator < baselineWorkflow.completed / baselineWorkflow.denominator)
|
|
511
|
+
if (calculation.workflowRegressed)
|
|
494
512
|
blockers.push('workflow_regression');
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
const
|
|
513
|
+
return blockers;
|
|
514
|
+
}
|
|
515
|
+
export function reduceEvaluation(protocolInput, runsInput, packetInput, mappingInput, ratingsInput, sealInput, releaseAuditInput) {
|
|
516
|
+
const protocol = assertCommittedProtocol(protocolInput);
|
|
517
|
+
const summary = validateRuns(protocol, runsInput);
|
|
518
|
+
const runs = runsInput;
|
|
519
|
+
const packet = parsePacket(packetInput);
|
|
520
|
+
validateEvaluationPacket(protocol, runs, summary, packet);
|
|
521
|
+
const mapping = parseMapping(mappingInput);
|
|
522
|
+
validateMappingOpening(protocol, mapping);
|
|
523
|
+
const ratings = validateReviewerLog(protocol, packet, ratingsInput, true);
|
|
524
|
+
const seal = sealRatings(packet, mapping, ratings);
|
|
525
|
+
if (sha256Canonical(seal) !== sha256Canonical(sealInput))
|
|
526
|
+
fail('ratings_seal_mismatch');
|
|
527
|
+
const audit = parseReleaseAudit(protocol, releaseAuditInput);
|
|
528
|
+
const calculation = calculateEvaluation(protocol, runs, ratings, mapping, summary);
|
|
529
|
+
const blockers = evaluationBlockers(protocol, runs, packet, ratings, mapping, calculation);
|
|
530
|
+
const intentToTreat = { ...summary, expectedRatings: calculation.preferenceDenominator, observedRatings: ratings.length, missingRatings: Math.max(0, calculation.preferenceDenominator - ratings.length), reconciled: summary.denominator === protocol.intentToTreat.expectedAssignments && ratings.length === calculation.preferenceDenominator };
|
|
531
|
+
const base = { kind: 'hyv-stage1-evaluation-report', version: '1', protocolDigest: protocol.protocolDigest, candidateDigest: candidateDigest(protocol), runsDigest: summary.runsDigest, packetDigest: packet.packetDigest, mappingDigest: packet.mappingDigest, sealDigest: seal.sealDigest, releaseAuditDigest: audit.digest, intentToTreat, metrics: calculation.metrics, decision: blockers.length ? 'BLOCKED' : 'PASS', promotable: blockers.length === 0, blockers };
|
|
506
532
|
return { ...base, reportDigest: sha256Canonical(base) };
|
|
507
533
|
}
|
|
508
534
|
export function preflight(protocolInput) {
|
package/dist/text.js
CHANGED
|
@@ -25,7 +25,14 @@ export function sentences(text) {
|
|
|
25
25
|
if (character === '.') {
|
|
26
26
|
const previous = text[index - 1] ?? '';
|
|
27
27
|
const next = text[index + 1] ?? '';
|
|
28
|
-
|
|
28
|
+
let previousWordStart = index;
|
|
29
|
+
while (previousWordStart > start) {
|
|
30
|
+
const previousLetter = text.slice(Math.max(start, previousWordStart - 2), previousWordStart).match(/\p{L}$/u)?.[0];
|
|
31
|
+
if (!previousLetter)
|
|
32
|
+
break;
|
|
33
|
+
previousWordStart -= previousLetter.length;
|
|
34
|
+
}
|
|
35
|
+
const previousWord = text.slice(previousWordStart, index).toLowerCase() || undefined;
|
|
29
36
|
if ((/\d/.test(previous) && /\d/.test(next)) || /\p{L}/u.test(next) || (previousWord && abbreviations.has(previousWord)))
|
|
30
37
|
continue;
|
|
31
38
|
}
|
package/dist/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const HYV_VERSION = '3.4.
|
|
1
|
+
export const HYV_VERSION = '3.4.2';
|
package/package.json
CHANGED