forge-workflow 0.1.0-beta.3 → 0.1.0-beta.5

package/scripts/lib/promotion-scorecard.js

@@ -0,0 +1,314 @@
+'use strict';
+
+const BOOTSTRAP_SAMPLES = 10000;
+const Z_95 = 1.959963984540054;
+const RISKS = Object.freeze(['low', 'medium', 'high']);
+const STATUSES = new Set(['PASS', 'FAIL', 'INCOMPLETE']);
+const TRIALS = Object.freeze([0, 1, 2]);
+const TIER_PHASES = Object.freeze({ 30: 'instrumentation', 100: 'decision', 300: 'confirmation' });
+
+function round(value) {
+  return Number(value.toFixed(12));
+}
+
+function wilsonInterval(successes, total) {
+  if (!Number.isInteger(successes) || !Number.isInteger(total)
+    || successes < 0 || total < 0 || successes > total) {
+    throw new Error('Wilson counts must be valid non-negative integers');
+  }
+  if (total === 0) return null;
+  const rate = successes / total;
+  const zSquared = Z_95 ** 2;
+  const denominator = 1 + zSquared / total;
+  const center = (rate + zSquared / (2 * total)) / denominator;
+  const margin = Z_95 * Math.sqrt(
+    (rate * (1 - rate) + zSquared / (4 * total)) / total
+  ) / denominator;
+  return { lower: round(center - margin), upper: round(center + margin) };
+}
+
+function isValidArm(value) {
+  return value && typeof value === 'object'
+    && STATUSES.has(value.status)
+    && typeof value.hardFailure === 'boolean'
+    && Number.isFinite(value.latencyMs) && value.latencyMs >= 0
+    && Number.isFinite(value.tokens) && value.tokens >= 0;
+}
+
+function isValidPair(pair) {
+  return pair && typeof pair === 'object'
+    && typeof pair.caseId === 'string' && pair.caseId.length > 0
+    && RISKS.includes(pair.risk)
+    && ['DEV', 'TEST'].includes(pair.split)
+    && typeof pair.model === 'string' && pair.model.length > 0
+    && TRIALS.includes(pair.trialIndex)
+    && isValidArm(pair.current)
+    && isValidArm(pair.bounded);
+}
+
+function evidenceShapeError(tier, pairs) {
+  if (!Object.hasOwn(TIER_PHASES, tier)) return { error: 'tier_invalid' };
+  if (!Array.isArray(pairs) || pairs.length === 0) return { error: 'evidence_missing' };
+  if (pairs.some((pair) => !isValidPair(pair))) return { error: 'evidence_malformed' };
+  if (pairs.some((pair) => pair.current.status === 'INCOMPLETE' || pair.bounded.status === 'INCOMPLETE')) {
+    return { error: 'evidence_incomplete' };
+  }
+  return null;
+}
+
+function collectMatrix(pairs) {
+  const models = [...new Set(pairs.map((pair) => pair.model))].sort();
+  const cases = new Map();
+  const seen = new Set();
+  for (const pair of pairs) {
+    const key = `${pair.caseId}\0${pair.model}\0${pair.trialIndex}`;
+    if (seen.has(key)) return { error: 'duplicate_pair' };
+    seen.add(key);
+    const existing = cases.get(pair.caseId);
+    if (existing && (existing.risk !== pair.risk || existing.split !== pair.split)) {
+      return { error: 'case_metadata_mismatch' };
+    }
+    cases.set(pair.caseId, { risk: pair.risk, split: pair.split });
+  }
+  return { models, cases, seen };
+}
+
+function pairingError(tier, models, cases, seen) {
+  if (models.length !== 2) return 'model_pair_incomplete';
+  if (cases.size !== tier) return 'case_count_incomplete';
+  const splitCounts = { DEV: 0, TEST: 0 };
+  for (const { split } of cases.values()) splitCounts[split] += 1;
+  if (splitCounts.DEV !== tier * 0.6 || splitCounts.TEST !== tier * 0.4) {
+    return 'split_count_incomplete';
+  }
+
+  for (const caseId of cases.keys()) {
+    for (const model of models) {
+      for (const trial of TRIALS) {
+        if (!seen.has(`${caseId}\0${model}\0${trial}`)) return 'pairing_incomplete';
+      }
+    }
+  }
+  return null;
+}
+
+function validatePairs(tier, pairs) {
+  const shapeError = evidenceShapeError(tier, pairs);
+  if (shapeError) return shapeError;
+  const matrix = collectMatrix(pairs);
+  if (matrix.error) return matrix;
+  const incompletePairing = pairingError(tier, matrix.models, matrix.cases, matrix.seen);
+  if (incompletePairing) return { error: incompletePairing };
+  const testRisks = new Set(pairs.filter((pair) => pair.split === 'TEST').map((pair) => pair.risk));
+  if (RISKS.some((risk) => !testRisks.has(risk))) return { error: 'risk_stratum_incomplete' };
+  return { models: matrix.models };
+}
+
+function passStats(pairs, armName) {
+  const passes = pairs.filter((pair) => pair[armName].status === 'PASS').length;
+  const total = pairs.length;
+  return {
+    passes,
+    total,
+    rate: round(passes / total),
+    interval: wilsonInterval(passes, total),
+  };
+}
+
+function absoluteStats(testPairs, models) {
+  const byModel = Object.create(null);
+  for (const model of models) {
+    const modelPairs = testPairs.filter((pair) => pair.model === model);
+    byModel[model] = {
+      current: passStats(modelPairs, 'current'),
+      bounded: passStats(modelPairs, 'bounded'),
+    };
+  }
+  return {
+    pooled: {
+      current: passStats(testPairs, 'current'),
+      bounded: passStats(testPairs, 'bounded'),
+    },
+    byModel,
+  };
+}
+
+function createPrng() {
+  let state = 0x5eed1234;
+  return () => {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let value = state;
+    value = Math.imul(value ^ (value >>> 15), value | 1);
+    value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
+    return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function riskClusters(testPairs) {
+  const strata = Object.fromEntries(RISKS.map((risk) => [risk, new Map()]));
+  for (const pair of testPairs) {
+    const clusters = strata[pair.risk];
+    if (!clusters.has(pair.caseId)) clusters.set(pair.caseId, []);
+    clusters.get(pair.caseId).push(pair);
+  }
+  return RISKS.map((risk) => (
+    [...strata[risk].entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([, rows]) => rows)
+  ));
+}
+
+function pairedDelta(pairs) {
+  const difference = pairs.reduce((sum, pair) => (
+    sum + Number(pair.bounded.status === 'PASS') - Number(pair.current.status === 'PASS')
+  ), 0);
+  return round(difference / pairs.length);
+}
+
+function quantile(sorted, probability) {
+  const position = (sorted.length - 1) * probability;
+  const lowerIndex = Math.floor(position);
+  const weight = position - lowerIndex;
+  const lower = sorted[lowerIndex];
+  const upper = sorted[Math.min(lowerIndex + 1, sorted.length - 1)];
+  return round(lower + (upper - lower) * weight);
+}
+
+function pairedBootstrap(testPairs) {
+  const strata = riskClusters(testPairs);
+  const random = createPrng();
+  const deltas = [];
+  for (let sample = 0; sample < BOOTSTRAP_SAMPLES; sample += 1) {
+    const sampled = [];
+    for (const clusters of strata) {
+      for (let index = 0; index < clusters.length; index += 1) {
+        sampled.push(...clusters[Math.floor(random() * clusters.length)]);
+      }
+    }
+    deltas.push(pairedDelta(sampled));
+  }
+  deltas.sort((left, right) => left - right);
+  return {
+    samples: BOOTSTRAP_SAMPLES,
+    delta: pairedDelta(testPairs),
+    interval: { lower: quantile(deltas, 0.025), upper: quantile(deltas, 0.975) },
+  };
+}
+
+function percentile(values, probability) {
+  const sorted = [...values].sort((left, right) => left - right);
+  return sorted[Math.ceil(probability * sorted.length) - 1];
+}
+
+function median(values) {
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1
+    ? sorted[middle]
+    : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+function safeRatio(numerator, denominator) {
+  if (denominator === 0) return numerator === 0 ? 1 : Number.POSITIVE_INFINITY;
+  return round(numerator / denominator);
+}
+
+function operationalRatios(testPairs) {
+  const currentLatency = testPairs.map((pair) => pair.current.latencyMs);
+  const boundedLatency = testPairs.map((pair) => pair.bounded.latencyMs);
+  const currentTokens = testPairs.map((pair) => pair.current.tokens);
+  const boundedTokens = testPairs.map((pair) => pair.bounded.tokens);
+  return {
+    latencyP95: safeRatio(percentile(boundedLatency, 0.95), percentile(currentLatency, 0.95)),
+    tokenMedian: safeRatio(median(boundedTokens), median(currentTokens)),
+  };
+}
+
+function evaluateThresholds({ pairs, testPairs, models, absolute, bootstrap, operations }) {
+  const perModelDelta = models.every((model) => {
+    const stats = absolute.byModel[model];
+    return stats.bounded.rate - stats.current.rate >= 0;
+  });
+  const highRisk = pairs.filter((pair) => pair.risk === 'high');
+  const highRiskRegression = models.every((model) => (
+    pairedDelta(highRisk.filter((pair) => pair.model === model)) >= -0.02
+  ));
+  return {
+    zeroHardFailures: pairs.every((pair) => !pair.current.hardFailure && !pair.bounded.hardFailure),
+    highRiskTestPass: testPairs
+      .filter((pair) => pair.risk === 'high')
+      .every((pair) => pair.bounded.status === 'PASS'),
+    pooledDelta: bootstrap.delta >= 0.05 && bootstrap.interval.lower > 0,
+    perModelDelta,
+    highRiskRegression,
+    latency: operations.latencyP95 <= 1.25,
+    tokens: operations.tokenMedian <= 1.20,
+  };
+}
+
+function thresholdReasons(thresholds) {
+  const reasonByThreshold = {
+    zeroHardFailures: 'hard_failure',
+    highRiskTestPass: 'high_risk_test_pass',
+    pooledDelta: 'pooled_delta',
+    perModelDelta: 'per_model_delta',
+    highRiskRegression: 'high_risk_regression',
+    latency: 'latency_cap',
+    tokens: 'token_cap',
+  };
+  return Object.entries(reasonByThreshold)
+    .filter(([threshold]) => !thresholds[threshold])
+    .map(([, reason]) => reason);
+}
+
+function incomplete(tier, reason) {
+  return {
+    status: 'INCOMPLETE',
+    winner: null,
+    phase: TIER_PHASES[tier] ?? null,
+    mergeAuthorized: false,
+    reasons: [reason],
+  };
+}
+
+function scorePromotion({ tier, pairs } = {}) {
+  const validation = validatePairs(tier, pairs);
+  if (validation.error) return incomplete(tier, validation.error);
+
+  const testPairs = pairs.filter((pair) => pair.split === 'TEST');
+  const absolute = absoluteStats(testPairs, validation.models);
+  const bootstrap = pairedBootstrap(testPairs);
+  const operations = operationalRatios(testPairs);
+  const thresholds = evaluateThresholds({
+    pairs,
+    testPairs,
+    models: validation.models,
+    absolute,
+    bootstrap,
+    operations,
+  });
+  if (tier === 30) {
+    return {
+      status: 'INCOMPLETE', winner: null, phase: TIER_PHASES[tier], mergeAuthorized: false,
+      reasons: ['instrumentation_only'], absolute, bootstrap, operations, thresholds,
+    };
+  }
+  const reasons = thresholdReasons(thresholds);
+  return {
+    status: reasons.length === 0 ? 'PASS' : 'FAIL',
+    winner: reasons.length === 0 ? 'bounded' : null,
+    phase: TIER_PHASES[tier],
+    mergeAuthorized: false,
+    reasons,
+    absolute,
+    bootstrap,
+    operations,
+    thresholds,
+  };
+}
+
+module.exports = {
+  BOOTSTRAP_SAMPLES,
+  scorePromotion,
+  wilsonInterval,
+};

package/scripts/npm-release-receipt.js

@@ -0,0 +1,134 @@
+#!/usr/bin/env node
+'use strict';
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const { execFileSync } = require('node:child_process');
+
+function requiredString(value, name) {
+	if (typeof value !== 'string' || value.length === 0 || /[\r\n]/.test(value)) {
+		throw new Error(`missing:${name}`);
+	}
+	return value;
+}
+
+function createReleaseSuiteReceipt(input) {
+	const repository = requiredString(input?.repository, 'repository');
+	const workflowRef = requiredString(input?.workflowRef, 'workflowRef');
+	const runId = requiredString(input?.runId, 'runId');
+	const runAttempt = requiredString(input?.runAttempt, 'runAttempt');
+	const sha = requiredString(input?.sha, 'sha');
+	const receiptSubject = [
+		repository,
+		workflowRef,
+		runId,
+		runAttempt,
+		'build',
+		sha,
+		'release-full',
+		'true',
+	].join('|');
+	const receipt = crypto.createHash('sha256').update(receiptSubject, 'utf8').digest('hex');
+	return { receipt, receiptSubject };
+}
+
+function verifyReleaseSuiteReceipt(input) {
+	const required = [
+		'repository',
+		'workflowRef',
+		'runId',
+		'runAttempt',
+		'expectedSha',
+		'verifiedSha',
+		'checkoutSha',
+		'receipt',
+		'receiptSubject',
+	];
+	for (const field of required) {
+		try {
+			requiredString(input?.[field], field);
+		} catch (_error) {
+			return { allowed: false, reason: `missing:${field}` };
+		}
+	}
+
+	if (input.expectedSha !== input.verifiedSha || input.expectedSha !== input.checkoutSha) {
+		return { allowed: false, reason: 'sha_mismatch' };
+	}
+	const expected = createReleaseSuiteReceipt({
+		repository: input.repository,
+		workflowRef: input.workflowRef,
+		runId: input.runId,
+		runAttempt: input.runAttempt,
+		sha: input.expectedSha,
+	});
+	if (input.receipt !== expected.receipt || input.receiptSubject !== expected.receiptSubject) {
+		return { allowed: false, reason: 'receipt_mismatch' };
+	}
+	return { allowed: true, reason: null };
+}
+
+function resolveCheckoutSha() {
+	return execFileSync('git', ['rev-parse', 'HEAD'], {
+		encoding: 'utf8',
+		stdio: ['ignore', 'pipe', 'pipe'],
+	}).trim();
+}
+
+function attributionFromEnv(env) {
+	return {
+		repository: env.GITHUB_REPOSITORY,
+		workflowRef: env.GITHUB_WORKFLOW_REF,
+		runId: env.GITHUB_RUN_ID,
+		runAttempt: env.GITHUB_RUN_ATTEMPT,
+	};
+}
+
+function main(argv = process.argv.slice(2), env = process.env, options = {}) {
+	const mode = argv[0];
+	const checkoutSha = (options.resolveCheckoutSha || resolveCheckoutSha)();
+	const attribution = attributionFromEnv(env);
+	if (mode === 'emit') {
+		const expectedSha = requiredString(env.EXPECTED_SHA, 'expectedSha');
+		if (checkoutSha !== expectedSha) return { success: false, reason: 'sha_mismatch' };
+		const outputPath = requiredString(env.GITHUB_OUTPUT, 'githubOutput');
+		const evidence = createReleaseSuiteReceipt({ ...attribution, sha: checkoutSha });
+		fs.appendFileSync(
+			outputPath,
+			`commitSha=${checkoutSha}\nreceipt=${evidence.receipt}\nreceiptSubject=${evidence.receiptSubject}\n`,
+			'utf8',
+		);
+		return { success: true, ...evidence, commitSha: checkoutSha };
+	}
+	if (mode === 'verify') {
+		const verification = verifyReleaseSuiteReceipt({
+			...attribution,
+			expectedSha: env.EXPECTED_SHA,
+			verifiedSha: env.VERIFIED_SHA,
+			checkoutSha,
+			receipt: env.RECEIPT,
+			receiptSubject: env.RECEIPT_SUBJECT,
+		});
+		return { success: verification.allowed, ...verification };
+	}
+	return { success: false, reason: 'mode_invalid' };
+}
+
+if (require.main === module) {
+	try {
+		const result = main();
+		if (!result.success) {
+			console.error(`Release suite receipt denied: ${result.reason}`);
+			process.exitCode = 1;
+		}
+	} catch (error) {
+		console.error(`Release suite receipt denied: ${error.message}`);
+		process.exitCode = 1;
+	}
+}
+
+module.exports = {
+	createReleaseSuiteReceipt,
+	verifyReleaseSuiteReceipt,
+	main,
+};