@aiwg/cli 2026.10.16 → 2026.10.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/capability-matrix.yaml +51 -0
- package/agentic/code/providers/flue/README.md +62 -0
- package/agentic/code/providers/flue/templates/aiwg/tools.ts +120 -0
- package/agentic/code/providers/model-capabilities.v1.json +23 -0
- package/agentic/code/providers/model-catalog.v1.json +9 -0
- package/dist/src/artifacts/corpus-tools/resolve.js +28 -6
- package/dist/src/artifacts/query-engine-corpus.js +217 -0
- package/dist/src/artifacts/query-engine-discover-output.js +229 -0
- package/dist/src/artifacts/query-engine-discover.js +285 -0
- package/dist/src/artifacts/query-engine-entries.js +188 -0
- package/dist/src/artifacts/query-engine-index.js +249 -0
- package/dist/src/artifacts/query-engine-matching.js +143 -0
- package/dist/src/artifacts/query-engine-options.js +2 -0
- package/dist/src/artifacts/query-engine-rules.js +202 -0
- package/dist/src/artifacts/query-engine-scoring.js +436 -0
- package/dist/src/artifacts/query-engine-show-card.js +194 -0
- package/dist/src/artifacts/query-engine-show-load.js +211 -0
- package/dist/src/artifacts/query-engine-show.js +300 -0
- package/dist/src/artifacts/query-engine.js +7 -2576
- package/dist/src/cli/handlers/help.js +1 -1
- package/dist/src/cli/handlers/init.js +1 -0
- package/dist/src/cli/handlers/mc-agents.js +111 -0
- package/dist/src/cli/handlers/mc-args.js +54 -0
- package/dist/src/cli/handlers/mc-control.js +227 -0
- package/dist/src/cli/handlers/mc-dispatch.js +203 -0
- package/dist/src/cli/handlers/mc-help.js +171 -0
- package/dist/src/cli/handlers/mc-run.js +343 -0
- package/dist/src/cli/handlers/mc-session.js +138 -0
- package/dist/src/cli/handlers/mc-status.js +258 -0
- package/dist/src/cli/handlers/mc-summary.js +43 -0
- package/dist/src/cli/handlers/mc-types.js +3 -0
- package/dist/src/cli/handlers/mc-worker.js +205 -0
- package/dist/src/cli/handlers/mc.js +12 -1665
- package/dist/src/cli/handlers/models-refusal.js +13 -0
- package/dist/src/cli/handlers/models.js +2 -1
- package/dist/src/cli/handlers/remove-bundled.js +133 -0
- package/dist/src/cli/handlers/remove-options.js +31 -0
- package/dist/src/cli/handlers/remove-user.js +174 -0
- package/dist/src/cli/handlers/remove.js +201 -0
- package/dist/src/cli/handlers/subcommands-bundles.js +257 -0
- package/dist/src/cli/handlers/subcommands-discovery.js +165 -0
- package/dist/src/cli/handlers/subcommands-list-format.js +129 -0
- package/dist/src/cli/handlers/subcommands-list.js +250 -0
- package/dist/src/cli/handlers/subcommands-plugins.js +213 -0
- package/dist/src/cli/handlers/subcommands-quickref.js +58 -0
- package/dist/src/cli/handlers/subcommands-rlm.js +113 -0
- package/dist/src/cli/handlers/subcommands-services.js +66 -0
- package/dist/src/cli/handlers/subcommands-storage.js +259 -0
- package/dist/src/cli/handlers/subcommands.js +18 -1984
- package/dist/src/cli/handlers/use.js +11 -5
- package/dist/src/cli/scope-resolver.js +1 -142
- package/dist/src/cli/services/deployment-verification.js +3 -0
- package/dist/src/cli/user-scope-mirror.js +173 -0
- package/dist/src/config/dispatch-host-thresholds.js +2 -0
- package/dist/src/config/dispatch-thresholds-runtime.mjs +4 -0
- package/dist/src/config/host-resource-report.d.mts +5 -0
- package/dist/src/config/host-resource-report.mjs +36 -6
- package/dist/src/config/host-resource-runtime.d.mts +3 -0
- package/dist/src/config/host-resource-runtime.mjs +9 -6
- package/dist/src/decision/ensemble-study/artifacts.js +7 -0
- package/dist/src/decision/ensemble-study/gate-policy.js +58 -0
- package/dist/src/decision/ensemble-study/gates.js +78 -0
- package/dist/src/decision/ensemble-study/protocol.js +18 -21
- package/dist/src/decision/ensemble-study/score.js +3 -2
- package/dist/src/decision/ensemble-study/statistics.js +7 -26
- package/dist/src/decision/sensitivity/comparative-gates.js +135 -0
- package/dist/src/decision/sensitivity/comparative.js +18 -41
- package/dist/src/extensions/claude-hooks-installer.js +18 -0
- package/dist/src/gates/evaluate.js +2 -358
- package/dist/src/gates/observations.js +363 -0
- package/dist/src/gates/pack-composition.js +270 -0
- package/dist/src/gates/pack-tightening.js +197 -0
- package/dist/src/gates/pack-validation.js +262 -0
- package/dist/src/gates/registry-error.js +5 -0
- package/dist/src/gates/registry.js +13 -685
- package/dist/src/mcp/json-document.mjs +29 -0
- package/dist/src/mcp/muse-native-config.mjs +1 -28
- package/dist/src/memory/provider-memory-adapter.js +1 -0
- package/dist/src/mission-protocol/worker-review-gate.js +7 -0
- package/dist/src/models/model-capabilities.v1.json +23 -0
- package/dist/src/models/model-catalog.v1.json +9 -0
- package/dist/src/models/policy-resources.js +1 -1
- package/dist/src/models/worker-plan.js +5 -5
- package/dist/src/plugin/command-ownership.js +34 -0
- package/dist/src/plugin/skill-command-frontmatter.js +197 -0
- package/dist/src/plugin/skill-command-translator.js +6 -232
- package/dist/src/providers/capability-matrix.yaml +51 -0
- package/dist/src/providers/context-output-ownership.js +26 -0
- package/dist/src/providers/deployment-ownership.d.mts +20 -0
- package/dist/src/providers/deployment-ownership.mjs +236 -0
- package/dist/src/providers/deployment-removal.js +222 -0
- package/dist/src/providers/provider-context-contracts.js +8 -0
- package/dist/src/providers/provider-definition-schema.js +1 -0
- package/dist/src/providers/provider-definitions.js +1 -0
- package/dist/src/providers/provider-seeds-3.js +18 -0
- package/dist/src/providers/removal-transaction.js +86 -0
- package/dist/src/providers/settings-document.d.mts +3 -0
- package/dist/src/providers/settings-document.mjs +47 -0
- package/dist/src/providers/settings-removal.js +190 -0
- package/dist/src/providers/skill-ownership.d.mts +20 -2
- package/dist/src/providers/skill-ownership.mjs +12 -4
- package/dist/src/providers/transformation-receipt-integration.js +15 -271
- package/dist/src/providers/transformation-receipt-outputs.js +99 -0
- package/dist/src/providers/transformation-receipt-source.js +194 -0
- package/dist/src/research/identity.js +8 -2
- package/dist/src/research/reference-match.js +66 -8
- package/dist/src/rules/activation.mjs +15 -6
- package/dist/src/rules/core-startup.mjs +2 -2
- package/dist/src/serve/mission-conductor.js +4 -0
- package/dist/src/serve/mission-summary.js +56 -0
- package/dist/src/skills/deployment-ownership.js +2 -2
- package/dist/src/skills/deployment-policy.js +9 -0
- package/dist/src/smiths/context-pipeline/aiwg-md.js +23 -0
- package/dist/src/smiths/context-pipeline/context-file-safety.js +3 -0
- package/dist/src/smiths/context-pipeline/flue-context.js +25 -0
- package/dist/src/smiths/context-pipeline/generator.js +7 -2
- package/dist/src/smiths/context-pipeline/workspace-bootstrap.js +9 -7
- package/dist/src/smiths/hook-bridge/index.js +9 -0
- package/package.json +5 -1
- package/schemas/decision/D17StudyProtocol.v1.schema.json +39 -67
- package/tools/agents/deploy-agents.mjs +21 -680
- package/tools/agents/deploy-command-mirrors.mjs +229 -0
- package/tools/agents/deploy-options.mjs +267 -0
- package/tools/agents/deploy-support.mjs +165 -0
- package/tools/agents/providers/base-addon-discovery.mjs +139 -0
- package/tools/agents/providers/base-behaviors.mjs +168 -0
- package/tools/agents/providers/base-context.mjs +205 -0
- package/tools/agents/providers/base-contract.mjs +116 -0
- package/tools/agents/providers/base-files.mjs +252 -0
- package/tools/agents/providers/base-flat-artifacts.mjs +233 -0
- package/tools/agents/providers/base-framework-discovery.mjs +309 -0
- package/tools/agents/providers/base-frontmatter.mjs +196 -0
- package/tools/agents/providers/base-inventory.mjs +269 -0
- package/tools/agents/providers/base-kernel-skills.mjs +354 -0
- package/tools/agents/providers/base-migrations.mjs +258 -0
- package/tools/agents/providers/base-models-platforms.mjs +273 -0
- package/tools/agents/providers/base-rule-index.mjs +450 -0
- package/tools/agents/providers/base-runtime.mjs +60 -0
- package/tools/agents/providers/base-skill-copy.mjs +273 -0
- package/tools/agents/providers/base.mjs +16 -3364
- package/tools/agents/providers/claude-post-deploy.mjs +10 -2
- package/tools/agents/providers/flue-project.mjs +79 -0
- package/tools/agents/providers/flue-specialists.mjs +64 -0
- package/tools/agents/providers/flue.mjs +50 -0
- package/tools/agents/providers/hermes-startup.mjs +0 -5
- package/tools/agents/providers/muse.mjs +7 -2
- package/tools/agents/providers/rule-delivery-surface.mjs +33 -5
- package/tools/agents/providers/rule-deployment-commit.mjs +19 -15
- package/tools/agents/providers/rule-deployment-prepare.mjs +13 -3
- package/tools/agents/providers/rule-projection.mjs +3 -0
- package/tools/agents/providers/rule-startup.mjs +3 -3
- package/tools/agents/use-rule-selection.mjs +17 -0
- package/tools/ralph-external/lib/worker-invocation-processes.mjs +4 -2
- package/tools/ralph-external/lib/worker-termination.mjs +15 -8
- package/vscode-extension/schemas/aiwg.config.v1.json +2 -4
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { scalarProvider } from '../../gates/providers/index.js';
|
|
2
|
+
export const D17_PACK_ID = 'project:d17-study';
|
|
3
|
+
export const D17_SLICES = ['direct-facts', 'multi-fact', 'negation', 'authority'];
|
|
4
|
+
const base = {
|
|
5
|
+
minimumPairs: 1200, minimumPerSlice: 300, maximumMissingPairs: 0,
|
|
6
|
+
nonInferiorityMarginBps: -300, benefitLowerExclusiveBps: 0,
|
|
7
|
+
maximumBrierUpperBps: 200, maximumAcceptedErrorUpperBps: 500,
|
|
8
|
+
minimumCoverageLowerBps: 6000, maximumAbstentionUpperBps: 500,
|
|
9
|
+
maximumP95LatencyIncreaseMs: 15000, maximumDisagreementBps: 1000,
|
|
10
|
+
minimumSuccessfulMembers: 3, highAgreementWarningBps: 100,
|
|
11
|
+
nativeRiskCoverage: 0.01, nativeLatency: 10000, nativeTokens: 8000, nativeCost: 800,
|
|
12
|
+
};
|
|
13
|
+
export const D17_NATIVE = [
|
|
14
|
+
{ metric: 'quality', comparison: 'delta-at-least', param: 'nonInferiorityMarginBps', divisor: 10000 },
|
|
15
|
+
{ metric: 'calibration', comparison: 'delta-at-most', param: 'maximumBrierUpperBps', divisor: 10000 },
|
|
16
|
+
{ metric: 'risk-coverage', comparison: 'delta-at-most', param: 'nativeRiskCoverage', divisor: 1 },
|
|
17
|
+
{ metric: 'abstention', comparison: 'delta-at-most', param: 'maximumAbstentionUpperBps', divisor: 10000 },
|
|
18
|
+
{ metric: 'latency', comparison: 'delta-at-most', param: 'nativeLatency', divisor: 1 },
|
|
19
|
+
{ metric: 'tokens', comparison: 'delta-at-most', param: 'nativeTokens', divisor: 1 },
|
|
20
|
+
{ metric: 'cost', comparison: 'delta-at-most', param: 'nativeCost', divisor: 1 },
|
|
21
|
+
{ metric: 'slice', comparison: 'delta-at-least', param: 'nonInferiorityMarginBps', divisor: 10000 },
|
|
22
|
+
];
|
|
23
|
+
export const d17NativeParameter = (metric) => `native_${metric.replaceAll('-', '_')}`;
|
|
24
|
+
export const D17_NATIVE_BOUNDS = Object.fromEntries(D17_NATIVE.map(row => [d17NativeParameter(row.metric), base[row.param] / row.divisor]));
|
|
25
|
+
export const D17_GATE_PARAMETERS = { ...base, ...D17_NATIVE_BOUNDS };
|
|
26
|
+
/** Scalar targets are the historical rounded endpoints, not newly recomputed intervals. */
|
|
27
|
+
export function d17GatePack() {
|
|
28
|
+
const definitions = [
|
|
29
|
+
['incomplete-pairs', 'minimumPairs', 'gte'], ['missing-pairs', 'maximumMissingPairs', 'lte'],
|
|
30
|
+
...D17_SLICES.map(slice => [`slice-support-${slice}`, 'minimumPerSlice', 'gte']),
|
|
31
|
+
['quality-ni', 'nonInferiorityMarginBps', 'gte'], ['brier-upper', 'maximumBrierUpperBps', 'lte'],
|
|
32
|
+
['abstention-upper', 'maximumAbstentionUpperBps', 'lte'],
|
|
33
|
+
['accepted-risk-upper', 'maximumAcceptedErrorUpperBps', 'lte'],
|
|
34
|
+
['coverage-lower', 'minimumCoverageLowerBps', 'gte'],
|
|
35
|
+
['latency-p95', 'maximumP95LatencyIncreaseMs', 'lte'],
|
|
36
|
+
['benefit', 'benefitLowerExclusiveBps', 'gte'],
|
|
37
|
+
...D17_NATIVE.map(row => [`native-${row.metric}`, d17NativeParameter(row.metric),
|
|
38
|
+
row.comparison === 'delta-at-least' ? 'gte' : 'lte']),
|
|
39
|
+
];
|
|
40
|
+
const direction = (name) => name.startsWith('minimum') || name === 'nonInferiorityMarginBps'
|
|
41
|
+
|| name === 'benefitLowerExclusiveBps' || name === 'native_quality' || name === 'native_slice' ? 'higher-is-stricter' : 'lower-is-stricter';
|
|
42
|
+
const gates = definitions.map(([id, param, op]) => {
|
|
43
|
+
const support = param === 'minimumPairs' || param === 'minimumPerSlice';
|
|
44
|
+
return { id, kind: support ? 'minimum-n' : 'value-threshold',
|
|
45
|
+
metric: { provider: scalarProvider.id, name: 'measurement' },
|
|
46
|
+
...(support ? { minimumN: { param } } : { threshold: { op, param } }),
|
|
47
|
+
scope: { mode: 'listed', slices: [id] },
|
|
48
|
+
...(id.startsWith('native-') ? { minimumN: { param: 'minimumPairs' } } : {}),
|
|
49
|
+
direction: op === 'gte' ? 'higher-is-stricter' : 'lower-is-stricter', onFail: 'HOLD' };
|
|
50
|
+
});
|
|
51
|
+
return { apiVersion: 'gates.aiwg.io/v1alpha1', kind: 'GatePack',
|
|
52
|
+
metadata: { id: D17_PACK_ID, version: '1.0.0', description: 'D17 historical diagnostic endpoints.' },
|
|
53
|
+
spec: { parameters: Object.fromEntries(Object.keys(D17_GATE_PARAMETERS).map(name => [name,
|
|
54
|
+
{ type: name === 'minimumPairs' || name === 'minimumPerSlice' ? 'count' : 'value', direction: direction(name) }])),
|
|
55
|
+
metrics: { measurement: { provider: scalarProvider.id, kind: 'scalar' } }, gates,
|
|
56
|
+
combine: { rule: 'max-severity' } } };
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=gate-policy.js.map
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// Adapt validated D17 measurements to gate bindings without changing retained report bytes.
|
|
2
|
+
import { artifactDigest } from '../validate.js';
|
|
3
|
+
import { evaluateGates, sealGateHoldout, sealUpstream } from '../../gates/evaluate.js';
|
|
4
|
+
import { ceilingDecision } from '../../gates/lattice.js';
|
|
5
|
+
import { GateRegistry } from '../../gates/registry.js';
|
|
6
|
+
import { createCoreProviderRegistry, scalarProvider } from '../../gates/providers/index.js';
|
|
7
|
+
import { qualifyGateParameter } from '../../gates/types.js';
|
|
8
|
+
import { D17_GATE_PARAMETERS, D17_PACK_ID, d17GatePack } from './gate-policy.js';
|
|
9
|
+
const registry = new GateRegistry(createCoreProviderRegistry());
|
|
10
|
+
registry.registerPack(d17GatePack(), { namespace: 'project' });
|
|
11
|
+
const pack = registry.resolvePack(D17_PACK_ID);
|
|
12
|
+
const stamp = '2026-10-01T00:00:00.000Z';
|
|
13
|
+
export function d17GateBinding() {
|
|
14
|
+
return { apiVersion: 'gates.aiwg.io/v1alpha1', kind: 'GateBinding',
|
|
15
|
+
metadata: { id: 'd17-study', version: '1.0.0', description: 'Legacy D17 diagnostic replay policy.' },
|
|
16
|
+
spec: { packs: [{ id: D17_PACK_ID, version: '1.0.0', digest: pack.digest, resolvedDigest: pack.resolvedDigest }],
|
|
17
|
+
parameters: Object.fromEntries(Object.entries(D17_GATE_PARAMETERS).map(([name, value]) => [qualifyGateParameter(D17_PACK_ID, name), value])),
|
|
18
|
+
slices: pack.resolved.spec.gates.map(gate => gate.id), references: [],
|
|
19
|
+
metricProviders: [{ id: scalarProvider.id, version: scalarProvider.version, sourceDigest: scalarProvider.sourceDigest }],
|
|
20
|
+
registeredAt: stamp, frozenAt: stamp } };
|
|
21
|
+
}
|
|
22
|
+
/** Historical core metrics are caller-trusted; this adapter grants no new freeze/access attestation. */
|
|
23
|
+
export function d17GateInput(measurements, binding = d17GateBinding(), integrity) {
|
|
24
|
+
const m = measurements;
|
|
25
|
+
const bySlice = {};
|
|
26
|
+
const add = (id, value, n = m.sampleN) => { bySlice[id] = { n, value }; };
|
|
27
|
+
add('incomplete-pairs', m.sampleN);
|
|
28
|
+
add('missing-pairs', m.missingIds.length);
|
|
29
|
+
for (const row of m.slices)
|
|
30
|
+
add(`slice-support-${row.slice}`, row.n, row.n);
|
|
31
|
+
add('quality-ni', m.quality?.lowerBps ?? null);
|
|
32
|
+
add('brier-upper', m.brier?.upperBps ?? null);
|
|
33
|
+
add('abstention-upper', m.abstention?.upperBps ?? null);
|
|
34
|
+
add('accepted-risk-upper', m.acceptedErrorUpperBps);
|
|
35
|
+
add('coverage-lower', m.coverageLowerBps);
|
|
36
|
+
add('latency-p95', m.p95LatencyIncreaseMs);
|
|
37
|
+
// Integer basis-point lower bounds: subtract one to retain the strict benefit comparison.
|
|
38
|
+
add('benefit', m.quality ? m.quality.lowerBps - 1 : null);
|
|
39
|
+
for (const row of m.pairedDeltas) {
|
|
40
|
+
add(`native-${row.metric}`, row.delta, row.pairs);
|
|
41
|
+
}
|
|
42
|
+
const trustedBindingDigest = artifactDigest(binding);
|
|
43
|
+
return { binding, registry, trustedBindingDigest, metrics: { providers: {
|
|
44
|
+
[scalarProvider.id]: { version: scalarProvider.version, sourceDigest: scalarProvider.sourceDigest,
|
|
45
|
+
metrics: { measurement: { bySlice, pooled: null } } },
|
|
46
|
+
} }, holdout: sealGateHoldout({ frozenDigest: trustedBindingDigest, firstAccessedAt: null }),
|
|
47
|
+
upstream: integrity ? sealUpstream(integrity) : null, now: stamp,
|
|
48
|
+
// Legacy D17 replay must not pick up ambient project policy or claim a new operator approval.
|
|
49
|
+
floors: 'none-explicit-opt-out' };
|
|
50
|
+
}
|
|
51
|
+
/** Translate evaluated statuses only; threshold comparisons belong exclusively to evaluateGates. */
|
|
52
|
+
export function d17GateFindings(report) {
|
|
53
|
+
const findings = new Set();
|
|
54
|
+
for (const gate of report.gateEvidence) {
|
|
55
|
+
if (gate.status === 'pass' || gate.gateId === 'benefit')
|
|
56
|
+
continue;
|
|
57
|
+
if (gate.gateId === 'missing-pairs')
|
|
58
|
+
findings.add('incomplete-pairs');
|
|
59
|
+
else if (gate.gateId.startsWith('slice-support-'))
|
|
60
|
+
findings.add(`slice-insufficient:${gate.gateId.slice(14)}`);
|
|
61
|
+
else if (gate.gateId.startsWith('native-'))
|
|
62
|
+
findings.add(`paired-delta-${gate.n === 0 ? 'unknown' : gate.status === 'insufficient'
|
|
63
|
+
? 'insufficient' : 'failed'}:${gate.gateId.slice(7)}`);
|
|
64
|
+
else
|
|
65
|
+
findings.add(gate.gateId);
|
|
66
|
+
}
|
|
67
|
+
return { findings: [...findings].sort(), statisticalGate: findings.size ? 'HOLD' : 'pass',
|
|
68
|
+
benefitSupported: report.gateEvidence.some(gate => gate.gateId === 'benefit' && gate.status === 'pass') };
|
|
69
|
+
}
|
|
70
|
+
export function evaluateD17Statistics(measurements, binding, integrity) {
|
|
71
|
+
return evaluateGates(d17GateInput(measurements, binding, integrity));
|
|
72
|
+
}
|
|
73
|
+
/** Independent diagnostic ceiling: neither a binding nor the calibrated D09 route may promote a study. */
|
|
74
|
+
export function d17StudyDecision(report, integrity) {
|
|
75
|
+
// Preserve D17's original release/label policy rather than adopting additional upstream allowlist rules.
|
|
76
|
+
return ceilingDecision('HOLD', report.ceilings.packOutcome, report.ceilings.bindingCeiling, integrity.release_gate.decision, integrity.compromise_labels.length ? 'ROLLBACK' : 'PROMOTE');
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=gates.js.map
|
|
@@ -3,31 +3,28 @@ import { dirname, resolve } from 'node:path';
|
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import Ajv2020 from 'ajv/dist/2020.js';
|
|
5
5
|
import { admitEntry } from '../entry.js';
|
|
6
|
+
import { canonicalJson } from '../../security/artifact-trust.js';
|
|
7
|
+
import { D17_GATE_PARAMETERS as bounds, D17_NATIVE, D17_NATIVE_BOUNDS, d17NativeParameter, D17_SLICES } from './gate-policy.js';
|
|
6
8
|
/** Frozen synthetic-population protocol. A changed threshold requires a new preregistration. */
|
|
7
9
|
export const D17_ANALYSIS = {
|
|
8
10
|
schemaVersion: 'decision-d17-protocol/v1',
|
|
9
|
-
minimumPairs:
|
|
10
|
-
slices:
|
|
11
|
-
minimumPerSlice:
|
|
12
|
-
interval: { method: 'newcombe-10', levelBps: 9500, nonInferiorityMarginBps:
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
minimumPairs: bounds.minimumPairs,
|
|
12
|
+
slices: D17_SLICES,
|
|
13
|
+
minimumPerSlice: bounds.minimumPerSlice,
|
|
14
|
+
interval: { method: 'newcombe-10', levelBps: 9500, nonInferiorityMarginBps: bounds.nonInferiorityMarginBps,
|
|
15
|
+
benefitLowerExclusiveBps: bounds.benefitLowerExclusiveBps },
|
|
16
|
+
brier: { method: 'percentile-bootstrap', levelBps: 9500, seed: 2611, resamples: 20000, maximumUpperBps: bounds.maximumBrierUpperBps },
|
|
17
|
+
risk: { method: 'wilson', levelBps: 9500, maximumAcceptedErrorUpperBps: bounds.maximumAcceptedErrorUpperBps,
|
|
18
|
+
minimumCoverageLowerBps: bounds.minimumCoverageLowerBps },
|
|
19
|
+
abstention: { method: 'newcombe-10', levelBps: 9500, maximumUpperBps: bounds.maximumAbstentionUpperBps },
|
|
20
|
+
maximumP95LatencyIncreaseMs: bounds.maximumP95LatencyIncreaseMs,
|
|
21
|
+
acceptance: { tieRule: 'defer', disagreementMetric: 'jensen-shannon-v1', maximumDisagreementBps: bounds.maximumDisagreementBps,
|
|
22
|
+
minimumSuccessfulMembers: bounds.minimumSuccessfulMembers, highAgreementWarningBps: bounds.highAgreementWarningBps },
|
|
19
23
|
economics: { kind: 'additional-cost-tolerance', positiveNetSavingsRequired: false, separateTradeoffApprovalRequired: true },
|
|
20
|
-
native:
|
|
21
|
-
|
|
22
|
-
{ metric: 'calibration', comparison: 'delta-at-most', bound: 0.02, minimumPairs: 1200 },
|
|
23
|
-
{ metric: 'risk-coverage', comparison: 'delta-at-most', bound: 0.01, minimumPairs: 1200 },
|
|
24
|
-
{ metric: 'abstention', comparison: 'delta-at-most', bound: 0.05, minimumPairs: 1200 },
|
|
25
|
-
{ metric: 'latency', comparison: 'delta-at-most', bound: 10000, minimumPairs: 1200 },
|
|
26
|
-
{ metric: 'tokens', comparison: 'delta-at-most', bound: 8000, minimumPairs: 1200 },
|
|
27
|
-
{ metric: 'cost', comparison: 'delta-at-most', bound: 800, minimumPairs: 1200 },
|
|
28
|
-
{ metric: 'slice', comparison: 'delta-at-least', bound: -0.03, minimumPairs: 1200 },
|
|
29
|
-
],
|
|
24
|
+
native: D17_NATIVE.map(row => ({ metric: row.metric, comparison: row.comparison,
|
|
25
|
+
bound: D17_NATIVE_BOUNDS[d17NativeParameter(row.metric)], minimumPairs: bounds.minimumPairs })),
|
|
30
26
|
};
|
|
27
|
+
const frozenProtocol = canonicalJson(D17_ANALYSIS);
|
|
31
28
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
32
29
|
const schema = [resolve(here, '../../../schemas/decision'), resolve(here, '../../../../schemas/decision')]
|
|
33
30
|
.map(directory => resolve(directory, 'D17StudyProtocol.v1.schema.json')).find(existsSync);
|
|
@@ -36,7 +33,7 @@ if (!schema)
|
|
|
36
33
|
const validator = new Ajv2020({ strict: true }).compile(JSON.parse(readFileSync(schema, 'utf8')));
|
|
37
34
|
export function validateD17Analysis(value) {
|
|
38
35
|
admitEntry(value);
|
|
39
|
-
if (!validator(value))
|
|
36
|
+
if (!validator(value) || canonicalJson(value) !== frozenProtocol)
|
|
40
37
|
throw new Error('D17 protocol differs from the frozen v1 design');
|
|
41
38
|
}
|
|
42
39
|
/**
|
|
@@ -8,6 +8,7 @@ import { championChallengerInputSetDigest, championChallengerShadowBaseline, run
|
|
|
8
8
|
import { qualificationIntegrityAllowlistProblems } from '../qualification/release.js';
|
|
9
9
|
import { evaluateBinaryHeldout, freezeQualificationSplit } from '../qualification/quality.js';
|
|
10
10
|
import { D17_ANALYSIS } from './protocol.js';
|
|
11
|
+
import { d17StudyDecision, evaluateD17Statistics } from './gates.js';
|
|
11
12
|
import { d17Statistics } from './statistics.js';
|
|
12
13
|
import { validateD17Artifact } from './artifacts.js';
|
|
13
14
|
import { applyD17Mapping, assertD17Staged, d17AttemptsByRow, d17RawProbabilities, verifyD17CalibrationSet } from './calibration.js';
|
|
@@ -146,7 +147,7 @@ export async function scoreD17Study(input, prepared) {
|
|
|
146
147
|
calibrationDiagnostics: { champion: binary('champion'), challenger: binary('challenger'), calibrated: false },
|
|
147
148
|
pending: ['D09 compatible member and aggregate calibration', 'blind gold audit and delayed repeats', 'separate extra-cost tradeoff approval',
|
|
148
149
|
'protected native report approval and eligibility'],
|
|
149
|
-
decision:
|
|
150
|
+
decision: d17StudyDecision(evaluateD17Statistics(statistics, undefined, input.integrity), input.integrity) };
|
|
150
151
|
validateD17Artifact('report', report);
|
|
151
152
|
return report;
|
|
152
153
|
}
|
|
@@ -326,7 +327,7 @@ export async function scoreD17StagedStudy(input, prepared, context) {
|
|
|
326
327
|
mappingDigest: heldoutDigest(verified[role].mapping), compatibility: { action: 'allow',
|
|
327
328
|
state: verified[role].compatibility.state, pinId: verified[role].compatibility.pinId } });
|
|
328
329
|
const approved = input.approvedCalibration;
|
|
329
|
-
const decision =
|
|
330
|
+
const decision = d17StudyDecision(evaluateD17Statistics(statistics, undefined, input.integrity), input.integrity);
|
|
330
331
|
const report = { schemaVersion: 'decision-d17-study-report/v2', scope: 'synthetic-staged-calibrated',
|
|
331
332
|
calibrated: true, d09Qualified: true, calibratedGate: true,
|
|
332
333
|
corpusDigest: heldoutDigest(prepared.corpus), preregistrationDigest: heldoutDigest(prepared.preregistration),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { pairedBinaryDifferenceInterval, pairedMeanDifferenceBootstrap,
|
|
2
|
-
import { pairedDeltaFinding } from '../ensemble/contract.js';
|
|
1
|
+
import { pairedBinaryDifferenceInterval, pairedMeanDifferenceBootstrap, wilsonScoreInterval } from '../qualification/quality.js';
|
|
3
2
|
import { D17_ANALYSIS, validateD17Analysis } from './protocol.js';
|
|
3
|
+
import { d17GateFindings, evaluateD17Statistics } from './gates.js';
|
|
4
4
|
const mean = (values) => values.reduce((sum, n) => sum + n, 0) / values.length;
|
|
5
5
|
const p95 = (values) => [...values].sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1];
|
|
6
6
|
/** Abstentions are false for correctness, but true events in their own paired endpoint. */
|
|
@@ -13,7 +13,7 @@ function interval(pairs, endpoint, levelBps) {
|
|
|
13
13
|
return pairedBinaryDifferenceInterval({ counts, levelBps, method: 'newcombe-10' });
|
|
14
14
|
}
|
|
15
15
|
/** Local descriptive statistics, never a release authorization. Missing endpoints withhold native reporting. */
|
|
16
|
-
export function d17Statistics(pairs, expectedIds, protocol = D17_ANALYSIS) {
|
|
16
|
+
export function d17Statistics(pairs, expectedIds, protocol = D17_ANALYSIS, binding) {
|
|
17
17
|
validateD17Analysis(protocol);
|
|
18
18
|
const p = protocol;
|
|
19
19
|
if (new Set(expectedIds).size !== expectedIds.length || new Set(pairs.map(row => row.id)).size !== pairs.length
|
|
@@ -27,14 +27,9 @@ export function d17Statistics(pairs, expectedIds, protocol = D17_ANALYSIS) {
|
|
|
27
27
|
|| arm.tokens !== null && (!Number.isSafeInteger(arm.tokens) || arm.tokens < 0))
|
|
28
28
|
throw new Error('D17 unknown or invalid measurement');
|
|
29
29
|
}
|
|
30
|
-
const findings = [];
|
|
31
|
-
if (pairs.length !== expectedIds.length || pairs.length < p.minimumPairs)
|
|
32
|
-
findings.push('incomplete-pairs');
|
|
33
30
|
const slices = p.slices.map(slice => {
|
|
34
31
|
const rows = pairs.filter(row => row.slice === slice);
|
|
35
32
|
const delta = rows.length ? mean(rows.map(row => Number(row.challenger.correct) - Number(row.champion.correct))) : null;
|
|
36
|
-
if (rows.length < p.minimumPerSlice)
|
|
37
|
-
findings.push(`slice-insufficient:${slice}`);
|
|
38
33
|
return { slice, n: rows.length, qualityDelta: delta, harm: delta !== null && delta < 0, descriptiveOnly: true };
|
|
39
34
|
});
|
|
40
35
|
const metrics = (arm) => ({ quality: Number(arm.correct), calibration: arm.brier,
|
|
@@ -43,9 +38,6 @@ export function d17Statistics(pairs, expectedIds, protocol = D17_ANALYSIS) {
|
|
|
43
38
|
const pairedDeltas = p.native.map(threshold => {
|
|
44
39
|
const complete = pairs.filter(row => metrics(row.champion)[threshold.metric] !== null && metrics(row.challenger)[threshold.metric] !== null);
|
|
45
40
|
const item = { metric: threshold.metric, pairs: complete.length, delta: complete.length ? mean(complete.map(row => metrics(row.challenger)[threshold.metric] - metrics(row.champion)[threshold.metric])) : null };
|
|
46
|
-
const finding = pairedDeltaFinding(threshold, item);
|
|
47
|
-
if (finding)
|
|
48
|
-
findings.push(finding);
|
|
49
41
|
return item;
|
|
50
42
|
});
|
|
51
43
|
const quality = pairs.length ? interval(pairs, arm => arm.correct, p.interval.levelBps) : null;
|
|
@@ -58,26 +50,15 @@ export function d17Statistics(pairs, expectedIds, protocol = D17_ANALYSIS) {
|
|
|
58
50
|
const acceptedErrorUpperBps = risk ? Math.ceil(risk[1] * 10000) : null;
|
|
59
51
|
const coverageLowerBps = coverage ? Math.floor(coverage[0] * 10000) : null;
|
|
60
52
|
const p95LatencyIncreaseMs = pairs.length ? p95(pairs.map(row => row.challenger.latencyMs)) - p95(pairs.map(row => row.champion.latencyMs)) : null;
|
|
61
|
-
if (!quality || pairedNonInferiority({ interval: quality, marginBps: p.interval.nonInferiorityMarginBps }).decision !== 'non-inferior')
|
|
62
|
-
findings.push('quality-ni');
|
|
63
|
-
if (!brier || brier.upperBps > p.brier.maximumUpperBps)
|
|
64
|
-
findings.push('brier-upper');
|
|
65
|
-
if (!abstention || abstention.upperBps > p.abstention.maximumUpperBps)
|
|
66
|
-
findings.push('abstention-upper');
|
|
67
|
-
if (acceptedErrorUpperBps === null || acceptedErrorUpperBps > p.risk.maximumAcceptedErrorUpperBps)
|
|
68
|
-
findings.push('accepted-risk-upper');
|
|
69
|
-
if (coverageLowerBps === null || coverageLowerBps < p.risk.minimumCoverageLowerBps)
|
|
70
|
-
findings.push('coverage-lower');
|
|
71
|
-
if (p95LatencyIncreaseMs === null || p95LatencyIncreaseMs > p.maximumP95LatencyIncreaseMs)
|
|
72
|
-
findings.push('latency-p95');
|
|
73
53
|
const observed = new Set(pairs.map(row => row.id));
|
|
74
|
-
|
|
54
|
+
const measurements = { sampleN: pairs.length, missingIds: expectedIds.filter(id => !observed.has(id)), slices, pairedDeltas,
|
|
75
55
|
quality, brier, abstention, acceptedN: accepted.length, acceptedErrors: errors, acceptedErrorUpperBps, coverageLowerBps,
|
|
76
56
|
p95LatencyIncreaseMs, latencyDifferencesMs: pairs.map(row => row.challenger.latencyMs - row.champion.latencyMs),
|
|
77
57
|
netSavingsMicros: pairs.reduce((sum, row) => sum + row.champion.reservedCostMicros - row.challenger.reservedCostMicros, 0),
|
|
78
58
|
costBasis: 'nonrefundable-reservations-including-failed-attempts',
|
|
79
|
-
|
|
80
|
-
|
|
59
|
+
};
|
|
60
|
+
const gates = d17GateFindings(evaluateD17Statistics(measurements, binding));
|
|
61
|
+
return { ...measurements, benefitSupported: gates.benefitSupported, findings: gates.findings, statisticalGate: gates.statisticalGate,
|
|
81
62
|
worstCaseFailureAsError: { n: expectedIds.length, championCorrect: pairs.filter(row => row.champion.correct).length,
|
|
82
63
|
challengerCorrect: pairs.filter(row => row.challenger.correct).length, missingPairsCountedAsErrors: expectedIds.length - pairs.length } };
|
|
83
64
|
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// Adapt the frozen D23 policy and validated observations to the shipped gate evaluator.
|
|
2
|
+
import { artifactDigest } from '../validate.js';
|
|
3
|
+
import { evaluateGates, sealGateHoldout, sealUpstream } from '../../gates/evaluate.js';
|
|
4
|
+
import { GateRegistry } from '../../gates/registry.js';
|
|
5
|
+
import { createCoreProviderRegistry, evidenceProvider, pairedProvider, proportionProvider } from '../../gates/providers/index.js';
|
|
6
|
+
import { qualifyGateParameter } from '../../gates/types.js';
|
|
7
|
+
export const COMPARATIVE_REPLAY_POLICY = {
|
|
8
|
+
minimumTestRoots: 400, minimumSliceRoots: 100, levelBps: 9500,
|
|
9
|
+
pairedMethod: 'newcombe-10', errorMethod: 'wilson', nonInferiorityMarginBps: -100,
|
|
10
|
+
maxErrorUpperBps: 100, maxSliceErrorUpperBps: 500,
|
|
11
|
+
maxBackendCalls: 0, maxTokens: 0, maxCostMicros: 0, requireExactReproduction: true,
|
|
12
|
+
};
|
|
13
|
+
const PACK_ID = 'project:d23-comparative-replay';
|
|
14
|
+
const SLICES = ['threshold-boundary', 'priority-loss', 'unchanged-control', 'unreplayable'];
|
|
15
|
+
const ALL = { mode: 'all' };
|
|
16
|
+
const LOWER = 'lower-is-stricter';
|
|
17
|
+
const HIGHER = 'higher-is-stricter';
|
|
18
|
+
const CHECKS = ['reports-valid', 'zero-call-budget-violated', 'reproduction-mismatch',
|
|
19
|
+
'integrity-sample-mismatch', 'reviewer-audit-pending'];
|
|
20
|
+
const NON_DIAGNOSTIC = new Set(['integrity-ceiling', 'integrity-sample-mismatch', 'reviewer-audit-pending']);
|
|
21
|
+
function replayPack() {
|
|
22
|
+
const errors = { provider: proportionProvider.id, name: 'false-ready' };
|
|
23
|
+
const gate = (id, kind, rest) => ({ id, kind, scope: ALL, direction: LOWER, onFail: 'HOLD', ...rest });
|
|
24
|
+
return {
|
|
25
|
+
apiVersion: 'gates.aiwg.io/v1alpha1', kind: 'GatePack',
|
|
26
|
+
metadata: { id: PACK_ID, version: '1.0.0', description: 'Frozen D23 comparative policy replay criteria.' },
|
|
27
|
+
spec: {
|
|
28
|
+
parameters: {
|
|
29
|
+
minimumTestRoots: { type: 'count', direction: HIGHER },
|
|
30
|
+
minimumSliceRoots: { type: 'count', direction: HIGHER },
|
|
31
|
+
levelBps: { type: 'level', direction: HIGHER },
|
|
32
|
+
nonInferiorityMarginBps: { type: 'value', direction: HIGHER },
|
|
33
|
+
maxErrorUpperBps: { type: 'bps', direction: LOWER },
|
|
34
|
+
maxSliceErrorUpperBps: { type: 'bps', direction: LOWER },
|
|
35
|
+
},
|
|
36
|
+
metrics: {
|
|
37
|
+
'false-ready': { provider: proportionProvider.id, kind: 'proportion' },
|
|
38
|
+
coverage: { provider: proportionProvider.id, kind: 'proportion' },
|
|
39
|
+
contrast: { provider: pairedProvider.id, kind: 'paired' },
|
|
40
|
+
attestations: { provider: evidenceProvider.id, kind: 'evidence' },
|
|
41
|
+
},
|
|
42
|
+
gates: [
|
|
43
|
+
gate('test-support-insufficient', 'minimum-n', { metric: errors, direction: HIGHER,
|
|
44
|
+
minimumN: { param: 'minimumTestRoots' } }),
|
|
45
|
+
gate('diagnostic-errors', 'count-max', { metric: errors, threshold: { op: 'lte', value: 0 } }),
|
|
46
|
+
gate('false-no-change', 'count-max', { metric: { provider: proportionProvider.id, name: 'coverage' },
|
|
47
|
+
threshold: { op: 'lte', value: 0 } }),
|
|
48
|
+
gate('non-inferiority-failed', 'paired-difference', {
|
|
49
|
+
metric: { provider: pairedProvider.id, name: 'contrast' }, direction: HIGHER,
|
|
50
|
+
statistic: { kind: 'paired-difference', method: 'newcombe', levelBps: { param: 'levelBps' },
|
|
51
|
+
mode: 'non-inferiority', marginBps: { param: 'nonInferiorityMarginBps' } },
|
|
52
|
+
}),
|
|
53
|
+
gate('error-bound-failed', 'interval-bound', { metric: errors,
|
|
54
|
+
statistic: { kind: 'interval-bound', method: 'wilson', bound: 'upper', levelBps: { param: 'levelBps' } },
|
|
55
|
+
threshold: { op: 'lte', param: 'maxErrorUpperBps' } }),
|
|
56
|
+
gate('slice-support', 'minimum-n', { metric: errors, direction: HIGHER, scope: { mode: 'each' },
|
|
57
|
+
minimumN: { param: 'minimumSliceRoots' } }),
|
|
58
|
+
gate('slice-errors', 'count-max', { metric: errors, scope: { mode: 'each' }, threshold: { op: 'lte', value: 0 } }),
|
|
59
|
+
gate('slice-error-bound', 'interval-bound', { metric: errors, scope: { mode: 'each' },
|
|
60
|
+
statistic: { kind: 'interval-bound', method: 'wilson', bound: 'upper', levelBps: { param: 'levelBps' } },
|
|
61
|
+
threshold: { op: 'lte', param: 'maxSliceErrorUpperBps' } }),
|
|
62
|
+
...CHECKS.map(id => gate(id, 'evidence', {
|
|
63
|
+
metric: { provider: evidenceProvider.id, name: 'attestations' }, evidence: { required: [id] },
|
|
64
|
+
})),
|
|
65
|
+
gate('integrity-ceiling', 'upstream-ceiling', { direction: HIGHER, floor: true }),
|
|
66
|
+
],
|
|
67
|
+
combine: { rule: 'max-severity' },
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/** Pure adapter; historical D23 trust pins remain validated by the comparative report owner. */
|
|
72
|
+
export function comparativeReplayGateInput(input, rows, usage, reportFindings) {
|
|
73
|
+
const plan = input.preregistration;
|
|
74
|
+
if (plan.pairedMethod !== COMPARATIVE_REPLAY_POLICY.pairedMethod
|
|
75
|
+
|| plan.errorMethod !== COMPARATIVE_REPLAY_POLICY.errorMethod) {
|
|
76
|
+
throw new Error('D23 comparative replay requires pairedMethod newcombe-10 and errorMethod wilson');
|
|
77
|
+
}
|
|
78
|
+
const providers = createCoreProviderRegistry();
|
|
79
|
+
const registry = new GateRegistry(providers);
|
|
80
|
+
registry.registerPack(replayPack(), { namespace: 'project' });
|
|
81
|
+
const pack = registry.resolvePack(PACK_ID);
|
|
82
|
+
const metricProviders = [proportionProvider, pairedProvider, evidenceProvider];
|
|
83
|
+
const binding = {
|
|
84
|
+
apiVersion: 'gates.aiwg.io/v1alpha1', kind: 'GateBinding',
|
|
85
|
+
metadata: { id: plan.id, version: '1.0.0', description: 'D23 frozen synthetic policy replay.' },
|
|
86
|
+
spec: {
|
|
87
|
+
packs: [{ id: PACK_ID, version: '1.0.0', digest: pack.digest, resolvedDigest: pack.resolvedDigest }],
|
|
88
|
+
parameters: Object.fromEntries(['minimumTestRoots', 'minimumSliceRoots', 'levelBps',
|
|
89
|
+
'nonInferiorityMarginBps', 'maxErrorUpperBps', 'maxSliceErrorUpperBps'].map(name => [qualifyGateParameter(PACK_ID, name), plan[name]])),
|
|
90
|
+
slices: [...SLICES], references: [],
|
|
91
|
+
metricProviders: metricProviders.map(({ id, version, sourceDigest }) => ({ id, version, sourceDigest })),
|
|
92
|
+
registeredAt: plan.frozenAt, frozenAt: plan.frozenAt,
|
|
93
|
+
corpusDigest: plan.corpusDigest, goldDigest: plan.goldDigest, splitDigest: plan.splitDigest,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
const testRows = rows.filter(row => row.split === 'test');
|
|
97
|
+
const checks = {
|
|
98
|
+
'reports-valid': reportFindings.size === 0,
|
|
99
|
+
'zero-call-budget-violated': usage.backendCalls <= plan.maxBackendCalls
|
|
100
|
+
&& usage.tokens <= plan.maxTokens && usage.costMicros <= plan.maxCostMicros,
|
|
101
|
+
'reproduction-mismatch': usage.exactReproduction,
|
|
102
|
+
'integrity-sample-mismatch': input.integrity.sample_n === testRows.length,
|
|
103
|
+
'reviewer-audit-pending': false,
|
|
104
|
+
};
|
|
105
|
+
const trustedBindingDigest = artifactDigest(binding);
|
|
106
|
+
return {
|
|
107
|
+
binding, registry, trustedBindingDigest,
|
|
108
|
+
holdout: sealGateHoldout({ frozenDigest: trustedBindingDigest, firstAccessedAt: input.accessedAt }),
|
|
109
|
+
upstream: sealUpstream(input.integrity), now: input.accessedAt,
|
|
110
|
+
// Legacy retained D23 policy: no ambient project configuration may change historical replay.
|
|
111
|
+
floors: 'none-explicit-opt-out',
|
|
112
|
+
metrics: { providers: {
|
|
113
|
+
[proportionProvider.id]: proportionProvider.compute(testRows.flatMap(row => [
|
|
114
|
+
{ slice: row.slice, metric: 'false-ready', event: !row.correct },
|
|
115
|
+
{ slice: row.slice, metric: 'coverage', event: row.falseNoChange },
|
|
116
|
+
])),
|
|
117
|
+
[pairedProvider.id]: pairedProvider.compute(testRows.map(row => ({ slice: row.slice, candidate: row.correct, baseline: true }))),
|
|
118
|
+
[evidenceProvider.id]: evidenceProvider.compute(CHECKS.map(id => ({ id, passed: checks[id] }))),
|
|
119
|
+
} },
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
export function evaluateComparativeReplayGates(...args) {
|
|
123
|
+
const context = comparativeReplayGateInput(...args);
|
|
124
|
+
return { binding: context.binding, gateReport: evaluateGates(context) };
|
|
125
|
+
}
|
|
126
|
+
/** Translate gate evidence to the retained D23 finding vocabulary without re-evaluating thresholds. */
|
|
127
|
+
export function comparativeReplayGateFindings(report) {
|
|
128
|
+
const failed = report.gateEvidence.filter(gate => gate.status !== 'pass');
|
|
129
|
+
const findings = new Set(failed.filter(gate => gate.gateId !== 'reports-valid' && gate.gateId !== 'integrity-ceiling')
|
|
130
|
+
.map(gate => gate.gateId.startsWith('slice-') ? `slice-failed:${gate.slice}` : gate.gateId));
|
|
131
|
+
for (const reason of report.upstream?.reasons ?? [])
|
|
132
|
+
findings.add(reason.replace(/^upstream:/, ''));
|
|
133
|
+
return { findings: [...findings], diagnosticDecision: failed.some(gate => !NON_DIAGNOSTIC.has(gate.gateId)) ? 'fail' : 'pass' };
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=comparative-gates.js.map
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { assertDecisionCeiling
|
|
2
|
-
import {
|
|
3
|
-
import { qualificationIntegrityAllowlistProblems } from '../qualification/release.js';
|
|
1
|
+
import { assertDecisionCeiling } from '../../gates/lattice.js';
|
|
2
|
+
import { comparativeReplayGateFindings, evaluateComparativeReplayGates } from './comparative-gates.js';
|
|
4
3
|
import { checkSensitivitySchema, sensitivityDigest, validateSensitivityReport } from './contract.js';
|
|
5
4
|
export const REPLAY_SLICES = ['threshold-boundary', 'priority-loss', 'unchanged-control', 'unreplayable'];
|
|
6
5
|
const DIGEST = /^sha256:[0-9a-f]{64}$/;
|
|
7
|
-
const bpsUpper = (events, n, levelBps) => n ? Math.ceil(wilsonScoreInterval({ events, n, levelBps })[1] * 10_000) : null;
|
|
8
6
|
export function validateComparativeReplayPreregistration(value) {
|
|
9
7
|
checkSensitivitySchema('comparativePlan', value);
|
|
10
8
|
if (value.minimumTestRoots < value.minimumSliceRoots * REPLAY_SLICES.length) {
|
|
@@ -15,7 +13,7 @@ export function validateComparativeReplayPreregistration(value) {
|
|
|
15
13
|
throw new Error('comparative replay preregistration digest mismatch');
|
|
16
14
|
}
|
|
17
15
|
/** Rebuild from trusted preregistration, stored reports and independently authored gold. */
|
|
18
|
-
export function
|
|
16
|
+
export function buildComparativeReplayEvaluation(input) {
|
|
19
17
|
const plan = input.preregistration;
|
|
20
18
|
validateComparativeReplayPreregistration(plan);
|
|
21
19
|
if (plan.digest !== input.trustedPreregistrationDigest)
|
|
@@ -92,45 +90,21 @@ export function buildComparativeReplayReport(input) {
|
|
|
92
90
|
const testRows = rows.filter(row => row.split === 'test');
|
|
93
91
|
const errors = testRows.filter(row => !row.correct).length;
|
|
94
92
|
const falseNoChange = testRows.filter(row => row.falseNoChange).length;
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
93
|
+
if (input.reviewerAuditDigest !== null)
|
|
94
|
+
throw new Error('operator review ingestion is not implemented');
|
|
95
|
+
const { binding, gateReport: gates } = evaluateComparativeReplayGates(input, rows, { backendCalls, tokens, costMicros, exactReproduction }, findings);
|
|
96
|
+
const { diagnosticDecision, findings: gateFindings } = comparativeReplayGateFindings(gates);
|
|
97
|
+
for (const finding of gateFindings)
|
|
98
|
+
findings.add(finding);
|
|
99
|
+
const pairedLowerBps = gates.gateEvidence.find(gate => gate.gateId === 'non-inferiority-failed')?.statistic?.lowerBps ?? null;
|
|
100
|
+
const errorUpperBps = gates.gateEvidence.find(gate => gate.gateId === 'error-bound-failed')?.statistic?.upperBps ?? null;
|
|
98
101
|
const slices = Object.fromEntries(REPLAY_SLICES.map(slice => {
|
|
99
102
|
const items = testRows.filter(row => row.slice === slice);
|
|
100
|
-
|
|
101
|
-
|
|
103
|
+
return [slice, { roots: items.length, errors: items.filter(row => !row.correct).length,
|
|
104
|
+
errorUpperBps: gates.gateEvidence.find(gate => gate.gateId === 'slice-error-bound' && gate.slice === slice)?.statistic?.upperBps ?? null }];
|
|
102
105
|
}));
|
|
103
|
-
if (testRows.length < plan.minimumTestRoots)
|
|
104
|
-
findings.add('test-support-insufficient');
|
|
105
|
-
if (errors > 0)
|
|
106
|
-
findings.add('diagnostic-errors');
|
|
107
|
-
if (falseNoChange > 0)
|
|
108
|
-
findings.add('false-no-change');
|
|
109
|
-
if (pairedLowerBps === null || pairedLowerBps < plan.nonInferiorityMarginBps)
|
|
110
|
-
findings.add('non-inferiority-failed');
|
|
111
|
-
if (errorUpperBps === null || errorUpperBps > plan.maxErrorUpperBps)
|
|
112
|
-
findings.add('error-bound-failed');
|
|
113
|
-
for (const slice of REPLAY_SLICES) {
|
|
114
|
-
const metric = slices[slice];
|
|
115
|
-
if (metric.roots < plan.minimumSliceRoots || metric.errors > 0
|
|
116
|
-
|| metric.errorUpperBps === null || metric.errorUpperBps > plan.maxSliceErrorUpperBps)
|
|
117
|
-
findings.add(`slice-failed:${slice}`);
|
|
118
|
-
}
|
|
119
|
-
if (backendCalls > plan.maxBackendCalls || tokens > plan.maxTokens || costMicros > plan.maxCostMicros)
|
|
120
|
-
findings.add('zero-call-budget-violated');
|
|
121
|
-
if (!exactReproduction)
|
|
122
|
-
findings.add('reproduction-mismatch');
|
|
123
|
-
const diagnosticDecision = findings.size === 0 ? 'pass' : 'fail';
|
|
124
|
-
for (const problem of qualificationIntegrityAllowlistProblems(input.integrity))
|
|
125
|
-
findings.add(problem);
|
|
126
|
-
if (input.integrity.sample_n !== testRows.length)
|
|
127
|
-
findings.add('integrity-sample-mismatch');
|
|
128
|
-
if (input.reviewerAuditDigest !== null)
|
|
129
|
-
throw new Error('operator review ingestion is not implemented');
|
|
130
|
-
findings.add('reviewer-audit-pending');
|
|
131
106
|
const upstreamDecision = input.integrity.release_gate.decision;
|
|
132
|
-
const decision =
|
|
133
|
-
|| input.integrity.compromise_labels.length > 0 ? 'ROLLBACK' : 'PROMOTE', findings.size > 0 ? 'HOLD' : 'PROMOTE');
|
|
107
|
+
const decision = gates.decision;
|
|
134
108
|
const payload = {
|
|
135
109
|
schemaVersion: 'decision-comparative-replay-report/v1', mode: 'shadow',
|
|
136
110
|
semantics: 'associative-sensitivity-not-causal', actionAuthorization: 'not-authorized',
|
|
@@ -143,7 +117,10 @@ export function buildComparativeReplayReport(input) {
|
|
|
143
117
|
};
|
|
144
118
|
const report = { ...payload, digest: sensitivityDigest(payload) };
|
|
145
119
|
checkSensitivitySchema('comparativeReport', report);
|
|
146
|
-
return report;
|
|
120
|
+
return { report, binding, gateReport: gates };
|
|
121
|
+
}
|
|
122
|
+
export function buildComparativeReplayReport(input) {
|
|
123
|
+
return buildComparativeReplayEvaluation(input).report;
|
|
147
124
|
}
|
|
148
125
|
export function validateComparativeReplayReport(report, input) {
|
|
149
126
|
checkSensitivitySchema('comparativeReport', report);
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
*/
|
|
29
29
|
import { promises as fs } from 'node:fs';
|
|
30
30
|
import * as path from 'node:path';
|
|
31
|
+
import { captureDeploymentOutputs, recordDeploymentOutputs } from '../providers/deployment-ownership.mjs';
|
|
31
32
|
import { mergeClaudeAttribution, undoClaudeAttribution } from './claude-attribution.js';
|
|
32
33
|
/**
|
|
33
34
|
* Hook script → Claude Code hook event mapping.
|
|
@@ -111,6 +112,8 @@ async function atomicWrite(filePath, content) {
|
|
|
111
112
|
* written.
|
|
112
113
|
*/
|
|
113
114
|
export async function installAiwgHooks(opts) {
|
|
115
|
+
const ownershipRoots = [path.join(opts.projectPath, '.claude')];
|
|
116
|
+
const ownershipBefore = opts.dryRun ? undefined : captureDeploymentOutputs(ownershipRoots);
|
|
114
117
|
const sourceHooksDir = path.join(opts.frameworkRoot, 'agentic', 'code', 'addons', 'aiwg-hooks', 'hooks');
|
|
115
118
|
let sourceFiles;
|
|
116
119
|
try {
|
|
@@ -259,6 +262,21 @@ export async function installAiwgHooks(opts) {
|
|
|
259
262
|
await fs.mkdir(path.dirname(result.settingsPath), { recursive: true });
|
|
260
263
|
await atomicWrite(result.settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
261
264
|
}
|
|
265
|
+
if (ownershipBefore) {
|
|
266
|
+
const sidecar = ownershipBefore.get(path.join(opts.projectPath, '.claude/.aiwg-manifest.json'));
|
|
267
|
+
let recordedOwners = [];
|
|
268
|
+
try {
|
|
269
|
+
recordedOwners = JSON.parse(sidecar?.toString() || '{}').managed?.['settings.json']?.owners || [];
|
|
270
|
+
}
|
|
271
|
+
catch { /* Standalone installs retain the addon owner when no receipt verifies users. */ }
|
|
272
|
+
const owners = opts.frameworkSlug ? [opts.frameworkSlug] : recordedOwners;
|
|
273
|
+
let snapshot = ownershipBefore;
|
|
274
|
+
for (const owner of owners.length ? owners : ['aiwg-hooks']) {
|
|
275
|
+
recordDeploymentOutputs(snapshot, ownershipRoots, { frameworkSlug: owner, claimUnchangedSettings: opts.claimUnchangedSettings,
|
|
276
|
+
writtenFiles: [...result.installedScripts, result.settingsPath] });
|
|
277
|
+
snapshot = captureDeploymentOutputs(ownershipRoots);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
262
280
|
return result;
|
|
263
281
|
}
|
|
264
282
|
/**
|