@openprose/reactor-cradle 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +270 -0
- package/dist/assert/index.d.ts +35 -0
- package/dist/assert/index.d.ts.map +1 -0
- package/dist/assert/index.js +398 -0
- package/dist/baselines/cost-thesis/index.d.ts +103 -0
- package/dist/baselines/cost-thesis/index.d.ts.map +1 -0
- package/dist/baselines/cost-thesis/index.js +337 -0
- package/dist/baselines/naive-loop/index.d.ts +46 -0
- package/dist/baselines/naive-loop/index.d.ts.map +1 -0
- package/dist/baselines/naive-loop/index.js +188 -0
- package/dist/baselines/no-memo/index.d.ts +84 -0
- package/dist/baselines/no-memo/index.d.ts.map +1 -0
- package/dist/baselines/no-memo/index.js +226 -0
- package/dist/doubles/clock.d.ts +9 -0
- package/dist/doubles/clock.d.ts.map +1 -0
- package/dist/doubles/clock.js +42 -0
- package/dist/doubles/storage.d.ts +24 -0
- package/dist/doubles/storage.d.ts.map +1 -0
- package/dist/doubles/storage.js +191 -0
- package/dist/eval/index.d.ts +204 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +596 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/policy-author/index.d.ts +103 -0
- package/dist/policy-author/index.d.ts.map +1 -0
- package/dist/policy-author/index.js +358 -0
- package/dist/policy-drift/index.d.ts +64 -0
- package/dist/policy-drift/index.d.ts.map +1 -0
- package/dist/policy-drift/index.js +495 -0
- package/dist/policy-replay/index.d.ts +106 -0
- package/dist/policy-replay/index.d.ts.map +1 -0
- package/dist/policy-replay/index.js +361 -0
- package/dist/provider-parity/index.d.ts +91 -0
- package/dist/provider-parity/index.d.ts.map +1 -0
- package/dist/provider-parity/index.js +439 -0
- package/dist/recompile/index.d.ts +161 -0
- package/dist/recompile/index.d.ts.map +1 -0
- package/dist/recompile/index.js +690 -0
- package/dist/release-candidate/index.d.ts +139 -0
- package/dist/release-candidate/index.d.ts.map +1 -0
- package/dist/release-candidate/index.js +553 -0
- package/dist/release-parity/index.d.ts +80 -0
- package/dist/release-parity/index.d.ts.map +1 -0
- package/dist/release-parity/index.js +1035 -0
- package/dist/replay/model-gateway.d.ts +25 -0
- package/dist/replay/model-gateway.d.ts.map +1 -0
- package/dist/replay/model-gateway.js +166 -0
- package/dist/replay/parity.d.ts +110 -0
- package/dist/replay/parity.d.ts.map +1 -0
- package/dist/replay/parity.js +232 -0
- package/dist/rollback/index.d.ts +106 -0
- package/dist/rollback/index.d.ts.map +1 -0
- package/dist/rollback/index.js +694 -0
- package/dist/scenario/parser.d.ts +11 -0
- package/dist/scenario/parser.d.ts.map +1 -0
- package/dist/scenario/parser.js +490 -0
- package/dist/scenario/runner.d.ts +12 -0
- package/dist/scenario/runner.d.ts.map +1 -0
- package/dist/scenario/runner.js +345 -0
- package/dist/scenario/synthetic-world-adapter.d.ts +4 -0
- package/dist/scenario/synthetic-world-adapter.d.ts.map +1 -0
- package/dist/scenario/synthetic-world-adapter.js +82 -0
- package/dist/scenario/time.d.ts +4 -0
- package/dist/scenario/time.d.ts.map +1 -0
- package/dist/scenario/time.js +45 -0
- package/dist/scenario/types.d.ts +149 -0
- package/dist/scenario/types.d.ts.map +1 -0
- package/dist/scenario/types.js +5 -0
- package/dist/spikes/index.d.ts +10 -0
- package/dist/spikes/index.d.ts.map +1 -0
- package/dist/spikes/index.js +29 -0
- package/dist/spikes/k1-ensemble-spread.d.ts +88 -0
- package/dist/spikes/k1-ensemble-spread.d.ts.map +1 -0
- package/dist/spikes/k1-ensemble-spread.js +396 -0
- package/dist/spikes/k2-policy-author.d.ts +25 -0
- package/dist/spikes/k2-policy-author.d.ts.map +1 -0
- package/dist/spikes/k2-policy-author.js +503 -0
- package/dist/spikes/live-refresh.d.ts +150 -0
- package/dist/spikes/live-refresh.d.ts.map +1 -0
- package/dist/spikes/live-refresh.js +511 -0
- package/dist/world/index.d.ts +2 -0
- package/dist/world/index.d.ts.map +1 -0
- package/dist/world/index.js +17 -0
- package/dist/world/synthetic-world.d.ts +104 -0
- package/dist/world/synthetic-world.d.ts.map +1 -0
- package/dist/world/synthetic-world.js +449 -0
- package/package.json +139 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.REACTOR_CRADLE_SPIKE_INTEGRATION_NOTES_V0 = exports.REACTOR_CRADLE_SPIKE_PUBLIC_EXPORTS_V0 = void 0;
|
|
18
|
+
__exportStar(require("./k1-ensemble-spread"), exports);
|
|
19
|
+
__exportStar(require("./k2-policy-author"), exports);
|
|
20
|
+
__exportStar(require("./live-refresh"), exports);
|
|
21
|
+
exports.REACTOR_CRADLE_SPIKE_PUBLIC_EXPORTS_V0 = Object.freeze({
|
|
22
|
+
live_refresh: "./spikes/live-refresh",
|
|
23
|
+
k1_ensemble_spread: "./spikes/k1-ensemble-spread",
|
|
24
|
+
k2_policy_author: "./spikes/k2-policy-author",
|
|
25
|
+
});
|
|
26
|
+
exports.REACTOR_CRADLE_SPIKE_INTEGRATION_NOTES_V0 = Object.freeze([
|
|
27
|
+
"K1/K2 spike evaluators are recorded-only by default; normal tests must not invoke live model refresh.",
|
|
28
|
+
"Live OpenRouter refresh is an explicit opt-in guard path with cap/accounting input and secret redaction.",
|
|
29
|
+
]);
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export declare const K1_ENSEMBLE_SPREAD_SCHEMA_V0: "openprose.reactor-cradle.k1-ensemble-spread";
|
|
2
|
+
export declare const K1_ENSEMBLE_SPREAD_VERSION_V0: 0;
|
|
3
|
+
export type K1AnchorCalibrationGradeV0 = "authored" | "accrued";
|
|
4
|
+
export type K1EnsembleVerdictStatusV0 = "up" | "drifting" | "down" | "blocked";
|
|
5
|
+
export type K1DiversityFloorComponentV0 = "model-family" | "provider" | "size-boundary";
|
|
6
|
+
export type K1CalibrationStatusV0 = "calibrated" | "degraded";
|
|
7
|
+
export type K1IssueCodeV0 = "fixture-not-object" | "schema-invalid" | "version-invalid" | "field-invalid" | "anchor-invalid" | "calibration-bar-invalid" | "outputs-invalid" | "ensemble-too-small" | "diversity-floor-unmet" | "mean-error-too-high" | "spread-error-gap-too-high" | "low-spread-anchor-error";
|
|
8
|
+
export interface K1CalibrationAnchorV0 {
|
|
9
|
+
readonly label_source: string;
|
|
10
|
+
readonly calibration_grade: K1AnchorCalibrationGradeV0;
|
|
11
|
+
readonly verdict: K1EnsembleVerdictStatusV0;
|
|
12
|
+
readonly score: number;
|
|
13
|
+
}
|
|
14
|
+
export interface K1CalibrationBarV0 {
|
|
15
|
+
readonly max_mean_error: number;
|
|
16
|
+
readonly max_spread_error_gap: number;
|
|
17
|
+
readonly low_spread_threshold: number;
|
|
18
|
+
readonly max_error_when_low_spread: number;
|
|
19
|
+
}
|
|
20
|
+
export interface K1RecordedModelOutputV0 {
|
|
21
|
+
readonly output_id: string;
|
|
22
|
+
readonly provider: string;
|
|
23
|
+
readonly model: string;
|
|
24
|
+
readonly family: string;
|
|
25
|
+
readonly size_class: string;
|
|
26
|
+
readonly verdict: K1EnsembleVerdictStatusV0;
|
|
27
|
+
readonly score: number;
|
|
28
|
+
readonly confidence: number;
|
|
29
|
+
readonly text: string;
|
|
30
|
+
}
|
|
31
|
+
export interface K1EnsembleSpreadFixtureV0 {
|
|
32
|
+
readonly schema: typeof K1_ENSEMBLE_SPREAD_SCHEMA_V0;
|
|
33
|
+
readonly v: typeof K1_ENSEMBLE_SPREAD_VERSION_V0;
|
|
34
|
+
readonly fixture_id: string;
|
|
35
|
+
readonly responsibility_id: string;
|
|
36
|
+
readonly recorded_at: string;
|
|
37
|
+
readonly anchor: K1CalibrationAnchorV0;
|
|
38
|
+
readonly calibration_bar: K1CalibrationBarV0;
|
|
39
|
+
readonly outputs: readonly K1RecordedModelOutputV0[];
|
|
40
|
+
}
|
|
41
|
+
export interface K1DiversityFloorEvaluationV0 {
|
|
42
|
+
readonly floor_met: boolean;
|
|
43
|
+
readonly families: readonly string[];
|
|
44
|
+
readonly providers: readonly string[];
|
|
45
|
+
readonly size_classes: readonly string[];
|
|
46
|
+
readonly missing: readonly K1DiversityFloorComponentV0[];
|
|
47
|
+
}
|
|
48
|
+
export interface K1OutputAnchorScoreV0 {
|
|
49
|
+
readonly output_id: string;
|
|
50
|
+
readonly provider: string;
|
|
51
|
+
readonly model: string;
|
|
52
|
+
readonly family: string;
|
|
53
|
+
readonly size_class: string;
|
|
54
|
+
readonly verdict: K1EnsembleVerdictStatusV0;
|
|
55
|
+
readonly score: number;
|
|
56
|
+
readonly anchor_error: number;
|
|
57
|
+
readonly matches_anchor_verdict: boolean;
|
|
58
|
+
}
|
|
59
|
+
export interface K1SpreadMetricsV0 {
|
|
60
|
+
readonly output_count: number;
|
|
61
|
+
readonly spread: number;
|
|
62
|
+
readonly mean_anchor_error: number;
|
|
63
|
+
readonly max_anchor_error: number;
|
|
64
|
+
readonly spread_error_gap: number;
|
|
65
|
+
readonly calibrated_confidence: number | null;
|
|
66
|
+
}
|
|
67
|
+
export interface K1EvaluationIssueV0 {
|
|
68
|
+
readonly path: string;
|
|
69
|
+
readonly code: K1IssueCodeV0;
|
|
70
|
+
readonly message: string;
|
|
71
|
+
readonly observed: unknown;
|
|
72
|
+
}
|
|
73
|
+
export interface K1EnsembleSpreadEvaluationV0 {
|
|
74
|
+
readonly ok: boolean;
|
|
75
|
+
readonly status: K1CalibrationStatusV0;
|
|
76
|
+
readonly accepted_as_calibrated_confidence: boolean;
|
|
77
|
+
readonly summary: string;
|
|
78
|
+
readonly fixture_id: string | null;
|
|
79
|
+
readonly responsibility_id: string | null;
|
|
80
|
+
readonly anchor: K1CalibrationAnchorV0 | null;
|
|
81
|
+
readonly diversity: K1DiversityFloorEvaluationV0;
|
|
82
|
+
readonly metrics: K1SpreadMetricsV0;
|
|
83
|
+
readonly output_scores: readonly K1OutputAnchorScoreV0[];
|
|
84
|
+
readonly issues: readonly K1EvaluationIssueV0[];
|
|
85
|
+
}
|
|
86
|
+
export declare function evaluateK1EnsembleSpreadV0(input: unknown): K1EnsembleSpreadEvaluationV0;
|
|
87
|
+
export declare function evaluateK1DiversityFloorV0(outputs: readonly K1RecordedModelOutputV0[]): K1DiversityFloorEvaluationV0;
|
|
88
|
+
//# sourceMappingURL=k1-ensemble-spread.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"k1-ensemble-spread.d.ts","sourceRoot":"","sources":["../../src/spikes/k1-ensemble-spread.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,4BAA4B,EACvC,6CAAsD,CAAC;AACzD,eAAO,MAAM,6BAA6B,EAAG,CAAU,CAAC;AAExD,MAAM,MAAM,0BAA0B,GAAG,UAAU,GAAG,SAAS,CAAC;AAChE,MAAM,MAAM,yBAAyB,GACjC,IAAI,GACJ,UAAU,GACV,MAAM,GACN,SAAS,CAAC;AACd,MAAM,MAAM,2BAA2B,GACnC,cAAc,GACd,UAAU,GACV,eAAe,CAAC;AACpB,MAAM,MAAM,qBAAqB,GAAG,YAAY,GAAG,UAAU,CAAC;AAE9D,MAAM,MAAM,aAAa,GACrB,oBAAoB,GACpB,gBAAgB,GAChB,iBAAiB,GACjB,eAAe,GACf,gBAAgB,GAChB,yBAAyB,GACzB,iBAAiB,GACjB,oBAAoB,GACpB,uBAAuB,GACvB,qBAAqB,GACrB,2BAA2B,GAC3B,yBAAyB,CAAC;AAE9B,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,iBAAiB,EAAE,0BAA0B,CAAC;IACvD,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,CAAC;CAC5C;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,4BAA4B,CAAC;IACrD,QAAQ,CAAC,CAAC,EAAE,OAAO,6BAA6B,CAAC;IACjD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;IACvC,QAAQ,CAAC,eAAe,EAAE,kBAAkB,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,SAAS,uBAAuB,EAAE,CAAC;CACtD;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAE,SAAS,2BAA2B,EAAE,CAAC;CAC1D;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,sBAAsB,EAAE,OAAO,CAAC;CAC1C;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;IACvC,QAAQ,CAAC,iCAAiC,EAAE,OAAO,CAAC;IACpD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC9C,QAAQ,CAAC,SAAS,EAAE,4BAA4B,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACpC,QAAQ,CAAC,aAAa,EAAE,SAAS,qBAAqB,EAAE,CAAC;IACzD,QAAQ,CAAC,MAAM,EAAE,SAAS,mBAAmB,EAAE,CAAC;CACjD;AAkBD,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,OAAO,GACb,4BAA4B,CAqJ9B;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,SAAS,uBAAuB,EAAE,GAC1C,4BAA4B,CAuB9B"}
|
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.K1_ENSEMBLE_SPREAD_VERSION_V0 = exports.K1_ENSEMBLE_SPREAD_SCHEMA_V0 = void 0;
|
|
4
|
+
exports.evaluateK1EnsembleSpreadV0 = evaluateK1EnsembleSpreadV0;
|
|
5
|
+
exports.evaluateK1DiversityFloorV0 = evaluateK1DiversityFloorV0;
|
|
6
|
+
exports.K1_ENSEMBLE_SPREAD_SCHEMA_V0 = "openprose.reactor-cradle.k1-ensemble-spread";
|
|
7
|
+
exports.K1_ENSEMBLE_SPREAD_VERSION_V0 = 0;
|
|
8
|
+
const VERDICT_STATUSES = new Set([
|
|
9
|
+
"up",
|
|
10
|
+
"drifting",
|
|
11
|
+
"down",
|
|
12
|
+
"blocked",
|
|
13
|
+
]);
|
|
14
|
+
const ANCHOR_GRADES = new Set([
|
|
15
|
+
"authored",
|
|
16
|
+
"accrued",
|
|
17
|
+
]);
|
|
18
|
+
const EPSILON = 1e-12;
|
|
19
|
+
function evaluateK1EnsembleSpreadV0(input) {
|
|
20
|
+
const read = readFixture(input);
|
|
21
|
+
if (!read.ok) {
|
|
22
|
+
return degradedResult({
|
|
23
|
+
summary: "K1 ensemble spread fixture failed validation",
|
|
24
|
+
fixture_id: null,
|
|
25
|
+
responsibility_id: null,
|
|
26
|
+
anchor: null,
|
|
27
|
+
diversity: emptyDiversity(),
|
|
28
|
+
metrics: emptyMetrics(),
|
|
29
|
+
output_scores: [],
|
|
30
|
+
issues: read.issues,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const fixture = read.fixture;
|
|
34
|
+
const diversity = evaluateK1DiversityFloorV0(fixture.outputs);
|
|
35
|
+
const outputScores = fixture.outputs.map((output) => scoreOutputAgainstAnchor(output, fixture.anchor));
|
|
36
|
+
const rawErrors = outputScores.map((score) => score.anchor_error);
|
|
37
|
+
const rawSpread = meanPairwiseDistance(fixture.outputs.map((output) => output.score));
|
|
38
|
+
const rawMeanError = mean(rawErrors);
|
|
39
|
+
const rawMaxError = rawErrors.length === 0 ? 0 : Math.max(...rawErrors);
|
|
40
|
+
const rawSpreadErrorGap = Math.abs(rawMeanError - rawSpread);
|
|
41
|
+
const issues = [];
|
|
42
|
+
if (fixture.outputs.length < 2) {
|
|
43
|
+
issues.push(issue("outputs", "ensemble-too-small", "K1 ensemble spread requires at least two recorded outputs", { outputs: fixture.outputs.length }));
|
|
44
|
+
}
|
|
45
|
+
if (!diversity.floor_met) {
|
|
46
|
+
issues.push(issue("outputs", "diversity-floor-unmet", "spread cannot be treated as calibrated confidence until the ensemble spans at least two families, two providers, and two size classes", {
|
|
47
|
+
families: diversity.families,
|
|
48
|
+
providers: diversity.providers,
|
|
49
|
+
size_classes: diversity.size_classes,
|
|
50
|
+
missing: diversity.missing,
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
if (rawMeanError > fixture.calibration_bar.max_mean_error + EPSILON) {
|
|
54
|
+
issues.push(issue("outputs", "mean-error-too-high", "ensemble outputs exceed the anchor mean-error calibration bar", {
|
|
55
|
+
mean_anchor_error: roundMetric(rawMeanError),
|
|
56
|
+
max_mean_error: fixture.calibration_bar.max_mean_error,
|
|
57
|
+
}));
|
|
58
|
+
}
|
|
59
|
+
if (rawSpreadErrorGap >
|
|
60
|
+
fixture.calibration_bar.max_spread_error_gap + EPSILON) {
|
|
61
|
+
issues.push(issue("outputs", "spread-error-gap-too-high", "spread does not track anchor error within the recorded calibration bar", {
|
|
62
|
+
spread: roundMetric(rawSpread),
|
|
63
|
+
mean_anchor_error: roundMetric(rawMeanError),
|
|
64
|
+
max_spread_error_gap: fixture.calibration_bar.max_spread_error_gap,
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
if (rawSpread <= fixture.calibration_bar.low_spread_threshold + EPSILON &&
|
|
68
|
+
rawMeanError >
|
|
69
|
+
fixture.calibration_bar.max_error_when_low_spread + EPSILON) {
|
|
70
|
+
issues.push(issue("outputs", "low-spread-anchor-error", "low inter-model spread disagrees with the correctness anchor and cannot license calibrated confidence", {
|
|
71
|
+
spread: roundMetric(rawSpread),
|
|
72
|
+
mean_anchor_error: roundMetric(rawMeanError),
|
|
73
|
+
low_spread_threshold: fixture.calibration_bar.low_spread_threshold,
|
|
74
|
+
max_error_when_low_spread: fixture.calibration_bar.max_error_when_low_spread,
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
const accepted = issues.length === 0;
|
|
78
|
+
const metrics = {
|
|
79
|
+
output_count: fixture.outputs.length,
|
|
80
|
+
spread: roundMetric(rawSpread),
|
|
81
|
+
mean_anchor_error: roundMetric(rawMeanError),
|
|
82
|
+
max_anchor_error: roundMetric(rawMaxError),
|
|
83
|
+
spread_error_gap: roundMetric(rawSpreadErrorGap),
|
|
84
|
+
calibrated_confidence: accepted
|
|
85
|
+
? roundMetric(1 - Math.min(1, rawSpread))
|
|
86
|
+
: null,
|
|
87
|
+
};
|
|
88
|
+
if (!accepted) {
|
|
89
|
+
return degradedResult({
|
|
90
|
+
summary: "K1 ensemble spread is degraded; spread is not accepted as calibrated confidence",
|
|
91
|
+
fixture_id: fixture.fixture_id,
|
|
92
|
+
responsibility_id: fixture.responsibility_id,
|
|
93
|
+
anchor: fixture.anchor,
|
|
94
|
+
diversity,
|
|
95
|
+
metrics,
|
|
96
|
+
output_scores: outputScores,
|
|
97
|
+
issues,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
ok: true,
|
|
102
|
+
status: "calibrated",
|
|
103
|
+
accepted_as_calibrated_confidence: true,
|
|
104
|
+
summary: "K1 ensemble spread is calibrated against the anchor with the diversity floor met",
|
|
105
|
+
fixture_id: fixture.fixture_id,
|
|
106
|
+
responsibility_id: fixture.responsibility_id,
|
|
107
|
+
anchor: fixture.anchor,
|
|
108
|
+
diversity,
|
|
109
|
+
metrics,
|
|
110
|
+
output_scores: Object.freeze([...outputScores]),
|
|
111
|
+
issues: Object.freeze([]),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function evaluateK1DiversityFloorV0(outputs) {
|
|
115
|
+
const families = uniqueSorted(outputs.map((output) => output.family));
|
|
116
|
+
const providers = uniqueSorted(outputs.map((output) => output.provider));
|
|
117
|
+
const sizeClasses = uniqueSorted(outputs.map((output) => output.size_class));
|
|
118
|
+
const missing = [];
|
|
119
|
+
if (families.length < 2) {
|
|
120
|
+
missing.push("model-family");
|
|
121
|
+
}
|
|
122
|
+
if (providers.length < 2) {
|
|
123
|
+
missing.push("provider");
|
|
124
|
+
}
|
|
125
|
+
if (sizeClasses.length < 2) {
|
|
126
|
+
missing.push("size-boundary");
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
floor_met: missing.length === 0,
|
|
130
|
+
families,
|
|
131
|
+
providers,
|
|
132
|
+
size_classes: sizeClasses,
|
|
133
|
+
missing: Object.freeze([...missing]),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
function readFixture(input) {
|
|
137
|
+
const issues = [];
|
|
138
|
+
if (!isRecord(input)) {
|
|
139
|
+
return {
|
|
140
|
+
ok: false,
|
|
141
|
+
issues: [
|
|
142
|
+
issue("fixture", "fixture-not-object", "K1 ensemble spread fixture must be an object", input),
|
|
143
|
+
],
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
if (input["schema"] !== exports.K1_ENSEMBLE_SPREAD_SCHEMA_V0) {
|
|
147
|
+
issues.push(issue("schema", "schema-invalid", "K1 fixture schema must be openprose.reactor-cradle.k1-ensemble-spread", input["schema"]));
|
|
148
|
+
}
|
|
149
|
+
if (input["v"] !== exports.K1_ENSEMBLE_SPREAD_VERSION_V0) {
|
|
150
|
+
issues.push(issue("v", "version-invalid", "K1 fixture version must be 0", input["v"]));
|
|
151
|
+
}
|
|
152
|
+
const fixtureId = readNonEmptyString(input, "fixture_id", "fixture", issues);
|
|
153
|
+
const responsibilityId = readNonEmptyString(input, "responsibility_id", "fixture", issues);
|
|
154
|
+
const recordedAt = readNonEmptyString(input, "recorded_at", "fixture", issues);
|
|
155
|
+
const anchor = readAnchor(input["anchor"], issues);
|
|
156
|
+
const calibrationBar = readCalibrationBar(input["calibration_bar"], issues);
|
|
157
|
+
const outputs = readOutputs(input["outputs"], issues);
|
|
158
|
+
if (issues.length > 0 ||
|
|
159
|
+
fixtureId === undefined ||
|
|
160
|
+
responsibilityId === undefined ||
|
|
161
|
+
recordedAt === undefined ||
|
|
162
|
+
anchor === undefined ||
|
|
163
|
+
calibrationBar === undefined ||
|
|
164
|
+
outputs === undefined) {
|
|
165
|
+
return { ok: false, issues: Object.freeze([...issues]) };
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
ok: true,
|
|
169
|
+
fixture: {
|
|
170
|
+
schema: exports.K1_ENSEMBLE_SPREAD_SCHEMA_V0,
|
|
171
|
+
v: exports.K1_ENSEMBLE_SPREAD_VERSION_V0,
|
|
172
|
+
fixture_id: fixtureId,
|
|
173
|
+
responsibility_id: responsibilityId,
|
|
174
|
+
recorded_at: recordedAt,
|
|
175
|
+
anchor,
|
|
176
|
+
calibration_bar: calibrationBar,
|
|
177
|
+
outputs,
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function readAnchor(value, issues) {
|
|
182
|
+
if (!isRecord(value)) {
|
|
183
|
+
issues.push(issue("anchor", "anchor-invalid", "K1 fixture must include a correctness anchor object", value));
|
|
184
|
+
return undefined;
|
|
185
|
+
}
|
|
186
|
+
const labelSource = readNonEmptyString(value, "label_source", "anchor", issues);
|
|
187
|
+
const verdict = readVerdict(value, "verdict", "anchor", issues);
|
|
188
|
+
const score = readUnitInterval(value, "score", "anchor", issues);
|
|
189
|
+
const grade = value["calibration_grade"];
|
|
190
|
+
if (typeof grade !== "string" ||
|
|
191
|
+
!ANCHOR_GRADES.has(grade)) {
|
|
192
|
+
issues.push(issue("anchor.calibration_grade", "anchor-invalid", "K1 correctness anchor must be authored or accrued", grade));
|
|
193
|
+
}
|
|
194
|
+
if (labelSource === undefined ||
|
|
195
|
+
verdict === undefined ||
|
|
196
|
+
score === undefined ||
|
|
197
|
+
typeof grade !== "string" ||
|
|
198
|
+
!ANCHOR_GRADES.has(grade)) {
|
|
199
|
+
return undefined;
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
label_source: labelSource,
|
|
203
|
+
calibration_grade: grade,
|
|
204
|
+
verdict,
|
|
205
|
+
score,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
function readCalibrationBar(value, issues) {
|
|
209
|
+
if (!isRecord(value)) {
|
|
210
|
+
issues.push(issue("calibration_bar", "calibration-bar-invalid", "K1 fixture must include a calibration bar object", value));
|
|
211
|
+
return undefined;
|
|
212
|
+
}
|
|
213
|
+
const maxMeanError = readUnitInterval(value, "max_mean_error", "calibration_bar", issues);
|
|
214
|
+
const maxSpreadErrorGap = readUnitInterval(value, "max_spread_error_gap", "calibration_bar", issues);
|
|
215
|
+
const lowSpreadThreshold = readUnitInterval(value, "low_spread_threshold", "calibration_bar", issues);
|
|
216
|
+
const maxErrorWhenLowSpread = readUnitInterval(value, "max_error_when_low_spread", "calibration_bar", issues);
|
|
217
|
+
if (maxMeanError === undefined ||
|
|
218
|
+
maxSpreadErrorGap === undefined ||
|
|
219
|
+
lowSpreadThreshold === undefined ||
|
|
220
|
+
maxErrorWhenLowSpread === undefined) {
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
return {
|
|
224
|
+
max_mean_error: maxMeanError,
|
|
225
|
+
max_spread_error_gap: maxSpreadErrorGap,
|
|
226
|
+
low_spread_threshold: lowSpreadThreshold,
|
|
227
|
+
max_error_when_low_spread: maxErrorWhenLowSpread,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
function readOutputs(value, issues) {
|
|
231
|
+
if (!Array.isArray(value)) {
|
|
232
|
+
issues.push(issue("outputs", "outputs-invalid", "K1 fixture outputs must be an array of recorded model outputs", value));
|
|
233
|
+
return undefined;
|
|
234
|
+
}
|
|
235
|
+
const outputs = [];
|
|
236
|
+
for (const [index, item] of value.entries()) {
|
|
237
|
+
const output = readOutput(item, index, issues);
|
|
238
|
+
if (output !== undefined) {
|
|
239
|
+
outputs.push(output);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return Object.freeze([...outputs]);
|
|
243
|
+
}
|
|
244
|
+
function readOutput(value, index, issues) {
|
|
245
|
+
const path = `outputs[${index}]`;
|
|
246
|
+
if (!isRecord(value)) {
|
|
247
|
+
issues.push(issue(path, "outputs-invalid", "K1 recorded model output must be an object", value));
|
|
248
|
+
return undefined;
|
|
249
|
+
}
|
|
250
|
+
const outputId = readNonEmptyString(value, "output_id", path, issues);
|
|
251
|
+
const provider = readNonEmptyString(value, "provider", path, issues);
|
|
252
|
+
const model = readNonEmptyString(value, "model", path, issues);
|
|
253
|
+
const family = readNonEmptyString(value, "family", path, issues);
|
|
254
|
+
const sizeClass = readNonEmptyString(value, "size_class", path, issues);
|
|
255
|
+
const verdict = readVerdict(value, "verdict", path, issues);
|
|
256
|
+
const score = readUnitInterval(value, "score", path, issues);
|
|
257
|
+
const confidence = readUnitInterval(value, "confidence", path, issues);
|
|
258
|
+
const text = readNonEmptyString(value, "text", path, issues);
|
|
259
|
+
if (outputId === undefined ||
|
|
260
|
+
provider === undefined ||
|
|
261
|
+
model === undefined ||
|
|
262
|
+
family === undefined ||
|
|
263
|
+
sizeClass === undefined ||
|
|
264
|
+
verdict === undefined ||
|
|
265
|
+
score === undefined ||
|
|
266
|
+
confidence === undefined ||
|
|
267
|
+
text === undefined) {
|
|
268
|
+
return undefined;
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
output_id: outputId,
|
|
272
|
+
provider,
|
|
273
|
+
model,
|
|
274
|
+
family,
|
|
275
|
+
size_class: sizeClass,
|
|
276
|
+
verdict,
|
|
277
|
+
score,
|
|
278
|
+
confidence,
|
|
279
|
+
text,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
function readNonEmptyString(record, key, path, issues) {
|
|
283
|
+
const value = record[key];
|
|
284
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
285
|
+
issues.push(issue(`${path}.${key}`, "field-invalid", `${path}.${key} must be a non-empty string`, value));
|
|
286
|
+
return undefined;
|
|
287
|
+
}
|
|
288
|
+
return value;
|
|
289
|
+
}
|
|
290
|
+
function readUnitInterval(record, key, path, issues) {
|
|
291
|
+
const value = record[key];
|
|
292
|
+
if (typeof value !== "number" ||
|
|
293
|
+
!Number.isFinite(value) ||
|
|
294
|
+
value < 0 ||
|
|
295
|
+
value > 1) {
|
|
296
|
+
issues.push(issue(`${path}.${key}`, "field-invalid", `${path}.${key} must be a finite number in [0, 1]`, value));
|
|
297
|
+
return undefined;
|
|
298
|
+
}
|
|
299
|
+
return value;
|
|
300
|
+
}
|
|
301
|
+
function readVerdict(record, key, path, issues) {
|
|
302
|
+
const value = record[key];
|
|
303
|
+
if (typeof value !== "string" ||
|
|
304
|
+
!VERDICT_STATUSES.has(value)) {
|
|
305
|
+
issues.push(issue(`${path}.${key}`, "field-invalid", `${path}.${key} must be one of up, drifting, down, blocked`, value));
|
|
306
|
+
return undefined;
|
|
307
|
+
}
|
|
308
|
+
return value;
|
|
309
|
+
}
|
|
310
|
+
function scoreOutputAgainstAnchor(output, anchor) {
|
|
311
|
+
return {
|
|
312
|
+
output_id: output.output_id,
|
|
313
|
+
provider: output.provider,
|
|
314
|
+
model: output.model,
|
|
315
|
+
family: output.family,
|
|
316
|
+
size_class: output.size_class,
|
|
317
|
+
verdict: output.verdict,
|
|
318
|
+
score: output.score,
|
|
319
|
+
anchor_error: roundMetric(Math.abs(output.score - anchor.score)),
|
|
320
|
+
matches_anchor_verdict: output.verdict === anchor.verdict,
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
function degradedResult(input) {
|
|
324
|
+
return {
|
|
325
|
+
ok: false,
|
|
326
|
+
status: "degraded",
|
|
327
|
+
accepted_as_calibrated_confidence: false,
|
|
328
|
+
summary: input.summary,
|
|
329
|
+
fixture_id: input.fixture_id,
|
|
330
|
+
responsibility_id: input.responsibility_id,
|
|
331
|
+
anchor: input.anchor,
|
|
332
|
+
diversity: input.diversity,
|
|
333
|
+
metrics: input.metrics,
|
|
334
|
+
output_scores: Object.freeze([...input.output_scores]),
|
|
335
|
+
issues: Object.freeze([...input.issues]),
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
function emptyDiversity() {
|
|
339
|
+
return {
|
|
340
|
+
floor_met: false,
|
|
341
|
+
families: Object.freeze([]),
|
|
342
|
+
providers: Object.freeze([]),
|
|
343
|
+
size_classes: Object.freeze([]),
|
|
344
|
+
missing: Object.freeze(["model-family", "provider", "size-boundary"]),
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
function emptyMetrics() {
|
|
348
|
+
return {
|
|
349
|
+
output_count: 0,
|
|
350
|
+
spread: 0,
|
|
351
|
+
mean_anchor_error: 0,
|
|
352
|
+
max_anchor_error: 0,
|
|
353
|
+
spread_error_gap: 0,
|
|
354
|
+
calibrated_confidence: null,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
function meanPairwiseDistance(values) {
|
|
358
|
+
if (values.length < 2) {
|
|
359
|
+
return 0;
|
|
360
|
+
}
|
|
361
|
+
let total = 0;
|
|
362
|
+
let count = 0;
|
|
363
|
+
for (let left = 0; left < values.length; left += 1) {
|
|
364
|
+
const leftValue = values[left];
|
|
365
|
+
if (leftValue === undefined) {
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
for (let right = left + 1; right < values.length; right += 1) {
|
|
369
|
+
const rightValue = values[right];
|
|
370
|
+
if (rightValue === undefined) {
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
total += Math.abs(leftValue - rightValue);
|
|
374
|
+
count += 1;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
return count === 0 ? 0 : total / count;
|
|
378
|
+
}
|
|
379
|
+
function mean(values) {
|
|
380
|
+
if (values.length === 0) {
|
|
381
|
+
return 0;
|
|
382
|
+
}
|
|
383
|
+
return values.reduce((total, value) => total + value, 0) / values.length;
|
|
384
|
+
}
|
|
385
|
+
function roundMetric(value) {
|
|
386
|
+
return Number(value.toFixed(6));
|
|
387
|
+
}
|
|
388
|
+
function uniqueSorted(values) {
|
|
389
|
+
return Object.freeze(Array.from(new Set(values)).sort());
|
|
390
|
+
}
|
|
391
|
+
function issue(path, code, message, observed) {
|
|
392
|
+
return { path, code, message, observed };
|
|
393
|
+
}
|
|
394
|
+
function isRecord(value) {
|
|
395
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
396
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { KernelPolicyArtifactValidation } from "@openprose/reactor/kernel";
|
|
2
|
+
export declare const K2_POLICY_AUTHOR_FIXTURE_SCHEMA_V0: "openprose.reactor-cradle.k2-policy-author-fixture";
|
|
3
|
+
export declare const K2_POLICY_AUTHOR_FIXTURE_VERSION_V0: 0;
|
|
4
|
+
export declare const K2_KNOWN_RESPONSIBILITY_ID: "incident-channel-current-briefing";
|
|
5
|
+
export type K2PolicyAuthorEvaluationStatusV0 = "pass" | "fail";
|
|
6
|
+
export interface K2PolicyAuthorEvaluationOptionsV0 {
|
|
7
|
+
readonly expected_responsibility_id?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface K2PolicyAuthorEvidenceV0 {
|
|
10
|
+
readonly path: string;
|
|
11
|
+
readonly message: string;
|
|
12
|
+
readonly observed?: unknown;
|
|
13
|
+
}
|
|
14
|
+
export interface K2PolicyAuthorEvaluationV0 {
|
|
15
|
+
readonly ok: boolean;
|
|
16
|
+
readonly status: K2PolicyAuthorEvaluationStatusV0;
|
|
17
|
+
readonly summary: string;
|
|
18
|
+
readonly responsibility_id?: string;
|
|
19
|
+
readonly live_observable_refs: readonly string[];
|
|
20
|
+
readonly evidence: readonly K2PolicyAuthorEvidenceV0[];
|
|
21
|
+
readonly failures: readonly K2PolicyAuthorEvidenceV0[];
|
|
22
|
+
readonly kernel_validation?: KernelPolicyArtifactValidation;
|
|
23
|
+
}
|
|
24
|
+
export declare function evaluateK2PolicyAuthorFixtureV0(fixture: unknown, options?: K2PolicyAuthorEvaluationOptionsV0): K2PolicyAuthorEvaluationV0;
|
|
25
|
+
//# sourceMappingURL=k2-policy-author.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"k2-policy-author.d.ts","sourceRoot":"","sources":["../../src/spikes/k2-policy-author.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,8BAA8B,EAE/B,MAAM,2BAA2B,CAAC;AAOnC,eAAO,MAAM,kCAAkC,EAC7C,mDAA4D,CAAC;AAC/D,eAAO,MAAM,mCAAmC,EAAG,CAAU,CAAC;AAC9D,eAAO,MAAM,0BAA0B,EACrC,mCAA4C,CAAC;AAE/C,MAAM,MAAM,gCAAgC,GAAG,MAAM,GAAG,MAAM,CAAC;AAE/D,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CAC9C;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,gCAAgC,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACvD,QAAQ,CAAC,QAAQ,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACvD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,8BAA8B,CAAC;CAC7D;AAUD,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,iCAAsC,GAC9C,0BAA0B,CAmG5B"}
|