@runbooks/schema 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +22 -0
- package/dist/agent-skills.test.d.ts +1 -0
- package/dist/agent-skills.test.js +71 -0
- package/dist/capabilities.d.ts +40 -0
- package/dist/capabilities.js +211 -0
- package/dist/capabilities.test.d.ts +1 -0
- package/dist/capabilities.test.js +119 -0
- package/dist/content-classes.test.d.ts +1 -0
- package/dist/content-classes.test.js +77 -0
- package/dist/generate.cli.d.ts +2 -0
- package/dist/generate.cli.js +27 -0
- package/dist/generate.d.ts +54 -0
- package/dist/generate.js +133 -0
- package/dist/generate.test.d.ts +1 -0
- package/dist/generate.test.js +85 -0
- package/dist/generated.d.ts +101 -0
- package/dist/generated.js +9 -0
- package/dist/generated.ts +82 -0
- package/dist/hash.d.ts +36 -0
- package/dist/hash.js +102 -0
- package/dist/hash.test.d.ts +1 -0
- package/dist/hash.test.js +54 -0
- package/dist/index.d.ts +58 -0
- package/dist/index.js +61 -0
- package/dist/index.test.d.ts +1 -0
- package/dist/index.test.js +236 -0
- package/dist/interpolation.d.ts +11 -0
- package/dist/interpolation.js +34 -0
- package/dist/node.d.ts +41 -0
- package/dist/node.js +111 -0
- package/dist/p1.test.d.ts +1 -0
- package/dist/p1.test.js +87 -0
- package/dist/run-report.test.d.ts +1 -0
- package/dist/run-report.test.js +145 -0
- package/dist/served.d.ts +36 -0
- package/dist/served.js +128 -0
- package/dist/stats.d.ts +17 -0
- package/dist/stats.js +23 -0
- package/dist/stats.test.d.ts +1 -0
- package/dist/stats.test.js +32 -0
- package/dist/validate.d.ts +16 -0
- package/dist/validate.js +46 -0
- package/dist/validate.test.d.ts +1 -0
- package/dist/validate.test.js +61 -0
- package/dist/vocabularies.test.d.ts +1 -0
- package/dist/vocabularies.test.js +77 -0
- package/package.json +50 -0
- package/spec/v1/schema/attestation.json +62 -0
- package/spec/v1/schema/p0.json +383 -0
- package/spec/v1/schema/p1.json +408 -0
- package/spec/v1/schema/record.json +908 -0
- package/spec/v1/schema/run-report-envelope.json +25 -0
- package/spec/v1/schema/run-report.json +121 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { loadRunReportSchema, loadRunReportEnvelopeSchema, validateRunReport, validateRunReportEnvelope, } from "./node.js";
|
|
3
|
+
const report = {
|
|
4
|
+
report_schema: "runbook-run-report/v1",
|
|
5
|
+
schema_version: "v1",
|
|
6
|
+
runbook: "std/k8s-node-not-ready-drain",
|
|
7
|
+
version: "1.0.0",
|
|
8
|
+
content_hash: `sha256:${"a".repeat(64)}`,
|
|
9
|
+
profile: "P1",
|
|
10
|
+
outcome: "failed",
|
|
11
|
+
failed_step_id: "s4",
|
|
12
|
+
failure_class: "permission_denied",
|
|
13
|
+
executor_kind: "autonomous",
|
|
14
|
+
runtime_profile: "R2",
|
|
15
|
+
deviation_count: 2,
|
|
16
|
+
duration_bucket: "1-5m",
|
|
17
|
+
env_class: "staging",
|
|
18
|
+
reported_at: "2026-09-03",
|
|
19
|
+
};
|
|
20
|
+
describe("the normative form validates", () => {
|
|
21
|
+
it("accepts §14's example", () => {
|
|
22
|
+
expect(validateRunReport(report).valid).toBe(true);
|
|
23
|
+
});
|
|
24
|
+
it("accepts a success with no failure fields", () => {
|
|
25
|
+
const { failed_step_id: _s, failure_class: _c, ...ok } = report;
|
|
26
|
+
expect(validateRunReport({ ...ok, outcome: "success" }).valid).toBe(true);
|
|
27
|
+
});
|
|
28
|
+
/**
|
|
29
|
+
* A reporter cannot be required to have run an indexed version, and refusing those
|
|
30
|
+
* reports would silently exclude exactly the runs worth hearing about.
|
|
31
|
+
*/
|
|
32
|
+
it("accepts a content hash the catalog has never seen", () => {
|
|
33
|
+
expect(validateRunReport({ ...report, content_hash: `sha256:${"f".repeat(64)}` }).valid).toBe(true);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
/**
|
|
37
|
+
* The exclusions are the point of this schema. The pressure to add "just an error
|
|
38
|
+
* message field" is constant, and each such field is a secret leak waiting to happen.
|
|
39
|
+
*/
|
|
40
|
+
describe("nothing accepts free text", () => {
|
|
41
|
+
function walk(schema, path = "$") {
|
|
42
|
+
const found = [];
|
|
43
|
+
if (schema.type === "string" || (schema.type === undefined && (schema.pattern || schema.format))) {
|
|
44
|
+
found.push({ path, node: schema });
|
|
45
|
+
}
|
|
46
|
+
for (const [key, child] of Object.entries(schema.properties ?? {})) {
|
|
47
|
+
found.push(...walk(child, `${path}.${key}`));
|
|
48
|
+
}
|
|
49
|
+
for (const [key, child] of Object.entries(schema.$defs ?? {})) {
|
|
50
|
+
found.push(...walk(child, `${path}.$defs.${key}`));
|
|
51
|
+
}
|
|
52
|
+
if (schema.items)
|
|
53
|
+
found.push(...walk(schema.items, `${path}[]`));
|
|
54
|
+
return found;
|
|
55
|
+
}
|
|
56
|
+
function objects(schema, path = "$") {
|
|
57
|
+
if (schema === null || typeof schema !== "object")
|
|
58
|
+
return [];
|
|
59
|
+
const node = schema;
|
|
60
|
+
const found = [];
|
|
61
|
+
if (node["type"] === "object")
|
|
62
|
+
found.push({ path, node: node });
|
|
63
|
+
for (const [key, child] of Object.entries(node)) {
|
|
64
|
+
found.push(...objects(child, `${path}.${key}`));
|
|
65
|
+
}
|
|
66
|
+
return found;
|
|
67
|
+
}
|
|
68
|
+
it.each([
|
|
69
|
+
["run-report.json", loadRunReportSchema()],
|
|
70
|
+
["run-report-envelope.json", loadRunReportEnvelopeSchema()],
|
|
71
|
+
])("%s constrains every string", (_name, schema) => {
|
|
72
|
+
for (const { path, node } of walk(schema)) {
|
|
73
|
+
const constrained = node.enum !== undefined || node.const !== undefined
|
|
74
|
+
|| node.pattern !== undefined || node.format !== undefined;
|
|
75
|
+
expect(constrained, `${path} is an unconstrained string, which is a free-text field`).toBe(true);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
it.each([
|
|
79
|
+
["run-report.json", loadRunReportSchema()],
|
|
80
|
+
["run-report-envelope.json", loadRunReportEnvelopeSchema()],
|
|
81
|
+
])("%s closes every object", (_name, schema) => {
|
|
82
|
+
for (const { path, node } of objects(schema)) {
|
|
83
|
+
expect(node.additionalProperties, `${path} accepts undeclared fields`).toBe(false);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
it("rejects a report carrying an error message", () => {
|
|
87
|
+
const result = validateRunReport({ ...report, error: "psql: FATAL: password authentication failed for user postgres" });
|
|
88
|
+
expect(result.valid).toBe(false);
|
|
89
|
+
});
|
|
90
|
+
it.each([
|
|
91
|
+
["a precise duration", { duration: "3m14s" }],
|
|
92
|
+
["a hostname", { host: "db-3.prod.internal" }],
|
|
93
|
+
["a sender identity", { reporter: "acme-corp" }],
|
|
94
|
+
["a region", { region: "eu-central-1" }],
|
|
95
|
+
])("rejects %s", (_case, extra) => {
|
|
96
|
+
expect(validateRunReport({ ...report, ...extra }).valid).toBe(false);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
describe("the closed vocabularies stay closed", () => {
|
|
100
|
+
it.each([
|
|
101
|
+
["outcome", "mostly-worked"],
|
|
102
|
+
["failure_class", "something_else"],
|
|
103
|
+
["executor_kind", "cron"],
|
|
104
|
+
["runtime_profile", "R3"],
|
|
105
|
+
["duration_bucket", "2m"],
|
|
106
|
+
["env_class", "prod-eu"],
|
|
107
|
+
["profile", "P2"],
|
|
108
|
+
])("rejects an unlisted %s", (field, value) => {
|
|
109
|
+
expect(validateRunReport({ ...report, [field]: value }).valid).toBe(false);
|
|
110
|
+
});
|
|
111
|
+
it("rejects a timestamp where a date belongs", () => {
|
|
112
|
+
expect(validateRunReport({ ...report, reported_at: "2026-09-03T14:22:07Z" }).valid).toBe(false);
|
|
113
|
+
});
|
|
114
|
+
it("requires a failure to name its class", () => {
|
|
115
|
+
const { failure_class: _c, ...unnamed } = report;
|
|
116
|
+
expect(validateRunReport(unnamed).valid).toBe(false);
|
|
117
|
+
});
|
|
118
|
+
it("refuses a failed step on a successful run", () => {
|
|
119
|
+
const { failure_class: _c, ...rest } = report;
|
|
120
|
+
expect(validateRunReport({ ...rest, outcome: "success" }).valid).toBe(false);
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
describe("the CloudEvents envelope", () => {
|
|
124
|
+
const event = {
|
|
125
|
+
specversion: "1.0",
|
|
126
|
+
type: "directory.runbooks.run-report.v1",
|
|
127
|
+
source: "urn:runbooks:hook",
|
|
128
|
+
id: "0b7f5f4e-9c2a-4a5b-8f1e-2d3c4b5a6978",
|
|
129
|
+
datacontenttype: "application/json",
|
|
130
|
+
data: report,
|
|
131
|
+
};
|
|
132
|
+
it("carries a valid report", () => {
|
|
133
|
+
expect(validateRunReportEnvelope(event).valid).toBe(true);
|
|
134
|
+
});
|
|
135
|
+
it("rejects an envelope whose data is not a valid report", () => {
|
|
136
|
+
expect(validateRunReportEnvelope({ ...event, data: { ...report, error: "boom" } }).valid).toBe(false);
|
|
137
|
+
});
|
|
138
|
+
it("keeps the source a kind of producer, not an installation", () => {
|
|
139
|
+
expect(validateRunReportEnvelope({ ...event, source: "https://runner.acme.internal" }).valid).toBe(false);
|
|
140
|
+
});
|
|
141
|
+
/** A precise emission time reintroduces the movement log that bucketing avoided. */
|
|
142
|
+
it("does not require a time, and refuses an undeclared one", () => {
|
|
143
|
+
expect(validateRunReportEnvelope({ ...event, time: "2026-09-03T14:22:07Z" }).valid).toBe(false);
|
|
144
|
+
});
|
|
145
|
+
});
|
package/dist/served.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The schema of a record **as the catalog serves it** (P-12 audit, §5, §12).
|
|
3
|
+
*
|
|
4
|
+
* `p0.json` and `p1.json` describe a record as an author writes it: `trust` is forbidden
|
|
5
|
+
* there, because trust is computed and never declared, and so is anything else the
|
|
6
|
+
* catalog assigns. What `/v1/runbooks/{publisher}/{slug}.json` contains is that document
|
|
7
|
+
* **plus** what the build concluded about it — the id, the hash, the trust level with its
|
|
8
|
+
* evidence, and any signature verdict.
|
|
9
|
+
*
|
|
10
|
+
* Those are two different shapes, and the catalog was publishing the second while
|
|
11
|
+
* offering only the first as its schema. A third party validating what we serve against
|
|
12
|
+
* what we publish got "must NOT have additional properties" — which makes "a third party
|
|
13
|
+
* can check conformance without our code" false at exactly the point where it is being
|
|
14
|
+
* relied on.
|
|
15
|
+
*
|
|
16
|
+
* So this derives the served schema from the authored one rather than restating it: the
|
|
17
|
+
* fields the catalog assigns are lifted from forbidden to defined, everything else is
|
|
18
|
+
* whatever the authored schema says, and there is no second copy of the record's shape
|
|
19
|
+
* to keep in step.
|
|
20
|
+
*/
|
|
21
|
+
export interface SchemaObject {
|
|
22
|
+
[key: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
export declare const SERVED_SCHEMA_ID = "https://runbooks.directory/spec/v1/schema/record.json";
|
|
25
|
+
/**
|
|
26
|
+
* What the catalog adds to a record on the way out, and nothing else may be added.
|
|
27
|
+
*
|
|
28
|
+
* Load-bearing rather than descriptive. It used to be neither: the constant said what the
|
|
29
|
+
* catalog assigns and `servedSchema` listed the same four fields again by hand, so the
|
|
30
|
+
* sentence "nothing else may be added" was a claim nothing could check and the two copies
|
|
31
|
+
* were free to drift. The schema below is built from this list, and a test asserts the
|
|
32
|
+
* served schema defines exactly these fields beyond the authored one.
|
|
33
|
+
*/
|
|
34
|
+
export declare const CATALOG_ASSIGNED: readonly ["trust", "trust_evidence", "signature", "attestations"];
|
|
35
|
+
export type CatalogAssigned = (typeof CATALOG_ASSIGNED)[number];
|
|
36
|
+
export declare function servedSchema(p0: SchemaObject, p1: SchemaObject): SchemaObject;
|
package/dist/served.js
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
export const SERVED_SCHEMA_ID = "https://runbooks.directory/spec/v1/schema/record.json";
|
|
2
|
+
/**
|
|
3
|
+
* What the catalog adds to a record on the way out, and nothing else may be added.
|
|
4
|
+
*
|
|
5
|
+
* Load-bearing rather than descriptive. It used to be neither: the constant said what the
|
|
6
|
+
* catalog assigns and `servedSchema` listed the same four fields again by hand, so the
|
|
7
|
+
* sentence "nothing else may be added" was a claim nothing could check and the two copies
|
|
8
|
+
* were free to drift. The schema below is built from this list, and a test asserts the
|
|
9
|
+
* served schema defines exactly these fields beyond the authored one.
|
|
10
|
+
*/
|
|
11
|
+
export const CATALOG_ASSIGNED = ["trust", "trust_evidence", "signature", "attestations"];
|
|
12
|
+
const TRUST_EVIDENCE = {
|
|
13
|
+
type: "object",
|
|
14
|
+
required: ["level", "date", "issuer", "self_attested"],
|
|
15
|
+
additionalProperties: false,
|
|
16
|
+
description: "Why the level reads as it does (§18.5): a badge without a date and an issuer is a claim nobody can check.",
|
|
17
|
+
properties: {
|
|
18
|
+
level: { enum: ["T0", "T1", "T2", "T3", "T4"] },
|
|
19
|
+
date: { type: "string", format: "date" },
|
|
20
|
+
issuer: { type: "string" },
|
|
21
|
+
self_attested: { type: "boolean" },
|
|
22
|
+
/** Present for the levels that decay. Past it, this record is T2 (§8, P-13). */
|
|
23
|
+
expires_on: { type: "string", format: "date" },
|
|
24
|
+
reasons: { type: "array", items: { type: "string" } },
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* The evidence behind the badge (W-18).
|
|
29
|
+
*
|
|
30
|
+
* Forbidden in an authored document — "attestations are issued and verified, never
|
|
31
|
+
* self-declared" — and present on a served one, because that is where they reach a record
|
|
32
|
+
* from: the catalog, having checked them.
|
|
33
|
+
*/
|
|
34
|
+
const ATTESTATIONS = {
|
|
35
|
+
type: "array",
|
|
36
|
+
items: {
|
|
37
|
+
type: "object",
|
|
38
|
+
required: ["issuer", "issued_at", "content_hash", "status", "self_attested", "counts"],
|
|
39
|
+
additionalProperties: false,
|
|
40
|
+
properties: {
|
|
41
|
+
issuer: { type: "string" },
|
|
42
|
+
issued_at: { type: "string", format: "date" },
|
|
43
|
+
content_hash: { type: "string", pattern: "^sha256:[0-9a-f]{64}$" },
|
|
44
|
+
status: { enum: ["valid", "invalid", "unverifiable"] },
|
|
45
|
+
self_attested: { type: "boolean" },
|
|
46
|
+
/** Whether it contributes to the level today: expiry and issuer standing decide. */
|
|
47
|
+
counts: { type: "boolean" },
|
|
48
|
+
why: { type: "string" },
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
const SIGNATURE = {
|
|
53
|
+
type: "object",
|
|
54
|
+
required: ["status", "key_id"],
|
|
55
|
+
additionalProperties: false,
|
|
56
|
+
description: "What checking the publisher's signature concluded. `unverifiable` states are published as themselves: a signature nobody could check must not look like one that passed, nor like a record nobody signed.",
|
|
57
|
+
properties: {
|
|
58
|
+
status: { enum: ["valid", "invalid", "unknown-key", "revoked"] },
|
|
59
|
+
key_id: { type: "string" },
|
|
60
|
+
why: { type: "string" },
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Derive the served schema from the authored ones.
|
|
65
|
+
*
|
|
66
|
+
* `p1.json` composes `p0.json` by `$ref`, and a `$ref` cannot relax an
|
|
67
|
+
* `additionalProperties: false` in the schema it refers to — so the P0 half is inlined
|
|
68
|
+
* and patched rather than referenced. That is the only reason this is a transform and not
|
|
69
|
+
* three lines of `allOf`.
|
|
70
|
+
*/
|
|
71
|
+
/** One fragment per assigned field. Keyed by the list, so a name with no shape fails to compile. */
|
|
72
|
+
const ASSIGNED_SCHEMA = {
|
|
73
|
+
trust: {
|
|
74
|
+
enum: ["T0", "T1", "T2", "T3", "T4"],
|
|
75
|
+
description: "Computed by the catalog at build time from evidence plus a clock, never asserted by the record. Forbidden in an authored document (p0.json); present in every served one.",
|
|
76
|
+
},
|
|
77
|
+
trust_evidence: TRUST_EVIDENCE,
|
|
78
|
+
signature: SIGNATURE,
|
|
79
|
+
attestations: ATTESTATIONS,
|
|
80
|
+
};
|
|
81
|
+
export function servedSchema(p0, p1) {
|
|
82
|
+
const authored = structuredClone(p0);
|
|
83
|
+
const runbook = (authored["properties"]["runbook"] ?? {});
|
|
84
|
+
const properties = (runbook["properties"] ?? {});
|
|
85
|
+
// Computed, not declared — on the way in. On the way out they are the point. Added by
|
|
86
|
+
// walking CATALOG_ASSIGNED, so that list is what decides rather than what comments.
|
|
87
|
+
for (const field of CATALOG_ASSIGNED)
|
|
88
|
+
properties[field] = ASSIGNED_SCHEMA[field];
|
|
89
|
+
const required = new Set([...(runbook["required"] ?? []), "content_hash", "trust"]);
|
|
90
|
+
const steps = (p1["allOf"] ?? []).find((part) => part["properties"]?.["runbook"]?.["properties"] !== undefined);
|
|
91
|
+
return {
|
|
92
|
+
$schema: authored["$schema"],
|
|
93
|
+
$id: SERVED_SCHEMA_ID,
|
|
94
|
+
// P1's own definitions travel with the half that uses them: `#/$defs/step` resolves
|
|
95
|
+
// against the schema it is written in, and inlining the steps block without them
|
|
96
|
+
// would produce a schema that cannot compile.
|
|
97
|
+
$defs: p1["$defs"] ?? {},
|
|
98
|
+
title: "Runbook, as the catalog serves it",
|
|
99
|
+
description: "A published record: the authored document plus what the build concluded about it. Validate /v1/runbooks/{publisher}/{slug}.json against this; validate a document you are about to submit against p0.json or p1.json, which forbid the fields the catalog assigns.",
|
|
100
|
+
allOf: [
|
|
101
|
+
{
|
|
102
|
+
...authored,
|
|
103
|
+
$id: undefined,
|
|
104
|
+
$schema: undefined,
|
|
105
|
+
properties: {
|
|
106
|
+
...authored["properties"],
|
|
107
|
+
runbook: { ...runbook, properties, required: [...required] },
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
// A served P1 record still has to satisfy P1's own additions. A P0 record has no
|
|
111
|
+
// steps, so this half is applied conditionally rather than required outright.
|
|
112
|
+
steps
|
|
113
|
+
? {
|
|
114
|
+
// Typed at every level: ajv runs in strict mode here, which refuses a
|
|
115
|
+
// `properties` without a `type` — a good rule, since a schema that forgets it
|
|
116
|
+
// silently accepts a string where an object was meant.
|
|
117
|
+
if: {
|
|
118
|
+
type: "object",
|
|
119
|
+
properties: {
|
|
120
|
+
runbook: { type: "object", properties: { profile: { const: "P1" } } },
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
then: steps,
|
|
124
|
+
}
|
|
125
|
+
: {},
|
|
126
|
+
],
|
|
127
|
+
};
|
|
128
|
+
}
|
package/dist/stats.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One median, because there were two and they disagreed.
|
|
3
|
+
*
|
|
4
|
+
* Five places computed `sorted[Math.floor(n / 2)]` — the upper of the two middles for an
|
|
5
|
+
* even count — and the moderation queue computed the same statistic as
|
|
6
|
+
* `percentile(ages, 0.5)`, which is `sorted[Math.floor((n - 1) * 0.5)]`, the lower one.
|
|
7
|
+
* Over thirty records those are index 15 and index 14: two different published numbers for
|
|
8
|
+
* one word. They agree for every odd count, which is why nothing noticed.
|
|
9
|
+
*
|
|
10
|
+
* The convention here is nearest-rank on the upper side: the answer is always a value that
|
|
11
|
+
* actually occurred rather than the average of two that did not. For a duration in days or
|
|
12
|
+
* hours that matters — "2.5 days since the trust judgement" is a number no record has, and
|
|
13
|
+
* a reader who goes looking for it finds nothing.
|
|
14
|
+
*/
|
|
15
|
+
export declare function percentile(sorted: readonly number[], p: number): number | undefined;
|
|
16
|
+
/** The middle value, upper of the two when the count is even. Undefined over nothing. */
|
|
17
|
+
export declare function median(sorted: readonly number[]): number | undefined;
|
package/dist/stats.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One median, because there were two and they disagreed.
|
|
3
|
+
*
|
|
4
|
+
* Five places computed `sorted[Math.floor(n / 2)]` — the upper of the two middles for an
|
|
5
|
+
* even count — and the moderation queue computed the same statistic as
|
|
6
|
+
* `percentile(ages, 0.5)`, which is `sorted[Math.floor((n - 1) * 0.5)]`, the lower one.
|
|
7
|
+
* Over thirty records those are index 15 and index 14: two different published numbers for
|
|
8
|
+
* one word. They agree for every odd count, which is why nothing noticed.
|
|
9
|
+
*
|
|
10
|
+
* The convention here is nearest-rank on the upper side: the answer is always a value that
|
|
11
|
+
* actually occurred rather than the average of two that did not. For a duration in days or
|
|
12
|
+
* hours that matters — "2.5 days since the trust judgement" is a number no record has, and
|
|
13
|
+
* a reader who goes looking for it finds nothing.
|
|
14
|
+
*/
|
|
15
|
+
export function percentile(sorted, p) {
|
|
16
|
+
if (sorted.length === 0)
|
|
17
|
+
return undefined;
|
|
18
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(sorted.length * p)))];
|
|
19
|
+
}
|
|
20
|
+
/** The middle value, upper of the two when the count is even. Undefined over nothing. */
|
|
21
|
+
export function median(sorted) {
|
|
22
|
+
return percentile(sorted, 0.5);
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { median, percentile } from "./stats.js";
|
|
3
|
+
describe("one middle value", () => {
|
|
4
|
+
it("takes the upper of the two middles when the count is even", () => {
|
|
5
|
+
// The disagreement this exists to end. The two middles of this list are 20 and 30:
|
|
6
|
+
// `sorted[Math.floor(n / 2)]` answers 30 and `sorted[Math.floor((n - 1) * 0.5)]`
|
|
7
|
+
// answers 20, and both were in the repository, under one word.
|
|
8
|
+
expect(median([10, 20, 30, 40])).toBe(30);
|
|
9
|
+
expect(median([10, 20, 30, 40])).not.toBe(20);
|
|
10
|
+
});
|
|
11
|
+
it("takes the middle when the count is odd", () => {
|
|
12
|
+
expect(median([1, 2, 3])).toBe(2);
|
|
13
|
+
expect(median([5])).toBe(5);
|
|
14
|
+
});
|
|
15
|
+
it("has no answer over nothing, rather than zero", () => {
|
|
16
|
+
// Zero is a duration a caller would publish. Undefined is the caller's to turn into
|
|
17
|
+
// "not measured", which is what /status does with it.
|
|
18
|
+
expect(median([])).toBeUndefined();
|
|
19
|
+
expect(percentile([], 0.9)).toBeUndefined();
|
|
20
|
+
});
|
|
21
|
+
it("clamps at both ends", () => {
|
|
22
|
+
expect(percentile([1, 2, 3], 1)).toBe(3);
|
|
23
|
+
expect(percentile([1, 2, 3], 0)).toBe(1);
|
|
24
|
+
});
|
|
25
|
+
it("is the same statistic as the fiftieth percentile", () => {
|
|
26
|
+
// They were two functions and disagreed. A median that is not p50 is a bug in one of
|
|
27
|
+
// them, and which one it is cannot be decided by reading either.
|
|
28
|
+
for (const xs of [[1, 2], [1, 2, 3], [1, 2, 3, 4], [4, 4, 4, 9]]) {
|
|
29
|
+
expect(median(xs)).toBe(percentile(xs, 0.5));
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ValidateFunction } from "ajv";
|
|
2
|
+
export interface ValidationError {
|
|
3
|
+
/** JSON Pointer into the document, precise enough for the constructor to highlight a
|
|
4
|
+
* node (tasks/v0/P-02-schema-package.md). */
|
|
5
|
+
readonly path: string;
|
|
6
|
+
readonly message: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ValidationResult {
|
|
9
|
+
readonly valid: boolean;
|
|
10
|
+
readonly errors: readonly ValidationError[];
|
|
11
|
+
}
|
|
12
|
+
/** Compile once per `$id`: ajv registers a schema by its identifier, and a failed
|
|
13
|
+
* compile still registers it — so a naive retry reports a duplicate and hides the
|
|
14
|
+
* real error. */
|
|
15
|
+
export declare function createValidator(schema: object, refs?: readonly object[]): ValidateFunction;
|
|
16
|
+
export declare function validate(doc: unknown, schema: object, refs?: readonly object[]): ValidationResult;
|
package/dist/validate.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema validation — pure. Takes a schema object; never reads one.
|
|
3
|
+
*
|
|
4
|
+
* The split is load-bearing, not stylistic. `packages/supervise` must acquire no
|
|
5
|
+
* filesystem or network dependency anywhere in its tree (R-01), and `packages/graph`
|
|
6
|
+
* must run in a browser for the live editor (P-07). Both depend on this package, so a
|
|
7
|
+
* single `node:fs` import here would break them at a distance. Loading from disk lives
|
|
8
|
+
* in `./node.js`, which only Node consumers import.
|
|
9
|
+
*
|
|
10
|
+
* Stock ajv, no custom keywords: publishing a JSON Schema is pointless if conformance
|
|
11
|
+
* cannot be checked without our code (RUNBOOK.md G3).
|
|
12
|
+
*/
|
|
13
|
+
import Ajv2020 from "ajv/dist/2020.js";
|
|
14
|
+
import addFormats from "ajv-formats";
|
|
15
|
+
const ajv = addFormats(new Ajv2020({ allErrors: true, strict: true }));
|
|
16
|
+
const compiled = new Map();
|
|
17
|
+
/** Compile once per `$id`: ajv registers a schema by its identifier, and a failed
|
|
18
|
+
* compile still registers it — so a naive retry reports a duplicate and hides the
|
|
19
|
+
* real error. */
|
|
20
|
+
export function createValidator(schema, refs = []) {
|
|
21
|
+
for (const ref of refs) {
|
|
22
|
+
const refId = ref.$id;
|
|
23
|
+
if (refId && !ajv.getSchema(refId))
|
|
24
|
+
ajv.addSchema(ref);
|
|
25
|
+
}
|
|
26
|
+
const id = schema.$id;
|
|
27
|
+
if (!id)
|
|
28
|
+
return ajv.compile(schema);
|
|
29
|
+
let check = compiled.get(id);
|
|
30
|
+
if (!check) {
|
|
31
|
+
check = ajv.getSchema(id) ?? ajv.compile(schema);
|
|
32
|
+
compiled.set(id, check);
|
|
33
|
+
}
|
|
34
|
+
return check;
|
|
35
|
+
}
|
|
36
|
+
export function validate(doc, schema, refs = []) {
|
|
37
|
+
const check = createValidator(schema, refs);
|
|
38
|
+
const valid = check(doc);
|
|
39
|
+
return {
|
|
40
|
+
valid,
|
|
41
|
+
errors: (check.errors ?? []).map((e) => ({
|
|
42
|
+
path: e.instancePath || "/",
|
|
43
|
+
message: e.message ?? "invalid",
|
|
44
|
+
})),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join, dirname, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { validateProfile as validate } from "./node.js";
|
|
6
|
+
const FIXTURES = resolve(dirname(fileURLToPath(import.meta.url)), "..", "fixtures");
|
|
7
|
+
function load(kind) {
|
|
8
|
+
const dir = join(FIXTURES, kind);
|
|
9
|
+
return readdirSync(dir).map((name) => ({
|
|
10
|
+
name,
|
|
11
|
+
doc: JSON.parse(readFileSync(join(dir, name), "utf8")),
|
|
12
|
+
}));
|
|
13
|
+
}
|
|
14
|
+
describe("P0 schema", () => {
|
|
15
|
+
/**
|
|
16
|
+
* Guards the guard.
|
|
17
|
+
*
|
|
18
|
+
* Both suites below are `it.each(load(...))`, and `it.each([])` registers no tests and
|
|
19
|
+
* reports success. An empty fixture directory would turn "the schema accepts every
|
|
20
|
+
* valid document and rejects every invalid one" — the claim the whole catalog's
|
|
21
|
+
* conformance rests on — into two sentences that checked nothing.
|
|
22
|
+
*/
|
|
23
|
+
it("has fixtures on both sides to check against", () => {
|
|
24
|
+
expect(load("valid").length).toBeGreaterThan(0);
|
|
25
|
+
expect(load("invalid").length).toBeGreaterThan(0);
|
|
26
|
+
});
|
|
27
|
+
it.each(load("valid"))("accepts $name", ({ doc }) => {
|
|
28
|
+
const result = validate(doc, "P0");
|
|
29
|
+
expect(result.errors).toEqual([]);
|
|
30
|
+
expect(result.valid).toBe(true);
|
|
31
|
+
});
|
|
32
|
+
// Each invalid fixture carries __expect naming the rule it must trip, so a fixture
|
|
33
|
+
// that starts passing for the wrong reason is still a visible failure.
|
|
34
|
+
it.each(load("invalid"))("rejects $name", ({ doc }) => {
|
|
35
|
+
expect(doc.__expect, "fixture must state which rule it trips").toBeTypeOf("string");
|
|
36
|
+
expect(validate(doc, "P0").valid).toBe(false);
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
describe("rules that must not silently regress", () => {
|
|
40
|
+
const base = JSON.parse(readFileSync(join(FIXTURES, "valid", "minimal.json"), "utf8"));
|
|
41
|
+
it("refuses a document that asserts its own trust", () => {
|
|
42
|
+
const doc = { ...base, runbook: { ...base.runbook, trust: "T4" } };
|
|
43
|
+
expect(validate(doc, "P0").valid).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
it("refuses provenance without upstream_state", () => {
|
|
46
|
+
const doc = {
|
|
47
|
+
...base,
|
|
48
|
+
runbook: {
|
|
49
|
+
...base.runbook,
|
|
50
|
+
source: { url: "https://example.com/r", fetched_at: "2026-09-02" },
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
expect(validate(doc, "P0").valid).toBe(false);
|
|
54
|
+
});
|
|
55
|
+
// The Q2 decision: our fields live under one namespace key, never flat alongside
|
|
56
|
+
// Agent Skills fields. A flat document is not a runbook document.
|
|
57
|
+
it("does not accept our fields at the top level", () => {
|
|
58
|
+
const { runbook, ...rest } = base;
|
|
59
|
+
expect(validate({ ...rest, ...runbook }, "P0").valid).toBe(false);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The vocabularies the schema declares are one list each (§15, §3).
|
|
3
|
+
*
|
|
4
|
+
* They were written out three times: here as prose in the schema, in `@runbooks/lint` as a
|
|
5
|
+
* type beside a table of what each class means, and in `@runbooks/moderation` as a type
|
|
6
|
+
* beside a table of what the queue does about it. Three copies is two chances to disagree,
|
|
7
|
+
* and they had already taken one — the linter said a `curated` record does not auto-publish
|
|
8
|
+
* while the policy said silence may publish it, and nothing read the linter's copy, so the
|
|
9
|
+
* wrong one was also the invisible one.
|
|
10
|
+
*
|
|
11
|
+
* The names live in this package with `Risk` and `StepKind`. What holds them to the
|
|
12
|
+
* schema itself is this file: an enum in `record.json` that gains a fifth value, or loses
|
|
13
|
+
* one, fails here rather than in whichever consumer meets the value first.
|
|
14
|
+
*/
|
|
15
|
+
import { describe, it, expect } from "vitest";
|
|
16
|
+
import { readFileSync } from "node:fs";
|
|
17
|
+
import { join, dirname, resolve } from "node:path";
|
|
18
|
+
import { fileURLToPath } from "node:url";
|
|
19
|
+
import { CONTENT_CLASSES, UPSTREAM_STATES } from "./index.js";
|
|
20
|
+
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
|
21
|
+
function enumIn(file, field = "content_class") {
|
|
22
|
+
const schema = JSON.parse(readFileSync(join(ROOT, "spec", "v1", "schema", file), "utf8"));
|
|
23
|
+
const found = [];
|
|
24
|
+
const walk = (node) => {
|
|
25
|
+
if (Array.isArray(node))
|
|
26
|
+
return node.forEach(walk);
|
|
27
|
+
if (typeof node !== "object" || node === null)
|
|
28
|
+
return;
|
|
29
|
+
for (const [key, value] of Object.entries(node)) {
|
|
30
|
+
if (key === field && typeof value === "object" && value !== null) {
|
|
31
|
+
const candidate = value.enum;
|
|
32
|
+
if (Array.isArray(candidate))
|
|
33
|
+
found.push(...candidate);
|
|
34
|
+
}
|
|
35
|
+
walk(value);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
walk(schema);
|
|
39
|
+
return found;
|
|
40
|
+
}
|
|
41
|
+
describe("the content class vocabulary", () => {
|
|
42
|
+
it.each(["record.json", "p0.json"])("matches the enum in %s", (file) => {
|
|
43
|
+
const declared = enumIn(file);
|
|
44
|
+
expect(declared.length, `${file} declares no content_class enum`).toBeGreaterThan(0);
|
|
45
|
+
expect([...declared].sort()).toEqual([...CONTENT_CLASSES].sort());
|
|
46
|
+
});
|
|
47
|
+
it("is in the order §15 lists them, least claimed first", () => {
|
|
48
|
+
expect(CONTENT_CLASSES).toEqual(["ingested", "adapted", "submitted", "curated"]);
|
|
49
|
+
});
|
|
50
|
+
/** The type is the list, so a class cannot be added to one and not the other. */
|
|
51
|
+
it("has a type that admits exactly these", () => {
|
|
52
|
+
const every = {
|
|
53
|
+
ingested: true,
|
|
54
|
+
adapted: true,
|
|
55
|
+
submitted: true,
|
|
56
|
+
curated: true,
|
|
57
|
+
};
|
|
58
|
+
expect(Object.keys(every).sort()).toEqual([...CONTENT_CLASSES].sort());
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
/**
|
|
62
|
+
* The same, for the state of the source a record came from.
|
|
63
|
+
*
|
|
64
|
+
* Written out three times — the reindexer that sets it, the resolver that warns on it, the
|
|
65
|
+
* page that shows it — and shared by none of them.
|
|
66
|
+
*/
|
|
67
|
+
describe("the upstream state vocabulary", () => {
|
|
68
|
+
it.each(["record.json", "p0.json"])("matches the enum in %s", (file) => {
|
|
69
|
+
const declared = enumIn(file, "upstream_state");
|
|
70
|
+
expect(declared.length, `${file} declares no upstream_state enum`).toBeGreaterThan(0);
|
|
71
|
+
expect([...declared].sort()).toEqual([...UPSTREAM_STATES].sort());
|
|
72
|
+
});
|
|
73
|
+
it("has a type that admits exactly these", () => {
|
|
74
|
+
const every = { current: true, changed: true, gone: true };
|
|
75
|
+
expect(Object.keys(every).sort()).toEqual([...UPSTREAM_STATES].sort());
|
|
76
|
+
});
|
|
77
|
+
});
|
package/package.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@runbooks/schema",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"description": "The published runbook schemas, and a validator that uses stock ajv.",
|
|
7
|
+
"main": "./dist/index.js",
|
|
8
|
+
"types": "./dist/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./node": {
|
|
15
|
+
"types": "./dist/node.d.ts",
|
|
16
|
+
"default": "./dist/node.js"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"dependencies": {
|
|
20
|
+
"ajv": "^8.20.0",
|
|
21
|
+
"ajv-formats": "^3.0.1"
|
|
22
|
+
},
|
|
23
|
+
"license": "MIT",
|
|
24
|
+
"repository": {
|
|
25
|
+
"type": "git",
|
|
26
|
+
"url": "https://github.com/runbooks-directory/runbooks.directory"
|
|
27
|
+
},
|
|
28
|
+
"engines": {
|
|
29
|
+
"node": ">=20.11"
|
|
30
|
+
},
|
|
31
|
+
"publishConfig": {
|
|
32
|
+
"access": "public"
|
|
33
|
+
},
|
|
34
|
+
"files": [
|
|
35
|
+
"dist",
|
|
36
|
+
"README.md",
|
|
37
|
+
"spec"
|
|
38
|
+
],
|
|
39
|
+
"runbooks": {
|
|
40
|
+
"schemaVersions": [
|
|
41
|
+
"v1"
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
"scripts": {
|
|
45
|
+
"build": "tsc -b",
|
|
46
|
+
"test": "vitest run --passWithNoTests",
|
|
47
|
+
"lint": "eslint src",
|
|
48
|
+
"generate": "node dist/generate.cli.js"
|
|
49
|
+
}
|
|
50
|
+
}
|