overseer-testing-protocol 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +75 -0
  4. data/SOURCE.json +6 -0
  5. data/bin/overseer-testing-conformance +53 -0
  6. data/docs/testing-control-implementation-guide.md +116 -0
  7. data/docs/testing-control-protocol.md +432 -0
  8. data/lib/overseer/testing_control/conformance/case_file.rb +120 -0
  9. data/lib/overseer/testing_control/conformance/http_transport.rb +114 -0
  10. data/lib/overseer/testing_control/conformance/report.rb +104 -0
  11. data/lib/overseer/testing_control/conformance/runner.rb +783 -0
  12. data/lib/overseer/testing_control/discovery.rb +79 -0
  13. data/lib/overseer/testing_control/json_subset_matcher.rb +120 -0
  14. data/lib/overseer/testing_control/protocol_v3.rb +298 -0
  15. data/lib/overseer/testing_control/redaction.rb +111 -0
  16. data/lib/overseer/testing_protocol.rb +9 -0
  17. data/protocol/testing-control/v3/conformance-case.schema.json +151 -0
  18. data/protocol/testing-control/v3/conformance-report.schema.json +255 -0
  19. data/protocol/testing-control/v3/fixtures/capabilities-response.json +147 -0
  20. data/protocol/testing-control/v3/fixtures/conformance-case.json +23 -0
  21. data/protocol/testing-control/v3/fixtures/conformance-report.json +88 -0
  22. data/protocol/testing-control/v3/fixtures/error-response.json +20 -0
  23. data/protocol/testing-control/v3/fixtures/manifest.json +13 -0
  24. data/protocol/testing-control/v3/fixtures/probe-request.json +20 -0
  25. data/protocol/testing-control/v3/fixtures/probe-response.json +22 -0
  26. data/protocol/testing-control/v3/fixtures/reset-response.json +15 -0
  27. data/protocol/testing-control/v3/fixtures/sink-query-request.json +26 -0
  28. data/protocol/testing-control/v3/fixtures/sink-query-response.json +39 -0
  29. data/protocol/testing-control/v3/fixtures/state-request.json +20 -0
  30. data/protocol/testing-control/v3/fixtures/state-response.json +23 -0
  31. data/protocol/testing-control/v3/openapi.yaml +343 -0
  32. data/protocol/testing-control/v3/schema.json +772 -0
  33. metadata +85 -0
@@ -0,0 +1,20 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "1ccafac8-0f96-4277-a2d1-21fa4300ec16",
9
+ "stepId": "setup-active-user"
10
+ },
11
+ "errors": [
12
+ {
13
+ "code": "schema-validation-failed",
14
+ "title": "State input is invalid",
15
+ "detail": "The requested state input does not satisfy its advertised schema.",
16
+ "pointer": "/data/input/email",
17
+ "retryable": false
18
+ }
19
+ ]
20
+ }
@@ -0,0 +1,13 @@
1
+ {
2
+ "capabilities-response": "capabilities-response.json",
3
+ "reset-response": "reset-response.json",
4
+ "state-request": "state-request.json",
5
+ "state-response": "state-response.json",
6
+ "probe-request": "probe-request.json",
7
+ "probe-response": "probe-response.json",
8
+ "sink-query-request": "sink-query-request.json",
9
+ "sink-query-response": "sink-query-response.json",
10
+ "error-response": "error-response.json",
11
+ "conformance-case": "conformance-case.json",
12
+ "conformance-report": "conformance-report.json"
13
+ }
@@ -0,0 +1,20 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "e4508a64-0177-4941-a55c-a94402a83765",
9
+ "stepId": "deactivate-user"
10
+ },
11
+ "data": {
12
+ "capability": {
13
+ "id": "user-projection",
14
+ "version": "1"
15
+ },
16
+ "input": {
17
+ "userId": "user-001"
18
+ }
19
+ }
20
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "e4508a64-0177-4941-a55c-a94402a83765",
9
+ "stepId": "deactivate-user"
10
+ },
11
+ "data": {
12
+ "capability": {
13
+ "id": "user-projection",
14
+ "version": "1"
15
+ },
16
+ "observedAt": "2026-07-31T10:01:00.000Z",
17
+ "projection": {
18
+ "id": "user-001",
19
+ "status": "inactive"
20
+ }
21
+ }
22
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "3ad2eb88-1eba-4fe2-802d-88caff52a87e"
9
+ },
10
+ "data": {
11
+ "resetAt": "2026-07-31T10:00:00.000Z",
12
+ "strategy": "in-process",
13
+ "sinksCleared": true
14
+ }
15
+ }
@@ -0,0 +1,26 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "f2e6bc89-3e8d-4cb5-b49b-7151c798a249",
9
+ "stepId": "deactivate-user"
10
+ },
11
+ "data": {
12
+ "capability": {
13
+ "id": "transactional-email",
14
+ "version": "1"
15
+ },
16
+ "filter": {
17
+ "correlationId": "public-request-001",
18
+ "stepId": "deactivate-user",
19
+ "effectKinds": ["email"]
20
+ },
21
+ "input": {
22
+ "template": "user-deactivated"
23
+ },
24
+ "limit": 25
25
+ }
26
+ }
@@ -0,0 +1,39 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "f2e6bc89-3e8d-4cb5-b49b-7151c798a249",
9
+ "stepId": "deactivate-user"
10
+ },
11
+ "data": {
12
+ "capability": {
13
+ "id": "transactional-email",
14
+ "version": "1"
15
+ },
16
+ "records": [
17
+ {
18
+ "id": "effect-001",
19
+ "capability": {
20
+ "id": "transactional-email",
21
+ "version": "1"
22
+ },
23
+ "effectKind": "email",
24
+ "outcome": "success",
25
+ "correlationId": "public-request-001",
26
+ "stepId": "deactivate-user",
27
+ "capturedAt": "2026-07-31T10:01:01.000Z",
28
+ "summary": "user-deactivated email captured for synthetic recipient",
29
+ "metadata": {
30
+ "attempt": 1
31
+ },
32
+ "payload": {
33
+ "template": "user-deactivated",
34
+ "recipientAlias": "user-001"
35
+ }
36
+ }
37
+ ]
38
+ }
39
+ }
@@ -0,0 +1,20 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "33a02fd0-02e0-4df6-a309-2880a0196049",
9
+ "stepId": "setup-active-user"
10
+ },
11
+ "data": {
12
+ "capability": {
13
+ "id": "active-user",
14
+ "version": "1"
15
+ },
16
+ "input": {
17
+ "email": "user-001@example.invalid"
18
+ }
19
+ }
20
+ }
@@ -0,0 +1,23 @@
1
+ {
2
+ "protocol": {
3
+ "name": "overseer-testing-control",
4
+ "version": "3"
5
+ },
6
+ "meta": {
7
+ "runId": "run-2026-07-31-001",
8
+ "correlationId": "33a02fd0-02e0-4df6-a309-2880a0196049",
9
+ "stepId": "setup-active-user"
10
+ },
11
+ "data": {
12
+ "capability": {
13
+ "id": "active-user",
14
+ "version": "1"
15
+ },
16
+ "identifiers": {
17
+ "userId": "user-001"
18
+ },
19
+ "output": {
20
+ "status": "active"
21
+ }
22
+ }
23
+ }
@@ -0,0 +1,343 @@
1
+ openapi: 3.1.0
2
+ info:
3
+ title: Overseer testing-control protocol
4
+ version: '3'
5
+ description: |
6
+ Product-agnostic control contract for an isolated target backend.
7
+ Product applications own the advertised state, probe, and sink definitions
8
+ and implement this shared wire contract.
9
+ jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema
10
+ servers:
11
+ - url: '{testingBaseUrl}'
12
+ variables:
13
+ testingBaseUrl:
14
+ default: http://127.0.0.1/testing
15
+ description: Loopback testing-control base URL verified by the test runner.
16
+ security: []
17
+ paths:
18
+ /capabilities:
19
+ get:
20
+ operationId: getTestingControlCapabilities
21
+ summary: Negotiate protocol and product-owned capabilities
22
+ parameters:
23
+ - $ref: '#/components/parameters/ProtocolVersion'
24
+ - $ref: '#/components/parameters/RunId'
25
+ - $ref: '#/components/parameters/CorrelationId'
26
+ - $ref: '#/components/parameters/StepId'
27
+ responses:
28
+ '200':
29
+ description: Compatible capabilities
30
+ headers:
31
+ Overseer-Testing-Control-Version:
32
+ $ref: '#/components/headers/ProtocolVersion'
33
+ Overseer-Correlation-Id:
34
+ $ref: '#/components/headers/CorrelationId'
35
+ content:
36
+ application/json:
37
+ schema:
38
+ $ref: './schema.json#/$defs/capabilitiesResponse'
39
+ '400':
40
+ $ref: '#/components/responses/InvalidRequest'
41
+ '409':
42
+ $ref: '#/components/responses/IncompatibleVersion'
43
+ '500':
44
+ $ref: '#/components/responses/InternalError'
45
+ '503':
46
+ $ref: '#/components/responses/RuntimeUnavailable'
47
+ /reset:
48
+ put:
49
+ operationId: resetTestingControlRuntime
50
+ summary: Reset the isolated runtime and clear fake-sink records
51
+ parameters:
52
+ - $ref: '#/components/parameters/ProtocolVersion'
53
+ - $ref: '#/components/parameters/RunId'
54
+ - $ref: '#/components/parameters/CorrelationId'
55
+ - $ref: '#/components/parameters/StepId'
56
+ - $ref: '#/components/parameters/IdempotencyKey'
57
+ responses:
58
+ '200':
59
+ description: Reset completed
60
+ headers:
61
+ Overseer-Testing-Control-Version:
62
+ $ref: '#/components/headers/ProtocolVersion'
63
+ Overseer-Correlation-Id:
64
+ $ref: '#/components/headers/CorrelationId'
65
+ content:
66
+ application/json:
67
+ schema:
68
+ $ref: './schema.json#/$defs/resetResponse'
69
+ '400':
70
+ $ref: '#/components/responses/InvalidRequest'
71
+ '409':
72
+ description: Protocol is incompatible or the runtime must be restarted
73
+ headers:
74
+ Overseer-Testing-Control-Version:
75
+ $ref: '#/components/headers/ProtocolVersion'
76
+ Overseer-Correlation-Id:
77
+ $ref: '#/components/headers/CorrelationId'
78
+ content:
79
+ application/json:
80
+ schema:
81
+ $ref: './schema.json#/$defs/errorResponse'
82
+ '413':
83
+ $ref: '#/components/responses/PayloadTooLarge'
84
+ '500':
85
+ $ref: '#/components/responses/InternalError'
86
+ '503':
87
+ $ref: '#/components/responses/RuntimeUnavailable'
88
+ /states:
89
+ post:
90
+ operationId: arrangeTestingControlState
91
+ summary: Arrange one allowlisted product-owned precondition
92
+ parameters:
93
+ - $ref: '#/components/parameters/ProtocolVersion'
94
+ - $ref: '#/components/parameters/RunId'
95
+ - $ref: '#/components/parameters/CorrelationId'
96
+ - $ref: '#/components/parameters/StepId'
97
+ - $ref: '#/components/parameters/IdempotencyKey'
98
+ requestBody:
99
+ required: true
100
+ content:
101
+ application/json:
102
+ schema:
103
+ $ref: './schema.json#/$defs/stateRequest'
104
+ responses:
105
+ '200':
106
+ description: State arranged
107
+ headers:
108
+ Overseer-Testing-Control-Version:
109
+ $ref: '#/components/headers/ProtocolVersion'
110
+ Overseer-Correlation-Id:
111
+ $ref: '#/components/headers/CorrelationId'
112
+ content:
113
+ application/json:
114
+ schema:
115
+ $ref: './schema.json#/$defs/stateResponse'
116
+ '400':
117
+ $ref: '#/components/responses/InvalidRequest'
118
+ '404':
119
+ $ref: '#/components/responses/CapabilityNotFound'
120
+ '409':
121
+ $ref: '#/components/responses/IncompatibleVersion'
122
+ '413':
123
+ $ref: '#/components/responses/PayloadTooLarge'
124
+ '422':
125
+ $ref: '#/components/responses/ValidationFailed'
126
+ '500':
127
+ $ref: '#/components/responses/InternalError'
128
+ '503':
129
+ $ref: '#/components/responses/RuntimeUnavailable'
130
+ /probes:
131
+ post:
132
+ operationId: evaluateTestingControlProbe
133
+ summary: Evaluate one allowlisted read-only product-owned projection
134
+ parameters:
135
+ - $ref: '#/components/parameters/ProtocolVersion'
136
+ - $ref: '#/components/parameters/RunId'
137
+ - $ref: '#/components/parameters/CorrelationId'
138
+ - $ref: '#/components/parameters/StepId'
139
+ requestBody:
140
+ required: true
141
+ content:
142
+ application/json:
143
+ schema:
144
+ $ref: './schema.json#/$defs/probeRequest'
145
+ responses:
146
+ '200':
147
+ description: Probe evaluated
148
+ headers:
149
+ Overseer-Testing-Control-Version:
150
+ $ref: '#/components/headers/ProtocolVersion'
151
+ Overseer-Correlation-Id:
152
+ $ref: '#/components/headers/CorrelationId'
153
+ content:
154
+ application/json:
155
+ schema:
156
+ $ref: './schema.json#/$defs/probeResponse'
157
+ '400':
158
+ $ref: '#/components/responses/InvalidRequest'
159
+ '404':
160
+ $ref: '#/components/responses/CapabilityNotFound'
161
+ '409':
162
+ $ref: '#/components/responses/IncompatibleVersion'
163
+ '413':
164
+ $ref: '#/components/responses/PayloadTooLarge'
165
+ '422':
166
+ $ref: '#/components/responses/ValidationFailed'
167
+ '500':
168
+ $ref: '#/components/responses/InternalError'
169
+ '503':
170
+ $ref: '#/components/responses/RuntimeUnavailable'
171
+ /sinks/query:
172
+ post:
173
+ operationId: queryTestingControlSink
174
+ summary: Query bounded sanitized records from one fake sink
175
+ parameters:
176
+ - $ref: '#/components/parameters/ProtocolVersion'
177
+ - $ref: '#/components/parameters/RunId'
178
+ - $ref: '#/components/parameters/CorrelationId'
179
+ - $ref: '#/components/parameters/StepId'
180
+ requestBody:
181
+ required: true
182
+ content:
183
+ application/json:
184
+ schema:
185
+ $ref: './schema.json#/$defs/sinkQueryRequest'
186
+ responses:
187
+ '200':
188
+ description: Matching sanitized effect records
189
+ headers:
190
+ Overseer-Testing-Control-Version:
191
+ $ref: '#/components/headers/ProtocolVersion'
192
+ Overseer-Correlation-Id:
193
+ $ref: '#/components/headers/CorrelationId'
194
+ content:
195
+ application/json:
196
+ schema:
197
+ $ref: './schema.json#/$defs/sinkQueryResponse'
198
+ '400':
199
+ $ref: '#/components/responses/InvalidRequest'
200
+ '404':
201
+ $ref: '#/components/responses/CapabilityNotFound'
202
+ '409':
203
+ $ref: '#/components/responses/IncompatibleVersion'
204
+ '413':
205
+ $ref: '#/components/responses/PayloadTooLarge'
206
+ '422':
207
+ $ref: '#/components/responses/ValidationFailed'
208
+ '500':
209
+ $ref: '#/components/responses/InternalError'
210
+ '503':
211
+ $ref: '#/components/responses/RuntimeUnavailable'
212
+ components:
213
+ schemas:
214
+ Identifier:
215
+ type: string
216
+ minLength: 1
217
+ maxLength: 128
218
+ pattern: ^[A-Za-z0-9][A-Za-z0-9._:-]*$
219
+ parameters:
220
+ ProtocolVersion:
221
+ name: Overseer-Testing-Control-Version
222
+ in: header
223
+ required: true
224
+ description: Exact protocol major version. Version negotiation never silently downgrades.
225
+ schema:
226
+ const: '3'
227
+ RunId:
228
+ name: Overseer-Run-Id
229
+ in: header
230
+ required: true
231
+ description: Opaque identifier for the current Overseer run.
232
+ schema:
233
+ $ref: '#/components/schemas/Identifier'
234
+ CorrelationId:
235
+ name: Overseer-Correlation-Id
236
+ in: header
237
+ required: true
238
+ description: Opaque identifier echoed in the response and propagated to effects.
239
+ schema:
240
+ $ref: '#/components/schemas/Identifier'
241
+ StepId:
242
+ name: Overseer-Step-Id
243
+ in: header
244
+ required: false
245
+ description: Opaque setup or public-step identifier when the call is attributable to one.
246
+ schema:
247
+ $ref: '#/components/schemas/Identifier'
248
+ IdempotencyKey:
249
+ name: Overseer-Idempotency-Key
250
+ in: header
251
+ required: false
252
+ description: Stable key for safely retrying reset or a state that advertises idempotency.
253
+ schema:
254
+ $ref: '#/components/schemas/Identifier'
255
+ headers:
256
+ ProtocolVersion:
257
+ required: true
258
+ description: Exact protocol major version used by the response.
259
+ schema:
260
+ const: '3'
261
+ CorrelationId:
262
+ required: true
263
+ description: Echo of the request correlation identifier.
264
+ schema:
265
+ $ref: '#/components/schemas/Identifier'
266
+ responses:
267
+ InvalidRequest:
268
+ description: Headers or request envelope are malformed
269
+ headers:
270
+ Overseer-Testing-Control-Version:
271
+ $ref: '#/components/headers/ProtocolVersion'
272
+ Overseer-Correlation-Id:
273
+ $ref: '#/components/headers/CorrelationId'
274
+ content:
275
+ application/json:
276
+ schema:
277
+ $ref: './schema.json#/$defs/errorResponse'
278
+ CapabilityNotFound:
279
+ description: The requested capability ID and version are not advertised
280
+ headers:
281
+ Overseer-Testing-Control-Version:
282
+ $ref: '#/components/headers/ProtocolVersion'
283
+ Overseer-Correlation-Id:
284
+ $ref: '#/components/headers/CorrelationId'
285
+ content:
286
+ application/json:
287
+ schema:
288
+ $ref: './schema.json#/$defs/errorResponse'
289
+ IncompatibleVersion:
290
+ description: The requested protocol version is not supported
291
+ headers:
292
+ Overseer-Testing-Control-Version:
293
+ $ref: '#/components/headers/ProtocolVersion'
294
+ Overseer-Correlation-Id:
295
+ $ref: '#/components/headers/CorrelationId'
296
+ content:
297
+ application/json:
298
+ schema:
299
+ $ref: './schema.json#/$defs/errorResponse'
300
+ PayloadTooLarge:
301
+ description: The request exceeds an advertised or server limit
302
+ headers:
303
+ Overseer-Testing-Control-Version:
304
+ $ref: '#/components/headers/ProtocolVersion'
305
+ Overseer-Correlation-Id:
306
+ $ref: '#/components/headers/CorrelationId'
307
+ content:
308
+ application/json:
309
+ schema:
310
+ $ref: './schema.json#/$defs/errorResponse'
311
+ ValidationFailed:
312
+ description: Input violates the advertised product-owned JSON Schema or a limit
313
+ headers:
314
+ Overseer-Testing-Control-Version:
315
+ $ref: '#/components/headers/ProtocolVersion'
316
+ Overseer-Correlation-Id:
317
+ $ref: '#/components/headers/CorrelationId'
318
+ content:
319
+ application/json:
320
+ schema:
321
+ $ref: './schema.json#/$defs/errorResponse'
322
+ RuntimeUnavailable:
323
+ description: The isolated runtime cannot currently service the request
324
+ headers:
325
+ Overseer-Testing-Control-Version:
326
+ $ref: '#/components/headers/ProtocolVersion'
327
+ Overseer-Correlation-Id:
328
+ $ref: '#/components/headers/CorrelationId'
329
+ content:
330
+ application/json:
331
+ schema:
332
+ $ref: './schema.json#/$defs/errorResponse'
333
+ InternalError:
334
+ description: A sanitized internal failure
335
+ headers:
336
+ Overseer-Testing-Control-Version:
337
+ $ref: '#/components/headers/ProtocolVersion'
338
+ Overseer-Correlation-Id:
339
+ $ref: '#/components/headers/CorrelationId'
340
+ content:
341
+ application/json:
342
+ schema:
343
+ $ref: './schema.json#/$defs/errorResponse'