@byollm/conformance 0.1.0-alpha.2 → 0.1.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,880 +0,0 @@
1
- // src/checks.ts
2
- import { AUDIENCES, OFFER_SCOPES } from "@byollm/protocol";
3
-
4
- // src/harness.ts
5
- import { mkdtemp, rm } from "fs/promises";
6
- import { tmpdir } from "os";
7
- import { join } from "path";
8
- import {
9
- Allowlist,
10
- Budgets,
11
- IngressLog,
12
- ProtocolClient,
13
- Runner,
14
- connect,
15
- resolveConfig,
16
- DaemonConfig
17
- } from "byollm";
18
- var EchoBackend = class {
19
- id = "openai-http";
20
- class = "http";
21
- /** Prompts this backend was asked to run, in order. */
22
- seen = [];
23
- /** Set to make the next call hang, for lease and cancel checks. */
24
- hangMs = 0;
25
- health() {
26
- return Promise.resolve({ healthy: true, models: ["echo-model"] });
27
- }
28
- async execute(request) {
29
- this.seen.push(request.prompt);
30
- const started = Date.now();
31
- if (this.hangMs > 0) {
32
- const hung = request.signal.aborted ? "aborted" : await new Promise((resolve) => {
33
- const timer = setTimeout(() => {
34
- resolve("done");
35
- }, this.hangMs);
36
- request.signal.addEventListener(
37
- "abort",
38
- () => {
39
- clearTimeout(timer);
40
- resolve("aborted");
41
- },
42
- { once: true }
43
- );
44
- });
45
- if (hung === "aborted") {
46
- return {
47
- ok: false,
48
- code: "canceled",
49
- message: "the job was canceled",
50
- retryable: false,
51
- durationMs: Date.now() - started
52
- };
53
- }
54
- }
55
- return {
56
- ok: true,
57
- text: `echo: ${request.prompt}`,
58
- durationMs: Date.now() - started
59
- };
60
- }
61
- };
62
- function daemonConfig(options) {
63
- const backendId = options.subscription ? "claude-cli" : "openai-http";
64
- return resolveConfig(
65
- DaemonConfig.parse({
66
- backends: {
67
- primary: {
68
- backend: backendId,
69
- ...options.subscription ? {} : { baseUrl: "http://127.0.0.1:11434/v1" },
70
- offer: options.offer
71
- }
72
- },
73
- routes: {
74
- "llm.generate": { backend: "primary", model: "echo-model" },
75
- "llm.chat": { backend: "primary", model: "echo-model" }
76
- },
77
- concurrency: 4
78
- })
79
- );
80
- }
81
- async function pairDaemon(target, options) {
82
- const home = await mkdtemp(join(tmpdir(), "byollm-conformance-"));
83
- const loaded = daemonConfig({
84
- offer: options.offer ?? "self",
85
- subscription: options.subscription ?? false
86
- });
87
- const allowlist = new Allowlist(join(home, "allow.json"));
88
- await allowlist.load();
89
- const budgets = new Budgets(
90
- join(home, "budgets.json"),
91
- loaded.config.community
92
- );
93
- await budgets.load(Date.now());
94
- const ingress = new IngressLog({
95
- path: join(home, "ingress.log"),
96
- communityPromptDays: 7,
97
- keepSelfPrompts: true
98
- });
99
- const backend = new EchoBackend();
100
- const fetchImpl = (input, init) => target.fetch(new Request(input, init));
101
- const capabilities = loaded.routes.map((route) => ({
102
- kind: route.kind,
103
- backendId: route.backendId,
104
- backendClass: route.backendClass,
105
- model: route.model,
106
- offerScope: route.offerScope
107
- }));
108
- const pairingClient = new ProtocolClient({
109
- origin: target.origin,
110
- fetch: fetchImpl
111
- });
112
- let userCode = "";
113
- const pairingAbort = new AbortController();
114
- let pairingError;
115
- const pairing = connect({
116
- client: pairingClient,
117
- daemonVersion: "conformance",
118
- label: options.label ?? `daemon-${options.owner}`,
119
- capabilities,
120
- onCode: (info) => {
121
- userCode = info.userCode;
122
- },
123
- // A real macrotask, not `Promise.resolve()`: a zero-delay microtask loop
124
- // never yields to the event loop, so the approval below could never run
125
- // and the poll would spin until the process died.
126
- sleep: () => sleep(1),
127
- signal: pairingAbort.signal
128
- }).catch((error) => {
129
- pairingError = error;
130
- return { ok: false, reason: "aborted", message: "" };
131
- });
132
- try {
133
- await waitFor(() => userCode !== "", { what: "a pairing code" });
134
- await target.approvePairing(userCode, options.owner);
135
- } catch (error) {
136
- pairingAbort.abort();
137
- await pairing;
138
- await rm(home, { recursive: true, force: true });
139
- throw error;
140
- }
141
- const result = await pairing;
142
- if (!result.ok) {
143
- pairingAbort.abort();
144
- await rm(home, { recursive: true, force: true });
145
- throw new Error(
146
- `conformance harness could not pair: ${pairingError instanceof Error ? pairingError.message : result.message}`
147
- );
148
- }
149
- const runner = new Runner({
150
- client: new ProtocolClient({
151
- origin: target.origin,
152
- token: result.pairing.token,
153
- fetch: fetchImpl
154
- }),
155
- runnerId: result.pairing.runnerId,
156
- owner: result.pairing.owner,
157
- daemonVersion: "conformance",
158
- loaded,
159
- allowlist,
160
- budgets,
161
- ingress,
162
- backendFactory: () => backend
163
- });
164
- return {
165
- runner,
166
- backend,
167
- allowlist,
168
- runnerId: result.pairing.runnerId,
169
- owner: result.pairing.owner,
170
- home,
171
- ingress,
172
- dispose: async () => {
173
- runner.cancelAll();
174
- await waitFor(() => runner.status().activeJobs === 0, {
175
- timeoutMs: 2e3,
176
- what: "in-flight jobs to unwind"
177
- }).catch(() => void 0);
178
- await rm(home, { recursive: true, force: true });
179
- },
180
- abandon: async () => {
181
- await rm(home, { recursive: true, force: true });
182
- }
183
- };
184
- }
185
- async function ownerIdFor(target, name) {
186
- return target.ownerId ? target.ownerId(name) : name;
187
- }
188
- async function waitFor(predicate, options = {}) {
189
- const timeoutMs = options.timeoutMs ?? 5e3;
190
- const intervalMs = options.intervalMs ?? 10;
191
- const deadline = Date.now() + timeoutMs;
192
- for (; ; ) {
193
- if (await predicate()) return;
194
- if (Date.now() >= deadline) {
195
- throw new Error(
196
- `timed out after ${String(timeoutMs)}ms waiting for ${options.what ?? "a condition"}`
197
- );
198
- }
199
- await sleep(intervalMs);
200
- }
201
- }
202
- function sleep(ms) {
203
- return new Promise((resolve) => setTimeout(resolve, ms));
204
- }
205
- async function advance(target, ms) {
206
- if (target.advanceTime) {
207
- await target.advanceTime(ms);
208
- } else {
209
- await sleep(ms);
210
- }
211
- await target.sweep();
212
- }
213
-
214
- // src/checks.ts
215
- function assert(condition, message) {
216
- if (!condition) throw new Error(message);
217
- }
218
- var prompt = (text = "hello") => ({ prompt: text });
219
- var CHECKS = [
220
- {
221
- id: "C001_PAIRING_BINDS_ONE_USER",
222
- title: "a runner token is bound to exactly the approving user",
223
- musts: ["PAIR_ONE_USER", "PAIR_INTERACTIVE"],
224
- async run(target) {
225
- const alice = await pairDaemon(target, { owner: "alice" });
226
- try {
227
- assert(
228
- alice.owner === await ownerIdFor(target, "alice"),
229
- `runner was bound to "${alice.owner}", not to the approving user`
230
- );
231
- const bob = await pairDaemon(target, { owner: "bob" });
232
- assert(
233
- bob.owner !== alice.owner,
234
- "two different approvers produced the same runner owner"
235
- );
236
- try {
237
- const job = await target.enqueue({
238
- kind: "llm.generate",
239
- payload: prompt("alice's private prompt"),
240
- owner: "alice",
241
- audience: "self"
242
- });
243
- await bob.runner.tick();
244
- await sleep(50);
245
- const state = await target.job(job.id);
246
- assert(
247
- state?.state === "queued",
248
- `another user's daemon took a self job (state: ${String(state?.state)})`
249
- );
250
- } finally {
251
- await bob.dispose();
252
- }
253
- } finally {
254
- await alice.dispose();
255
- }
256
- }
257
- },
258
- {
259
- id: "C002_JOB_ROUND_TRIP",
260
- title: "an enqueued job runs on the owner's daemon and the result comes back",
261
- musts: ["CLAIM_REQUIRES_CAPABILITY", "RESULT_IDEMPOTENT"],
262
- async run(target) {
263
- const daemon = await pairDaemon(target, { owner: "alice" });
264
- try {
265
- const job = await target.enqueue({
266
- kind: "llm.generate",
267
- payload: prompt("summarise this"),
268
- owner: "alice"
269
- });
270
- await daemon.runner.tick();
271
- await waitFor(async () => (await target.job(job.id))?.state === "ok", {
272
- what: "the job to complete"
273
- });
274
- const finished = await target.job(job.id);
275
- assert(
276
- finished?.outcome?.text === "echo: summarise this",
277
- "the result text did not survive the round trip"
278
- );
279
- assert(
280
- daemon.backend.seen[0] === "summarise this",
281
- "the prompt did not reach the model verbatim"
282
- );
283
- } finally {
284
- await daemon.dispose();
285
- }
286
- }
287
- },
288
- {
289
- id: "C003_UNKNOWN_KIND_REFUSED",
290
- title: "a daemon is never handed a kind it did not advertise",
291
- musts: ["KIND_TYPED_ONLY", "CLAIM_REQUIRES_CAPABILITY"],
292
- async run(target) {
293
- const daemon = await pairDaemon(target, { owner: "alice" });
294
- try {
295
- const job = await target.enqueue({
296
- kind: "llm.chat",
297
- payload: { messages: [{ role: "user", content: "hi" }] },
298
- owner: "alice"
299
- });
300
- const before = daemon.backend.seen.length;
301
- await daemon.runner.tick();
302
- await sleep(50);
303
- const state = await target.job(job.id);
304
- assert(
305
- state?.state === "ok" || state?.state === "running" || state?.state === "claimed",
306
- `a job for an advertised kind was not taken (state: ${String(state?.state)})`
307
- );
308
- assert(
309
- daemon.backend.seen.length > before,
310
- "the advertised kind never reached the backend"
311
- );
312
- } finally {
313
- await daemon.dispose();
314
- }
315
- }
316
- },
317
- {
318
- id: "C004_LEASE_RECLAIM",
319
- title: "a job whose runner vanished is offered again, losing nothing",
320
- musts: ["LEASE_RECLAIMABLE", "LEASE_HONORED"],
321
- async run(target) {
322
- const dead = await pairDaemon(target, { owner: "alice", label: "dead" });
323
- const job = await target.enqueue({
324
- kind: "llm.generate",
325
- payload: prompt("work"),
326
- owner: "alice"
327
- });
328
- dead.backend.hangMs = 6e4;
329
- await dead.runner.tick();
330
- await waitFor(
331
- async () => {
332
- const state = await target.job(job.id);
333
- return state?.state === "claimed" || state?.state === "running";
334
- },
335
- { what: "the job to be claimed" }
336
- );
337
- await dead.abandon();
338
- await advance(target, target.leaseMs + 500);
339
- const alive = await pairDaemon(target, {
340
- owner: "alice",
341
- label: "alive"
342
- });
343
- try {
344
- await alive.runner.tick();
345
- await waitFor(async () => (await target.job(job.id))?.state === "ok", {
346
- what: "the reclaimed job to complete"
347
- });
348
- } finally {
349
- await alive.dispose();
350
- }
351
- }
352
- },
353
- {
354
- id: "C005_AUDIENCE_MATRIX",
355
- title: "all nine audience \xD7 offer-scope combinations behave as specified",
356
- musts: ["AUDIENCE_BOTH_SIDES", "NAMED_LOCAL_ALLOWLIST"],
357
- async run(target) {
358
- const expected = {
359
- "self:self": false,
360
- "self:named": false,
361
- "self:public": false,
362
- "named:self": false,
363
- "named:named": false,
364
- // refused locally — allowlist is empty
365
- "named:public": true,
366
- "public:self": false,
367
- "public:named": false,
368
- // refused locally — allowlist is empty
369
- "public:public": true
370
- };
371
- for (const audience of AUDIENCES) {
372
- for (const offer of OFFER_SCOPES) {
373
- await target.reset();
374
- const bob = await pairDaemon(target, { owner: "bob", offer });
375
- try {
376
- const job = await target.enqueue({
377
- kind: "llm.generate",
378
- payload: prompt("community work"),
379
- owner: "alice",
380
- audience
381
- });
382
- await bob.runner.tick();
383
- await sleep(80);
384
- const state = await target.job(job.id);
385
- const ran = state?.state === "ok";
386
- const shouldRun = expected[`${audience}:${offer}`] ?? false;
387
- assert(
388
- ran === shouldRun,
389
- `audience=${audience} offer=${offer}: expected ${shouldRun ? "to run" : "to be refused"}, got state "${String(state?.state)}"`
390
- );
391
- } finally {
392
- await bob.dispose();
393
- }
394
- }
395
- }
396
- }
397
- },
398
- {
399
- id: "C006_NAMED_LOCAL_ALLOWLIST",
400
- title: "a named job runs only once the daemon's own allowlist admits it",
401
- musts: ["NAMED_LOCAL_ALLOWLIST", "REFUSAL_NOT_REOFFERED"],
402
- async run(target) {
403
- const bob = await pairDaemon(target, { owner: "bob", offer: "named" });
404
- try {
405
- const refused = await target.enqueue({
406
- kind: "llm.generate",
407
- payload: prompt("before"),
408
- owner: "alice",
409
- audience: "named"
410
- });
411
- await bob.runner.tick();
412
- await sleep(80);
413
- assert(
414
- (await target.job(refused.id))?.state !== "ok",
415
- "a named job ran without the daemon's local allowlist admitting it"
416
- );
417
- const before = bob.backend.seen.length;
418
- await bob.runner.tick();
419
- await sleep(50);
420
- assert(
421
- bob.backend.seen.length === before,
422
- "a refused job was re-offered to the runner that refused it"
423
- );
424
- await bob.allowlist.add(
425
- { origin: target.origin, owner: await ownerIdFor(target, "alice") },
426
- Date.now()
427
- );
428
- const allowed = await target.enqueue({
429
- kind: "llm.generate",
430
- payload: prompt("after"),
431
- owner: "alice",
432
- audience: "named"
433
- });
434
- await bob.runner.tick();
435
- await waitFor(
436
- async () => (await target.job(allowed.id))?.state === "ok",
437
- { what: "the allowed named job to run" }
438
- );
439
- } finally {
440
- await bob.dispose();
441
- }
442
- }
443
- },
444
- {
445
- id: "C007_SUBSCRIPTION_SELF_LOCK",
446
- title: "a subscription backend refuses another user's work at any configured scope",
447
- musts: ["SUBSCRIPTION_SELF_LOCK"],
448
- async run(target) {
449
- const bob = await pairDaemon(target, {
450
- owner: "bob",
451
- offer: "public",
452
- subscription: true
453
- });
454
- try {
455
- await bob.allowlist.add(
456
- { origin: target.origin, owner: await ownerIdFor(target, "alice") },
457
- Date.now()
458
- );
459
- const job = await target.enqueue({
460
- kind: "llm.generate",
461
- payload: prompt("someone else's work"),
462
- owner: "alice",
463
- audience: "public"
464
- });
465
- await bob.runner.tick();
466
- await sleep(80);
467
- const state = await target.job(job.id);
468
- assert(
469
- state?.state !== "ok",
470
- "a subscription backend ran another user's job"
471
- );
472
- assert(
473
- bob.backend.seen.length === 0,
474
- "another user's prompt reached a subscription backend"
475
- );
476
- const own = await target.enqueue({
477
- kind: "llm.generate",
478
- payload: prompt("my own work"),
479
- owner: "bob",
480
- audience: "self"
481
- });
482
- await bob.runner.tick();
483
- await waitFor(async () => (await target.job(own.id))?.state === "ok", {
484
- what: "the owner's own subscription job to run"
485
- });
486
- } finally {
487
- await bob.dispose();
488
- }
489
- }
490
- },
491
- {
492
- id: "C008_REVOCATION",
493
- title: "a revoked daemon stops mid-queue",
494
- musts: ["REVOCATION_HONORED"],
495
- async run(target) {
496
- const daemon = await pairDaemon(target, { owner: "alice" });
497
- try {
498
- await target.revokeRunner(daemon.runnerId);
499
- const job = await target.enqueue({
500
- kind: "llm.generate",
501
- payload: prompt("after revocation"),
502
- owner: "alice"
503
- });
504
- await daemon.runner.tick();
505
- await sleep(80);
506
- assert(
507
- daemon.runner.status().revoked,
508
- "the daemon did not learn it was revoked"
509
- );
510
- assert(
511
- (await target.job(job.id))?.state === "queued",
512
- "a revoked daemon took new work"
513
- );
514
- } finally {
515
- await daemon.dispose();
516
- }
517
- }
518
- },
519
- {
520
- id: "C009_CANCEL_MID_FLIGHT",
521
- title: "cancel aborts a running job's backend call",
522
- musts: ["CANCEL_HONORED"],
523
- async run(target) {
524
- const daemon = await pairDaemon(target, { owner: "alice" });
525
- try {
526
- daemon.backend.hangMs = 3e4;
527
- const job = await target.enqueue({
528
- kind: "llm.generate",
529
- payload: prompt("long job"),
530
- owner: "alice"
531
- });
532
- await daemon.runner.tick();
533
- await waitFor(() => daemon.backend.seen.length > 0, {
534
- what: "the job to start running"
535
- });
536
- await target.cancelJob(job.id);
537
- await daemon.runner.tick();
538
- await waitFor(
539
- async () => (await target.job(job.id))?.state === "canceled",
540
- { what: "the job to report canceled", timeoutMs: 1e4 }
541
- );
542
- } finally {
543
- await daemon.dispose();
544
- }
545
- }
546
- },
547
- {
548
- id: "C010_RESULT_IDEMPOTENT",
549
- title: "the first terminal outcome wins",
550
- musts: ["RESULT_IDEMPOTENT"],
551
- async run(target) {
552
- const daemon = await pairDaemon(target, { owner: "alice" });
553
- try {
554
- const job = await target.enqueue({
555
- kind: "llm.generate",
556
- payload: prompt("once"),
557
- owner: "alice"
558
- });
559
- await daemon.runner.tick();
560
- await waitFor(async () => (await target.job(job.id))?.state === "ok", {
561
- what: "the job to complete"
562
- });
563
- const first = await target.job(job.id);
564
- const response = await target.fetch(
565
- new Request(`${target.origin}/byollm/result`, {
566
- method: "POST",
567
- headers: { "content-type": "application/json" },
568
- body: JSON.stringify({
569
- protocolVersion: "0",
570
- runnerId: daemon.runnerId,
571
- jobId: job.id,
572
- outcome: { outcome: "ok", text: "SECOND ANSWER" },
573
- model: "echo-model",
574
- backendClass: "http",
575
- durationMs: 1
576
- })
577
- })
578
- );
579
- void response;
580
- const after = await target.job(job.id);
581
- assert(
582
- after?.outcome?.text === first?.outcome?.text,
583
- "a second result overwrote the first"
584
- );
585
- } finally {
586
- await daemon.dispose();
587
- }
588
- }
589
- },
590
- {
591
- id: "C011_DEPENDENCY_ORDER",
592
- title: "a dependent job waits for its dependency, across two daemons",
593
- musts: ["DEPENDS_ON_GATING", "TTL_EXPIRY"],
594
- async run(target) {
595
- const alice = await pairDaemon(target, { owner: "alice" });
596
- const bob = await pairDaemon(target, { owner: "bob" });
597
- try {
598
- const first = await target.enqueue({
599
- kind: "llm.generate",
600
- payload: prompt("step one"),
601
- owner: "bob",
602
- audience: "self"
603
- });
604
- const second = await target.enqueue({
605
- kind: "llm.generate",
606
- payload: prompt("step two"),
607
- owner: "alice",
608
- audience: "self",
609
- dependsOn: [first.id]
610
- });
611
- await alice.runner.tick();
612
- await sleep(80);
613
- assert(
614
- alice.backend.seen.length === 0,
615
- "a dependent job ran before its dependency completed"
616
- );
617
- assert(
618
- (await target.job(second.id))?.state === "queued",
619
- "a dependent job left the queue early"
620
- );
621
- await bob.runner.tick();
622
- await waitFor(
623
- async () => (await target.job(first.id))?.state === "ok",
624
- { what: "the dependency to complete" }
625
- );
626
- await alice.runner.tick();
627
- await waitFor(
628
- async () => (await target.job(second.id))?.state === "ok",
629
- { what: "the dependent job to complete" }
630
- );
631
- } finally {
632
- await alice.dispose();
633
- await bob.dispose();
634
- }
635
- }
636
- },
637
- {
638
- id: "C012_TTL_AND_NO_RUNNER",
639
- title: "an unclaimed job expires and no-runner is surfaced, but not while blocked",
640
- musts: ["TTL_EXPIRY", "NO_RUNNER_SIGNAL"],
641
- async run(target) {
642
- const availability = await target.runnerAvailability({
643
- kind: "llm.generate",
644
- owner: "alice"
645
- });
646
- assert(
647
- !availability.available,
648
- "no-runner was not surfaced with nothing paired"
649
- );
650
- const job = await target.enqueue({
651
- kind: "llm.generate",
652
- payload: prompt("nobody will run this"),
653
- owner: "alice",
654
- ttlMs: target.ttlMs
655
- });
656
- await advance(target, target.ttlMs + 500);
657
- const state = await target.job(job.id);
658
- assert(
659
- state?.state === "expired",
660
- `an unclaimed job past its TTL was "${String(state?.state)}", not expired`
661
- );
662
- }
663
- },
664
- {
665
- id: "C013_TTL_CLOCK_STARTS_WHEN_CLAIMABLE",
666
- title: "a dependent job's TTL starts when it becomes claimable, not at enqueue",
667
- musts: ["TTL_EXPIRY"],
668
- async run(target) {
669
- const daemon = await pairDaemon(target, { owner: "alice" });
670
- try {
671
- daemon.backend.hangMs = target.ttlMs * 2;
672
- const first = await target.enqueue({
673
- kind: "llm.generate",
674
- payload: prompt("slow step"),
675
- owner: "alice"
676
- });
677
- const second = await target.enqueue({
678
- kind: "llm.generate",
679
- payload: prompt("waiting step"),
680
- owner: "alice",
681
- dependsOn: [first.id],
682
- ttlMs: target.ttlMs
683
- });
684
- await daemon.runner.tick();
685
- await advance(target, target.ttlMs + 200);
686
- const blocked = await target.job(second.id);
687
- assert(
688
- blocked?.state === "queued",
689
- `a blocked job expired while waiting on its dependency (state: ${String(blocked?.state)}) \u2014 the TTL clock started too early`
690
- );
691
- } finally {
692
- await daemon.dispose();
693
- }
694
- }
695
- },
696
- {
697
- id: "C014_RESULT_PROVENANCE",
698
- title: "a community result arrives marked untrusted, a self result does not",
699
- musts: ["RESULT_PROVENANCE"],
700
- async run(target) {
701
- const bob = await pairDaemon(target, { owner: "bob", offer: "public" });
702
- try {
703
- const community = await target.enqueue({
704
- kind: "llm.generate",
705
- payload: prompt("run this anywhere"),
706
- owner: "alice",
707
- audience: "public"
708
- });
709
- await bob.runner.tick();
710
- await waitFor(
711
- async () => (await target.job(community.id))?.state === "ok",
712
- { what: "the community job to complete" }
713
- );
714
- const delivered = await target.job(community.id);
715
- assert(
716
- delivered?.provenance?.untrusted === true,
717
- "a public result was not marked untrusted"
718
- );
719
- assert(
720
- delivered.provenance.runnerOwner === "bob",
721
- "the result did not carry the runner's owner"
722
- );
723
- const own = await target.enqueue({
724
- kind: "llm.generate",
725
- payload: prompt("my own"),
726
- owner: "bob",
727
- audience: "self"
728
- });
729
- await bob.runner.tick();
730
- await waitFor(async () => (await target.job(own.id))?.state === "ok", {
731
- what: "the self job to complete"
732
- });
733
- assert(
734
- (await target.job(own.id))?.provenance?.untrusted === false,
735
- "a self result was marked untrusted"
736
- );
737
- } finally {
738
- await bob.dispose();
739
- }
740
- }
741
- },
742
- {
743
- id: "C015_INGRESS_BEFORE_EXECUTION",
744
- title: "every executed prompt is in the ingress log",
745
- musts: ["INGRESS_LOGGED_BEFORE_EXECUTION"],
746
- async run(target) {
747
- const daemon = await pairDaemon(target, { owner: "alice" });
748
- try {
749
- const job = await target.enqueue({
750
- kind: "llm.generate",
751
- payload: prompt("logged prompt"),
752
- owner: "alice"
753
- });
754
- await daemon.runner.tick();
755
- await waitFor(async () => (await target.job(job.id))?.state === "ok", {
756
- what: "the job to complete"
757
- });
758
- const entries = await daemon.ingress.read();
759
- const logged = entries.find(
760
- (entry) => entry.type === "prompt" && entry.jobId === job.id
761
- );
762
- assert(
763
- logged !== void 0,
764
- "the executed prompt is not in the ingress log"
765
- );
766
- assert(
767
- logged.type === "prompt" && logged.prompt === "logged prompt",
768
- "the ingress log did not record the prompt text"
769
- );
770
- } finally {
771
- await daemon.dispose();
772
- }
773
- }
774
- },
775
- {
776
- id: "C016_UNAUTHENTICATED_REFUSED",
777
- title: "the protocol endpoints refuse an unknown token",
778
- musts: ["PAIR_ONE_USER"],
779
- async run(target) {
780
- for (const endpoint of ["claim", "heartbeat", "result", "release"]) {
781
- const response = await target.fetch(
782
- new Request(`${target.origin}/byollm/${endpoint}`, {
783
- method: "POST",
784
- headers: {
785
- "content-type": "application/json",
786
- authorization: "Bearer definitely-not-a-real-token"
787
- },
788
- body: JSON.stringify({ protocolVersion: "0" })
789
- })
790
- );
791
- assert(
792
- response.status === 401,
793
- `${endpoint} answered ${String(response.status)} to an unknown token, not 401`
794
- );
795
- }
796
- }
797
- }
798
- ];
799
-
800
- // src/certify.ts
801
- import { MUSTS, MUST_IDS } from "@byollm/protocol";
802
- async function certify(target, options = {}) {
803
- const only = options.only;
804
- const selected = only ? CHECKS.filter((check) => only.includes(check.id)) : CHECKS;
805
- const results = [];
806
- for (const check of selected) {
807
- await target.reset();
808
- const started = Date.now();
809
- try {
810
- await check.run(target);
811
- const result = {
812
- check,
813
- passed: true,
814
- durationMs: Date.now() - started
815
- };
816
- results.push(result);
817
- options.onProgress?.(result);
818
- } catch (error) {
819
- const result = {
820
- check,
821
- passed: false,
822
- durationMs: Date.now() - started,
823
- error: error instanceof Error ? error.message : String(error)
824
- };
825
- results.push(result);
826
- options.onProgress?.(result);
827
- }
828
- }
829
- return {
830
- target: target.name,
831
- passed: results.every((result) => result.passed),
832
- results,
833
- uncoveredMusts: uncoveredMusts(selected)
834
- };
835
- }
836
- function uncoveredMusts(checks = CHECKS) {
837
- const covered = new Set(checks.flatMap((check) => check.musts));
838
- return MUST_IDS.filter((id) => !covered.has(id));
839
- }
840
- function formatReport(report) {
841
- const lines = [];
842
- lines.push(`byollm conformance \u2014 ${report.target}`);
843
- lines.push("");
844
- for (const result of report.results) {
845
- lines.push(
846
- ` ${result.passed ? "\u2713" : "\u2717"} ${result.check.id} ${result.check.title} (${String(result.durationMs)}ms)`
847
- );
848
- if (!result.passed && result.error !== void 0) {
849
- lines.push(` ${result.error}`);
850
- }
851
- }
852
- const failed = report.results.filter((result) => !result.passed).length;
853
- lines.push("");
854
- lines.push(
855
- report.passed ? ` ${String(report.results.length)} checks passed \u2014 ${report.target} is byollm-compatible.` : ` ${String(failed)} of ${String(report.results.length)} checks failed \u2014 not compatible.`
856
- );
857
- if (report.uncoveredMusts.length > 0) {
858
- lines.push("");
859
- lines.push(" MUSTs with no check yet (the kit is honest about its gaps):");
860
- for (const id of report.uncoveredMusts) {
861
- lines.push(` - ${id}: ${MUSTS[id].statement}`);
862
- }
863
- }
864
- return `${lines.join("\n")}
865
- `;
866
- }
867
-
868
- export {
869
- EchoBackend,
870
- pairDaemon,
871
- ownerIdFor,
872
- waitFor,
873
- sleep,
874
- advance,
875
- CHECKS,
876
- certify,
877
- uncoveredMusts,
878
- formatReport
879
- };
880
- //# sourceMappingURL=chunk-PDQJJ3Q2.js.map