@volter/twin-standard 1.0.149 → 1.0.150

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +12 -3
  2. package/client-sdks/bun.lock +0 -32
  3. package/client-sdks/package.json +0 -2
  4. package/dist/client-sdks/bun.lock +0 -32
  5. package/dist/client-sdks/package.json +0 -2
  6. package/dist/src/assessment.d.ts +5 -4
  7. package/dist/src/assessment.js +2 -1
  8. package/dist/src/cli.js +21 -6
  9. package/dist/src/clients/harness.d.ts +3 -1
  10. package/dist/src/clients/harness.js +4 -2
  11. package/dist/src/clients/jira.d.ts +2 -0
  12. package/dist/src/clients/jira.js +71 -0
  13. package/dist/src/clients/polar.d.ts +2 -0
  14. package/dist/src/clients/polar.js +73 -0
  15. package/dist/src/clients/resend.js +38 -0
  16. package/dist/src/clients/run.js +1 -1
  17. package/dist/src/clients/sentry.js +1 -26
  18. package/dist/src/clients/smtp.d.ts +2 -0
  19. package/dist/src/clients/smtp.js +54 -0
  20. package/dist/src/clients/stripe.js +1 -1
  21. package/dist/src/clients/volteridentity.js +4 -3
  22. package/dist/src/clients/x.d.ts +2 -0
  23. package/dist/src/clients/x.js +64 -0
  24. package/dist/src/conformance.d.ts +4 -1
  25. package/dist/src/conformance.js +256 -6
  26. package/dist/src/consumer.js +5 -1
  27. package/dist/src/create-pack.d.ts +5 -0
  28. package/dist/src/create-pack.js +31 -3
  29. package/dist/src/protocol-3.js +30 -2
  30. package/dist/src/spec-ir.d.ts +2 -0
  31. package/dist/src/spec-ir.js +43 -0
  32. package/dist/src/tools/judge-brief.d.ts +1 -0
  33. package/dist/src/tools/judge-brief.js +67 -0
  34. package/dist/src/tools/life-coverage-walk.d.ts +2 -0
  35. package/dist/src/tools/life-coverage-walk.js +220 -0
  36. package/dist/src/tools/life-coverage.d.ts +2 -0
  37. package/dist/src/tools/life-coverage.js +271 -0
  38. package/dist/src/tools/packs-root.d.ts +9 -0
  39. package/dist/src/tools/packs-root.js +17 -0
  40. package/dist/src/tools/run-tool.d.ts +1 -0
  41. package/dist/src/tools/run-tool.js +18 -0
  42. package/dist/src/tools/score-pack.d.ts +2 -0
  43. package/dist/src/tools/score-pack.js +264 -0
  44. package/dist/src/tools/vendor-examples.d.ts +2 -0
  45. package/dist/src/tools/vendor-examples.js +1493 -0
  46. package/dist/src/verdict.js +1 -1
  47. package/dist/src/walk/behavior-journey.d.ts +6 -0
  48. package/dist/src/walk/behavior-journey.js +8 -6
  49. package/dist/src/walk/journey-kit.d.ts +2 -0
  50. package/dist/src/walk/journey-kit.js +6 -2
  51. package/dist/src/walk/rendered-scalar.d.ts +3 -0
  52. package/dist/src/walk/rendered-scalar.js +23 -0
  53. package/dist/src/walk/sse-example.js +45 -0
  54. package/package.json +2 -2
  55. package/src/assessment.ts +2 -1
  56. package/src/cli.ts +15 -2
  57. package/src/clients/harness.ts +4 -2
  58. package/src/clients/jira.ts +73 -0
  59. package/src/clients/polar.ts +73 -0
  60. package/src/clients/resend.ts +32 -0
  61. package/src/clients/run.ts +2 -2
  62. package/src/clients/sentry.ts +1 -25
  63. package/src/clients/smtp.ts +49 -0
  64. package/src/clients/stripe.ts +1 -1
  65. package/src/clients/volteridentity.ts +4 -3
  66. package/src/clients/x.ts +66 -0
  67. package/src/conformance.ts +192 -8
  68. package/src/consumer.ts +5 -1
  69. package/src/create-pack.ts +27 -3
  70. package/src/protocol-3.ts +24 -2
  71. package/src/spec-ir.ts +31 -0
  72. package/src/tools/judge-brief.ts +63 -0
  73. package/src/tools/life-coverage-walk.ts +173 -0
  74. package/src/tools/life-coverage.ts +237 -0
  75. package/src/tools/packs-root.ts +21 -0
  76. package/src/tools/run-tool.ts +10 -0
  77. package/src/tools/score-pack.ts +244 -0
  78. package/src/tools/vendor-examples.ts +1250 -0
  79. package/src/verdict.ts +1 -1
  80. package/src/walk/behavior-journey.ts +6 -2
  81. package/src/walk/journey-kit.ts +7 -2
  82. package/src/walk/rendered-scalar.ts +17 -0
  83. package/src/walk/sse-example.ts +37 -0
package/src/verdict.ts CHANGED
@@ -7,7 +7,7 @@
7
7
  /** Committed attribution names people or author roles, never execution identities. */
8
8
  export function attributionName(name: string | undefined): string | undefined {
9
9
  if (name === undefined) return undefined;
10
- if (/\bcx-|\bt_[0-9a-f]{8}|wt\/|\/root(?:\/|$|[: )])|\b(?:pane|session)[ -:]|[0-9a-f]{8}-[0-9a-f-]{27}|\b[0-9a-f]{16,64}\b/i.test(name)) throw new Error('attribution must use an author name, not an execution identifier');
10
+ if (/\bcx-|\bt_[0-9a-f]{8}|wt\/|\/root(?:\/|$|[: )])|\b(?:pane|session)[ :-]|[0-9a-f]{8}-[0-9a-f-]{27}|\b[0-9a-f]{16,64}\b/i.test(name)) throw new Error('attribution must use an author name, not an execution identifier');
11
11
  return name;
12
12
  }
13
13
  export type Verdict = { judge?: { agent?: string; name?: string; session?: string; review?: string }; authors?: string[] };
@@ -142,7 +142,7 @@ export type Failure = { step: string; beat: string; detail: string };
142
142
  * reads (an event's delivery, a socket frame), the kernel's own `updatedAt`, which a raw row filtered on `_` alone
143
143
  * carries (Slack's message events once did). The kernel's `deleted` is not read there: vendors send their own
144
144
  * (Stripe's deleted customer's `deleted: true`). */
145
- function leaked(v: unknown, vendorFields: Set<string>, at = '', unshaped = false): string[] {
145
+ export function leaked(v: unknown, vendorFields: Set<string>, at = '', unshaped = false): string[] {
146
146
  if (Array.isArray(v)) return v.flatMap((x, i) => leaked(x, vendorFields, `${at}[${i}]`, unshaped)).slice(0, 5);
147
147
  if (!v || typeof v !== 'object') return [];
148
148
  return Object.entries(v).flatMap(([k, x]) => ((k.startsWith('_') || (unshaped && k === 'updatedAt')) && !vendorFields.has(k) ? [`${at}${at ? '.' : ''}${k}`] : leaked(x, vendorFields, `${at}${at ? '.' : ''}${k}`, unshaped))).slice(0, 5);
@@ -823,7 +823,11 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
823
823
  let res: Answer;
824
824
  if (deferred) {
825
825
  pending.delete(step.await!);
826
- try { res = await deferred.answer; } catch (error) { collect.push({ step: step.name, beat: 'PENDING', detail: String(error) }); continue; }
826
+ const got = await deferred.answer.then((a) => ({ a }), (error: unknown) => ({ error }));
827
+ // a body the vendor broke off, as the collect expects (`breaks`): its words are the answer
828
+ if ('error' in got) { if (step.breaks === undefined || !String(got.error).includes(step.breaks)) collect.push({ step: step.name, beat: 'PENDING', detail: String(got.error) }); continue; }
829
+ // over a listener, which ends a broken stream as a short body, the ordinary checks hold the part that arrived
830
+ res = got.a;
827
831
  } else res = await send({ ...step, body, headers, ...(formed ? { formed } : {}) }, path);
828
832
  answered = { status: res.status, body: res.body, headers: res.headers };
829
833
  if (opts.verbose) console.log(` ${String(res.status).padEnd(4)} ${step.method.padEnd(6)} ${path.slice(0, 70).padEnd(70)} ${step.name}`);
@@ -52,6 +52,8 @@ export type JourneyStep = {
52
52
  defer?: string;
53
53
  /** Collect a previously deferred request once, with all ordinary response checks. */
54
54
  await?: string;
55
+ /** A collect whose answer's body breaks off with these words (a stream the vendor ends in an error), not arriving whole. */
56
+ breaks?: string;
55
57
  body?: unknown;
56
58
  /** A body of raw bytes (a gzip'd package, an image, an audio file), base64 in the journey and sent decoded: a
57
59
  * string body is sent as UTF-8, which no binary format survives. A step carries `body` or `bodyBase64`, never both. */
@@ -215,7 +217,8 @@ export function expectFailures(expect: NonNullable<JourneyStep['expect']>, body:
215
217
  if (e.contains !== undefined) {
216
218
  const want = fill(e.contains, captured);
217
219
  const text = typeof v === 'string' ? v : JSON.stringify(v ?? '');
218
- if (!text.includes(want)) out.push(`${e.path || 'the answer'} does not contain ${JSON.stringify(want)}`);
220
+ // what it holds instead, its first characters, so a failure says what the twin answered
221
+ if (!text.includes(want)) out.push(`${e.path || 'the answer'} does not contain ${JSON.stringify(want)} (it begins ${JSON.stringify(text.replace(/\s+/g, ' ').slice(0, 160))})`);
219
222
  }
220
223
  // `notContains`: the value does not hold the text (a list that omits what the caller may not see)
221
224
  if (e.notContains !== undefined) {
@@ -543,7 +546,9 @@ export function fill(template: string, captured: Captured, graphqlDocument = fal
543
546
  }
544
547
  // a doubled brace is the vendor's own template syntax ({{sample.output_text}} in an OpenAI grader), not a capture; a
545
548
  // capture's name starts with a letter, so a pattern's count (`[A-Za-z0-9]{40}` in an expectation) is no capture
546
- return encoded.replace(/\{\{[^{}]*\}\}|\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (m, key: string | undefined, offset: number) => {
549
+ // `{literal:word}` is `{word}` as the vendor's page prints it (a Redis hash tag, `{queue}`), never a capture
550
+ return encoded.replace(/\{literal:([^{}]*)\}|\{\{[^{}]*\}\}|\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (m, literal: string | undefined, key: string | undefined, offset: number) => {
551
+ if (literal !== undefined) return `{${literal}}`;
547
552
  if (selections.has(offset)) return m;
548
553
  if (key === undefined) return m;
549
554
  const v = captured[key];
@@ -0,0 +1,17 @@
1
+ /** Unfilled scalar type words in schema-rendered examples. Real values and enum words remain literal. */
2
+ const BOOLEAN = Symbol('rendered boolean');
3
+ const INTEGER = Symbol('rendered integer');
4
+ const NUMBER = Symbol('rendered number');
5
+ export function renderedScalar(value: unknown, types: unknown[], enumerated: boolean): symbol | undefined {
6
+ if (enumerated || typeof value !== 'string') return undefined;
7
+ if (value === 'boolean' && types.includes('boolean')) return BOOLEAN;
8
+ if (value === 'integer' && types.includes('integer')) return INTEGER;
9
+ if (value === 'number' && types.includes('number')) return NUMBER;
10
+ return undefined;
11
+ }
12
+ export function isRenderedScalar(value: unknown): boolean { return value === BOOLEAN || value === INTEGER || value === NUMBER; }
13
+ export function matchesRenderedScalar(marker: unknown, value: unknown): boolean {
14
+ if (marker === BOOLEAN) return typeof value === 'boolean';
15
+ if (marker === INTEGER) return typeof value === 'bigint' || (typeof value === 'number' && Number.isInteger(value));
16
+ return marker === NUMBER && typeof value === 'number' && Number.isFinite(value);
17
+ }
@@ -1,6 +1,7 @@
1
1
  import { parseExactJson } from '@volter/world-core/exact-json';
2
2
  /** Match a published SDK-decoded SSE sample to a real HTTP frame; preserve non-SSE answers and mismatches. */
3
3
  export function sseExampleAnswer(example: unknown, answer: unknown, matches: (frame: unknown) => boolean): unknown {
4
+ if (typeof example === 'string' && typeof answer === 'string') return streamSample(example, answer);
4
5
  if (typeof answer !== 'string' || !example || typeof example !== 'object') return answer;
5
6
  const wrapped = 'data' in example;
6
7
  if (!wrapped && !('type' in example) && !('object' in example)) return answer;
@@ -23,3 +24,39 @@ export function sseExampleAnswer(example: unknown, answer: unknown, matches: (fr
23
24
  const candidates = relevant.map((frame) => wrapped ? frame : frame.data);
24
25
  return candidates.find(matches) ?? candidates[0] ?? answer;
25
26
  }
27
+
28
+ type Frame = { data: unknown; event?: string };
29
+ /** A stream's frames: each block's `event:` and joined `data:` lines, or a block that is JSON alone (a page printing the
30
+ * chunks without their framing); `[DONE]`, an elision (`...`, `....`) and anything else not JSON are no frame. */
31
+ function framesOf(text: string): Frame[] {
32
+ const frames: Frame[] = [];
33
+ for (const block of text.replace(/\r\n/g, '\n').split(/\n\s*\n/)) {
34
+ const lines = block.split('\n').map((l) => l.trim()).filter(Boolean);
35
+ const framed = lines.some((l) => l.startsWith('data:'));
36
+ const data = framed ? lines.filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join('\n') : lines.join('\n');
37
+ if (!data || data === '[DONE]' || /^\.{3,}$/.test(data)) continue;
38
+ const event = lines.find((l) => l.startsWith('event:'))?.slice(6).trim();
39
+ try { frames.push({ data: parseExactJson(data), ...(event ? { event } : {}) }); } catch { /* not a JSON frame */ }
40
+ }
41
+ return frames;
42
+ }
43
+ const kindOf = (v: unknown): string => (v === null ? 'null' : Array.isArray(v) ? 'array' : typeof v);
44
+ /** Whether an answer's frame holds a published frame's keys with the same kinds of value (a null published value admits
45
+ * any), its type or object the same. */
46
+ function holds(published: unknown, got: unknown): boolean {
47
+ if (!published || typeof published !== 'object' || Array.isArray(published)) return kindOf(published) === kindOf(got) || published === null;
48
+ if (!got || typeof got !== 'object' || Array.isArray(got)) return false;
49
+ const p = published as Record<string, unknown>, g = got as Record<string, unknown>;
50
+ for (const key of ['type', 'object']) if (typeof p[key] === 'string' && p[key] !== g[key]) return false;
51
+ return Object.entries(p).every(([k, v]) => k in g && (v === null || kindOf(v) === kindOf(g[k]) || (kindOf(v) === 'object' && holds(v, g[k]))));
52
+ }
53
+ /** A published stream printed as text (a page's sample of a stream, its middle elided): each frame it shows has a frame
54
+ * of the answer holding it, the answer then standing for the sample; else the answer, unchanged, which the literal
55
+ * comparison reports. */
56
+ function streamSample(example: string, answer: string): unknown {
57
+ const published = framesOf(example);
58
+ if (published.length < 2) return answer;
59
+ const frames = framesOf(answer);
60
+ const held = published.every((p) => frames.some((f) => (!p.event || p.event === f.event) && holds(p.data, f.data)));
61
+ return held ? example : answer;
62
+ }