@openwop/openwop-conformance 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.md +2 -2
- package/api/asyncapi.yaml +8 -3
- package/api/openapi.yaml +305 -0
- package/coverage.md +35 -10
- package/fixtures/conformance-phase4-nondet-tool.json +53 -0
- package/fixtures/conformance-phase4-replay-divergence.json +40 -0
- package/fixtures.md +5 -3
- package/package.json +1 -1
- package/schemas/README.md +2 -0
- package/schemas/capabilities.schema.json +176 -3
- package/schemas/credential-reference.schema.json +21 -0
- package/schemas/node-pack-manifest.schema.json +112 -1
- package/schemas/run-diff-response.schema.json +64 -0
- package/schemas/run-event-payloads.schema.json +104 -2
- package/schemas/run-event.schema.json +8 -1
- package/schemas/run-snapshot.schema.json +11 -0
- package/src/lib/behavior-gate.ts +51 -0
- package/src/lib/driver.ts +13 -1
- package/src/lib/saml-idp.ts +179 -0
- package/src/scenarios/approval-gate-events.test.ts +61 -0
- package/src/scenarios/approval-gate-flow.test.ts +68 -0
- package/src/scenarios/auth-saml-profile.test.ts +119 -0
- package/src/scenarios/auth-scim-profile.test.ts +65 -0
- package/src/scenarios/authorization-fail-closed.test.ts +80 -0
- package/src/scenarios/authorization-roles-shape.test.ts +83 -0
- package/src/scenarios/connector-manifest-validity.test.ts +142 -0
- package/src/scenarios/credential-payload-redaction.test.ts +93 -0
- package/src/scenarios/credentials-capability-shape.test.ts +90 -0
- package/src/scenarios/cross-engine-append-behavior.test.ts +204 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +13 -6
- package/src/scenarios/cross-workspace-isolation.test.ts +72 -0
- package/src/scenarios/deadletter-capability-shape.test.ts +59 -0
- package/src/scenarios/deadletter-retry-exhaustion.test.ts +62 -0
- package/src/scenarios/experimental-tier-shape.test.ts +192 -0
- package/src/scenarios/identity-owner-shape.test.ts +64 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +59 -21
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +87 -12
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +203 -0
- package/src/scenarios/oauth-capability-shape.test.ts +97 -0
- package/src/scenarios/oauth-connector-redaction.test.ts +91 -0
- package/src/scenarios/pack-registry-isolation.test.ts +108 -0
- package/src/scenarios/pack-registry-publish.test.ts +1 -1
- package/src/scenarios/prompt-mutation-workspace-membership-enforced.test.ts +126 -0
- package/src/scenarios/prompt-read-workspace-membership-enforced.test.ts +183 -0
- package/src/scenarios/replay-divergence-at-refusal.test.ts +187 -7
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +20 -6
- package/src/scenarios/run-diff.test.ts +143 -0
- package/src/scenarios/sandbox-capability-gate-respected.test.ts +15 -13
- package/src/scenarios/sandbox-memory-cap.test.ts +7 -8
- package/src/scenarios/sandbox-mvp-behavior.test.ts +280 -0
- package/src/scenarios/sandbox-no-cross-pack-mutation.test.ts +14 -13
- package/src/scenarios/sandbox-no-host-env-leak.test.ts +14 -21
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +20 -15
- package/src/scenarios/sandbox-no-host-process-escape.test.ts +18 -13
- package/src/scenarios/sandbox-no-network-escape.test.ts +14 -31
- package/src/scenarios/sandbox-timeout-cap.test.ts +7 -8
- package/src/scenarios/scheduling-capability-shape.test.ts +81 -0
- package/src/scenarios/scheduling-cron-fires-once.test.ts +66 -0
- package/src/scenarios/secret-leakage-otel-attribute.test.ts +241 -0
- package/src/scenarios/spec-corpus-validity.test.ts +2 -2
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* connector-manifest-validity — RFC 0045 §A/§B verification.
|
|
3
|
+
*
|
|
4
|
+
* Status: DRAFT. RFC 0045 (connector pack manifest & action model) is `Draft`.
|
|
5
|
+
* The optional `connector` block + `Connector` / `ConnectorAuth` $defs have
|
|
6
|
+
* landed in `schemas/node-pack-manifest.schema.json`.
|
|
7
|
+
*
|
|
8
|
+
* Server-free schema + semantic validation. Two contracts:
|
|
9
|
+
* 1. Schema validity (§A): a well-formed `connector` block validates; a
|
|
10
|
+
* block missing `id`/`displayName` or an action missing `typeId` is
|
|
11
|
+
* rejected. Both ConnectorAuth variants (oauth2 / credential) validate.
|
|
12
|
+
* 2. Action resolution (§B): every `connector.actions[].typeId` and
|
|
13
|
+
* `connector.triggers[]` entry MUST resolve to a `nodes[].typeId` in the
|
|
14
|
+
* same manifest; an unresolved reference is `connector_action_unresolved`.
|
|
15
|
+
*
|
|
16
|
+
* The Connector subschema is extracted self-contained (Connector + its
|
|
17
|
+
* ConnectorAuth + NodeAuth $defs) so ajv compiles it without resolving the
|
|
18
|
+
* parent manifest's external `agent-manifest.schema.json` $ref.
|
|
19
|
+
*
|
|
20
|
+
* @see RFCS/0045-connector-pack-manifest-action-model.md
|
|
21
|
+
* @see schemas/node-pack-manifest.schema.json §Connector
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { describe, it, expect } from 'vitest';
|
|
25
|
+
import { readFileSync } from 'node:fs';
|
|
26
|
+
import { join } from 'node:path';
|
|
27
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
28
|
+
import addFormats from 'ajv-formats';
|
|
29
|
+
import { SCHEMAS_DIR } from '../lib/paths.js';
|
|
30
|
+
|
|
31
|
+
interface SchemaDefs {
|
|
32
|
+
$defs: Record<string, unknown>;
|
|
33
|
+
$schema: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const manifestSchema = JSON.parse(
|
|
37
|
+
readFileSync(join(SCHEMAS_DIR, 'node-pack-manifest.schema.json'), 'utf8'),
|
|
38
|
+
) as SchemaDefs;
|
|
39
|
+
|
|
40
|
+
// Self-contained Connector schema: root = Connector, carrying the $defs it
|
|
41
|
+
// references (ConnectorAuth → NodeAuth). No external $ref resolution needed.
|
|
42
|
+
const connectorSchema = {
|
|
43
|
+
$schema: manifestSchema.$schema,
|
|
44
|
+
...(manifestSchema.$defs.Connector as Record<string, unknown>),
|
|
45
|
+
$defs: {
|
|
46
|
+
ConnectorAuth: manifestSchema.$defs.ConnectorAuth,
|
|
47
|
+
NodeAuth: manifestSchema.$defs.NodeAuth,
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
// §B action/trigger resolution — the semantic check the registry applies
|
|
52
|
+
// beyond pure JSON Schema (typeIds must resolve to real nodes).
|
|
53
|
+
interface ConnectorAction {
|
|
54
|
+
typeId: string;
|
|
55
|
+
}
|
|
56
|
+
interface ConnectorBlock {
|
|
57
|
+
actions?: ConnectorAction[];
|
|
58
|
+
triggers?: string[];
|
|
59
|
+
}
|
|
60
|
+
function unresolvedReferences(nodeTypeIds: string[], connector: ConnectorBlock): string[] {
|
|
61
|
+
const known = new Set(nodeTypeIds);
|
|
62
|
+
const refs = [
|
|
63
|
+
...(connector.actions ?? []).map((a) => a.typeId),
|
|
64
|
+
...(connector.triggers ?? []),
|
|
65
|
+
];
|
|
66
|
+
return refs.filter((t) => !known.has(t));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
describe('category: connector-manifest validity — schema shape (RFC 0045 §A)', () => {
|
|
70
|
+
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
71
|
+
addFormats(ajv);
|
|
72
|
+
const validate = ajv.compile(connectorSchema);
|
|
73
|
+
|
|
74
|
+
it('positive: a well-formed connector block (oauth2 auth) validates', () => {
|
|
75
|
+
const ok = validate({
|
|
76
|
+
id: 'salesforce',
|
|
77
|
+
displayName: 'Salesforce',
|
|
78
|
+
auth: { type: 'oauth2', provider: 'salesforce', scopes: ['api'] },
|
|
79
|
+
actions: [
|
|
80
|
+
{ typeId: 'vendor.acme.salesforce.upsert', displayName: 'Upsert', idempotent: true, rateLimit: { requests: 100, perSeconds: 60 } },
|
|
81
|
+
{ typeId: 'vendor.acme.salesforce.query', displayName: 'Query', paginated: true },
|
|
82
|
+
],
|
|
83
|
+
triggers: ['vendor.acme.salesforce.onRecordChange'],
|
|
84
|
+
});
|
|
85
|
+
expect(ok, JSON.stringify(validate.errors)).toBe(true);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('positive: credential-auth variant validates', () => {
|
|
89
|
+
const ok = validate({
|
|
90
|
+
id: 'stripe',
|
|
91
|
+
displayName: 'Stripe',
|
|
92
|
+
auth: { type: 'credential', key: 'stripe-secret-key', scope: 'workspace' },
|
|
93
|
+
actions: [{ typeId: 'vendor.acme.stripe.charge', displayName: 'Charge' }],
|
|
94
|
+
});
|
|
95
|
+
expect(ok, JSON.stringify(validate.errors)).toBe(true);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('negative: connector missing displayName is rejected', () => {
|
|
99
|
+
expect(validate({ id: 'salesforce' })).toBe(false);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('negative: an action missing typeId is rejected', () => {
|
|
103
|
+
const ok = validate({
|
|
104
|
+
id: 'salesforce',
|
|
105
|
+
displayName: 'Salesforce',
|
|
106
|
+
actions: [{ displayName: 'Upsert' }],
|
|
107
|
+
});
|
|
108
|
+
expect(ok).toBe(false);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('negative: an unknown ConnectorAuth type is rejected', () => {
|
|
112
|
+
const ok = validate({
|
|
113
|
+
id: 'salesforce',
|
|
114
|
+
displayName: 'Salesforce',
|
|
115
|
+
auth: { type: 'basic', user: 'x' },
|
|
116
|
+
});
|
|
117
|
+
expect(ok).toBe(false);
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
describe('category: connector-manifest validity — action resolution (RFC 0045 §B)', () => {
|
|
122
|
+
const nodeTypeIds = [
|
|
123
|
+
'vendor.acme.salesforce.upsert',
|
|
124
|
+
'vendor.acme.salesforce.query',
|
|
125
|
+
'vendor.acme.salesforce.onRecordChange',
|
|
126
|
+
];
|
|
127
|
+
|
|
128
|
+
it('all action + trigger typeIds resolve to nodes[] in the same manifest', () => {
|
|
129
|
+
const unresolved = unresolvedReferences(nodeTypeIds, {
|
|
130
|
+
actions: [{ typeId: 'vendor.acme.salesforce.upsert' }, { typeId: 'vendor.acme.salesforce.query' }],
|
|
131
|
+
triggers: ['vendor.acme.salesforce.onRecordChange'],
|
|
132
|
+
});
|
|
133
|
+
expect(unresolved, 'every connector.actions[].typeId + triggers[] MUST resolve to a nodes[].typeId').toEqual([]);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('an action typeId not in nodes[] is flagged (connector_action_unresolved)', () => {
|
|
137
|
+
const unresolved = unresolvedReferences(nodeTypeIds, {
|
|
138
|
+
actions: [{ typeId: 'vendor.acme.salesforce.does-not-exist' }],
|
|
139
|
+
});
|
|
140
|
+
expect(unresolved).toContain('vendor.acme.salesforce.does-not-exist');
|
|
141
|
+
});
|
|
142
|
+
});
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* credential-payload-redaction — RFC 0046 §C.1 / §E invariant verification.
|
|
3
|
+
*
|
|
4
|
+
* Status: DRAFT. RFC 0046 (`host.credentials`) is `Draft`. Backs the
|
|
5
|
+
* SECURITY invariant `credential-payload-redaction`.
|
|
6
|
+
*
|
|
7
|
+
* Capability-gated: skips when the host does not advertise
|
|
8
|
+
* `capabilities.credentials.supported = true`.
|
|
9
|
+
*
|
|
10
|
+
* What this scenario asserts:
|
|
11
|
+
* 1. Advertisement shape — `capabilities.credentials` is well-formed when
|
|
12
|
+
* supported (delegated in detail to credentials-capability-shape).
|
|
13
|
+
* 2. Redaction MUST-NOT — when the host exposes the optional
|
|
14
|
+
* `POST /v1/host/sample/credentials/echo` test seam, resolving a seeded
|
|
15
|
+
* credential whose plaintext is a known canary MUST NOT cause that
|
|
16
|
+
* canary to appear in ANY of the run's observable surfaces (events,
|
|
17
|
+
* inputs, persisted variables, channels, snapshot, debug bundle). The
|
|
18
|
+
* reference is the only credential artifact permitted on the wire.
|
|
19
|
+
*
|
|
20
|
+
* Hosts without the test seam soft-skip the redaction probe (404) and still
|
|
21
|
+
* assert the advertisement shape — mirroring fs-path-traversal.
|
|
22
|
+
*
|
|
23
|
+
* @see RFCS/0046-host-credentials-capability.md
|
|
24
|
+
* @see SECURITY/invariants.yaml id: credential-payload-redaction
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { describe, it, expect } from 'vitest';
|
|
28
|
+
import { driver } from '../lib/driver.js';
|
|
29
|
+
|
|
30
|
+
interface DiscoveryCredentials {
|
|
31
|
+
supported?: boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface DiscoveryDoc {
|
|
35
|
+
capabilities?: {
|
|
36
|
+
credentials?: DiscoveryCredentials;
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// A unique canary the seam seeds as the resolved credential's plaintext.
|
|
41
|
+
const CANARY = 'OPENWOP_CRED_CANARY_4f1c8a2e9b';
|
|
42
|
+
|
|
43
|
+
async function readCredentials(): Promise<DiscoveryCredentials | null> {
|
|
44
|
+
const res = await driver.get('/.well-known/openwop');
|
|
45
|
+
const body = res.json as DiscoveryDoc | undefined;
|
|
46
|
+
return body?.capabilities?.credentials ?? null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
describe('credential-payload-redaction: advertisement shape (RFC 0046 §A)', () => {
|
|
50
|
+
it('capabilities.credentials.supported is a boolean when advertised', async () => {
|
|
51
|
+
const cred = await readCredentials();
|
|
52
|
+
if (cred === null) return;
|
|
53
|
+
expect(
|
|
54
|
+
typeof cred.supported,
|
|
55
|
+
driver.describe(
|
|
56
|
+
'capabilities.schema.json §credentials',
|
|
57
|
+
'capabilities.credentials.supported MUST be a boolean when credentials is advertised',
|
|
58
|
+
),
|
|
59
|
+
).toBe('boolean');
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe('credential-payload-redaction: resolved material MUST NOT cross the wire (RFC 0046 §C.1)', () => {
|
|
64
|
+
it('canary plaintext is absent from every observable run surface', async () => {
|
|
65
|
+
const cred = await readCredentials();
|
|
66
|
+
if (!cred?.supported) return; // capability-gated
|
|
67
|
+
|
|
68
|
+
// Seam contract: resolve a seeded credential whose plaintext is CANARY,
|
|
69
|
+
// run an echo node, and return the run's observable surfaces.
|
|
70
|
+
const res = await driver.post('/v1/host/sample/credentials/echo', { canary: CANARY });
|
|
71
|
+
// 404 from a host that hasn't wired the test seam is a soft-skip.
|
|
72
|
+
if (res.status === 404) return;
|
|
73
|
+
|
|
74
|
+
expect(
|
|
75
|
+
res.status,
|
|
76
|
+
driver.describe(
|
|
77
|
+
'SECURITY/invariants.yaml credential-payload-redaction',
|
|
78
|
+
'the credentials echo seam MUST resolve the seeded credential and return its observable surfaces',
|
|
79
|
+
),
|
|
80
|
+
).toBeLessThan(400);
|
|
81
|
+
|
|
82
|
+
// The entire serialized observable surface (events + inputs + variables +
|
|
83
|
+
// channels + snapshot + debug bundle) MUST NOT contain the canary plaintext.
|
|
84
|
+
const serialized = JSON.stringify(res.json ?? {});
|
|
85
|
+
expect(
|
|
86
|
+
serialized.includes(CANARY),
|
|
87
|
+
driver.describe(
|
|
88
|
+
'SECURITY/invariants.yaml credential-payload-redaction',
|
|
89
|
+
'resolved credential material MUST NOT appear in inputs, variables, channels, events, snapshot, or debug bundle — only the reference may cross the wire',
|
|
90
|
+
),
|
|
91
|
+
).toBe(false);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* credentials-capability-shape — RFC 0046 §A advertisement-shape verification.
|
|
3
|
+
*
|
|
4
|
+
* Status: DRAFT. RFC 0046 (`host.credentials`) is `Draft`. The
|
|
5
|
+
* `capabilities.credentials` block has landed in
|
|
6
|
+
* `schemas/capabilities.schema.json` and the invariant row
|
|
7
|
+
* `credential-payload-redaction` is in `SECURITY/invariants.yaml`.
|
|
8
|
+
*
|
|
9
|
+
* Always runs (shape-only): when the host advertises `capabilities.credentials`,
|
|
10
|
+
* its fields MUST be well-formed; when it doesn't, the block is simply absent.
|
|
11
|
+
*
|
|
12
|
+
* What this scenario asserts:
|
|
13
|
+
* 1. `capabilities.credentials` is either absent or a well-formed object.
|
|
14
|
+
* 2. When `supported: true`, `scopes` (when present) is a subset of
|
|
15
|
+
* {user, workspace, tenant}, and `rotation` (when present) is one of
|
|
16
|
+
* {none, two-key-overlap} (RFC 0046 §A).
|
|
17
|
+
*
|
|
18
|
+
* @see RFCS/0046-host-credentials-capability.md
|
|
19
|
+
* @see SECURITY/invariants.yaml id: credential-payload-redaction
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { describe, it, expect } from 'vitest';
|
|
23
|
+
import { driver } from '../lib/driver.js';
|
|
24
|
+
|
|
25
|
+
interface DiscoveryCredentials {
|
|
26
|
+
supported?: boolean;
|
|
27
|
+
scopes?: string[];
|
|
28
|
+
encryptionAtRest?: boolean;
|
|
29
|
+
rotation?: string;
|
|
30
|
+
sharing?: boolean;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface DiscoveryDoc {
|
|
34
|
+
capabilities?: {
|
|
35
|
+
credentials?: DiscoveryCredentials;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const VALID_SCOPES: ReadonlySet<string> = new Set(['user', 'workspace', 'tenant']);
|
|
40
|
+
const VALID_ROTATION: ReadonlySet<string> = new Set(['none', 'two-key-overlap']);
|
|
41
|
+
|
|
42
|
+
async function readCredentials(): Promise<DiscoveryCredentials | null> {
|
|
43
|
+
const res = await driver.get('/.well-known/openwop');
|
|
44
|
+
const body = res.json as DiscoveryDoc | undefined;
|
|
45
|
+
return body?.capabilities?.credentials ?? null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
describe('credentials-capability-shape: advertisement shape (RFC 0046 §A)', () => {
|
|
49
|
+
it('capabilities.credentials is either absent or well-formed', async () => {
|
|
50
|
+
const cred = await readCredentials();
|
|
51
|
+
if (cred === null) return; // host doesn't advertise host.credentials at all
|
|
52
|
+
expect(
|
|
53
|
+
typeof cred.supported,
|
|
54
|
+
driver.describe(
|
|
55
|
+
'capabilities.schema.json §credentials',
|
|
56
|
+
'capabilities.credentials.supported MUST be a boolean when credentials is advertised',
|
|
57
|
+
),
|
|
58
|
+
).toBe('boolean');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('scopes is a subset of {user, workspace, tenant} when supported', async () => {
|
|
62
|
+
const cred = await readCredentials();
|
|
63
|
+
if (!cred?.supported || cred.scopes === undefined) return;
|
|
64
|
+
expect(
|
|
65
|
+
Array.isArray(cred.scopes),
|
|
66
|
+
driver.describe('RFC 0046 §A', 'capabilities.credentials.scopes MUST be an array'),
|
|
67
|
+
).toBe(true);
|
|
68
|
+
for (const scope of cred.scopes) {
|
|
69
|
+
expect(
|
|
70
|
+
VALID_SCOPES.has(scope),
|
|
71
|
+
driver.describe(
|
|
72
|
+
'RFC 0046 §A',
|
|
73
|
+
`capabilities.credentials.scopes entries MUST be one of {${[...VALID_SCOPES].join(', ')}}, got: ${scope}`,
|
|
74
|
+
),
|
|
75
|
+
).toBe(true);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('rotation is one of {none, two-key-overlap} when present', async () => {
|
|
80
|
+
const cred = await readCredentials();
|
|
81
|
+
if (!cred?.supported || cred.rotation === undefined) return;
|
|
82
|
+
expect(
|
|
83
|
+
VALID_ROTATION.has(cred.rotation),
|
|
84
|
+
driver.describe(
|
|
85
|
+
'RFC 0046 §A',
|
|
86
|
+
`capabilities.credentials.rotation MUST be one of {${[...VALID_ROTATION].join(', ')}}, got: ${cred.rotation}`,
|
|
87
|
+
),
|
|
88
|
+
).toBe(true);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cross-engine-append-behavior — RFC 0036 §B cross-engine append-ordering behavioral probe.
|
|
3
|
+
*
|
|
4
|
+
* Companion to `cross-engine-append-ordering.test.ts` which carries the
|
|
5
|
+
* advertisement-shape probes. This file exercises the canonical cross-engine
|
|
6
|
+
* append-ordering behavior specified by `spec/v1/channels-and-reducers.md`
|
|
7
|
+
* §"Cross-engine ordering" via the host-extension test seams:
|
|
8
|
+
*
|
|
9
|
+
* POST /v1/host/sample/test/cross-engine/append — single engine append
|
|
10
|
+
* GET /v1/host/sample/test/cross-engine/read — read ordered sequence
|
|
11
|
+
* POST /v1/host/sample/test/cross-engine/reset — clear log
|
|
12
|
+
*
|
|
13
|
+
* The seam is conformance-only (host-extension namespace), gated on the
|
|
14
|
+
* host's `OPENWOP_TEST_CROSS_ENGINE_HARNESS=true` env var. The seam itself
|
|
15
|
+
* is OPTIONAL — hosts that don't expose it soft-skip; hosts that DO expose
|
|
16
|
+
* it MUST honor the §B contract:
|
|
17
|
+
*
|
|
18
|
+
* 1. Multiple engines appending concurrently to the same channelId
|
|
19
|
+
* converge to a single globally-ordered linearization on read.
|
|
20
|
+
* 2. Per-engine order is preserved within each engine's local sequence
|
|
21
|
+
* (writes from engine A appear in A's submission order, ditto B).
|
|
22
|
+
* 3. The host's advertised `orderingModel` (lamport / vector-clock /
|
|
23
|
+
* global-sequencer) determines the cross-engine merge semantics.
|
|
24
|
+
* 4. Read after partition heal converges to the same total order
|
|
25
|
+
* regardless of which engine's view we read from.
|
|
26
|
+
*
|
|
27
|
+
* @see RFCS/0036-multi-region-and-cross-engine-guarantees.md §B
|
|
28
|
+
* @see spec/v1/channels-and-reducers.md §"Cross-engine ordering"
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { describe, it, expect } from 'vitest';
|
|
32
|
+
import { driver } from '../lib/driver.js';
|
|
33
|
+
|
|
34
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
35
|
+
|
|
36
|
+
interface AppendEntry {
|
|
37
|
+
engineId: string;
|
|
38
|
+
value: unknown;
|
|
39
|
+
lamport: number;
|
|
40
|
+
seq: number;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function appendEntry(
|
|
44
|
+
engineId: string,
|
|
45
|
+
channelId: string,
|
|
46
|
+
value: unknown,
|
|
47
|
+
lamport?: number,
|
|
48
|
+
): Promise<{ status: number; entry?: AppendEntry }> {
|
|
49
|
+
const body: Record<string, unknown> = { engineId, channelId, value };
|
|
50
|
+
if (lamport !== undefined) body.lamport = lamport;
|
|
51
|
+
const res = await driver.post('/v1/host/sample/test/cross-engine/append', body);
|
|
52
|
+
if (res.status === 200) {
|
|
53
|
+
return { status: res.status, entry: res.json as AppendEntry };
|
|
54
|
+
}
|
|
55
|
+
return { status: res.status };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function readEntries(channelId: string): Promise<{ status: number; entries: AppendEntry[] }> {
|
|
59
|
+
const res = await driver.get(`/v1/host/sample/test/cross-engine/read?channelId=${encodeURIComponent(channelId)}`);
|
|
60
|
+
return {
|
|
61
|
+
status: res.status,
|
|
62
|
+
entries: res.status === 200 ? (res.json as { entries: AppendEntry[] }).entries : [],
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async function resetLog(): Promise<number> {
|
|
67
|
+
const res = await driver.post('/v1/host/sample/test/cross-engine/reset', {});
|
|
68
|
+
return res.status;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine ordering (RFC 0036)', () => {
|
|
72
|
+
it('interleaved appends from two engines converge to a single globally-ordered sequence', async (ctx) => {
|
|
73
|
+
const resetStatus = await resetLog();
|
|
74
|
+
if (resetStatus === 404) {
|
|
75
|
+
ctx.skip(); // host doesn't expose the cross-engine harness seam
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
expect(resetStatus).toBe(200);
|
|
79
|
+
|
|
80
|
+
const ch = 'channel-A';
|
|
81
|
+
|
|
82
|
+
// Engine A: 3 appends. Engine B: 2 appends. Interleaved.
|
|
83
|
+
const a1 = await appendEntry('engine-A', ch, 'a-1');
|
|
84
|
+
const b1 = await appendEntry('engine-B', ch, 'b-1');
|
|
85
|
+
const a2 = await appendEntry('engine-A', ch, 'a-2');
|
|
86
|
+
const a3 = await appendEntry('engine-A', ch, 'a-3');
|
|
87
|
+
const b2 = await appendEntry('engine-B', ch, 'b-2');
|
|
88
|
+
|
|
89
|
+
for (const r of [a1, b1, a2, a3, b2]) {
|
|
90
|
+
expect(r.status).toBe(200);
|
|
91
|
+
expect(r.entry).toBeDefined();
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const read = await readEntries(ch);
|
|
95
|
+
expect(read.status).toBe(200);
|
|
96
|
+
|
|
97
|
+
expect(
|
|
98
|
+
read.entries.length,
|
|
99
|
+
driver.describe(
|
|
100
|
+
'channels-and-reducers.md §"Cross-engine ordering"',
|
|
101
|
+
'all appends across all engines MUST appear in the linearized read',
|
|
102
|
+
),
|
|
103
|
+
).toBe(5);
|
|
104
|
+
|
|
105
|
+
// Per-engine order MUST be preserved within each engine's submissions.
|
|
106
|
+
const engineAEntries = read.entries.filter((e) => e.engineId === 'engine-A').map((e) => e.value);
|
|
107
|
+
const engineBEntries = read.entries.filter((e) => e.engineId === 'engine-B').map((e) => e.value);
|
|
108
|
+
expect(
|
|
109
|
+
engineAEntries,
|
|
110
|
+
driver.describe(
|
|
111
|
+
'channels-and-reducers.md §"Cross-engine ordering"',
|
|
112
|
+
'engine-A submissions MUST appear in submission order within the linearization',
|
|
113
|
+
),
|
|
114
|
+
).toEqual(['a-1', 'a-2', 'a-3']);
|
|
115
|
+
expect(
|
|
116
|
+
engineBEntries,
|
|
117
|
+
driver.describe(
|
|
118
|
+
'channels-and-reducers.md §"Cross-engine ordering"',
|
|
119
|
+
'engine-B submissions MUST appear in submission order within the linearization',
|
|
120
|
+
),
|
|
121
|
+
).toEqual(['b-1', 'b-2']);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('lamport clocks monotonically advance across engines', async (ctx) => {
|
|
125
|
+
const resetStatus = await resetLog();
|
|
126
|
+
if (resetStatus === 404) {
|
|
127
|
+
ctx.skip();
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
expect(resetStatus).toBe(200);
|
|
131
|
+
|
|
132
|
+
const ch = 'channel-B';
|
|
133
|
+
const a1 = await appendEntry('engine-A', ch, 'a-1');
|
|
134
|
+
const b1 = await appendEntry('engine-B', ch, 'b-1');
|
|
135
|
+
const a2 = await appendEntry('engine-A', ch, 'a-2');
|
|
136
|
+
|
|
137
|
+
expect(a1.entry?.lamport).toBeDefined();
|
|
138
|
+
expect(b1.entry?.lamport).toBeDefined();
|
|
139
|
+
expect(a2.entry?.lamport).toBeDefined();
|
|
140
|
+
|
|
141
|
+
// Lamport invariant: each subsequent append on the same shared
|
|
142
|
+
// channel MUST have strictly-higher clock than the previous.
|
|
143
|
+
expect(
|
|
144
|
+
a2.entry!.lamport > b1.entry!.lamport && b1.entry!.lamport > a1.entry!.lamport,
|
|
145
|
+
driver.describe(
|
|
146
|
+
'channels-and-reducers.md §"Cross-engine ordering" — Lamport',
|
|
147
|
+
'lamport clocks MUST be strictly monotonic on the shared channel',
|
|
148
|
+
),
|
|
149
|
+
).toBe(true);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it('lamport hint from engine A advances engine B past it', async (ctx) => {
|
|
153
|
+
const resetStatus = await resetLog();
|
|
154
|
+
if (resetStatus === 404) {
|
|
155
|
+
ctx.skip();
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
expect(resetStatus).toBe(200);
|
|
159
|
+
|
|
160
|
+
const ch = 'channel-C';
|
|
161
|
+
// Engine A appends, gets lamport L. Engine B then appends with
|
|
162
|
+
// lamport hint == L (proxy for "B saw A's clock at L"); B's
|
|
163
|
+
// resulting clock MUST be > L per the lamport receive rule
|
|
164
|
+
// max(local, incoming) + 1.
|
|
165
|
+
const a1 = await appendEntry('engine-A', ch, 'a-1');
|
|
166
|
+
expect(a1.status).toBe(200);
|
|
167
|
+
const seen = a1.entry!.lamport;
|
|
168
|
+
const b1 = await appendEntry('engine-B', ch, 'b-1', seen);
|
|
169
|
+
expect(b1.status).toBe(200);
|
|
170
|
+
expect(
|
|
171
|
+
b1.entry!.lamport > seen,
|
|
172
|
+
driver.describe(
|
|
173
|
+
'channels-and-reducers.md §"Cross-engine ordering" — Lamport receive rule',
|
|
174
|
+
'when engine B sees engine A\'s clock at L, B\'s next append MUST have clock > L',
|
|
175
|
+
),
|
|
176
|
+
).toBe(true);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('linearization is deterministic — same appends → same total order', async (ctx) => {
|
|
180
|
+
const resetStatus = await resetLog();
|
|
181
|
+
if (resetStatus === 404) {
|
|
182
|
+
ctx.skip();
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
expect(resetStatus).toBe(200);
|
|
186
|
+
|
|
187
|
+
const ch = 'channel-D';
|
|
188
|
+
await appendEntry('engine-A', ch, 'a-1');
|
|
189
|
+
await appendEntry('engine-B', ch, 'b-1');
|
|
190
|
+
await appendEntry('engine-A', ch, 'a-2');
|
|
191
|
+
|
|
192
|
+
const r1 = await readEntries(ch);
|
|
193
|
+
const r2 = await readEntries(ch);
|
|
194
|
+
expect(r1.status).toBe(200);
|
|
195
|
+
expect(r2.status).toBe(200);
|
|
196
|
+
expect(
|
|
197
|
+
r1.entries.map((e) => `${e.engineId}:${String(e.value)}`),
|
|
198
|
+
driver.describe(
|
|
199
|
+
'channels-and-reducers.md §"Cross-engine ordering" — determinism',
|
|
200
|
+
'two reads MUST produce the same linearization (deterministic merge)',
|
|
201
|
+
),
|
|
202
|
+
).toEqual(r2.entries.map((e) => `${e.engineId}:${String(e.value)}`));
|
|
203
|
+
});
|
|
204
|
+
});
|
|
@@ -50,11 +50,18 @@ describe('cross-host-traceparent-propagation: behavioral (RFC 0040 §B)', () =>
|
|
|
50
50
|
// the format `00-{traceId}-{spanId}-{flags}` per W3C tracecontext.
|
|
51
51
|
// Until the peer harness lands, the assertion is surfaced as `todo` so
|
|
52
52
|
// test reporters track the gap rather than reporting a vacuous PASS.
|
|
53
|
-
|
|
53
|
+
// Marked out of stable profile via RFC 0042 §B (experimental tier):
|
|
54
|
+
// RFC 0040 remains Active. Hosts that wire Phase 3 cross-host causation
|
|
55
|
+
// before RFC 0040 graduates SHOULD advertise
|
|
56
|
+
// `multiAgent.executionModel.tier: 'experimental'` per RFC 0042 §A
|
|
57
|
+
// until cross-host evidence drives the promotion. Path-to-runnable
|
|
58
|
+
// requires the MCP peer harness (OPENWOP_MCP_REAL_SERVER_URL) +
|
|
59
|
+
// inbound-header recorder; flips to a real `it()` on first non-steward
|
|
60
|
+
// Phase 3 host advertising matching capabilities.
|
|
61
|
+
it.skip('Phase 3 host MUST inject parent run\'s traceparent into outbound MCP requests — out of stable profile via RFC 0042');
|
|
54
62
|
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
|
|
59
|
-
it.todo('Phase 3 host MUST inject parent run\'s traceparent into outbound A2A messages');
|
|
63
|
+
// Same routing — out of stable profile via RFC 0042 §B until RFC 0040
|
|
64
|
+
// graduates to Accepted; behavioral A2A test seam contract still to be
|
|
65
|
+
// designed alongside the corresponding peer harness.
|
|
66
|
+
it.skip('Phase 3 host MUST inject parent run\'s traceparent into outbound A2A messages — out of stable profile via RFC 0042');
|
|
60
67
|
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cross-workspace-isolation — RFC 0048 §D verification.
|
|
3
|
+
*
|
|
4
|
+
* Status: DRAFT. RFC 0048 (tenant·workspace·principal identity model) is
|
|
5
|
+
* `Draft`.
|
|
6
|
+
*
|
|
7
|
+
* What this scenario asserts:
|
|
8
|
+
* 1. Run-ownership echo shape — when a readable run snapshot carries
|
|
9
|
+
* `owner`, it MUST include a non-empty `tenant` (RFC 0048 §C).
|
|
10
|
+
* 2. Cross-workspace isolation MUST-NOT (§D) — when the host exposes the
|
|
11
|
+
* optional `POST /v1/host/sample/identity/cross-workspace-read` seam
|
|
12
|
+
* (a principal scoped to workspace A attempts to read a run owned by
|
|
13
|
+
* workspace B), the read MUST fail closed with `run_forbidden` (or a
|
|
14
|
+
* `404`/`403` that does not leak the other workspace's run contents).
|
|
15
|
+
*
|
|
16
|
+
* Hosts without the seam soft-skip the isolation probe (404). The
|
|
17
|
+
* advertisement/ownership-shape assertion still runs.
|
|
18
|
+
*
|
|
19
|
+
* @see RFCS/0048-tenant-workspace-principal-identity-model.md
|
|
20
|
+
* @see spec/v1/auth.md §"Identity claims — tenant · workspace · principal"
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { describe, it, expect } from 'vitest';
|
|
24
|
+
import { driver } from '../lib/driver.js';
|
|
25
|
+
|
|
26
|
+
const ISOLATION_CODES: ReadonlySet<string> = new Set(['run_forbidden', 'not_found']);
|
|
27
|
+
|
|
28
|
+
interface OwnerTriple {
|
|
29
|
+
tenant?: string;
|
|
30
|
+
workspace?: string;
|
|
31
|
+
principal?: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
describe('cross-workspace-isolation: run-ownership echo shape (RFC 0048 §C)', () => {
|
|
35
|
+
it('owner, when present on a run snapshot, carries a non-empty tenant', async () => {
|
|
36
|
+
// Best-effort: probe a sample run if the host exposes one; otherwise skip.
|
|
37
|
+
const res = await driver.get('/v1/host/sample/identity/owned-run');
|
|
38
|
+
if (res.status === 404) return; // no sample-run seam — soft-skip
|
|
39
|
+
const owner = (res.json as { owner?: OwnerTriple } | undefined)?.owner;
|
|
40
|
+
if (owner === undefined) return; // single-tenant host — owner omitted
|
|
41
|
+
expect(
|
|
42
|
+
typeof owner.tenant === 'string' && owner.tenant.length > 0,
|
|
43
|
+
driver.describe('RFC 0048 §C', 'RunSnapshot.owner MUST carry a non-empty tenant when present'),
|
|
44
|
+
).toBe(true);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe('cross-workspace-isolation: a principal MUST NOT read another workspace\'s run (RFC 0048 §D)', () => {
|
|
49
|
+
it('cross-workspace read fails closed with run_forbidden', async () => {
|
|
50
|
+
// Seam contract: a principal scoped to workspace A requests a run owned
|
|
51
|
+
// by workspace B. The host MUST refuse rather than return B's run.
|
|
52
|
+
const res = await driver.post('/v1/host/sample/identity/cross-workspace-read', {});
|
|
53
|
+
if (res.status === 404) return; // seam unwired — soft-skip
|
|
54
|
+
|
|
55
|
+
expect(
|
|
56
|
+
res.status,
|
|
57
|
+
driver.describe(
|
|
58
|
+
'spec/v1/auth.md §Identity claims',
|
|
59
|
+
'a cross-workspace read MUST fail closed (4xx), never return the other workspace\'s run',
|
|
60
|
+
),
|
|
61
|
+
).toBeGreaterThanOrEqual(400);
|
|
62
|
+
|
|
63
|
+
const code = (res.json as { error?: string } | undefined)?.error;
|
|
64
|
+
expect(
|
|
65
|
+
code !== undefined && ISOLATION_CODES.has(code),
|
|
66
|
+
driver.describe(
|
|
67
|
+
'spec/v1/rest-endpoints.md run_forbidden',
|
|
68
|
+
`error MUST be one of {${[...ISOLATION_CODES].join(', ')}} (fail-closed, no existence leak), got: ${code ?? '(absent)'}`,
|
|
69
|
+
),
|
|
70
|
+
).toBe(true);
|
|
71
|
+
});
|
|
72
|
+
});
|