harness-everything 0.36.4 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +8 -0
- package/ci/fixtures/doc-obligations/providers-s3/cases.json +535 -0
- package/ci/fixtures/doc-obligations/reconciliation-prep/provider-contract.json +1 -1
- package/ci/mechanism-325-3b-s3-providers.test.js +378 -0
- package/docs/doc-workflow-reconciliation.md +116 -2
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/doc-providers.js +113 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/doc-providers.js +113 -0
- package/sbom.cdx.json +7 -7
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// #325 Phase 3b S3 RED suite: advisory provider suggestions (pure suggestDocProviders).
|
|
5
|
+
//
|
|
6
|
+
// Every assertion runs against both legs: the canonical harness-everything/scripts/doc-providers.js and the packaged
|
|
7
|
+
// plugins/harness-everything/skills/harness-everything/scripts/doc-providers.js. A missing or unloadable module is reported
|
|
8
|
+
// per assertion and per leg as a module-missing (or module-load) reason; the harness itself does not crash. Byte parity
|
|
9
|
+
// between the two copies is one assertion over both files. Expected outputs are hand-derived from the S3 spec in
|
|
10
|
+
// docs/doc-workflow-reconciliation.md and from reconciliation-prep/provider-contract.json, never from an implementation.
|
|
11
|
+
// Exit 1 means at least one assertion FAILED; exit 2 means an infrastructure fault in this suite (unreadable fixture).
|
|
12
|
+
|
|
13
|
+
const crypto = require('crypto');
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const path = require('path');
|
|
16
|
+
const http = require('http');
|
|
17
|
+
const https = require('https');
|
|
18
|
+
const net = require('net');
|
|
19
|
+
const childProcess = require('child_process');
|
|
20
|
+
|
|
21
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
22
|
+
const FIXTURE_DIR = path.join(__dirname, 'fixtures', 'doc-obligations');
|
|
23
|
+
const CASES_FILE = path.join(FIXTURE_DIR, 'providers-s3', 'cases.json');
|
|
24
|
+
const CONTRACT_FILE = path.join(FIXTURE_DIR, 'reconciliation-prep', 'provider-contract.json');
|
|
25
|
+
const DERIVE_DIR = path.join(FIXTURE_DIR, 'derive');
|
|
26
|
+
const MODULE = 'doc-providers.js';
|
|
27
|
+
const LEGS = [
|
|
28
|
+
{ name: 'source', dir: path.join(ROOT, 'harness-everything', 'scripts') },
|
|
29
|
+
{ name: 'packaged', dir: path.join(ROOT, 'plugins', 'harness-everything', 'skills', 'harness-everything', 'scripts') }
|
|
30
|
+
];
|
|
31
|
+
const INVALID = 'invalid-input';
|
|
32
|
+
const R_OK = 'active-step capability and host requirements match';
|
|
33
|
+
const NO_META = 'no available provider metadata for the active-step capability';
|
|
34
|
+
const VALID_KEYS = ['suggestions', 'unavailable'];
|
|
35
|
+
const INVALID_KEYS = ['diagnostics', 'suggestions', 'unavailable'];
|
|
36
|
+
const SUGGESTION_KEYS = ['availability', 'capability', 'providerId', 'reason'];
|
|
37
|
+
const UNAVAILABLE_KEYS = ['capability', 'reason'];
|
|
38
|
+
const AVAILABILITY_ENUM = ['available', 'unavailable', 'unknown'];
|
|
39
|
+
|
|
40
|
+
let corpus;
|
|
41
|
+
let contract;
|
|
42
|
+
let deriveConfig;
|
|
43
|
+
try {
|
|
44
|
+
corpus = JSON.parse(fs.readFileSync(CASES_FILE, 'utf8'));
|
|
45
|
+
contract = JSON.parse(fs.readFileSync(CONTRACT_FILE, 'utf8'));
|
|
46
|
+
deriveConfig = JSON.parse(fs.readFileSync(path.join(DERIVE_DIR, 'config.json'), 'utf8'));
|
|
47
|
+
} catch (error) {
|
|
48
|
+
console.error(`[s3-providers] infrastructure: fixture unreadable: ${error.message.split('\n')[0]}`);
|
|
49
|
+
process.exitCode = 2;
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
let passed = 0;
|
|
54
|
+
let failed = 0;
|
|
55
|
+
const failures = [];
|
|
56
|
+
const infrastructure = [];
|
|
57
|
+
|
|
58
|
+
function record(status, id, leg, detail) {
|
|
59
|
+
const where = leg ? ` [${leg}]` : '';
|
|
60
|
+
if (status === 'PASS') passed += 1;
|
|
61
|
+
else { failed += 1; failures.push(`${id}${where}`); }
|
|
62
|
+
console.log(` ${status} ${id}${where}${detail ? ` — ${detail}` : ''}`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Stable JSON: object keys sorted recursively, so comparisons do not depend on property order.
|
|
66
|
+
function stable(value) {
|
|
67
|
+
if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`;
|
|
68
|
+
if (value && typeof value === 'object') {
|
|
69
|
+
return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${stable(value[key])}`).join(',')}}`;
|
|
70
|
+
}
|
|
71
|
+
return JSON.stringify(value);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function truncate(text) {
|
|
75
|
+
return text.length > 300 ? `${text.slice(0, 300)}…` : text;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function same(actual, expected, label) {
|
|
79
|
+
const a = stable(actual);
|
|
80
|
+
const e = stable(expected);
|
|
81
|
+
if (a !== e) throw new Error(`${label}: got ${truncate(a)} expected ${truncate(e)}`);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Candidate-set comparison: order-insensitive, no ranking implied.
|
|
85
|
+
function sameSet(actual, expected, label) {
|
|
86
|
+
const a = (Array.isArray(actual) ? actual : []).map(stable).sort();
|
|
87
|
+
const e = expected.map(stable).sort();
|
|
88
|
+
if (JSON.stringify(a) !== JSON.stringify(e)) throw new Error(`${label}: set got ${truncate(a.join(' | '))} expected ${truncate(e.join(' | '))}`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function deepFreeze(value) {
|
|
92
|
+
if (value && typeof value === 'object' && !Object.isFrozen(value)) {
|
|
93
|
+
Object.freeze(value);
|
|
94
|
+
for (const item of Object.values(value)) deepFreeze(item);
|
|
95
|
+
}
|
|
96
|
+
return value;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function cloneJson(value) {
|
|
100
|
+
return value === undefined ? undefined : JSON.parse(JSON.stringify(value));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Module loading is per leg and cached; a load failure is an assertion FAIL for that leg, not a harness crash.
|
|
104
|
+
const loaded = {};
|
|
105
|
+
function loadLeg(leg) {
|
|
106
|
+
if (loaded[leg.name]) return loaded[leg.name];
|
|
107
|
+
const file = path.join(leg.dir, MODULE);
|
|
108
|
+
let result;
|
|
109
|
+
if (!fs.existsSync(file)) {
|
|
110
|
+
result = { error: `module-missing: ${path.relative(ROOT, file)} does not exist` };
|
|
111
|
+
} else {
|
|
112
|
+
try {
|
|
113
|
+
delete require.cache[require.resolve(file)];
|
|
114
|
+
const mod = require(file);
|
|
115
|
+
if (typeof mod.suggestDocProviders !== 'function') result = { error: 'module-missing-export: suggestDocProviders is not a function' };
|
|
116
|
+
else result = { mod };
|
|
117
|
+
} catch (error) {
|
|
118
|
+
result = { error: `module-load: ${error.message.split('\n')[0]}` };
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
loaded[leg.name] = result;
|
|
122
|
+
return result;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Runs one assertion on each leg. The callback receives the suggest function and the leg name.
|
|
126
|
+
function assertPerLeg(id, fn) {
|
|
127
|
+
for (const leg of LEGS) {
|
|
128
|
+
const { mod, error } = loadLeg(leg);
|
|
129
|
+
if (error) { record('FAIL', id, leg.name, error); continue; }
|
|
130
|
+
try {
|
|
131
|
+
fn(mod.suggestDocProviders, leg);
|
|
132
|
+
record('PASS', id, leg.name);
|
|
133
|
+
} catch (error) {
|
|
134
|
+
record('FAIL', id, leg.name, error.message.split('\n')[0]);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Runs one assertion over both legs at once (parity). Reports a missing leg as module-missing for that leg.
|
|
140
|
+
function assertPair(id, fn) {
|
|
141
|
+
const missing = LEGS.filter(leg => loadLeg(leg).error);
|
|
142
|
+
if (missing.length) {
|
|
143
|
+
for (const leg of missing) record('FAIL', id, leg.name, loadLeg(leg).error);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
fn();
|
|
148
|
+
record('PASS', id, 'both');
|
|
149
|
+
} catch (error) {
|
|
150
|
+
record('FAIL', id, 'both', error.message.split('\n')[0]);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Validity of one output for a valid input: exact keys, then exact ordered content.
|
|
155
|
+
function checkValidOutput(out, expected) {
|
|
156
|
+
same(Object.keys(out).sort(), VALID_KEYS, 'top-level keys (valid input has no diagnostics)');
|
|
157
|
+
same(out.suggestions, expected.suggestions, 'suggestions');
|
|
158
|
+
same(out.unavailable, expected.unavailable, 'unavailable');
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Validity of one structured invalid-input result.
|
|
162
|
+
function checkInvalidOutput(out, expected) {
|
|
163
|
+
same(Object.keys(out).sort(), INVALID_KEYS, 'top-level keys (invalid input)');
|
|
164
|
+
same(out.suggestions, [], 'suggestions must be empty for invalid input');
|
|
165
|
+
same(out.unavailable, expected.unavailable, 'unavailable (retained needed capabilities)');
|
|
166
|
+
if (!Array.isArray(out.diagnostics) || out.diagnostics.length === 0) throw new Error('diagnostics must be a nonempty array');
|
|
167
|
+
for (const item of out.diagnostics) {
|
|
168
|
+
same(item.code, INVALID, 'diagnostic code');
|
|
169
|
+
if (typeof item.path !== 'string' || !item.path.trim()) throw new Error('diagnostic path must be a nonempty string');
|
|
170
|
+
if (typeof item.message !== 'string' || !item.message.trim()) throw new Error('diagnostic message must be a nonempty string');
|
|
171
|
+
}
|
|
172
|
+
for (const prefix of expected.diagnosticPathPrefixes) {
|
|
173
|
+
if (!out.diagnostics.some(item => item.path.startsWith(prefix))) throw new Error(`no diagnostic path starts with ${prefix}`);
|
|
174
|
+
}
|
|
175
|
+
if (expected.minDiagnostics !== undefined && out.diagnostics.length < expected.minDiagnostics) {
|
|
176
|
+
throw new Error(`expected at least ${expected.minDiagnostics} diagnostics, got ${out.diagnostics.length}`);
|
|
177
|
+
}
|
|
178
|
+
const sorted = out.diagnostics.map(item => [item.path, item.message]).slice().sort((x, y) => (x[0] < y[0] ? -1 : x[0] > y[0] ? 1 : x[1] < y[1] ? -1 : x[1] > y[1] ? 1 : 0));
|
|
179
|
+
same(out.diagnostics.map(item => [item.path, item.message]), sorted, 'diagnostics ordered by path then message');
|
|
180
|
+
if (expected.forbiddenText) {
|
|
181
|
+
const text = JSON.stringify(out);
|
|
182
|
+
for (const forbidden of expected.forbiddenText) {
|
|
183
|
+
if (text.includes(forbidden)) throw new Error('diagnostics expose an unrelated record value');
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
console.log('=== #325 3b S3: advisory provider suggestions (RED; canonical and packaged legs) ===');
|
|
189
|
+
|
|
190
|
+
// ---- Contract examples: candidate sets, both legs -----------------------------------------------------------------
|
|
191
|
+
for (const example of contract.examples) {
|
|
192
|
+
assertPerLeg(`contract:${example.id}`, suggest => {
|
|
193
|
+
const out = suggest(cloneJson(example.input));
|
|
194
|
+
same(Object.keys(out).sort(), VALID_KEYS, 'top-level keys');
|
|
195
|
+
sameSet(out.suggestions, example.expected.suggestions, 'suggestions');
|
|
196
|
+
sameSet(out.unavailable, example.expected.unavailable, 'unavailable');
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
assertPerLeg('contract:swap-provider-order-same-candidate-set', suggest => {
|
|
201
|
+
const example = contract.examples[0];
|
|
202
|
+
const swapped = cloneJson(example.input);
|
|
203
|
+
swapped.availableProviderMetadata.reverse();
|
|
204
|
+
const out = suggest(swapped);
|
|
205
|
+
sameSet(out.suggestions, example.expected.suggestions, 'suggestions after swapping provider order');
|
|
206
|
+
sameSet(out.unavailable, example.expected.unavailable, 'unavailable after swapping provider order');
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
// ---- Fixture cases: exact outputs (valid) and structured invalid-input results (malformed) -------------------------
|
|
210
|
+
for (const item of corpus.cases) {
|
|
211
|
+
assertPerLeg(`case:${item.id}`, suggest => {
|
|
212
|
+
const out = suggest(cloneJson(item.input));
|
|
213
|
+
if (item.kind === 'valid') checkValidOutput(out, item.expected);
|
|
214
|
+
else if (item.kind === 'malformed') checkInvalidOutput(out, item.expected);
|
|
215
|
+
else throw new Error(`fixture kind ${item.kind} is unknown`);
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// ---- H1: obligations are unchanged by any suggestion metadata ------------------------------------------------------
|
|
220
|
+
function readTree(root) {
|
|
221
|
+
const files = {};
|
|
222
|
+
if (!fs.existsSync(root)) return files;
|
|
223
|
+
const walk = dir => {
|
|
224
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
225
|
+
const full = path.join(dir, entry.name);
|
|
226
|
+
if (entry.isDirectory()) walk(full);
|
|
227
|
+
else {
|
|
228
|
+
const rel = path.relative(root, full).split(path.sep).join('/');
|
|
229
|
+
files[rel.endsWith('.fixture') ? rel.slice(0, -'.fixture'.length) : rel] = fs.readFileSync(full, 'utf8');
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
};
|
|
233
|
+
walk(root);
|
|
234
|
+
return files;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function obligationsFor(leg) {
|
|
238
|
+
const { deriveDocFacts, deriveDocObligations } = require(path.join(leg.dir, 'doc-obligations.js'));
|
|
239
|
+
const caseDir = path.join(DERIVE_DIR, 'mixed-docs-code');
|
|
240
|
+
const facts = deriveDocFacts({
|
|
241
|
+
base: { files: readTree(path.join(caseDir, 'base')) },
|
|
242
|
+
candidate: { files: readTree(path.join(caseDir, 'candidate')) },
|
|
243
|
+
config: deriveConfig
|
|
244
|
+
});
|
|
245
|
+
return stable(deriveDocObligations(facts.facts));
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
for (const leg of LEGS) {
|
|
249
|
+
const id = 'H1:suggestions-leave-derived-obligations-unchanged';
|
|
250
|
+
const { mod, error } = loadLeg(leg);
|
|
251
|
+
if (error) { record('FAIL', id, leg.name, error); continue; }
|
|
252
|
+
try {
|
|
253
|
+
const before = obligationsFor(leg);
|
|
254
|
+
for (const example of contract.examples) mod.suggestDocProviders(cloneJson(example.input));
|
|
255
|
+
for (const item of corpus.cases) mod.suggestDocProviders(cloneJson(item.input));
|
|
256
|
+
const after = obligationsFor(leg);
|
|
257
|
+
if (before !== after) throw new Error('deriveDocObligations output changed after suggestDocProviders calls');
|
|
258
|
+
record('PASS', id, leg.name);
|
|
259
|
+
} catch (error) {
|
|
260
|
+
record('FAIL', id, leg.name, error.message.split('\n')[0]);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// ---- Output shape: no channel that could remove or waive a capability -----------------------------------------------
|
|
265
|
+
assertPerLeg('output:exact-key-shapes-and-no-removal-channel', suggest => {
|
|
266
|
+
const outputs = [
|
|
267
|
+
...contract.examples.map(example => suggest(cloneJson(example.input))),
|
|
268
|
+
...corpus.cases.map(item => suggest(cloneJson(item.input)))
|
|
269
|
+
];
|
|
270
|
+
for (const out of outputs) {
|
|
271
|
+
const keys = Object.keys(out).sort();
|
|
272
|
+
if (JSON.stringify(keys) !== JSON.stringify(VALID_KEYS) && JSON.stringify(keys) !== JSON.stringify(INVALID_KEYS)) {
|
|
273
|
+
throw new Error(`unexpected top-level keys ${keys.join(',')}`);
|
|
274
|
+
}
|
|
275
|
+
for (const suggestion of out.suggestions) same(Object.keys(suggestion).sort(), SUGGESTION_KEYS, 'suggestion keys (no locator or extra metadata)');
|
|
276
|
+
for (const gap of out.unavailable) same(Object.keys(gap).sort(), UNAVAILABLE_KEYS, 'unavailable keys');
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
// ---- Host: availability enum; unknown is reserved and never emitted ------------------------------------------------
|
|
281
|
+
assertPerLeg('host:unknown-availability-never-emitted', suggest => {
|
|
282
|
+
const outputs = [
|
|
283
|
+
...contract.examples.map(example => suggest(cloneJson(example.input))),
|
|
284
|
+
...corpus.cases.map(item => suggest(cloneJson(item.input)))
|
|
285
|
+
];
|
|
286
|
+
for (const out of outputs) {
|
|
287
|
+
for (const suggestion of out.suggestions) {
|
|
288
|
+
if (!['available', 'unavailable'].includes(suggestion.availability)) {
|
|
289
|
+
throw new Error(`availability ${suggestion.availability} is outside the S3 emitted set`);
|
|
290
|
+
}
|
|
291
|
+
if (!AVAILABILITY_ENUM.includes(suggestion.availability)) throw new Error('availability outside the contract enum');
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
// ---- Purity: input unchanged (including frozen), determinism, no file/network/process activity -----------------------
|
|
297
|
+
const everyInput = [...contract.examples.map(example => example.input), ...corpus.cases.map(item => item.input)];
|
|
298
|
+
|
|
299
|
+
assertPerLeg('purity:input-unchanged-including-frozen', suggest => {
|
|
300
|
+
for (const input of everyInput) {
|
|
301
|
+
const copy = cloneJson(input);
|
|
302
|
+
const before = stable(copy);
|
|
303
|
+
suggest(copy);
|
|
304
|
+
same(stable(copy), before, 'input after the call');
|
|
305
|
+
const frozen = deepFreeze(cloneJson(input));
|
|
306
|
+
suggest(frozen);
|
|
307
|
+
same(stable(frozen), before, 'frozen input after the call');
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
assertPerLeg('purity:deterministic-two-calls', suggest => {
|
|
312
|
+
for (const input of everyInput) {
|
|
313
|
+
const first = stable(suggest(cloneJson(input)));
|
|
314
|
+
const second = stable(suggest(cloneJson(input)));
|
|
315
|
+
same(second, first, 'second call output');
|
|
316
|
+
}
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
assertPerLeg('purity:no-file-network-process-activity-during-calls', suggest => {
|
|
320
|
+
const calls = [];
|
|
321
|
+
const trap = (owner, name) => {
|
|
322
|
+
const original = owner[name];
|
|
323
|
+
owner[name] = (...args) => { calls.push(`${name}`); throw new Error(`forbidden ${name} call`); };
|
|
324
|
+
return () => { owner[name] = original; };
|
|
325
|
+
};
|
|
326
|
+
const restores = [
|
|
327
|
+
trap(fs, 'readFileSync'), trap(fs, 'readdirSync'), trap(fs, 'existsSync'), trap(fs, 'statSync'),
|
|
328
|
+
trap(fs, 'lstatSync'), trap(fs, 'openSync'), trap(childProcess, 'spawnSync'), trap(childProcess, 'execFileSync'),
|
|
329
|
+
trap(childProcess, 'execSync'), trap(childProcess, 'spawn'), trap(childProcess, 'exec'), trap(childProcess, 'fork'),
|
|
330
|
+
trap(net, 'connect'), trap(net, 'createConnection'), trap(http, 'request'), trap(https, 'request')
|
|
331
|
+
];
|
|
332
|
+
try {
|
|
333
|
+
for (const input of everyInput) suggest(cloneJson(input));
|
|
334
|
+
} finally {
|
|
335
|
+
for (const restore of restores) restore();
|
|
336
|
+
}
|
|
337
|
+
same(calls, [], 'forbidden calls during suggestDocProviders');
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
// ---- Static checks on both source copies --------------------------------------------------------------------------
|
|
341
|
+
const FORBIDDEN_REQUIRE = /require\(\s*['"](?:node:)?(?:fs|child_process|net|http|https)(?:\/promises)?['"]\s*\)/;
|
|
342
|
+
const FORBIDDEN_IMPORT = /\bfrom\s+['"](?:node:)?(?:fs|child_process|net|http|https)(?:\/promises)?['"]/;
|
|
343
|
+
const SKILL_LITERAL = /['"`][^'"`\n]*SKILL\.md[^'"`\n]*['"`]/;
|
|
344
|
+
|
|
345
|
+
function sourceText(leg) {
|
|
346
|
+
return fs.readFileSync(path.join(leg.dir, MODULE), 'utf8');
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
assertPerLeg('static:no-forbidden-requires', (suggest, leg) => {
|
|
350
|
+
const text = sourceText(leg);
|
|
351
|
+
if (FORBIDDEN_REQUIRE.test(text) || FORBIDDEN_IMPORT.test(text)) {
|
|
352
|
+
throw new Error('module requires fs, child_process, net, http or https');
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
assertPerLeg('static:never-references-SKILL.md-as-a-path', (suggest, leg) => {
|
|
357
|
+
if (SKILL_LITERAL.test(sourceText(leg))) throw new Error('module contains a SKILL.md path literal');
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
assertPair('parity:source-and-packaged-byte-identical', () => {
|
|
361
|
+
const digest = leg => crypto.createHash('sha256').update(fs.readFileSync(path.join(leg.dir, MODULE))).digest('hex');
|
|
362
|
+
const [source, packaged] = LEGS.map(digest);
|
|
363
|
+
if (source !== packaged) throw new Error(`source ${source.slice(0, 12)} differs from packaged ${packaged.slice(0, 12)}`);
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
// ---- Summary ------------------------------------------------------------------------------------------------------
|
|
367
|
+
console.log(`\nS3 provider suggestion summary: ${passed} PASS, ${failed} FAIL, ${infrastructure.length} INFRA across ${corpus.cases.length} fixture cases and ${contract.examples.length} contract examples`);
|
|
368
|
+
if (failures.length) console.log(`FAILED assertions (${failures.length}):\n ${failures.join('\n ')}`);
|
|
369
|
+
|
|
370
|
+
if (infrastructure.length) {
|
|
371
|
+
console.error('S3 suite: infrastructure faults present; these are not assertion results.');
|
|
372
|
+
process.exitCode = 2;
|
|
373
|
+
} else if (failed > 0) {
|
|
374
|
+
console.error('S3 suite: assertion FAIL present (expected on the RED base; GREEN must clear every FAIL).');
|
|
375
|
+
process.exitCode = 1;
|
|
376
|
+
} else {
|
|
377
|
+
console.log('S3 suite: all assertions PASS.');
|
|
378
|
+
}
|
|
@@ -96,8 +96,11 @@ the existing authorized verification path. Snapshot consistency is not VCS or re
|
|
|
96
96
|
|
|
97
97
|
## Advisory provider contract
|
|
98
98
|
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
**S3 implements the pure suggestion function below; it is not wired into workflow authorization,
|
|
100
|
+
bindings or reconciliation.** This Docs-stage contract precedes RED/GREEN and is not a claim of
|
|
101
|
+
completed runtime implementation. The
|
|
102
|
+
[provider data contract](../ci/fixtures/doc-obligations/reconciliation-prep/provider-contract.json)
|
|
103
|
+
specifies its interface:
|
|
101
104
|
|
|
102
105
|
```text
|
|
103
106
|
suggestDocProviders({ activeStep, neededCapabilities, availableProviderMetadata, hostCapabilities })
|
|
@@ -111,6 +114,117 @@ expected obligations, authorize a tool, or convert missing evidence to PASS. Exi
|
|
|
111
114
|
resolution handles a chosen provider; independent valid evidence can satisfy a MUST without it.
|
|
112
115
|
An unsupported host reports unavailable capability/reason. H1/H2 runtime proof remains pending.
|
|
113
116
|
|
|
117
|
+
## S3 — advisory provider suggestions
|
|
118
|
+
|
|
119
|
+
S3 delivers the pure named export `suggestDocProviders(input)` in
|
|
120
|
+
`harness-everything/scripts/doc-providers.js`, with the installed copy generated by
|
|
121
|
+
`npm run plugin:sync`. This is an independent slice from S1: it imports no S1 hook or pilot code
|
|
122
|
+
and changes no workflow disposition. The
|
|
123
|
+
[approved S3 row](https://github.com/dyphn1/Harness-everything/issues/325#issuecomment-6308606462),
|
|
124
|
+
[suggestion signature](https://github.com/dyphn1/Harness-everything/issues/325#issuecomment-6230246088)
|
|
125
|
+
and [H1/H2 boundary](https://github.com/dyphn1/Harness-everything/issues/325#issuecomment-6230191562)
|
|
126
|
+
are the scope; the provider data contract supplies the schema and examples.
|
|
127
|
+
|
|
128
|
+
### Purity and validation
|
|
129
|
+
|
|
130
|
+
Identical inputs produce identical output, including ordering and reason strings. The function
|
|
131
|
+
MUST NOT mutate input or retained caller objects, read files/SKILL.md, access the network,
|
|
132
|
+
spawn processes, execute tools/generators, discover host state or load future-step content.
|
|
133
|
+
All decisions use supplied metadata only; importing the module has no such side effects either.
|
|
134
|
+
|
|
135
|
+
Validate the complete input before producing suggestions. Required records are non-null objects,
|
|
136
|
+
not arrays; required strings must be strings with non-whitespace content, without coercion.
|
|
137
|
+
Matching uses exact, case-sensitive values; validation does not normalize identifiers.
|
|
138
|
+
|
|
139
|
+
| Input | Required shape |
|
|
140
|
+
| --- | --- |
|
|
141
|
+
| `input` | Object containing all four required fields below |
|
|
142
|
+
| `activeStep` | Object with nonempty `id` and `stepType`; identifies the caller's current authorized active step, not an instruction to authorize it |
|
|
143
|
+
| `neededCapabilities` | Array of distinct, nonempty capability strings; empty is valid |
|
|
144
|
+
| `availableProviderMetadata` | Array of metadata objects; empty is valid |
|
|
145
|
+
| Each metadata record | Nonempty stable `providerId`; `capabilities` is an array of distinct nonempty strings; `stepTypes` and `requiresHostCapabilities` are arrays of nonempty strings; empty host requirements means none |
|
|
146
|
+
| `locator` in each record | Object with `kind` exactly `binding` or `tool`, and nonempty `id`; a lookup locator, never an executable command |
|
|
147
|
+
| `hostCapabilities` | Required array of distinct, nonempty supported-host capability strings; empty means no supported capabilities are declared |
|
|
148
|
+
|
|
149
|
+
Extra fields confer no capability or authority and are not copied into suggestions. Validate
|
|
150
|
+
metadata structurally without exposing unrelated record values in diagnostics. This slice adds
|
|
151
|
+
no provider registry, skill-content parser or host-capability probe.
|
|
152
|
+
|
|
153
|
+
**Invalid input returns a structured result, not a validation exception.** Use
|
|
154
|
+
`{ suggestions: [], unavailable: [...], diagnostics: [{ code: "invalid-input", path, message }] }`.
|
|
155
|
+
Each diagnostic has nonempty string `path`/`message`; order diagnostics by field path, then
|
|
156
|
+
message. Emit no suggestions from a partly valid input. If `neededCapabilities` is structurally
|
|
157
|
+
valid, retain every needed capability exactly once in `unavailable`, with reason `invalid-input`;
|
|
158
|
+
otherwise leave that array empty and diagnose the malformed needed set. Empty error arrays are
|
|
159
|
+
not evidence that no obligations exist. Normal valid output has only the canonical
|
|
160
|
+
`suggestions` and `unavailable` arrays; diagnostics are the validation-error extension only.
|
|
161
|
+
|
|
162
|
+
This follows the data-result convention of `deriveDocFacts` (`doc-obligations.js:1375`, invalid
|
|
163
|
+
inputs yield `inputKnown: false`) and `deriveDocObligations` (`:1508`, invalid facts retain unknown
|
|
164
|
+
checks), plus the checker's structured diagnostics (`doc-evidence.js:88–127`). It does not copy
|
|
165
|
+
invalid evidence into a positive result or modify any of those APIs.
|
|
166
|
+
|
|
167
|
+
**Host uncertainty decision:** retain strict validation. Missing, null or malformed
|
|
168
|
+
`hostCapabilities` is an invalid input, not an alternate unknown-host encoding. For a valid
|
|
169
|
+
supported-capability array, a required capability absent from the array is unavailable. The
|
|
170
|
+
contract's `unknown` availability enum remains reserved: current inputs have no representation
|
|
171
|
+
of unknown host support, so S3 does not invent one or emit `unknown` from malformed input.
|
|
172
|
+
Any future unknown-host representation must preserve unknown as unknown rather than available.
|
|
173
|
+
|
|
174
|
+
### Filtering, results and ordering
|
|
175
|
+
|
|
176
|
+
For valid input, apply these rules without loading provider content:
|
|
177
|
+
|
|
178
|
+
1. A record is eligible only if its `stepTypes` contains `activeStep.stepType` and its
|
|
179
|
+
`capabilities` intersects `neededCapabilities`. Future/unrelated-step records and unrelated
|
|
180
|
+
capabilities do not appear in output; an eligible multi-capability record contributes one
|
|
181
|
+
suggestion for each matched needed capability.
|
|
182
|
+
2. A suggestion contains exactly `capability`, `providerId`, `availability`, `reason`, with
|
|
183
|
+
nonempty strings and availability from the contract enum. Locator values and other metadata
|
|
184
|
+
are not returned. If all `requiresHostCapabilities` are present in `hostCapabilities`, report
|
|
185
|
+
`available` with reason `active-step capability and host requirements match`.
|
|
186
|
+
3. If host requirements are unmet, keep the eligible suggestion as `unavailable`, with reason
|
|
187
|
+
`host lacks <missing capabilities>`. Sort missing names and join them with `, `; no missing
|
|
188
|
+
requirement may be treated as supported. Empty requirements always pass this host check.
|
|
189
|
+
4. For every needed capability with **no available suggestion**, emit exactly one
|
|
190
|
+
`{ capability, reason }` in `unavailable`. With no eligible metadata, use
|
|
191
|
+
`no available provider metadata for the active-step capability`. Otherwise join the distinct
|
|
192
|
+
blocking suggestion reasons in sorted order with `; `. An unavailable candidate remains
|
|
193
|
+
visible even if a different candidate is available; that capability is then absent from the
|
|
194
|
+
top-level `unavailable` list. This list reports capability gaps and never removes a MUST.
|
|
195
|
+
5. Output is a candidate set with **no ranking or default selection**. For deterministic display,
|
|
196
|
+
sort suggestions by capability, then providerId, then availability and reason; sort
|
|
197
|
+
`unavailable` by capability. Use case-sensitive text comparison, independent of locale.
|
|
198
|
+
Tests compare candidate sets; ordering grants no preference, authorization or binding.
|
|
199
|
+
|
|
200
|
+
The three provider-contract examples are executable acceptance inputs: two active alternatives
|
|
201
|
+
are available while the future outline provider is absent; omitting candidates retains a needed
|
|
202
|
+
capability as unavailable; a missing independent-review host capability keeps both the blocked
|
|
203
|
+
candidate and the top-level gap with reason `host lacks independent-review-events`.
|
|
204
|
+
|
|
205
|
+
### RED boundary and deferred integration
|
|
206
|
+
|
|
207
|
+
Independent RED covers the contract examples, active-step and capability intersection, multiple
|
|
208
|
+
host requirements, empty arrays, missing/null/wrong-type fields, invalid strings/locators and
|
|
209
|
+
explicit distinct-array violations. Assert complete output shape, stable order/reasons, input
|
|
210
|
+
non-mutation (including frozen inputs), no file/network/process/skill-content activity and
|
|
211
|
+
source/package parity. Unsupported host capabilities produce reasons rather than a silent empty
|
|
212
|
+
list; malformed inputs follow the structured-error contract above. No runtime host support is
|
|
213
|
+
inferred from a provider ID, locator, loaded skill or success-looking free text.
|
|
214
|
+
|
|
215
|
+
S3 is not wired into authorization, binding selection or evidence reconciliation. Obligations
|
|
216
|
+
still come exclusively from unchanged `deriveDocObligations`; swapping or omitting suggestions
|
|
217
|
+
cannot change their identity, strength, applicability, scope or acceptance. The suggestion module
|
|
218
|
+
neither invokes the checker nor grants PASS. Existing binding resolution applies only when a
|
|
219
|
+
caller later chooses a provider through its already-authorized workflow.
|
|
220
|
+
|
|
221
|
+
H1/H2 **workflow integration proof remains S2/close-out**, per the reviewed acceptance boundary:
|
|
222
|
+
provider swap/omission leaves expected obligations identical; independent qualifying alternate
|
|
223
|
+
evidence can satisfy a MUST without the suggestion; missing evidence stays unresolved; capture
|
|
224
|
+
active-step metadata and file-read behavior plus unsupported-host reasons. S3's pure-function
|
|
225
|
+
and package tests establish only that local mechanism, not reconciled-workflow H1/H2 proof,
|
|
226
|
+
live-host loading, E1 GO or #320 promotion. No S1 implementation is a dependency of this slice.
|
|
227
|
+
|
|
114
228
|
## One switch and rollback
|
|
115
229
|
|
|
116
230
|
Planned switch: `HARNESS_DOC_RECONCILIATION_PILOT=1`. Missing, `0`, or any other value is OFF.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Suggestions describe candidates only; they never select a provider or satisfy evidence.
|
|
4
|
+
const AVAILABLE = 'active-step capability and host requirements match';
|
|
5
|
+
const NO_METADATA = 'no available provider metadata for the active-step capability';
|
|
6
|
+
const compareText = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
7
|
+
const isRecord = value => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
8
|
+
const isText = value => typeof value === 'string' && value.trim().length > 0;
|
|
9
|
+
|
|
10
|
+
function validateInput(input) {
|
|
11
|
+
const diagnostics = [];
|
|
12
|
+
const diagnose = (path, message) => diagnostics.push({ code: 'invalid-input', path, message });
|
|
13
|
+
const text = (value, path) => {
|
|
14
|
+
if (!isText(value)) diagnose(path, 'Expected a nonempty string.');
|
|
15
|
+
};
|
|
16
|
+
const strings = (value, path, distinct = false) => {
|
|
17
|
+
const before = diagnostics.length;
|
|
18
|
+
if (!Array.isArray(value)) {
|
|
19
|
+
diagnose(path, 'Expected an array of nonempty strings.');
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
const seen = new Set();
|
|
23
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
24
|
+
text(value[i], `${path}[${i}]`);
|
|
25
|
+
if (distinct && seen.has(value[i])) diagnose(`${path}[${i}]`, 'Expected distinct strings.');
|
|
26
|
+
seen.add(value[i]);
|
|
27
|
+
}
|
|
28
|
+
return diagnostics.length === before;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
if (!isRecord(input)) {
|
|
32
|
+
diagnose('input', 'Expected an object.');
|
|
33
|
+
return { diagnostics, neededValid: false };
|
|
34
|
+
}
|
|
35
|
+
if (!isRecord(input.activeStep)) {
|
|
36
|
+
diagnose('activeStep', 'Expected an object.');
|
|
37
|
+
} else {
|
|
38
|
+
text(input.activeStep.id, 'activeStep.id');
|
|
39
|
+
text(input.activeStep.stepType, 'activeStep.stepType');
|
|
40
|
+
}
|
|
41
|
+
const neededValid = strings(input.neededCapabilities, 'neededCapabilities', true);
|
|
42
|
+
strings(input.hostCapabilities, 'hostCapabilities', true);
|
|
43
|
+
if (!Array.isArray(input.availableProviderMetadata)) {
|
|
44
|
+
diagnose('availableProviderMetadata', 'Expected an array of metadata objects.');
|
|
45
|
+
} else {
|
|
46
|
+
for (let i = 0; i < input.availableProviderMetadata.length; i += 1) {
|
|
47
|
+
const metadata = input.availableProviderMetadata[i];
|
|
48
|
+
const path = `availableProviderMetadata[${i}]`;
|
|
49
|
+
if (!isRecord(metadata)) {
|
|
50
|
+
diagnose(path, 'Expected a metadata object.');
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
text(metadata.providerId, `${path}.providerId`);
|
|
54
|
+
strings(metadata.capabilities, `${path}.capabilities`, true);
|
|
55
|
+
strings(metadata.stepTypes, `${path}.stepTypes`);
|
|
56
|
+
strings(metadata.requiresHostCapabilities, `${path}.requiresHostCapabilities`);
|
|
57
|
+
if (!isRecord(metadata.locator)) {
|
|
58
|
+
diagnose(`${path}.locator`, 'Expected a locator object.');
|
|
59
|
+
} else {
|
|
60
|
+
if (!['binding', 'tool'].includes(metadata.locator.kind)) {
|
|
61
|
+
diagnose(`${path}.locator.kind`, 'Expected binding or tool.');
|
|
62
|
+
}
|
|
63
|
+
text(metadata.locator.id, `${path}.locator.id`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
diagnostics.sort((a, b) => compareText(a.path, b.path) || compareText(a.message, b.message));
|
|
68
|
+
return { diagnostics, neededValid };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Pure advisory projection of caller-supplied active-step metadata. */
|
|
72
|
+
function suggestDocProviders(input) {
|
|
73
|
+
const { diagnostics, neededValid } = validateInput(input);
|
|
74
|
+
if (diagnostics.length) {
|
|
75
|
+
return {
|
|
76
|
+
suggestions: [],
|
|
77
|
+
unavailable: neededValid
|
|
78
|
+
? [...input.neededCapabilities].sort(compareText).map(capability => ({ capability, reason: 'invalid-input' }))
|
|
79
|
+
: [],
|
|
80
|
+
diagnostics
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const needed = new Set(input.neededCapabilities);
|
|
85
|
+
const host = new Set(input.hostCapabilities);
|
|
86
|
+
const suggestions = [];
|
|
87
|
+
for (const metadata of input.availableProviderMetadata) {
|
|
88
|
+
if (!metadata.stepTypes.includes(input.activeStep.stepType)) continue;
|
|
89
|
+
const missing = metadata.requiresHostCapabilities.filter(capability => !host.has(capability)).sort(compareText);
|
|
90
|
+
const availability = missing.length ? 'unavailable' : 'available';
|
|
91
|
+
const reason = missing.length ? `host lacks ${missing.join(', ')}` : AVAILABLE;
|
|
92
|
+
for (const capability of metadata.capabilities) {
|
|
93
|
+
if (needed.has(capability)) {
|
|
94
|
+
suggestions.push({ capability, providerId: metadata.providerId, availability, reason });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
suggestions.sort((a, b) => compareText(a.capability, b.capability)
|
|
99
|
+
|| compareText(a.providerId, b.providerId)
|
|
100
|
+
|| compareText(a.availability, b.availability)
|
|
101
|
+
|| compareText(a.reason, b.reason));
|
|
102
|
+
|
|
103
|
+
const unavailable = [];
|
|
104
|
+
for (const capability of [...needed].sort(compareText)) {
|
|
105
|
+
const candidates = suggestions.filter(suggestion => suggestion.capability === capability);
|
|
106
|
+
if (candidates.some(candidate => candidate.availability === 'available')) continue;
|
|
107
|
+
const reasons = [...new Set(candidates.map(candidate => candidate.reason))].sort(compareText);
|
|
108
|
+
unavailable.push({ capability, reason: reasons.length ? reasons.join('; ') : NO_METADATA });
|
|
109
|
+
}
|
|
110
|
+
return { suggestions, unavailable };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
module.exports = { suggestDocProviders };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "harness-everything",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.37.0",
|
|
5
5
|
"description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Miya Daniel",
|