@ruvector/ruvllm 2.7.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GPU-LOOP.md +229 -0
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +157 -263
- package/examples/gpu-loop.example.json +26 -0
- package/package.json +50 -120
- package/src/adapters-cli.mjs +23 -0
- package/src/adapters.mjs +91 -0
- package/src/cli.mjs +30 -0
- package/src/engine.mjs +238 -0
- package/src/fixture.mjs +38 -0
- package/src/gpu-host.mjs +47 -0
- package/src/gpu-loop-cli.mjs +67 -0
- package/src/gpu-loop.mjs +184 -0
- package/src/gpu-spool-adapter.mjs +80 -0
- package/src/vast-host-adapter.mjs +274 -0
- package/bin/cli.js +0 -1020
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts +0 -102
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts.map +0 -1
- package/dist/cjs/benchmarks/embedding-benchmark.js +0 -436
- package/dist/cjs/benchmarks/index.d.ts +0 -34
- package/dist/cjs/benchmarks/index.d.ts.map +0 -1
- package/dist/cjs/benchmarks/index.js +0 -132
- package/dist/cjs/benchmarks/model-comparison.d.ts +0 -71
- package/dist/cjs/benchmarks/model-comparison.d.ts.map +0 -1
- package/dist/cjs/benchmarks/model-comparison.js +0 -476
- package/dist/cjs/benchmarks/routing-benchmark.d.ts +0 -70
- package/dist/cjs/benchmarks/routing-benchmark.d.ts.map +0 -1
- package/dist/cjs/benchmarks/routing-benchmark.js +0 -289
- package/dist/cjs/contrastive.d.ts +0 -229
- package/dist/cjs/contrastive.d.ts.map +0 -1
- package/dist/cjs/contrastive.js +0 -589
- package/dist/cjs/engine.d.ts +0 -95
- package/dist/cjs/engine.d.ts.map +0 -1
- package/dist/cjs/engine.js +0 -372
- package/dist/cjs/export.d.ts +0 -182
- package/dist/cjs/export.d.ts.map +0 -1
- package/dist/cjs/export.js +0 -422
- package/dist/cjs/federated.d.ts +0 -233
- package/dist/cjs/federated.d.ts.map +0 -1
- package/dist/cjs/federated.js +0 -525
- package/dist/cjs/index.d.ts +0 -68
- package/dist/cjs/index.d.ts.map +0 -1
- package/dist/cjs/index.js +0 -104
- package/dist/cjs/intelligence.d.ts +0 -136
- package/dist/cjs/intelligence.d.ts.map +0 -1
- package/dist/cjs/intelligence.js +0 -226
- package/dist/cjs/lora.d.ts +0 -251
- package/dist/cjs/lora.d.ts.map +0 -1
- package/dist/cjs/lora.js +0 -509
- package/dist/cjs/models.d.ts +0 -129
- package/dist/cjs/models.d.ts.map +0 -1
- package/dist/cjs/models.js +0 -323
- package/dist/cjs/native.d.ts +0 -92
- package/dist/cjs/native.d.ts.map +0 -1
- package/dist/cjs/native.js +0 -77
- package/dist/cjs/session.d.ts +0 -80
- package/dist/cjs/session.d.ts.map +0 -1
- package/dist/cjs/session.js +0 -203
- package/dist/cjs/simd.d.ts +0 -90
- package/dist/cjs/simd.d.ts.map +0 -1
- package/dist/cjs/simd.js +0 -209
- package/dist/cjs/sona.d.ts +0 -223
- package/dist/cjs/sona.d.ts.map +0 -1
- package/dist/cjs/sona.js +0 -562
- package/dist/cjs/streaming.d.ts +0 -55
- package/dist/cjs/streaming.d.ts.map +0 -1
- package/dist/cjs/streaming.js +0 -131
- package/dist/cjs/training.d.ts +0 -312
- package/dist/cjs/training.d.ts.map +0 -1
- package/dist/cjs/training.js +0 -610
- package/dist/cjs/types.d.ts +0 -615
- package/dist/cjs/types.d.ts.map +0 -1
- package/dist/cjs/types.js +0 -6
- package/dist/esm/benchmarks/embedding-benchmark.d.ts +0 -102
- package/dist/esm/benchmarks/embedding-benchmark.d.ts.map +0 -1
- package/dist/esm/benchmarks/embedding-benchmark.js +0 -427
- package/dist/esm/benchmarks/index.d.ts +0 -34
- package/dist/esm/benchmarks/index.d.ts.map +0 -1
- package/dist/esm/benchmarks/index.js +0 -110
- package/dist/esm/benchmarks/model-comparison.d.ts +0 -71
- package/dist/esm/benchmarks/model-comparison.d.ts.map +0 -1
- package/dist/esm/benchmarks/model-comparison.js +0 -434
- package/dist/esm/benchmarks/routing-benchmark.d.ts +0 -70
- package/dist/esm/benchmarks/routing-benchmark.d.ts.map +0 -1
- package/dist/esm/benchmarks/routing-benchmark.js +0 -283
- package/dist/esm/contrastive.d.ts +0 -229
- package/dist/esm/contrastive.d.ts.map +0 -1
- package/dist/esm/contrastive.js +0 -578
- package/dist/esm/engine.d.ts +0 -95
- package/dist/esm/engine.d.ts.map +0 -1
- package/dist/esm/engine.js +0 -368
- package/dist/esm/export.d.ts +0 -182
- package/dist/esm/export.d.ts.map +0 -1
- package/dist/esm/export.js +0 -414
- package/dist/esm/federated.d.ts +0 -233
- package/dist/esm/federated.d.ts.map +0 -1
- package/dist/esm/federated.js +0 -520
- package/dist/esm/index.d.ts +0 -68
- package/dist/esm/index.d.ts.map +0 -1
- package/dist/esm/index.js +0 -84
- package/dist/esm/intelligence.d.ts +0 -136
- package/dist/esm/intelligence.d.ts.map +0 -1
- package/dist/esm/intelligence.js +0 -188
- package/dist/esm/lora.d.ts +0 -251
- package/dist/esm/lora.d.ts.map +0 -1
- package/dist/esm/lora.js +0 -504
- package/dist/esm/models.d.ts +0 -129
- package/dist/esm/models.d.ts.map +0 -1
- package/dist/esm/models.js +0 -282
- package/dist/esm/native.d.ts +0 -92
- package/dist/esm/native.d.ts.map +0 -1
- package/dist/esm/native.js +0 -72
- package/dist/esm/session.d.ts +0 -80
- package/dist/esm/session.d.ts.map +0 -1
- package/dist/esm/session.js +0 -199
- package/dist/esm/simd.d.ts +0 -90
- package/dist/esm/simd.d.ts.map +0 -1
- package/dist/esm/simd.js +0 -205
- package/dist/esm/sona.d.ts +0 -223
- package/dist/esm/sona.d.ts.map +0 -1
- package/dist/esm/sona.js +0 -556
- package/dist/esm/streaming.d.ts +0 -55
- package/dist/esm/streaming.d.ts.map +0 -1
- package/dist/esm/streaming.js +0 -126
- package/dist/esm/training.d.ts +0 -312
- package/dist/esm/training.d.ts.map +0 -1
- package/dist/esm/training.js +0 -603
- package/dist/esm/types.d.ts +0 -615
- package/dist/esm/types.d.ts.map +0 -1
- package/dist/esm/types.js +0 -5
- package/scripts/ensemble-model-compare.js +0 -287
- package/scripts/fix-esm-extensions.mjs +0 -181
- package/scripts/huggingface/README.md +0 -537
- package/scripts/huggingface/publish.sh +0 -112
- package/scripts/hybrid-model-compare.js +0 -373
- package/scripts/improved-model-compare.js +0 -288
- package/scripts/optimized-model-compare.js +0 -364
- package/scripts/real-model-compare.js +0 -280
- package/scripts/smoke-esm.mjs +0 -82
- package/scripts/training/agentic-flow-capabilities.json +0 -329
- package/scripts/training/claude-code-synth.js +0 -505
- package/scripts/training/claude-flow-capabilities.json +0 -1322
- package/scripts/training/claude-hard-negatives.js +0 -222
- package/scripts/training/contrastive-finetune.js +0 -517
- package/scripts/training/ecosystem-triplets.jsonl +0 -1467
- package/scripts/training/generate-ecosystem-triplets.js +0 -1324
- package/scripts/training/routing-dataset.js +0 -634
- package/scripts/training/ruvector-capabilities.json +0 -798
- package/scripts/training/validate-ecosystem.js +0 -381
- package/scripts/training/validation-results.json +0 -23
package/package.json
CHANGED
|
@@ -1,135 +1,65 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ruvector/ruvllm",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
|
|
9
|
-
"
|
|
10
|
-
|
|
11
|
-
"types": "./dist/esm/index.d.ts",
|
|
12
|
-
"default": "./dist/esm/index.js"
|
|
13
|
-
},
|
|
14
|
-
"require": {
|
|
15
|
-
"types": "./dist/cjs/index.d.ts",
|
|
16
|
-
"default": "./dist/cjs/index.js"
|
|
17
|
-
}
|
|
18
|
-
},
|
|
19
|
-
"./simd": {
|
|
20
|
-
"import": {
|
|
21
|
-
"types": "./dist/esm/simd.d.ts",
|
|
22
|
-
"default": "./dist/esm/simd.js"
|
|
23
|
-
},
|
|
24
|
-
"require": {
|
|
25
|
-
"types": "./dist/cjs/simd.d.ts",
|
|
26
|
-
"default": "./dist/cjs/simd.js"
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
},
|
|
30
|
-
"bin": {
|
|
31
|
-
"ruvllm": "./bin/cli.js"
|
|
32
|
-
},
|
|
33
|
-
"napi": {
|
|
34
|
-
"binaryName": "ruvllm",
|
|
35
|
-
"targets": [
|
|
36
|
-
"x86_64-unknown-linux-gnu",
|
|
37
|
-
"aarch64-unknown-linux-gnu",
|
|
38
|
-
"x86_64-apple-darwin",
|
|
39
|
-
"aarch64-apple-darwin",
|
|
40
|
-
"x86_64-pc-windows-msvc"
|
|
41
|
-
]
|
|
42
|
-
},
|
|
43
|
-
"scripts": {
|
|
44
|
-
"artifacts": "napi artifacts",
|
|
45
|
-
"build": "npm run build:cjs && npm run build:esm",
|
|
46
|
-
"build:cjs": "tsc",
|
|
47
|
-
"build:esm": "tsc -p tsconfig.esm.json && node scripts/fix-esm-extensions.mjs",
|
|
48
|
-
"verify:esm": "node scripts/fix-esm-extensions.mjs --verify",
|
|
49
|
-
"build:native": "napi build --platform --release -p ruvllm --manifest-path ../../../examples/ruvLLM/Cargo.toml -F napi",
|
|
50
|
-
"build:debug": "napi build --platform -p ruvllm --manifest-path ../../../examples/ruvLLM/Cargo.toml -F napi",
|
|
51
|
-
"prepublishOnly": "npm run build",
|
|
52
|
-
"test": "node --test test/*.test.js",
|
|
53
|
-
"test:esm-smoke": "node scripts/smoke-esm.mjs",
|
|
54
|
-
"universal": "napi universal",
|
|
55
|
-
"version": "napi version",
|
|
56
|
-
"typecheck": "tsc --noEmit",
|
|
57
|
-
"clean": "rm -rf dist"
|
|
58
|
-
},
|
|
59
|
-
"devDependencies": {
|
|
60
|
-
"@napi-rs/cli": "^2.18.0",
|
|
61
|
-
"@types/node": "^20.10.5",
|
|
62
|
-
"typescript": "^5.3.3"
|
|
63
|
-
},
|
|
64
|
-
"dependencies": {
|
|
65
|
-
"chalk": "^4.1.2",
|
|
66
|
-
"commander": "^12.0.0",
|
|
67
|
-
"ora": "^5.4.1"
|
|
3
|
+
"version": "4.0.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Bounded RuVLLM weight experiments, a durable GPU job loop with a Vast.ai host adapter, and verified paired measurements",
|
|
6
|
+
"license": "Apache-2.0",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/ruvnet/ruvllm.git",
|
|
10
|
+
"directory": "integrations/metaharness"
|
|
68
11
|
},
|
|
69
|
-
"
|
|
70
|
-
|
|
71
|
-
"
|
|
72
|
-
"@ruvector/ruvllm-darwin-x64": "2.0.1",
|
|
73
|
-
"@ruvector/ruvllm-darwin-arm64": "2.0.1",
|
|
74
|
-
"@ruvector/ruvllm-win32-x64-msvc": "2.0.1"
|
|
12
|
+
"homepage": "https://github.com/ruvnet/ruvllm/tree/main/integrations/metaharness#readme",
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/ruvnet/ruvllm/issues"
|
|
75
15
|
},
|
|
76
16
|
"keywords": [
|
|
77
17
|
"ruvllm",
|
|
78
|
-
"
|
|
79
|
-
"self-learning",
|
|
80
|
-
"adaptive-learning",
|
|
81
|
-
"sona",
|
|
18
|
+
"metaharness",
|
|
82
19
|
"lora",
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"router",
|
|
88
|
-
"simd",
|
|
89
|
-
"inference",
|
|
90
|
-
"federated-learning",
|
|
91
|
-
"continual-learning",
|
|
92
|
-
"machine-learning",
|
|
93
|
-
"ai",
|
|
94
|
-
"deep-learning",
|
|
95
|
-
"napi",
|
|
96
|
-
"rust",
|
|
97
|
-
"ruvector",
|
|
98
|
-
"turboquant",
|
|
99
|
-
"kv-cache",
|
|
100
|
-
"quantization",
|
|
101
|
-
"flash-attention",
|
|
102
|
-
"speculative-decoding",
|
|
103
|
-
"gguf",
|
|
104
|
-
"mamba",
|
|
105
|
-
"transformer",
|
|
106
|
-
"edge-ai",
|
|
107
|
-
"local-llm",
|
|
108
|
-
"model-compression"
|
|
20
|
+
"gpu",
|
|
21
|
+
"vast.ai",
|
|
22
|
+
"evaluation",
|
|
23
|
+
"weight-editing"
|
|
109
24
|
],
|
|
110
|
-
"
|
|
111
|
-
|
|
112
|
-
"repository": {
|
|
113
|
-
"type": "git",
|
|
114
|
-
"url": "https://github.com/ruvnet/ruvector.git",
|
|
115
|
-
"directory": "npm/packages/ruvllm"
|
|
25
|
+
"engines": {
|
|
26
|
+
"node": ">=22"
|
|
116
27
|
},
|
|
117
|
-
"
|
|
118
|
-
|
|
119
|
-
"
|
|
28
|
+
"bin": {
|
|
29
|
+
"ruvllm": "src/cli.mjs",
|
|
30
|
+
"ruvllm-gpu-loop": "src/gpu-loop-cli.mjs",
|
|
31
|
+
"ruvllm-adapters": "src/adapters-cli.mjs"
|
|
120
32
|
},
|
|
121
|
-
"
|
|
122
|
-
"
|
|
33
|
+
"exports": {
|
|
34
|
+
".": "./src/engine.mjs",
|
|
35
|
+
"./adapters": "./src/adapters.mjs",
|
|
36
|
+
"./fixture": "./src/fixture.mjs",
|
|
37
|
+
"./gpu-loop": "./src/gpu-loop.mjs",
|
|
38
|
+
"./gpu-host": "./src/gpu-host.mjs",
|
|
39
|
+
"./gpu-spool-adapter": "./src/gpu-spool-adapter.mjs",
|
|
40
|
+
"./vast-host-adapter": "./src/vast-host-adapter.mjs",
|
|
41
|
+
"./package.json": "./package.json"
|
|
42
|
+
},
|
|
43
|
+
"files": [
|
|
44
|
+
"src/",
|
|
45
|
+
"examples/",
|
|
46
|
+
"README.md",
|
|
47
|
+
"GPU-LOOP.md",
|
|
48
|
+
"LICENSE",
|
|
49
|
+
"NOTICE"
|
|
50
|
+
],
|
|
51
|
+
"scripts": {
|
|
52
|
+
"test": "node --test test/*.test.mjs",
|
|
53
|
+
"bench": "node bench/run.mjs",
|
|
54
|
+
"prepack": "cp ../../LICENSE ../../NOTICE .",
|
|
55
|
+
"postpack": "rm -f LICENSE NOTICE"
|
|
123
56
|
},
|
|
124
57
|
"publishConfig": {
|
|
125
|
-
"registry": "https://registry.npmjs.org/",
|
|
126
58
|
"access": "public"
|
|
127
59
|
},
|
|
128
|
-
"
|
|
129
|
-
"
|
|
130
|
-
"
|
|
131
|
-
"
|
|
132
|
-
|
|
133
|
-
"README.md"
|
|
134
|
-
]
|
|
60
|
+
"dependencies": {
|
|
61
|
+
"@metaharness/darwin": "0.12.0",
|
|
62
|
+
"@metaharness/flywheel": "0.1.13",
|
|
63
|
+
"@metaharness/weight-eft": "0.1.2"
|
|
64
|
+
}
|
|
135
65
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { exportArchive, trainingPlan, selectDevelopmentCandidate } from './adapters.mjs';
|
|
3
|
+
|
|
4
|
+
const operations = { export: exportArchive, 'training-plan': trainingPlan, select: selectDevelopmentCandidate };
|
|
5
|
+
if (process.argv.length !== 3 || !Object.hasOwn(operations, process.argv[2])) {
|
|
6
|
+
process.stderr.write('Usage: node src/adapters-cli.mjs export|training-plan|select < input.json\nJSON data only; no training, inference or activation is executed.\n');
|
|
7
|
+
process.exitCode = process.argv[2] === '--help' ? 0 : 2;
|
|
8
|
+
} else {
|
|
9
|
+
try {
|
|
10
|
+
const chunks = [];
|
|
11
|
+
let bytes = 0;
|
|
12
|
+
for await (const chunk of process.stdin) {
|
|
13
|
+
bytes += chunk.length;
|
|
14
|
+
if (bytes > 16 * 1024 * 1024) throw new Error('Input exceeds 16 MiB');
|
|
15
|
+
chunks.push(chunk);
|
|
16
|
+
}
|
|
17
|
+
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
18
|
+
process.stdout.write(JSON.stringify(await operations[process.argv[2]](input), null, 2) + '\n');
|
|
19
|
+
} catch (error) {
|
|
20
|
+
process.stderr.write(JSON.stringify({ ok: false, error: error.message }) + '\n');
|
|
21
|
+
process.exitCode = 1;
|
|
22
|
+
}
|
|
23
|
+
}
|
package/src/adapters.mjs
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// MetaHarness adapters are data-only: no provider requests, trainer launch or activation.
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { exportTrainingData, twoStagePlan } from '@metaharness/weight-eft';
|
|
4
|
+
import { runContrastive, digestDocument } from '@metaharness/darwin/contrastive';
|
|
5
|
+
|
|
6
|
+
const hash = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
7
|
+
const require = (condition, message) => { if (!condition) throw new Error(message); };
|
|
8
|
+
const identifier = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value);
|
|
9
|
+
|
|
10
|
+
/** Export reviewed caller-asserted trajectories; held-out instance IDs are always excluded. */
|
|
11
|
+
export function exportArchive({ archive, evalHoldout, maxTokens = 12000 }) {
|
|
12
|
+
require(Array.isArray(archive) && archive.length <= 10000, 'Archive limit is 10000 trajectories');
|
|
13
|
+
require(Array.isArray(evalHoldout) && evalHoldout.every(identifier), 'Explicit eval holdout IDs required');
|
|
14
|
+
require(Number.isInteger(maxTokens) && maxTokens >= 1 && maxTokens <= 32768, 'Invalid token budget');
|
|
15
|
+
require(Buffer.byteLength(JSON.stringify(archive)) <= 16 * 1024 * 1024, 'Archive exceeds 16 MiB');
|
|
16
|
+
for (const row of archive) {
|
|
17
|
+
require(row && identifier(row.instance_id) && typeof row.model === 'string' && row.model.length > 0,
|
|
18
|
+
'Invalid trajectory identity');
|
|
19
|
+
require(['cheap', 'frontier'].includes(row.tier) && typeof row.resolved === 'boolean', 'Invalid trajectory outcome');
|
|
20
|
+
require(typeof row.model_patch === 'string' && Array.isArray(row.messages) && row.messages.length <= 4096,
|
|
21
|
+
'Invalid trajectory content');
|
|
22
|
+
for (const message of row.messages) {
|
|
23
|
+
require(message && ['system', 'user', 'assistant', 'tool'].includes(message.role)
|
|
24
|
+
&& (typeof message.content === 'string' || message.content === null), 'Invalid message');
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const result = exportTrainingData(archive, {
|
|
28
|
+
evalHoldout, maxTokens, truncateOverLength: false, dropRewardHacked: true,
|
|
29
|
+
});
|
|
30
|
+
return {
|
|
31
|
+
schema: 'ruvllm-training-export/v1', ...result,
|
|
32
|
+
archiveSha256: hash(archive), holdoutSha256: hash(evalHoldout),
|
|
33
|
+
evidenceOrigin: 'caller-asserted-trajectories', trainingExecuted: false,
|
|
34
|
+
productionActivation: false,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Upstream command is informational; this checkout has no `ruvllm microlora` CLI. */
|
|
39
|
+
export function trainingPlan({ base, sftPath, dpoPath, adapterPrefix }) {
|
|
40
|
+
require(base && typeof base.id === 'string' && Number.isFinite(base.paramsB), 'Invalid model');
|
|
41
|
+
for (const path of [sftPath, dpoPath, adapterPrefix]) require(typeof path === 'string' && path.length > 0, 'Path required');
|
|
42
|
+
return {
|
|
43
|
+
schema: 'ruvllm-training-plan/v1',
|
|
44
|
+
plans: twoStagePlan(base, sftPath, dpoPath, adapterPrefix),
|
|
45
|
+
executionStatus: 'not_executed',
|
|
46
|
+
runnerCompatibility: 'adapter-required',
|
|
47
|
+
reason: 'The upstream ruvllm microlora command is not implemented by this standalone checkout. Use a separately validated trainer; DPO requires an additional trainer implementation.',
|
|
48
|
+
productionActivation: false,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Replay bounded measured scores through Darwin. This ranks development evidence only. */
|
|
53
|
+
export async function selectDevelopmentCandidate({ baseline, candidates, envelope }) {
|
|
54
|
+
require(envelope && typeof envelope === 'object' && !Array.isArray(envelope), 'Immutable experiment envelope required');
|
|
55
|
+
require(Array.isArray(candidates) && candidates.length > 0 && candidates.length <= 32, 'Expected 1..32 candidates');
|
|
56
|
+
const all = [baseline, ...candidates];
|
|
57
|
+
for (const candidate of all) {
|
|
58
|
+
require(candidate && identifier(candidate.id) && Number.isFinite(candidate.score)
|
|
59
|
+
&& candidate.score >= 0 && candidate.score <= 1
|
|
60
|
+
&& /^[a-f0-9]{64}$/.test(candidate.evidenceSha256), 'Invalid measured candidate');
|
|
61
|
+
}
|
|
62
|
+
require(new Set(all.map(row => row.id)).size === all.length, 'Duplicate candidate ID');
|
|
63
|
+
const byId = new Map(all.map(row => [row.id, { ...row }]));
|
|
64
|
+
const best = candidates.reduce((a, b) => b.score > a.score ? b : a);
|
|
65
|
+
const document = { components: [{ id: 'weight-candidate', scope: 'domain', content: { id: baseline.id } }] };
|
|
66
|
+
const result = await runContrastive({
|
|
67
|
+
tenantId: 'local', projectId: 'ruvllm', task: { purpose: 'development-candidate-selection' },
|
|
68
|
+
environment: { mode: 'evidence-replay' },
|
|
69
|
+
library: { tenantId: 'local', projectId: 'ruvllm', entries: [] },
|
|
70
|
+
envelope, baseline: document,
|
|
71
|
+
budget: { maxUsd: 0, maxCalls: 5, timeoutMs: 5000, probeUsd: 0, proposeUsd: 0, composeUsd: 0 },
|
|
72
|
+
evaluator: {
|
|
73
|
+
surfaceAware: true,
|
|
74
|
+
async probe({ candidate }) {
|
|
75
|
+
const row = byId.get(candidate.components[0].content.id);
|
|
76
|
+
require(row, 'Unmeasured candidate');
|
|
77
|
+
return { appliedDigest: digestDocument(candidate), score: row.score, costUsd: 0,
|
|
78
|
+
observations: [{ id: 'measured-failure', componentId: 'weight-candidate', message: `Bound evidence ${row.evidenceSha256}` }] };
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
proposer: {
|
|
82
|
+
async propose({ scope }) {
|
|
83
|
+
return { edits: scope === 'domain' ? [{ componentId: 'weight-candidate', content: { id: best.id },
|
|
84
|
+
citations: ['measured-failure'] }] : [], costUsd: 0 };
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
return { schema: 'ruvllm-development-selection/v1', selectedId: result.candidate.components[0].content.id,
|
|
89
|
+
envelopeSha256: hash(envelope), evidenceSha256: hash(all), evidenceMode: 'score-replay',
|
|
90
|
+
result, independentConfirmation: false, productionActivation: false };
|
|
91
|
+
}
|
package/src/cli.mjs
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { runExperiment } from './engine.mjs';
|
|
3
|
+
import { createFixture } from './fixture.mjs';
|
|
4
|
+
|
|
5
|
+
const [command, ...args] = process.argv.slice(2);
|
|
6
|
+
function options(allowed) {
|
|
7
|
+
const result = {};
|
|
8
|
+
for (let i = 0; i < args.length; i += 2) {
|
|
9
|
+
if (!allowed.includes(args[i]) || !args[i + 1] || Object.hasOwn(result, args[i])) throw new Error('Unknown, duplicate or missing argument');
|
|
10
|
+
result[args[i]] = args[i + 1];
|
|
11
|
+
}
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
let result;
|
|
16
|
+
if (command === 'run') {
|
|
17
|
+
const flags = options(['--config', '--plan']);
|
|
18
|
+
result = await runExperiment({ configPath: flags['--config'], planPath: flags['--plan'] });
|
|
19
|
+
} else if (command === 'fixture') {
|
|
20
|
+
const flags = options(['--dir', '--binary']);
|
|
21
|
+
if (!flags['--dir'] || !flags['--binary']) throw new Error('fixture requires --dir and --binary');
|
|
22
|
+
result = await runExperiment(await createFixture({ directory: flags['--dir'], binaryPath: flags['--binary'] }));
|
|
23
|
+
} else if (command === '--help' || command === 'help') {
|
|
24
|
+
console.log('Usage: ruvllm-experiment run --config ABS --plan ABS\n ruvllm-experiment fixture --dir NEW_ABS --binary ABS');
|
|
25
|
+
process.exit(0);
|
|
26
|
+
} else throw new Error('Expected run, fixture or --help');
|
|
27
|
+
console.log(JSON.stringify(result, null, 2));
|
|
28
|
+
} catch (error) {
|
|
29
|
+
console.error(JSON.stringify({ ok: false, error: error.message, productionActivation: false })); process.exitCode = 1;
|
|
30
|
+
}
|
package/src/engine.mjs
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { constants } from 'node:fs';
|
|
4
|
+
import { open, lstat, realpath, mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
5
|
+
import { resolve, dirname, join, relative, isAbsolute, parse } from 'node:path';
|
|
6
|
+
import { meetsPromotionRule, gateFingerprint } from '@metaharness/flywheel';
|
|
7
|
+
import { selectDevelopmentCandidate } from './adapters.mjs';
|
|
8
|
+
|
|
9
|
+
const MAX_JSON = 1024 * 1024;
|
|
10
|
+
const HASH = /^[a-f0-9]{64}$/;
|
|
11
|
+
const inside = (root, path) => path === root || (!relative(root, path).startsWith('..') && !isAbsolute(relative(root, path)));
|
|
12
|
+
const assert = (condition, message) => { if (!condition) throw new Error(message); };
|
|
13
|
+
export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
14
|
+
|
|
15
|
+
function keys(value, required, optional = []) {
|
|
16
|
+
assert(value && typeof value === 'object' && !Array.isArray(value), 'Expected JSON object');
|
|
17
|
+
assert(required.every(k => Object.hasOwn(value, k)), `Missing fields: ${required.join(', ')}`);
|
|
18
|
+
assert(Object.keys(value).every(k => [...required, ...optional].includes(k)), 'Unknown object fields');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Reject duplicate object keys and non-finite numbers before applying schemas.
|
|
22
|
+
export function parseJson(bytes) {
|
|
23
|
+
const text = bytes.toString('utf8'); let index = 0;
|
|
24
|
+
const ws = () => { while (/[ \t\r\n]/.test(text[index] ?? '') && index < text.length) index++; };
|
|
25
|
+
function string() {
|
|
26
|
+
const start = index++; let escaped = false;
|
|
27
|
+
while (index < text.length) {
|
|
28
|
+
const c = text[index++];
|
|
29
|
+
if (c === '"' && !escaped) return JSON.parse(text.slice(start, index));
|
|
30
|
+
if (c === '\\' && !escaped) escaped = true; else escaped = false;
|
|
31
|
+
}
|
|
32
|
+
throw new Error('Unterminated JSON string');
|
|
33
|
+
}
|
|
34
|
+
function value(depth = 0) {
|
|
35
|
+
assert(depth < 64, 'JSON nesting limit'); ws();
|
|
36
|
+
if (text[index] === '"') return string();
|
|
37
|
+
if (text[index] === '{') {
|
|
38
|
+
index++; ws(); const result = Object.create(null); const seen = new Set();
|
|
39
|
+
if (text[index] === '}') { index++; return result; }
|
|
40
|
+
while (true) {
|
|
41
|
+
ws(); assert(text[index] === '"', 'Invalid JSON key'); const key = string();
|
|
42
|
+
assert(!seen.has(key), 'Duplicate JSON key'); seen.add(key); ws();
|
|
43
|
+
assert(text[index++] === ':', 'Invalid JSON separator'); result[key] = value(depth + 1); ws();
|
|
44
|
+
const end = text[index++]; if (end === '}') return result; assert(end === ',', 'Invalid JSON object');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (text[index] === '[') {
|
|
48
|
+
index++; ws(); const result = []; if (text[index] === ']') { index++; return result; }
|
|
49
|
+
while (true) { result.push(value(depth + 1)); ws(); const end = text[index++]; if (end === ']') return result; assert(end === ',', 'Invalid JSON array'); }
|
|
50
|
+
}
|
|
51
|
+
const token = /^(?:true|false|null|-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)/.exec(text.slice(index));
|
|
52
|
+
assert(token, 'Invalid JSON value'); index += token[0].length; const result = JSON.parse(token[0]);
|
|
53
|
+
assert(typeof result !== 'number' || Number.isFinite(result), 'Non-finite JSON number'); return result;
|
|
54
|
+
}
|
|
55
|
+
const result = value(); ws(); assert(index === text.length, 'Trailing JSON data'); return result;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function checkedPath(path, { directory = false, roots } = {}) {
|
|
59
|
+
assert(typeof path === 'string' && isAbsolute(path) && !path.includes('\0'), 'Expected absolute path');
|
|
60
|
+
assert(path === resolve(path), 'Path must be canonical without traversal');
|
|
61
|
+
let cursor = parse(path).root;
|
|
62
|
+
for (const part of path.slice(cursor.length).split('/').filter(Boolean)) {
|
|
63
|
+
cursor = join(cursor, part); const stat = await lstat(cursor); assert(!stat.isSymbolicLink(), 'Symlinks are forbidden');
|
|
64
|
+
}
|
|
65
|
+
const canonical = await realpath(path); const stat = await lstat(canonical);
|
|
66
|
+
assert(directory ? stat.isDirectory() : stat.isFile(), directory ? 'Expected directory' : 'Expected regular file');
|
|
67
|
+
if (roots) assert(roots.some(root => inside(root, canonical)), 'Path outside configured read roots');
|
|
68
|
+
return canonical;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function boundedRead(path, limit = MAX_JSON) {
|
|
72
|
+
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
73
|
+
try {
|
|
74
|
+
const stat = await file.stat(); assert(stat.isFile() && stat.size <= limit, 'Control file too large or not regular');
|
|
75
|
+
const bytes = Buffer.alloc(stat.size); const { bytesRead } = await file.read(bytes, 0, bytes.length, 0);
|
|
76
|
+
assert(bytesRead === bytes.length, 'Control file changed during read'); return bytes;
|
|
77
|
+
} finally { await file.close(); }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export async function fileDigest(path) {
|
|
81
|
+
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
82
|
+
try {
|
|
83
|
+
assert((await file.stat()).isFile(), 'Expected regular file');
|
|
84
|
+
const hash = createHash('sha256'); const bytes = Buffer.alloc(1024 * 1024); let position = 0;
|
|
85
|
+
while (true) { const { bytesRead } = await file.read(bytes, 0, bytes.length, position); if (!bytesRead) break; hash.update(bytes.subarray(0, bytesRead)); position += bytesRead; }
|
|
86
|
+
return hash.digest('hex');
|
|
87
|
+
} finally { await file.close(); }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function identicalFiles(a, b) {
|
|
91
|
+
const first = await open(a, constants.O_RDONLY | constants.O_NOFOLLOW), second = await open(b, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
92
|
+
try {
|
|
93
|
+
const size = (await first.stat()).size; if (size !== (await second.stat()).size) return false;
|
|
94
|
+
const x = Buffer.alloc(Math.min(size, 1024 * 1024)), y = Buffer.alloc(x.length);
|
|
95
|
+
for (let position = 0; position < size; position += x.length) {
|
|
96
|
+
const length = Math.min(x.length, size - position);
|
|
97
|
+
const [rx, ry] = await Promise.all([first.read(x, 0, length, position), second.read(y, 0, length, position)]);
|
|
98
|
+
if (rx.bytesRead !== length || ry.bytesRead !== length || !x.subarray(0, length).equals(y.subarray(0, length))) return false;
|
|
99
|
+
}
|
|
100
|
+
return true;
|
|
101
|
+
} finally { await first.close(); await second.close(); }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function save(path, value) { await writeFile(path, JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); }
|
|
105
|
+
|
|
106
|
+
async function runtime(configPath) {
|
|
107
|
+
configPath = await checkedPath(configPath); const bytes = await boundedRead(configPath); const config = parseJson(bytes);
|
|
108
|
+
keys(config, ['schema', 'binaryPath', 'binarySha256', 'readRoots', 'writeRoot', 'policySha256s', 'maxCandidates', 'timeoutMs']);
|
|
109
|
+
assert(config.schema === 'ruvllm-experiment-runtime/v1', 'Unsupported runtime schema');
|
|
110
|
+
assert(HASH.test(config.binarySha256), 'Invalid binary digest');
|
|
111
|
+
assert(Array.isArray(config.readRoots) && config.readRoots.length > 0 && config.readRoots.length <= 32, 'Invalid read roots');
|
|
112
|
+
assert(Array.isArray(config.policySha256s) && config.policySha256s.length > 0 && config.policySha256s.length <= 128 && config.policySha256s.every(x => HASH.test(x)), 'Invalid approved policy digests');
|
|
113
|
+
assert(Number.isInteger(config.maxCandidates) && config.maxCandidates >= 1 && config.maxCandidates <= 16, 'Candidate limit must be 1..16');
|
|
114
|
+
assert(Number.isInteger(config.timeoutMs) && config.timeoutMs >= 100 && config.timeoutMs <= 300000, 'Invalid timeout');
|
|
115
|
+
config.readRoots = await Promise.all(config.readRoots.map(path => checkedPath(path, { directory: true })));
|
|
116
|
+
config.writeRoot = await checkedPath(config.writeRoot, { directory: true });
|
|
117
|
+
config.binaryPath = await checkedPath(config.binaryPath);
|
|
118
|
+
assert(!inside(config.writeRoot, configPath) && !inside(config.writeRoot, config.binaryPath), 'Runtime config and binary must be outside write root');
|
|
119
|
+
const stat = await lstat(configPath); assert((stat.mode & 0o077) === 0, 'Runtime config must be owner-only');
|
|
120
|
+
assert(((await lstat(config.writeRoot)).mode & 0o077) === 0, 'Write root must be private');
|
|
121
|
+
assert(await fileDigest(config.binaryPath) === config.binarySha256, 'Binary digest mismatch');
|
|
122
|
+
return { ...config, configSha256: sha256(bytes) };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function native(config, request) {
|
|
126
|
+
assert(await fileDigest(config.binaryPath) === config.binarySha256, 'Binary digest mismatch');
|
|
127
|
+
const input = JSON.stringify(request); assert(Buffer.byteLength(input) <= 64 * 1024, 'Native request limit');
|
|
128
|
+
return new Promise((accept, reject) => {
|
|
129
|
+
const environment = Object.fromEntries(['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL'].filter(k => process.env[k]).map(k => [k, process.env[k]]));
|
|
130
|
+
const child = spawn(config.binaryPath, [], { stdio: ['pipe', 'pipe', 'pipe'], env: environment, shell: false });
|
|
131
|
+
let stdout = [], stderr = [], outSize = 0, errSize = 0, failure;
|
|
132
|
+
const fail = message => { failure ??= new Error(message); child.kill('SIGKILL'); };
|
|
133
|
+
const timer = setTimeout(() => fail('Native operation timed out'), config.timeoutMs);
|
|
134
|
+
child.stdout.on('data', bytes => { outSize += bytes.length; if (outSize > MAX_JSON) fail('Native stdout limit'); else stdout.push(bytes); });
|
|
135
|
+
child.stderr.on('data', bytes => { errSize += bytes.length; if (errSize > 64 * 1024) fail('Native stderr limit'); else stderr.push(bytes); });
|
|
136
|
+
child.on('error', error => { clearTimeout(timer); reject(error); });
|
|
137
|
+
child.stdin.on('error', () => {});
|
|
138
|
+
child.on('close', code => {
|
|
139
|
+
clearTimeout(timer); if (failure) return reject(failure);
|
|
140
|
+
if (code !== 0) return reject(new Error(`Native ${request.op} rejected: ${Buffer.concat(stderr).toString('utf8').slice(0, 2048)}`));
|
|
141
|
+
try { accept(parseJson(Buffer.concat(stdout))); } catch (error) { reject(error); }
|
|
142
|
+
});
|
|
143
|
+
child.stdin.end(input);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export async function runExperiment({ configPath, planPath }) {
|
|
148
|
+
const config = await runtime(configPath); planPath = await checkedPath(planPath, { roots: config.readRoots });
|
|
149
|
+
const planBytes = await boundedRead(planPath), plan = parseJson(planBytes);
|
|
150
|
+
keys(plan, ['schema', 'evidenceMode', 'source', 'policy', 'candidates']);
|
|
151
|
+
assert(plan.schema === 'ruvllm-experiment-plan/v1', 'Unsupported plan schema');
|
|
152
|
+
assert(['fixture', 'external'].includes(plan.evidenceMode), 'Explicit fixture or external evidence mode required');
|
|
153
|
+
assert(Array.isArray(plan.candidates) && plan.candidates.length >= 1 && plan.candidates.length <= config.maxCandidates, 'Candidate limit exceeded');
|
|
154
|
+
const source = await checkedPath(plan.source, { roots: config.readRoots });
|
|
155
|
+
const policyPath = await checkedPath(plan.policy, { roots: config.readRoots });
|
|
156
|
+
const policyBytes = await boundedRead(policyPath), policy = parseJson(policyBytes), policySha256 = sha256(policyBytes);
|
|
157
|
+
assert(config.policySha256s.includes(policySha256), 'Policy is not approved');
|
|
158
|
+
assert(Array.isArray(policy.allowed_tensors) && policy.allowed_tensors.length > 0 && policy.allowed_tensors.length <= 256, 'Invalid tensor selection');
|
|
159
|
+
const ids = new Set(), candidates = [];
|
|
160
|
+
for (const candidate of plan.candidates) {
|
|
161
|
+
keys(candidate, ['id', 'patch'], ['evaluation']);
|
|
162
|
+
assert(typeof candidate.id === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/.test(candidate.id) && !ids.has(candidate.id), 'Invalid or duplicate candidate id'); ids.add(candidate.id);
|
|
163
|
+
const patchPath = await checkedPath(candidate.patch, { roots: config.readRoots }), patchBytes = await boundedRead(patchPath);
|
|
164
|
+
const entry = { ...candidate, patchBytes, patch: parseJson(patchBytes) };
|
|
165
|
+
if (candidate.evaluation) { const path = await checkedPath(candidate.evaluation, { roots: config.readRoots }); entry.evaluationBytes = await boundedRead(path); entry.evaluationValue = parseJson(entry.evaluationBytes); }
|
|
166
|
+
candidates.push(entry);
|
|
167
|
+
}
|
|
168
|
+
const runDirectory = join(config.writeRoot, `run-${randomUUID()}`); await mkdir(runDirectory, { mode: 0o700 });
|
|
169
|
+
await writeFile(join(runDirectory, 'plan.json'), planBytes, { flag: 'wx', mode: 0o600 });
|
|
170
|
+
const copiedPolicy = join(runDirectory, 'policy.json'); await writeFile(copiedPolicy, policyBytes, { flag: 'wx', mode: 0o600 });
|
|
171
|
+
const inspection = await native(config, { op: 'inspect', source, tensors: policy.allowed_tensors });
|
|
172
|
+
const reportPath = join(runDirectory, 'report.json');
|
|
173
|
+
const report = { schema: 'ruvllm-experiment-report/v1', evidenceMode: plan.evidenceMode, sourceSha256: inspection.source_sha256,
|
|
174
|
+
policySha256, planSha256: sha256(planBytes), binarySha256: config.binarySha256, configSha256: config.configSha256,
|
|
175
|
+
gateSha256: gateFingerprint(meetsPromotionRule), runDirectory, reportPath, productionActivation: false, activationAuthorized: false,
|
|
176
|
+
inferencePerformed: false, independentConfirmation: false, evaluationTrust: 'caller-supplied; not independent attestation', costMetered: false, candidates: [], selectedCandidateId: null };
|
|
177
|
+
let sharedEvaluation, baselineScore;
|
|
178
|
+
for (const candidate of candidates) {
|
|
179
|
+
const output = join(runDirectory, candidate.id); await mkdir(output, { mode: 0o700 });
|
|
180
|
+
const patch = join(output, 'patch.json'), checkpoint = join(output, 'candidate.safetensors'), receiptPath = join(output, 'receipt.json');
|
|
181
|
+
await writeFile(patch, candidate.patchBytes, { flag: 'wx', mode: 0o600 });
|
|
182
|
+
const result = { id: candidate.id, status: 'rejected', patchSha256: sha256(candidate.patchBytes), rollbackExact: false, rollbackVerified: false, rvfVerified: false, productionActivation: false };
|
|
183
|
+
let receipt;
|
|
184
|
+
try {
|
|
185
|
+
assert(candidate.patch.source_sha256 === report.sourceSha256, 'Patch source mismatch');
|
|
186
|
+
receipt = await native(config, { op: 'patch', source, output: checkpoint, patch, policy: copiedPolicy });
|
|
187
|
+
assert(receipt.source_sha256 === report.sourceSha256 && receipt.policy_sha256 === policySha256 && receipt.patch_sha256 === result.patchSha256 && receipt.activation_authorized === false, 'Native receipt binding mismatch');
|
|
188
|
+
await save(receiptPath, receipt); result.candidateSha256 = receipt.candidate_sha256; result.status = 'unqualified';
|
|
189
|
+
let evaluationPath;
|
|
190
|
+
if (candidate.evaluationBytes) {
|
|
191
|
+
evaluationPath = join(output, 'evaluation.json'); await writeFile(evaluationPath, candidate.evaluationBytes, { flag: 'wx', mode: 0o600 });
|
|
192
|
+
const value = candidate.evaluationValue;
|
|
193
|
+
result.evaluationSha256 = sha256(candidate.evaluationBytes);
|
|
194
|
+
// All candidates must use the same suite, baseline, runtime and protocol.
|
|
195
|
+
const signature = JSON.stringify([value.suite_sha256, value.protocol_sha256, value.baseline, value.total, value.baseline_correct, value.baseline_truncated, value.baseline_output_sha256]);
|
|
196
|
+
const validation = await native(config, { op: 'evaluate', evaluation: evaluationPath, receipt: receiptPath });
|
|
197
|
+
assert(validation.valid === true && validation.activation_authorized === false, 'Evaluation not valid');
|
|
198
|
+
if (sharedEvaluation) assert(sharedEvaluation === signature, 'Candidates evaluated against different baselines or suites'); else sharedEvaluation = signature;
|
|
199
|
+
const total = value.total;
|
|
200
|
+
assert(Number.isInteger(total) && total > 0, 'Evaluation total must be positive');
|
|
201
|
+
const score = (correct, truncated) => ({ primary: correct / total, noopRate: truncated / total, costPerWin: 0, regressed: false });
|
|
202
|
+
baselineScore = value.baseline_correct / total;
|
|
203
|
+
const gate = meetsPromotionRule({ baseline: score(value.baseline_correct, value.baseline_truncated), candidate: score(value.candidate_correct, value.candidate_truncated) });
|
|
204
|
+
if (value.candidate_correct <= value.baseline_correct) { gate.promote = false; gate.reasons.push('no_strict_accuracy_improvement'); }
|
|
205
|
+
result.gate = { ...gate, meaning: 'research qualification only; no activation authority' };
|
|
206
|
+
result.score = score(value.candidate_correct, value.candidate_truncated);
|
|
207
|
+
result.status = gate.promote ? 'qualified' : 'unqualified';
|
|
208
|
+
} else result.gate = { promote: false, reasons: ['missing_evaluation'] };
|
|
209
|
+
const artifact = join(output, 'evidence.rvf');
|
|
210
|
+
const exported = await native(config, { op: 'export', patch, policy: copiedPolicy, receipt: receiptPath, output: artifact, ...(evaluationPath ? { evaluation: evaluationPath } : {}) });
|
|
211
|
+
const verified = await native(config, { op: 'verify', artifact, expected_sha256: exported.sha256 });
|
|
212
|
+
assert(verified.integrity_verified === true && verified.activation_authorized === false, 'RVF verification failed'); result.rvfSha256 = exported.sha256; result.rvfVerified = true;
|
|
213
|
+
} catch (error) { result.status = 'rejected'; result.error = error.message; }
|
|
214
|
+
finally {
|
|
215
|
+
if (receipt) {
|
|
216
|
+
const restored = join(output, 'restored.safetensors');
|
|
217
|
+
const rollback = await native(config, { op: 'rollback', source: checkpoint, output: restored, patch, policy: copiedPolicy, receipt: receiptPath });
|
|
218
|
+
result.restoredSha256 = rollback.candidate_sha256;
|
|
219
|
+
result.rollbackExact = result.rollbackVerified = rollback.candidate_sha256 === report.sourceSha256 && await identicalFiles(source, restored);
|
|
220
|
+
assert(result.rollbackExact, 'Rollback did not restore original bytes');
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
report.candidates.push(result);
|
|
224
|
+
}
|
|
225
|
+
assert(await fileDigest(source) === report.sourceSha256, 'Source checkpoint changed during experiment');
|
|
226
|
+
const qualified = report.candidates.filter(candidate => candidate.status === 'qualified');
|
|
227
|
+
qualified.sort((a, b) => b.score.primary - a.score.primary || a.score.noopRate - b.score.noopRate || a.id.localeCompare(b.id));
|
|
228
|
+
if (qualified.length) {
|
|
229
|
+
report.developmentSelection = await selectDevelopmentCandidate({
|
|
230
|
+
baseline: { id: 'source.baseline', score: baselineScore, evidenceSha256: report.sourceSha256 },
|
|
231
|
+
candidates: qualified.map(candidate => ({ id: candidate.id, score: candidate.score.primary, evidenceSha256: candidate.evaluationSha256 })),
|
|
232
|
+
envelope: Object.fromEntries(['sourceSha256', 'policySha256', 'binarySha256', 'gateSha256', 'planSha256'].map(key => [key, report[key]])),
|
|
233
|
+
});
|
|
234
|
+
assert(qualified.some(candidate => candidate.id === report.developmentSelection.selectedId), 'Darwin selected an unqualified candidate');
|
|
235
|
+
report.selectedCandidateId = report.developmentSelection.selectedId;
|
|
236
|
+
}
|
|
237
|
+
await save(reportPath, report); return report;
|
|
238
|
+
}
|
package/src/fixture.mjs
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { fileDigest, sha256, checkedPath } from './engine.mjs';
|
|
4
|
+
|
|
5
|
+
// This deterministic fixture measures contracts and native I/O, never model quality.
|
|
6
|
+
export async function createFixture({ directory, binaryPath }) {
|
|
7
|
+
directory = resolve(directory); binaryPath = await checkedPath(resolve(binaryPath));
|
|
8
|
+
await mkdir(directory, { mode: 0o700 });
|
|
9
|
+
const inputs = join(directory, 'inputs'), outputs = join(directory, 'outputs');
|
|
10
|
+
await mkdir(inputs, { mode: 0o700 }); await mkdir(outputs, { mode: 0o700 });
|
|
11
|
+
async function json(name, value) {
|
|
12
|
+
const path = join(inputs, name), bytes = Buffer.from(JSON.stringify(value, null, 2) + '\n');
|
|
13
|
+
await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); return { path, sha256: sha256(bytes) };
|
|
14
|
+
}
|
|
15
|
+
const rawHeader = JSON.stringify({ 'layer.weight': { dtype: 'BF16', shape: [4], data_offsets: [0, 8] } });
|
|
16
|
+
const header = Buffer.from(rawHeader + ' '.repeat((8 - Buffer.byteLength(rawHeader) % 8) % 8));
|
|
17
|
+
const size = Buffer.alloc(8); size.writeBigUInt64LE(BigInt(header.length));
|
|
18
|
+
const values = Buffer.alloc(8); [0x3f80, 0x4000, 0x4040, 0x4080].forEach((value, i) => values.writeUInt16LE(value, i * 2));
|
|
19
|
+
const original = Buffer.concat([size, header, values]), source = join(inputs, 'original.safetensors');
|
|
20
|
+
await writeFile(source, original, { flag: 'wx', mode: 0o600 });
|
|
21
|
+
const candidate = Buffer.from(original); candidate.writeUInt16LE(0x4020, candidate.length - 6);
|
|
22
|
+
const policy = await json('policy.json', { allowed_tensors: ['layer.weight'], max_changed_values: 1, max_l2: 0.5, max_abs_delta: 0.5, max_relative_l2: 1, protected: [] });
|
|
23
|
+
const patch = await json('patch.json', { schema: 'ruvllm.weight_patch.v1', source_sha256: sha256(original), edits: [{ tensor: 'layer.weight', index: 1, old_bits: 0x4000, new_bits: 0x4020 }] });
|
|
24
|
+
const binding = { runtime_sha256: await fileDigest(binaryPath), topology_sha256: '2'.repeat(64), protocol_sha256: '3'.repeat(64), toolchain_sha256: '4'.repeat(64) };
|
|
25
|
+
const evaluation = await json('fixture-evaluation.json', {
|
|
26
|
+
schema: 'ruvllm.weight_evaluation.v1',
|
|
27
|
+
baseline: { ...binding, model_generation: 0, model_sha256: sha256(original), patch_sha256: '0'.repeat(64) },
|
|
28
|
+
candidate: { ...binding, model_generation: 1, model_sha256: sha256(candidate), patch_sha256: patch.sha256 },
|
|
29
|
+
suite_sha256: '5'.repeat(64), protocol_sha256: binding.protocol_sha256,
|
|
30
|
+
baseline_output_sha256: '6'.repeat(64), candidate_output_sha256: '7'.repeat(64),
|
|
31
|
+
evaluator_identity: 'synthetic-fixture-evaluator', proposer_identity: 'synthetic-fixture-proposer',
|
|
32
|
+
total: 2, baseline_correct: 0, candidate_correct: 1, baseline_truncated: 1, candidate_truncated: 0,
|
|
33
|
+
});
|
|
34
|
+
const plan = await json('plan.json', { schema: 'ruvllm-experiment-plan/v1', evidenceMode: 'fixture', source, policy: policy.path, candidates: [{ id: 'fixture-improvement', patch: patch.path, evaluation: evaluation.path }, { id: 'unevaluated-control', patch: patch.path }] });
|
|
35
|
+
const configPath = join(directory, 'runtime.json');
|
|
36
|
+
await writeFile(configPath, JSON.stringify({ schema: 'ruvllm-experiment-runtime/v1', binaryPath, binarySha256: binding.runtime_sha256, readRoots: [inputs], writeRoot: outputs, policySha256s: [policy.sha256], maxCandidates: 16, timeoutMs: 30000 }, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
|
|
37
|
+
return { configPath, planPath: plan.path };
|
|
38
|
+
}
|