claude-flow 3.24.0 → 3.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/.proven-config-version +1 -0
- package/.claude/proven-config.json +42 -0
- package/package.json +1 -1
- package/v3/@claude-flow/cli/dist/src/mcp-tools/neural-tools.js +22 -2
- package/v3/@claude-flow/cli/dist/src/ruvector/lattice-wasm.d.ts +14 -0
- package/v3/@claude-flow/cli/dist/src/ruvector/lattice-wasm.js +144 -0
- package/v3/@claude-flow/cli/dist/src/services/evolve-proof.d.ts +6 -0
- package/v3/@claude-flow/cli/dist/src/services/evolve-proof.js +5 -3
- package/v3/@claude-flow/cli/dist/src/services/harness-flywheel-generations.d.ts +4 -0
- package/v3/@claude-flow/cli/dist/src/services/harness-flywheel-generations.js +7 -1
- package/v3/@claude-flow/cli/dist/src/services/harness-flywheel-runtime.js +5 -14
- package/v3/@claude-flow/cli/dist/src/services/harness-frozen-eval.d.ts +22 -0
- package/v3/@claude-flow/cli/dist/src/services/harness-frozen-eval.js +66 -0
- package/v3/@claude-flow/cli/package.json +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sha256:6141a8ea990c5063b77e090ae8f37f9c539d8aa8f58dcceb30f3a82f97e57319
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"adoptedAt": 1783265769778,
|
|
3
|
+
"championId": "sha256:6141a8ea990c5063b77e090ae8f37f9c539d8aa8f58dcceb30f3a82f97e57319",
|
|
4
|
+
"manifest": {
|
|
5
|
+
"schema": "ruflo.proven-config/v1",
|
|
6
|
+
"policy": {
|
|
7
|
+
"ref": "sha256:6141a8ea990c5063b77e090ae8f37f9c539d8aa8f58dcceb30f3a82f97e57319",
|
|
8
|
+
"value": {
|
|
9
|
+
"alpha": 0.3,
|
|
10
|
+
"subjectWeight": 1,
|
|
11
|
+
"mmrLambda": 0.5,
|
|
12
|
+
"bodyWeight": 1.5,
|
|
13
|
+
"typePenaltyFactor": 0.5
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"layer": "framework/node-cli",
|
|
17
|
+
"compatibility": {
|
|
18
|
+
"ruflo": ">=3.24.0"
|
|
19
|
+
},
|
|
20
|
+
"benchmark": {
|
|
21
|
+
"corpus": "ADR-081-labelled-v1",
|
|
22
|
+
"corpusHash": "sha256:2f700b5c363e20a3bd88ce2bc9b87bbbbaa61732c6177894c6ec37890f888982"
|
|
23
|
+
},
|
|
24
|
+
"receipt": {
|
|
25
|
+
"heldOutDelta": 0.07381404928570845,
|
|
26
|
+
"redblue": "PASS",
|
|
27
|
+
"drift": 0,
|
|
28
|
+
"canary": {
|
|
29
|
+
"rollbackRate": 0,
|
|
30
|
+
"latencyP95": 244.612458000076,
|
|
31
|
+
"costPerTask": 0
|
|
32
|
+
},
|
|
33
|
+
"receiptCoverage": 1
|
|
34
|
+
},
|
|
35
|
+
"platform": [
|
|
36
|
+
"linux",
|
|
37
|
+
"macOS",
|
|
38
|
+
"windows"
|
|
39
|
+
]
|
|
40
|
+
},
|
|
41
|
+
"previous": ""
|
|
42
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-flow",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.25.0",
|
|
4
4
|
"description": "Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -89,14 +89,34 @@ function ensureEmbeddings() {
|
|
|
89
89
|
return embeddingsPromise;
|
|
90
90
|
embeddingsPromise = (async () => {
|
|
91
91
|
try {
|
|
92
|
+
// Tier -1 (PRIMARY): Lattice WASM — real multi-model semantic embeddings
|
|
93
|
+
// (miniLM / bge / paraphrase-miniLM / GPU qwen3-0.6b) ahead of everything.
|
|
94
|
+
// Optional + FAIL-CLOSED: absent/init-fail ⇒ fall straight through to the
|
|
95
|
+
// existing ruvector-ONNX → hash tiers with zero regression.
|
|
96
|
+
try {
|
|
97
|
+
const lat = await import('../ruvector/lattice-wasm.js').catch(() => null);
|
|
98
|
+
if (lat && await lat.latticeAvailable()) {
|
|
99
|
+
const model = lat.DEFAULT_LATTICE_MODEL;
|
|
100
|
+
realEmbeddings = {
|
|
101
|
+
embed: async (text) => {
|
|
102
|
+
const v = await lat.latticeEmbed(text, model);
|
|
103
|
+
if (!v || !v.length)
|
|
104
|
+
throw new Error('lattice embed failed'); // → generateEmbedding falls to next tier
|
|
105
|
+
return v;
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
embeddingServiceName = `lattice-wasm/${model} (${lat.latticeModels().length} model${lat.latticeModels().length === 1 ? '' : 's'})`;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
catch { /* lattice absent — fall through */ }
|
|
92
112
|
// Tier 0: ruvector@0.2.27 — bundled all-MiniLM-L6-v2 + parallel worker pool.
|
|
93
113
|
// Probe with isOnnxAvailable() and verify an actual embed succeeds (avoids
|
|
94
114
|
// the type-load-success-but-runtime-fails trap from ADR-086). The probe now
|
|
95
115
|
// runs on first embed request instead of at import time.
|
|
96
116
|
// NOTE: ruvector's embed() returns `{embedding, dimension, timeMs}` — we
|
|
97
117
|
// unwrap to plain number[] for the shared interface.
|
|
98
|
-
const rv = await import('ruvector').catch(() => null);
|
|
99
|
-
if (rv?.embed && typeof rv.embed === 'function' && rv.isOnnxAvailable?.()) {
|
|
118
|
+
const rv = !realEmbeddings ? await import('ruvector').catch(() => null) : null;
|
|
119
|
+
if (!realEmbeddings && rv?.embed && typeof rv.embed === 'function' && rv.isOnnxAvailable?.()) {
|
|
100
120
|
try {
|
|
101
121
|
if (typeof rv.initOnnxEmbedder === 'function')
|
|
102
122
|
await rv.initOnnxEmbedder();
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** Default package name (ruvnet WASM convention); override with RUFLO_LATTICE_WASM_PKG. */
|
|
2
|
+
export declare const LATTICE_WASM_PKG: string;
|
|
3
|
+
export type LatticeModel = 'minilm' | 'bge' | 'paraphrase-minilm' | 'qwen3-0.6b' | string;
|
|
4
|
+
export declare const DEFAULT_LATTICE_MODEL: LatticeModel;
|
|
5
|
+
/** Is the Lattice WASM embedder installed + initializable? Cached; never throws. */
|
|
6
|
+
export declare function latticeAvailable(): Promise<boolean>;
|
|
7
|
+
/** The models Lattice reports (empty until availability is probed). */
|
|
8
|
+
export declare function latticeModels(): LatticeModel[];
|
|
9
|
+
/**
|
|
10
|
+
* Embed `text` with `model` via Lattice WASM. Returns null on any failure so the
|
|
11
|
+
* caller can fall through to the next tier. Never throws.
|
|
12
|
+
*/
|
|
13
|
+
export declare function latticeEmbed(text: string, model?: LatticeModel): Promise<number[] | null>;
|
|
14
|
+
//# sourceMappingURL=lattice-wasm.d.ts.map
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lattice WASM embedder adapter — the primary embedding tier (ADR: embedding
|
|
3
|
+
* substrate upgrade). Replaces hash placeholders with real semantic embeddings
|
|
4
|
+
* and adds multiple models (miniLM, bge, multi-paraphrase-miniLM, GPU qwen3-0.6B).
|
|
5
|
+
*
|
|
6
|
+
* Built on the proven `@ruvector/ruvllm-wasm` optional-dependency convention:
|
|
7
|
+
* - dynamically imported (never a hard dependency),
|
|
8
|
+
* - WASM initialized from bundled bytes,
|
|
9
|
+
* - fully FAIL-CLOSED: if the package is absent, the WASM fails to init, or the
|
|
10
|
+
* embed API differs, `latticeAvailable()` returns false and callers fall
|
|
11
|
+
* through to the existing ruvector-ONNX → hash tiers with ZERO regression.
|
|
12
|
+
*
|
|
13
|
+
* The package specifier is configurable (`RUFLO_LATTICE_WASM_PKG`) so the exact
|
|
14
|
+
* published name can be set without a code change; the API is probed tolerantly.
|
|
15
|
+
* Opt-in for the GPU model: qwen3 is only selected when explicitly requested.
|
|
16
|
+
*/
|
|
17
|
+
import { createRequire } from 'module';
|
|
18
|
+
import { readFileSync } from 'fs';
|
|
19
|
+
const require_ = createRequire(import.meta.url);
|
|
20
|
+
/** Default package name (ruvnet WASM convention); override with RUFLO_LATTICE_WASM_PKG. */
|
|
21
|
+
export const LATTICE_WASM_PKG = process.env.RUFLO_LATTICE_WASM_PKG || '@ruvector/lattice-wasm';
|
|
22
|
+
export const DEFAULT_LATTICE_MODEL = process.env.RUFLO_EMBED_MODEL || 'minilm';
|
|
23
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
24
|
+
let _mod = null;
|
|
25
|
+
let _ready = false;
|
|
26
|
+
let _probed = false;
|
|
27
|
+
let _available = false;
|
|
28
|
+
let _models = [];
|
|
29
|
+
async function loadModule() {
|
|
30
|
+
if (_mod)
|
|
31
|
+
return _mod;
|
|
32
|
+
const spec = LATTICE_WASM_PKG;
|
|
33
|
+
_mod = await import(spec).catch(() => null); // optional — absent ⇒ null ⇒ unavailable
|
|
34
|
+
return _mod;
|
|
35
|
+
}
|
|
36
|
+
async function ensureInit() {
|
|
37
|
+
if (_ready)
|
|
38
|
+
return true;
|
|
39
|
+
const mod = await loadModule();
|
|
40
|
+
if (!mod)
|
|
41
|
+
return false;
|
|
42
|
+
try {
|
|
43
|
+
// ruvnet WASM convention: initSync({ module: <wasm bytes> }); tolerate variants.
|
|
44
|
+
if (!_ready && typeof mod.initSync === 'function') {
|
|
45
|
+
let wasmBytes;
|
|
46
|
+
for (const cand of ['lattice_wasm_bg.wasm', 'lattice_bg.wasm', 'index_bg.wasm']) {
|
|
47
|
+
try {
|
|
48
|
+
wasmBytes = readFileSync(require_.resolve(`${LATTICE_WASM_PKG}/${cand}`));
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
catch { /* try next */ }
|
|
52
|
+
}
|
|
53
|
+
mod.initSync(wasmBytes ? { module: wasmBytes } : undefined);
|
|
54
|
+
}
|
|
55
|
+
else if (typeof mod.default === 'function') {
|
|
56
|
+
await mod.default(); // some wasm-bindgen builds export an async default init
|
|
57
|
+
}
|
|
58
|
+
_ready = true;
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return false; // init failed ⇒ fail closed
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Extract a plain number[] from whatever shape the module's embed returns. */
|
|
66
|
+
function toVec(r) {
|
|
67
|
+
const v = (r && typeof r === 'object' && 'embedding' in r) ? r.embedding : r;
|
|
68
|
+
if (!v)
|
|
69
|
+
return null;
|
|
70
|
+
if (Array.isArray(v))
|
|
71
|
+
return v;
|
|
72
|
+
if (v.length !== undefined)
|
|
73
|
+
return Array.from(v);
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
/** Is the Lattice WASM embedder installed + initializable? Cached; never throws. */
|
|
77
|
+
export async function latticeAvailable() {
|
|
78
|
+
if (_probed)
|
|
79
|
+
return _available;
|
|
80
|
+
_probed = true;
|
|
81
|
+
try {
|
|
82
|
+
if (!(await ensureInit())) {
|
|
83
|
+
_available = false;
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
const mod = _mod;
|
|
87
|
+
// discover models (tolerant): listModels() / models / MODELS
|
|
88
|
+
try {
|
|
89
|
+
const list = typeof mod.listModels === 'function' ? mod.listModels() : (mod.models ?? mod.MODELS);
|
|
90
|
+
if (Array.isArray(list) && list.length)
|
|
91
|
+
_models = list;
|
|
92
|
+
}
|
|
93
|
+
catch { /* leave default */ }
|
|
94
|
+
if (!_models.length)
|
|
95
|
+
_models = [DEFAULT_LATTICE_MODEL];
|
|
96
|
+
// verify an actual embed succeeds (the ADR-086 "loads but runtime-fails" trap).
|
|
97
|
+
const probe = await latticeEmbedRaw('probe', DEFAULT_LATTICE_MODEL);
|
|
98
|
+
_available = !!probe && probe.length > 0;
|
|
99
|
+
return _available;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
_available = false;
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/** The models Lattice reports (empty until availability is probed). */
|
|
107
|
+
export function latticeModels() { return [..._models]; }
|
|
108
|
+
async function latticeEmbedRaw(text, model) {
|
|
109
|
+
const mod = _mod;
|
|
110
|
+
if (!mod)
|
|
111
|
+
return null;
|
|
112
|
+
// tolerant API probing across plausible wasm-bindgen surfaces.
|
|
113
|
+
const attempts = [
|
|
114
|
+
() => typeof mod.embed === 'function' ? mod.embed(text, model) : undefined,
|
|
115
|
+
() => typeof mod.embed === 'function' ? mod.embed(text) : undefined,
|
|
116
|
+
() => typeof mod.embedText === 'function' ? mod.embedText(text, model) : undefined,
|
|
117
|
+
() => typeof mod.Embedder === 'function' ? new mod.Embedder(model).embed(text) : undefined,
|
|
118
|
+
];
|
|
119
|
+
for (const a of attempts) {
|
|
120
|
+
try {
|
|
121
|
+
const r = await a();
|
|
122
|
+
const v = toVec(r);
|
|
123
|
+
if (v)
|
|
124
|
+
return v;
|
|
125
|
+
}
|
|
126
|
+
catch { /* try next surface */ }
|
|
127
|
+
}
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Embed `text` with `model` via Lattice WASM. Returns null on any failure so the
|
|
132
|
+
* caller can fall through to the next tier. Never throws.
|
|
133
|
+
*/
|
|
134
|
+
export async function latticeEmbed(text, model = DEFAULT_LATTICE_MODEL) {
|
|
135
|
+
try {
|
|
136
|
+
if (!(await latticeAvailable()))
|
|
137
|
+
return null;
|
|
138
|
+
return await latticeEmbedRaw(text, model);
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
//# sourceMappingURL=lattice-wasm.js.map
|
|
@@ -42,6 +42,7 @@ export interface ChangeDeltas {
|
|
|
42
42
|
benchmark: number;
|
|
43
43
|
security: number;
|
|
44
44
|
cost: number;
|
|
45
|
+
humanRelevance?: number;
|
|
45
46
|
}
|
|
46
47
|
/**
|
|
47
48
|
* Causal promotion record (not just provenance). Answers *why* a candidate won —
|
|
@@ -85,6 +86,7 @@ export interface EvolveReceiptBundle {
|
|
|
85
86
|
mutationClass: string;
|
|
86
87
|
mutationSummary: string;
|
|
87
88
|
deltas: ChangeDeltas;
|
|
89
|
+
humanEvalHash?: string;
|
|
88
90
|
promotion: PromotionRecord | null;
|
|
89
91
|
regression: RegressionRecord | null;
|
|
90
92
|
holdout: HoldoutTask[];
|
|
@@ -109,6 +111,8 @@ export interface AssembleOpts {
|
|
|
109
111
|
redblue?: 'PASS' | 'FAIL' | 'SKIPPED';
|
|
110
112
|
drift?: number;
|
|
111
113
|
canaryRollbackRate?: number;
|
|
114
|
+
humanRelevanceDelta?: number;
|
|
115
|
+
humanEvalHash?: string;
|
|
112
116
|
layer?: string;
|
|
113
117
|
corpus?: string;
|
|
114
118
|
}
|
|
@@ -137,6 +141,8 @@ export declare function runRealEvolveRound(opts: {
|
|
|
137
141
|
redblue?: 'PASS' | 'FAIL' | 'SKIPPED';
|
|
138
142
|
drift?: number;
|
|
139
143
|
canaryRollbackRate?: number;
|
|
144
|
+
humanRelevanceDelta?: number;
|
|
145
|
+
humanEvalHash?: string;
|
|
140
146
|
corpus: string;
|
|
141
147
|
}): EvolveReceiptBundle;
|
|
142
148
|
/**
|
|
@@ -95,7 +95,7 @@ export function assembleBundle(baseline, candidate, holdout, o) {
|
|
|
95
95
|
state: 'shadow', served: false, candidateManifestHash, registeredAt: o.now,
|
|
96
96
|
} : null;
|
|
97
97
|
const { mutationClass, mutationSummary } = classifyMutation(baseline, candidate);
|
|
98
|
-
const deltas = { benchmark: candidateHeldOut - baselineHeldOut, security: o.redblue === 'FAIL' ? -1 : 0, cost: 0 };
|
|
98
|
+
const deltas = { benchmark: candidateHeldOut - baselineHeldOut, security: o.redblue === 'FAIL' ? -1 : 0, cost: 0, humanRelevance: o.humanRelevanceDelta };
|
|
99
99
|
const promotion = promoted ? { parentManifestHash: o.parent, candidateManifestHash, mutationClass, mutationSummary, deltas, decisionReceipt } : null;
|
|
100
100
|
const regression = promoted ? null : {
|
|
101
101
|
candidateManifestHash, ancestor: o.parent ?? baselineManifestHash, mutationClass,
|
|
@@ -107,7 +107,7 @@ export function assembleBundle(baseline, candidate, holdout, o) {
|
|
|
107
107
|
meetsPromotionRule: { version: PROMOTION_RULE_VERSION, result: promoted },
|
|
108
108
|
decisionReceipt, shadow,
|
|
109
109
|
costReceipt: { usd: 0, llmCalls: 0, tier: o.cost.tier, notes: o.cost.notes },
|
|
110
|
-
mutationClass, mutationSummary, deltas, promotion, regression,
|
|
110
|
+
mutationClass, mutationSummary, deltas, humanEvalHash: o.humanEvalHash, promotion, regression,
|
|
111
111
|
holdout, baselineManifest, candidateManifest,
|
|
112
112
|
};
|
|
113
113
|
}
|
|
@@ -122,7 +122,9 @@ export function runRealEvolveRound(opts) {
|
|
|
122
122
|
return assembleBundle(opts.baseline, opts.candidate, opts.holdout, {
|
|
123
123
|
generation: opts.generation, parent: opts.parent, branch: opts.branch ?? 'main', now: opts.now, kind: 'real',
|
|
124
124
|
cost: { tier: 'real-local', notes: 'measured on the frozen anchor via live retrieval — no LLM, no network' },
|
|
125
|
-
redblue: opts.redblue, drift: opts.drift, canaryRollbackRate: opts.canaryRollbackRate,
|
|
125
|
+
redblue: opts.redblue, drift: opts.drift, canaryRollbackRate: opts.canaryRollbackRate,
|
|
126
|
+
humanRelevanceDelta: opts.humanRelevanceDelta, humanEvalHash: opts.humanEvalHash,
|
|
127
|
+
layer: 'real/retrieval', corpus: opts.corpus,
|
|
126
128
|
});
|
|
127
129
|
}
|
|
128
130
|
/**
|
|
@@ -12,6 +12,7 @@ export interface GenerationDeps {
|
|
|
12
12
|
getPatterns: () => HarvestPattern[] | Promise<HarvestPattern[]>;
|
|
13
13
|
search: (q: string, cfg: RetrievalConfig) => Promise<RankedItem[]> | RankedItem[];
|
|
14
14
|
anchorTasks: AnchorTask[];
|
|
15
|
+
humanEvalHash?: string;
|
|
15
16
|
sample?: number;
|
|
16
17
|
now: number;
|
|
17
18
|
applyFn?: (cfg: Record<string, number>, hash: string, generation: number) => void;
|
|
@@ -100,6 +101,9 @@ export interface FlywheelStatus {
|
|
|
100
101
|
plateau: PlateauReport;
|
|
101
102
|
mutation: MutationStat[];
|
|
102
103
|
axisEffectiveness: AxisStat[];
|
|
104
|
+
cumulativeBenchmarkDelta: number;
|
|
105
|
+
cumulativeHumanRelevanceDelta: number;
|
|
106
|
+
humanEvalHash: string | null;
|
|
103
107
|
served: ServedState;
|
|
104
108
|
champion: {
|
|
105
109
|
config: Record<string, number>;
|
|
@@ -267,7 +267,10 @@ export async function runFlywheelGeneration(root, deps) {
|
|
|
267
267
|
const canaryRollbackRate = CANARY.length ? cRoll / CANARY.length : 0;
|
|
268
268
|
const candAnchor = await anchorMean(cand);
|
|
269
269
|
const redblue = candAnchor >= baseAnchor - ANCHOR_TOL ? 'PASS' : 'FAIL';
|
|
270
|
-
|
|
270
|
+
// per-generation HUMAN-RELEVANCE delta on the frozen eval set (anti-overfitting
|
|
271
|
+
// visibility): if this stays ~0 while benchmark Δ > 0, the loop is overfitting.
|
|
272
|
+
const humanRelevanceDelta = candAnchor - baseAnchor;
|
|
273
|
+
const bundle = runRealEvolveRound({ baseline: baseline, candidate: cand, holdout, generation, parent, branch: 'main', now: deps.now, redblue, canaryRollbackRate, humanRelevanceDelta, humanEvalHash: deps.humanEvalHash, corpus: FROZEN_CORPUS });
|
|
271
274
|
appendAttempt(root, bundle);
|
|
272
275
|
if (bundle.decisionReceipt.promoted)
|
|
273
276
|
appendPromotion(root, bundle);
|
|
@@ -347,6 +350,9 @@ export function flywheelStatus(root) {
|
|
|
347
350
|
plateau: detectPlateau(attempts.length ? attempts : promotions, { window: 5 }),
|
|
348
351
|
mutation: mutationEffectiveness(attempts.length ? attempts : promotions),
|
|
349
352
|
axisEffectiveness: axisEffectiveness(promotions),
|
|
353
|
+
cumulativeBenchmarkDelta: promotions.reduce((s, b) => s + (b.deltas.benchmark ?? 0), 0),
|
|
354
|
+
cumulativeHumanRelevanceDelta: promotions.reduce((s, b) => s + (b.deltas.humanRelevance ?? 0), 0),
|
|
355
|
+
humanEvalHash: promotions.length ? (promotions[promotions.length - 1].humanEvalHash ?? null) : null,
|
|
350
356
|
served: servedChampion(root),
|
|
351
357
|
champion: { config: champ.config, hash: champ.hash },
|
|
352
358
|
};
|
|
@@ -7,19 +7,7 @@
|
|
|
7
7
|
import { harnessLoopOptedIn } from './harness-worker.js';
|
|
8
8
|
import { runFlywheelTick } from './harness-flywheel.js';
|
|
9
9
|
import { runFlywheelGeneration, checkServedChampionDrift } from './harness-flywheel-generations.js';
|
|
10
|
-
|
|
11
|
-
const GEN_ANCHOR = [
|
|
12
|
-
['how was the Opus model alias fixed', ['opus 4.8', 'opus alias', 'opus model alias', '#2232']],
|
|
13
|
-
['self-learning wiring task-completed pretrain', ['self-learning', 'adr-074', '#2245', 'task-completed']],
|
|
14
|
-
['deterministic codemod engine var-to-const', ['deterministic tier-1 codemod', 'adr-143', 'codemod', 'var-to-const']],
|
|
15
|
-
['MCP server orphan leak parent-death', ['mcp orphan', 'parent-death', '#2234', 'orphan on every claude']],
|
|
16
|
-
['unified learning stats aggregator', ['unified learning-stats', 'adr-075']],
|
|
17
|
-
['structured distillation 4-field schema', ['structured distillation', 'adr-076', '4-field schema']],
|
|
18
|
-
['SQL injection migrate.ts table identifier', ['sql injection', 'migrate.ts', 'agentdb', 'cve']],
|
|
19
|
-
['recall@k HNSW benchmark harness', ['hnsw', 'recall@k', 'benchmark intelligence']],
|
|
20
|
-
['Q-learning encoder keyword block', ['q-state encoder', 'keyword block', '#2239', 'q-encoder']],
|
|
21
|
-
['security hardening crypto random IDs', ['cwe-347', 'crypto.randomuuid', 'random id', 'crypto random']],
|
|
22
|
-
].map(([q, labels], i) => ({ id: `anchor-${i}`, q: q, labels: labels }));
|
|
10
|
+
import { loadFrozenHumanEval } from './harness-frozen-eval.js';
|
|
23
11
|
/** The human-labeled ADR-081 anchor — the never-regress relevance set. */
|
|
24
12
|
const ANCHOR = [
|
|
25
13
|
['how was the Opus model alias fixed', ['opus 4.8', 'opus alias', 'opus model alias', '#2232']],
|
|
@@ -79,13 +67,16 @@ export async function runFlywheelGenerationWorker(projectRoot, opts = {}) {
|
|
|
79
67
|
const tool = neural.neuralTools.find((t) => t.name === 'neural_patterns');
|
|
80
68
|
if (!tool)
|
|
81
69
|
return { ran: false, reason: 'neural_patterns tool unavailable', generation: 0 };
|
|
70
|
+
// Load the FROZEN, hashed, public human eval set (throws if it has drifted).
|
|
71
|
+
const frozen = loadFrozenHumanEval();
|
|
82
72
|
const deps = {
|
|
83
73
|
getPatterns: () => neural.getStorePatterns(),
|
|
84
74
|
search: async (query, cfg) => {
|
|
85
75
|
const r = await tool.handler({ action: 'search', query, mode: 'hybrid', limit: 5, rerank: false, ...cfg });
|
|
86
76
|
return (r.results || []).slice(0, 5).map((m) => ({ id: m?.id ?? '', name: m?.name ?? '' }));
|
|
87
77
|
},
|
|
88
|
-
anchorTasks:
|
|
78
|
+
anchorTasks: frozen.tasks,
|
|
79
|
+
humanEvalHash: frozen.corpusHash,
|
|
89
80
|
sample: opts.sample ?? 120,
|
|
90
81
|
now: opts.now ?? Date.now(),
|
|
91
82
|
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export declare const FROZEN_HUMAN_EVAL_VERSION = "human-relevance-frozen-v1";
|
|
2
|
+
export declare const FROZEN_HUMAN_EVAL_FILE: string;
|
|
3
|
+
/** Pinned canonical hash of the frozen eval tasks — the tamper-evidence anchor. */
|
|
4
|
+
export declare const FROZEN_HUMAN_EVAL_HASH = "sha256:6096e48ef8f2182e0f00348a953f0f00fe0415575b300234fe2316f37b768200";
|
|
5
|
+
export interface HumanEvalTask {
|
|
6
|
+
id: string;
|
|
7
|
+
q: string;
|
|
8
|
+
labels: string[];
|
|
9
|
+
}
|
|
10
|
+
export interface FrozenHumanEval {
|
|
11
|
+
version: string;
|
|
12
|
+
tasks: HumanEvalTask[];
|
|
13
|
+
corpusHash: string;
|
|
14
|
+
}
|
|
15
|
+
/** Canonical content hash over the tasks — order-independent (tasks sorted by id). */
|
|
16
|
+
export declare function humanEvalHash(tasks: HumanEvalTask[]): string;
|
|
17
|
+
/**
|
|
18
|
+
* Load + verify the frozen human eval set. Throws if missing or if its content
|
|
19
|
+
* hash != the pinned FROZEN_HUMAN_EVAL_HASH (the "frozen" guarantee).
|
|
20
|
+
*/
|
|
21
|
+
export declare function loadFrozenHumanEval(): FrozenHumanEval;
|
|
22
|
+
//# sourceMappingURL=harness-frozen-eval.d.ts.map
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Frozen public human-labeled eval set (ADR-176 — anti-overfitting).
|
|
3
|
+
*
|
|
4
|
+
* The flywheel's compounding gains are measured on a self-supervised retrieval
|
|
5
|
+
* benchmark, which could overfit — improving self-retrieval while human
|
|
6
|
+
* relevance stays merely *preserved*, not improved. The defense is a SINGLE,
|
|
7
|
+
* FROZEN, PUBLIC, hashed human-labeled eval set (`.claude/eval/…-v1.json`):
|
|
8
|
+
* - it is the red/blue anchor the flywheel must never regress, AND
|
|
9
|
+
* - the set against which a per-generation HUMAN-RELEVANCE DELTA is recorded
|
|
10
|
+
* in every receipt — so "self-retrieval up, human relevance flat" is
|
|
11
|
+
* observable in the lineage, not hidden.
|
|
12
|
+
*
|
|
13
|
+
* `loadFrozenHumanEval()` verifies the file's content hash against a PINNED
|
|
14
|
+
* constant and throws on mismatch — the set cannot silently drift; changing it
|
|
15
|
+
* means shipping a new versioned file, not editing this one.
|
|
16
|
+
*/
|
|
17
|
+
import * as fs from 'fs';
|
|
18
|
+
import * as path from 'path';
|
|
19
|
+
import { fileURLToPath } from 'url';
|
|
20
|
+
import { createRequire } from 'module';
|
|
21
|
+
import { createHash } from 'node:crypto';
|
|
22
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
export const FROZEN_HUMAN_EVAL_VERSION = 'human-relevance-frozen-v1';
|
|
24
|
+
export const FROZEN_HUMAN_EVAL_FILE = path.join('.claude', 'eval', `${FROZEN_HUMAN_EVAL_VERSION}.json`);
|
|
25
|
+
/** Pinned canonical hash of the frozen eval tasks — the tamper-evidence anchor. */
|
|
26
|
+
export const FROZEN_HUMAN_EVAL_HASH = 'sha256:6096e48ef8f2182e0f00348a953f0f00fe0415575b300234fe2316f37b768200';
|
|
27
|
+
function canon(v) {
|
|
28
|
+
return Array.isArray(v) ? v.map(canon)
|
|
29
|
+
: (v && typeof v === 'object') ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, canon(v[k])])) : v;
|
|
30
|
+
}
|
|
31
|
+
/** Canonical content hash over the tasks — order-independent (tasks sorted by id). */
|
|
32
|
+
export function humanEvalHash(tasks) {
|
|
33
|
+
const sorted = [...tasks].sort((a, b) => a.id.localeCompare(b.id));
|
|
34
|
+
return 'sha256:' + createHash('sha256').update(JSON.stringify(canon(sorted))).digest('hex');
|
|
35
|
+
}
|
|
36
|
+
function locate() {
|
|
37
|
+
const candidates = [];
|
|
38
|
+
try {
|
|
39
|
+
const req = createRequire(import.meta.url);
|
|
40
|
+
candidates.push(path.join(path.dirname(req.resolve('@claude-flow/cli/package.json')), FROZEN_HUMAN_EVAL_FILE));
|
|
41
|
+
}
|
|
42
|
+
catch { /* not resolvable in this context */ }
|
|
43
|
+
candidates.push(path.resolve(__dirname, '..', '..', '..', FROZEN_HUMAN_EVAL_FILE)); // dist/src/services → pkg root
|
|
44
|
+
candidates.push(path.resolve(__dirname, '..', '..', FROZEN_HUMAN_EVAL_FILE)); // src/services → pkg root
|
|
45
|
+
for (const c of candidates)
|
|
46
|
+
if (fs.existsSync(c))
|
|
47
|
+
return c;
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Load + verify the frozen human eval set. Throws if missing or if its content
|
|
52
|
+
* hash != the pinned FROZEN_HUMAN_EVAL_HASH (the "frozen" guarantee).
|
|
53
|
+
*/
|
|
54
|
+
export function loadFrozenHumanEval() {
|
|
55
|
+
const p = locate();
|
|
56
|
+
if (!p)
|
|
57
|
+
throw new Error(`frozen human eval set not found (${FROZEN_HUMAN_EVAL_FILE})`);
|
|
58
|
+
const parsed = JSON.parse(fs.readFileSync(p, 'utf-8'));
|
|
59
|
+
const tasks = parsed.tasks ?? [];
|
|
60
|
+
const corpusHash = humanEvalHash(tasks);
|
|
61
|
+
if (corpusHash !== FROZEN_HUMAN_EVAL_HASH) {
|
|
62
|
+
throw new Error(`frozen human eval hash mismatch — set has drifted (got ${corpusHash}, pinned ${FROZEN_HUMAN_EVAL_HASH}); supersede with a new versioned file, do not edit`);
|
|
63
|
+
}
|
|
64
|
+
return { version: parsed.version ?? FROZEN_HUMAN_EVAL_VERSION, tasks, corpusHash };
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=harness-frozen-eval.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@claude-flow/cli",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.25.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
|
|
6
6
|
"main": "dist/src/index.js",
|