claude-flow 3.20.0 → 3.21.0

package/v3/@claude-flow/cli/dist/src/services/swarm-memory-branches.js

@@ -0,0 +1,213 @@
+/**
+ * SwarmMemoryBranches — per-agent Copy-On-Write memory branching for swarms.
+ *
+ * ## Why this exists
+ *
+ * The v3.14.4 release uncovered a tarball-bloat regression: the Darwin
+ * git-worktree-per-agent pattern accumulated 3.3 GB of disk because each
+ * agent got a *full copy* of shared state. `agenticow` (a sibling RVF-based
+ * COW vector store by the same author) forks a branch in a measured **162
+ * bytes** regardless of base size — read-through semantics (parent ∪ edits,
+ * child wins) instead of a linear-growth snapshot.
+ *
+ * This service is the swarm-facing consumer of that primitive. The pattern
+ * mirrors `agenticow`'s own `examples/parallel-agents`:
+ *
+ *   shared base .rvf
+ *     → each agent forks a 162-byte COW branch (nativeAnn:true) at spawn
+ *     → the agent reads/writes its branch in isolation
+ *     → on success: promote branch → base (merge its edits back)
+ *     → on failure: discard the branch file (throw the edits away)
+ *
+ * ## Honest scope (the seam)
+ *
+ * The current `agent_spawn` MCP path (src/mcp-tools/agent-tools.ts) stores an
+ * agent as pure JSON metadata — it does **not** create, copy, or hold any
+ * per-agent `.rvf` workspace today. So there is no full-copy for a COW branch
+ * to "replace" inline. This service therefore wires in as an **opt-in**: a
+ * swarm/agent that actually wants an isolated memory workspace passes a base
+ * memory path, and only then does a branch get forked. Default behavior is
+ * unchanged.
+ *
+ * ## Non-fatal + kill-switch (ADR-150)
+ *
+ *   - `agenticow` is an optional dependency — every method degrades to
+ *     `{ degraded: true }` when it is absent (never throws MODULE_NOT_FOUND).
+ *   - The `CLAUDE_FLOW_NO_COW_MEMORY=1` env var hard-disables branching so an
+ *     operator can kill the feature without a redeploy.
+ *   - agenticow is loaded lazily (dynamic import inside `loadAgenticow`), so
+ *     importing this module does NOT pull agenticow onto the CLI startup path.
+ *
+ * @module @claude-flow/cli/services/swarm-memory-branches
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';
+import { dirname, join, basename } from 'node:path';
+import { getProjectCwd } from '../mcp-tools/types.js';
+import { loadAgenticow, resolveMemoryPath, manifestFor, validateLabel, openWithLineage, } from '../mcp-tools/agenticow-loader.js';
+/** Env var that hard-disables per-agent COW branching (operator kill switch). */
+export const COW_KILL_SWITCH_ENV = 'CLAUDE_FLOW_NO_COW_MEMORY';
+/** True unless the operator set the kill switch. */
+export function cowMemoryEnabled() {
+    const v = process.env[COW_KILL_SWITCH_ENV];
+    return !(v === '1' || v === 'true' || v === 'yes');
+}
+const REGISTRY_REL = join('.claude-flow', 'swarm', 'cow-branches.json');
+/**
+ * Manages the branch→base COW lifecycle for a swarm's agents. One instance per
+ * project cwd; state is persisted to `.claude-flow/swarm/cow-branches.json` so
+ * a branch forked in `branchForAgent` survives to a later `promoteAgent` /
+ * `discardAgent` call in a different process.
+ */
+export class SwarmMemoryBranches {
+    registryPath;
+    /**
+     * @param registryPath Override the registry location (tests point this at a
+     *   temp dir). Defaults to `<cwd>/.claude-flow/swarm/cow-branches.json`.
+     */
+    constructor(registryPath) {
+        this.registryPath = registryPath ?? join(getProjectCwd(), REGISTRY_REL);
+    }
+    // ---- registry persistence -------------------------------------------------
+    loadRegistry() {
+        try {
+            if (existsSync(this.registryPath)) {
+                const parsed = JSON.parse(readFileSync(this.registryPath, 'utf-8'));
+                if (parsed && typeof parsed === 'object' && parsed.branches)
+                    return parsed;
+            }
+        }
+        catch { /* fall through to empty */ }
+        return { version: '1.0.0', branches: {} };
+    }
+    saveRegistry(reg) {
+        mkdirSync(dirname(this.registryPath), { recursive: true });
+        writeFileSync(this.registryPath, JSON.stringify(reg, null, 2), 'utf-8');
+    }
+    /** Look up the branch record for an agent, if one was forked. */
+    getBranch(agentId) {
+        return this.loadRegistry().branches[agentId];
+    }
+    /**
+     * Deterministic per-agent branch path: `<baseDir>/.swarm-cow/<base>.<agentId>.rvf`.
+     * Keeps branch files namespaced next to their base so cleanup is obvious.
+     */
+    branchPathFor(basePath, agentId) {
+        const safeAgent = agentId.replace(/[^A-Za-z0-9_.\-]/g, '_');
+        const baseName = basename(basePath).replace(/\.rvf$/i, '');
+        return join(dirname(basePath), '.swarm-cow', `${baseName}.${safeAgent}.rvf`);
+    }
+    // ---- lifecycle ------------------------------------------------------------
+    /**
+     * Fork a 162-byte COW branch off `base` for `agentId`. The agent then owns
+     * an isolated read/write view (parent ∪ its own edits). Idempotent: a second
+     * call for the same agent returns the existing branch record.
+     *
+     * @param base      Path to the shared base `.rvf` (absolute or cwd-relative).
+     * @param agentId   Agent that owns the branch (used as the COW label).
+     * @param opts.dimension  Required only when `base` does not yet exist.
+     * @param opts.nativeAnn  Fork with the native Rust ANN path (default true —
+     *   agent branches are meant to be queried).
+     */
+    async branchForAgent(base, agentId, opts = {}) {
+        if (!cowMemoryEnabled()) {
+            return { success: true, agentId, degraded: true, reason: 'cow-memory-disabled' };
+        }
+        const api = await loadAgenticow();
+        if (!api)
+            return { success: true, agentId, degraded: true, reason: 'agenticow-not-found' };
+        const label = validateLabel(agentId);
+        const basePath = resolveMemoryPath(base);
+        // Idempotent — a retried spawn must not double-fork.
+        const existing = this.getBranch(agentId);
+        if (existing && existsSync(existing.branchPath)) {
+            return {
+                success: true, agentId,
+                basePath: existing.basePath, branchPath: existing.branchPath, label: existing.label,
+            };
+        }
+        const branchPath = this.branchPathFor(basePath, agentId);
+        mkdirSync(dirname(branchPath), { recursive: true });
+        const nativeAnn = opts.nativeAnn !== false;
+        const baseMem = await openWithLineage(api, basePath, opts.dimension);
+        try {
+            const branch = await baseMem.fork(label, branchPath, { nativeAnn });
+            // Persist lineage manifests so the branch + base reopen with the COW
+            // chain intact — without this, the fork is in-memory only.
+            await branch.save?.(manifestFor(branchPath));
+            await baseMem.save?.(manifestFor(basePath));
+            await branch.close?.();
+        }
+        finally {
+            await baseMem.close?.();
+        }
+        const reg = this.loadRegistry();
+        reg.branches[agentId] = {
+            agentId, basePath, branchPath, label, createdAt: new Date().toISOString(),
+        };
+        this.saveRegistry(reg);
+        return { success: true, agentId, basePath, branchPath, label };
+    }
+    /**
+     * Promote an agent's branch back into its base (merge its edits + tombstones
+     * atomically), then delete the branch file and drop it from the registry.
+     * Call on agent success. No-op (`promoted:false`) when the agent has no
+     * branch.
+     */
+    async promoteAgent(agentId) {
+        const rec = this.getBranch(agentId);
+        if (!rec)
+            return { success: true, agentId, promoted: false, reason: 'no-branch' };
+        const api = await loadAgenticow();
+        if (!api)
+            return { success: true, agentId, promoted: false, degraded: true, reason: 'agenticow-not-found' };
+        if (existsSync(manifestFor(rec.branchPath)) || existsSync(rec.branchPath)) {
+            const branch = await openWithLineage(api, rec.branchPath);
+            const baseMem = await openWithLineage(api, rec.basePath);
+            try {
+                await branch.promote(baseMem);
+                await branch.save?.(manifestFor(rec.branchPath));
+                await baseMem.save?.(manifestFor(rec.basePath));
+            }
+            finally {
+                await branch.close?.();
+                await baseMem.close?.();
+            }
+        }
+        this.removeBranchFiles(rec.branchPath);
+        this.forgetBranch(agentId);
+        return { success: true, agentId, promoted: true, branchPath: rec.branchPath, basePath: rec.basePath };
+    }
+    /**
+     * Discard an agent's branch — delete the branch file + lineage manifest and
+     * drop the registry entry, WITHOUT merging anything into base. Call on agent
+     * failure. No-op (`discarded:false`) when the agent has no branch. Never
+     * touches agenticow (pure filesystem), so it works even in the degraded path.
+     */
+    async discardAgent(agentId) {
+        const rec = this.getBranch(agentId);
+        if (!rec)
+            return { success: true, agentId, discarded: false, reason: 'no-branch' };
+        this.removeBranchFiles(rec.branchPath);
+        this.forgetBranch(agentId);
+        return { success: true, agentId, discarded: true, branchPath: rec.branchPath };
+    }
+    // ---- helpers --------------------------------------------------------------
+    removeBranchFiles(branchPath) {
+        try {
+            rmSync(branchPath, { recursive: true, force: true });
+        }
+        catch { /* best-effort */ }
+        try {
+            rmSync(manifestFor(branchPath), { force: true });
+        }
+        catch { /* best-effort */ }
+    }
+    forgetBranch(agentId) {
+        const reg = this.loadRegistry();
+        if (reg.branches[agentId]) {
+            delete reg.branches[agentId];
+            this.saveRegistry(reg);
+        }
+    }
+}
+//# sourceMappingURL=swarm-memory-branches.js.map

package/v3/@claude-flow/cli/dist/src/services/weight-eft.d.ts

@@ -0,0 +1,305 @@
+/**
+ * weight-eft.ts — Optional-dependency service wrapper around
+ * `@metaharness/weight-eft` (agenticow / ADR-150 weight-eft slice).
+ *
+ * WHAT THIS SHIPS (and what it deliberately does NOT)
+ * ---------------------------------------------------
+ * This service turns ruflo's captured run transcripts into AUDITED TRAINING
+ * DATA + a COST-PARETO measurement + a GPU TRAINING PLAN. It does NOT train a
+ * model and it does NOT "reduce escalation":
+ *   - `runExport`  → SFT (OpenAI chat) + DPO (TRL preference) JSONL + a guard
+ *                    report (contamination / reward-hack / long-context). $0.
+ *   - `runPlan`    → the GPU training plan + the exact `ruvllm microlora`
+ *                    command a GPU host would run. $0 dry-run — never spawns.
+ *   - `runEval`    → the cost-Pareto delta folded from two CascadeOutcome[]. $0.
+ *   - `runRemoteTrain` → an SSH-based remote-GPU invocation. DRY-RUN by default
+ *                    (prints commands + read-only preflight); real compute only
+ *                    behind explicit `execute && yes`. This is the ONLY path
+ *                    that can spend, and it spends on the USER's host, not $0.
+ *
+ * HARD HONESTY RULE (do not overclaim): weight-eft's own `train` never spawns
+ * (its README §Status), and no GPU tune has run here. `resolved` in the
+ * captured archive is a PROXY (ruflo has no SWE-bench gold oracle) — the SFT
+ * data-quality caveat stands. Nothing here claims ruflo "trains a model" as a
+ * $0/local capability.
+ *
+ * ADR-150 GRACEFUL DEGRADATION
+ * ----------------------------
+ * `@metaharness/weight-eft` is an OPTIONAL dependency. Every entry point loads
+ * it via a dynamic import through `loadWeightEft()` and returns a structured
+ * `{ degraded: true }` result when it is absent — never a throw. There is NO
+ * static `import … from '@metaharness/weight-eft'` anywhere in this file; the
+ * types below are LOCAL mirrors so tsc stays green with the package removed.
+ *
+ * @module services/weight-eft
+ */
+import type { RunTranscriptRecord } from '../ruvector/run-transcript-recorder.js';
+export interface WeftToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+export interface WeftChatMessage {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string | null;
+    tool_calls?: WeftToolCall[];
+    tool_call_id?: string;
+    name?: string;
+}
+export type WeftPolicyTier = 'cheap' | 'frontier';
+/** Input contract for the exporter. Mirrors weight-eft's DarwinTrajectory. */
+export interface DarwinTrajectory {
+    instance_id: string;
+    model: string;
+    tier: WeftPolicyTier;
+    resolved: boolean;
+    messages: WeftChatMessage[];
+    model_patch: string;
+    sample?: number;
+    source?: string;
+}
+export interface WeftExportReport {
+    totalTrajectories: number;
+    excludedByHoldout: number;
+    droppedOverLength: number;
+    truncatedOverLength: number;
+    droppedRewardHacked: number;
+    sftRows: number;
+    dpoRows: number;
+    sftInstanceIds: string[];
+    dpoInstanceIds: string[];
+    notes: string[];
+}
+export interface WeftCascadeOutcome {
+    instance_id: string;
+    cheapResolved: boolean;
+    escalated: boolean;
+    resolved: boolean;
+    costUsd: number;
+}
+export interface WeftCostParetoDelta {
+    base: unknown;
+    adapter: unknown;
+    cheapResolveLift: number;
+    escalationRateReduction: number;
+    costPerResolvedReduction: number;
+    resolveRateDelta: number;
+    verdict: string;
+}
+export interface WeftBaseModelSpec {
+    id: string;
+    paramsB: number;
+}
+export interface WeftTrainingPlan {
+    config: unknown;
+    command: string;
+    summary: string;
+}
+/** The subset of the weight-eft module surface this service calls. */
+interface WeightEftApi {
+    exportTrainingData: (t: DarwinTrajectory[], o: {
+        evalHoldout: string[];
+        maxTokens?: number;
+        truncateOverLength?: boolean;
+        dropRewardHacked?: boolean;
+    }) => {
+        sft: {
+            messages: WeftChatMessage[];
+        }[];
+        dpo: unknown[];
+        report: WeftExportReport;
+    };
+    sftToJsonl: (rows: {
+        messages: WeftChatMessage[];
+    }[]) => string;
+    dpoToJsonl: (rows: unknown[]) => string;
+    costParetoDelta: (base: WeftCascadeOutcome[], adapter: WeftCascadeOutcome[]) => WeftCostParetoDelta;
+    twoStagePlan: (base: WeftBaseModelSpec, sftPath: string, dpoPath: string, adapterPrefix: string) => {
+        sft: WeftTrainingPlan;
+        dpo: WeftTrainingPlan;
+    };
+}
+/** Injectable importer so tests can force the degraded path deterministically. */
+export type WeightEftImporter = () => Promise<unknown>;
+/**
+ * Load `@metaharness/weight-eft` (optional dep). Returns the module or `null`
+ * when it is absent/broken. Never throws. The specifier is held in a variable
+ * so tsc does not statically resolve the package (keeps the build green when
+ * the optional dep is removed — ADR-150).
+ */
+export declare function loadWeightEft(importer?: WeightEftImporter): Promise<WeightEftApi | null>;
+export interface ArchiveBuildStats {
+    total: number;
+    resolved: number;
+    byTier: Record<WeftPolicyTier, number>;
+    /** Honest breakdown: how many `resolved` booleans came from which proxy. */
+    byResolvedSource: Record<string, number>;
+    skipped: number;
+}
+export interface ArchiveBuildResult {
+    trajectories: DarwinTrajectory[];
+    stats: ArchiveBuildStats;
+    /** Load-bearing honesty banner surfaced to CLI/report output. */
+    proxyNote: string;
+}
+/**
+ * Map captured ruflo run transcripts to the DarwinTrajectory[] contract the
+ * weight-eft exporter codes against. PURE + synchronous — this is the seam the
+ * archive-builder unit test exercises.
+ *
+ * A record is skipped (not thrown) when it lacks the minimum an exporter needs:
+ * an instance_id and a non-empty messages array. `resolved` is copied verbatim
+ * and its proxy provenance is tallied so the caller can print an honest note.
+ */
+export declare function buildArchiveFromRecords(records: RunTranscriptRecord[]): ArchiveBuildResult;
+export type ExportOutcome = {
+    degraded: false;
+    report: WeftExportReport;
+    sftJsonl: string;
+    dpoJsonl: string;
+    sftRows: number;
+    dpoRows: number;
+} | {
+    degraded: true;
+    reason: string;
+};
+/** Run the weight-eft exporter over a DarwinTrajectory[] archive. $0. */
+export declare function runExport(opts: {
+    archive: DarwinTrajectory[];
+    evalHoldout?: string[];
+    maxTokens?: number;
+    truncateOverLength?: boolean;
+    dropRewardHacked?: boolean;
+    importer?: WeightEftImporter;
+}): Promise<ExportOutcome>;
+export type PlanOutcome = {
+    degraded: false;
+    base: WeftBaseModelSpec;
+    sft: WeftTrainingPlan;
+    dpo: WeftTrainingPlan;
+} | {
+    degraded: true;
+    reason: string;
+};
+/** Emit the two-stage (SFT → on-policy DPO) GPU training plan. $0 dry-run. */
+export declare function runPlan(opts: {
+    base?: WeftBaseModelSpec;
+    sftPath: string;
+    dpoPath: string;
+    adapterPrefix?: string;
+    importer?: WeightEftImporter;
+}): Promise<PlanOutcome>;
+export type EvalOutcome = {
+    degraded: false;
+    delta: WeftCostParetoDelta;
+} | {
+    degraded: true;
+    reason: string;
+};
+/** Fold base + adapter CascadeOutcome[] into the cost-Pareto delta. $0. */
+export declare function runEval(opts: {
+    baseOutcomes: WeftCascadeOutcome[];
+    adapterOutcomes: WeftCascadeOutcome[];
+    importer?: WeightEftImporter;
+}): Promise<EvalOutcome>;
+/** Default cheap-tier tune target — a 7B coder in the tunable [1,14]B band. */
+export declare const DEFAULT_BASE_MODEL: WeftBaseModelSpec;
+export interface RemoteTrainArgs {
+    /** SSH host or tailscale name (parameterized; never hard-coded). */
+    host: string;
+    /** Base model id to tune. Default DEFAULT_BASE_MODEL.id. */
+    base?: string;
+    /** Local path to the exported SFT jsonl. */
+    sftPath: string;
+    /** Local path to the exported DPO jsonl. */
+    dpoPath: string;
+    /** Local dir to fetch the trained LoRA adapter back into. Default .claude-flow/neural. */
+    adapterDir?: string;
+    /** Remote working dir. Default ~/.ruflo-weft/<runId>. */
+    remoteWorkdir?: string;
+    /** SSH user (default: current, i.e. no user@ prefix). */
+    sshUser?: string;
+    /** SSH port. Default 22. */
+    sshPort?: number;
+    /** Stable run id used in remote workdir + adapter names. Default derived. */
+    runId?: string;
+    /** Adapter name prefix. Default 'ruflo-weft'. */
+    adapterPrefix?: string;
+}
+export interface RemoteStep {
+    label: string;
+    argv: string[];
+}
+export interface RemoteTrainPlan {
+    host: string;
+    base: string;
+    runId: string;
+    remoteWorkdir: string;
+    adapterDir: string;
+    sftAdapter: string;
+    dpoAdapter: string;
+    /** Read-only reachability/capability probes (safe to run in dry-run). */
+    preflight: RemoteStep[];
+    /** The mutating steps: rsync data up, train, fetch adapter back. */
+    steps: RemoteStep[];
+    /** Rendered one-line commands for human display. */
+    humanCommands: string[];
+}
+/**
+ * Construct the exact ssh/rsync/ruvllm invocations a remote GPU tune would run.
+ * PURE — no spawning, no filesystem, deterministic. This is the seam the
+ * command-construction unit test exercises. The ruvllm commands mirror the
+ * canonical `ruvllm microlora sft … && ruvllm microlora dpo --init-from …`
+ * plan weight-eft emits, wrapped for SSH execution on the remote host.
+ */
+export declare function buildRemoteTrainInvocation(args: RemoteTrainArgs): RemoteTrainPlan;
+/** Minimal spawn result shape (subset of child_process.SpawnSyncReturns). */
+export interface SpawnLike {
+    status: number | null;
+    stdout?: string;
+    stderr?: string;
+    error?: Error;
+}
+export type SpawnFn = (cmd: string, argv: string[]) => SpawnLike;
+export type RemoteTrainOutcome = {
+    degraded: true;
+    reason: string;
+} | {
+    degraded: false;
+    mode: 'dry-run' | 'refused' | 'preflight-failed' | 'executed';
+    plan: RemoteTrainPlan;
+    preflight?: {
+        label: string;
+        ok: boolean;
+        detail: string;
+    }[];
+    steps?: {
+        label: string;
+        ok: boolean;
+        detail: string;
+    }[];
+    reason?: string;
+};
+/**
+ * Run (or, by default, DRY-RUN) a remote-GPU tune. SAFETY MODEL:
+ *   - DEFAULT (no execute): builds the plan, runs read-only preflight probes,
+ *     and returns the commands that WOULD run. NO rsync of data, NO training.
+ *   - execute && !yes → 'refused' (real GPU spend needs an explicit second gate).
+ *   - execute && yes  → runs preflight; if unreachable → 'preflight-failed'
+ *     WITHOUT training; else runs rsync + ssh ruvllm sft/dpo + fetch-back.
+ *   - No `ssh` binary / host unreachable / any spawn error → structured
+ *     degraded/preflight-failed. NEVER throws.
+ *
+ * `spawn` is injected for testing so CI never touches a live host.
+ */
+export declare function runRemoteTrain(args: RemoteTrainArgs & {
+    execute?: boolean;
+    yes?: boolean;
+    preflight?: boolean;
+    spawn?: SpawnFn;
+}): Promise<RemoteTrainOutcome>;
+export {};
+//# sourceMappingURL=weight-eft.d.ts.map