instar 0.28.83 → 0.28.85

package/dist/core/EvolutionManager.js

@@ -17,16 +17,497 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { SafeFsExecutor } from './SafeFsExecutor.js';
+import { TaskFlowError } from '../tasks/task-flow-types.js';
+/**
+ * TaskFlow controller identity for EvolutionManager (Phase 3a dual-write).
+ * Every proposal's flow is owned by this controllerId; the registry uses it
+ * for OCC scope checks. Single-instance per server process.
+ */
+const EVOLUTION_TASKFLOW_CONTROLLER_ID = 'EvolutionManager';
+function statusToFlowAction(prev, next, resolution) {
+    // Initial create-from-proposed is handled in addProposal directly.
+    if (prev === null && next === 'proposed')
+        return { kind: 'create' };
+    if (prev === next)
+        return { kind: 'noop' };
+    switch (next) {
+        case 'approved':
+            return { kind: 'start', step: 'approved' };
+        case 'in_progress':
+            return { kind: 'start', step: 'in_progress' };
+        case 'implemented':
+            return { kind: 'finish' };
+        case 'rejected':
+            return { kind: 'fail', reason: resolution ?? 'rejected' };
+        case 'deferred':
+            return { kind: 'cancel', reason: resolution ?? 'deferred' };
+        case 'proposed':
+            // Re-opening is not a legal transition; treat as no-op for v1.
+            return { kind: 'noop' };
+        default:
+            return { kind: 'noop' };
+    }
+}
+/**
+ * Owner-key for a proposal-flow. Deterministic — used as both the TaskFlow
+ * `ownerKey` and the basis of the `idempotencyKey`. Phase 3b's cutover
+ * gate relies on `evolution:cluster:<id>` being a stable bidirectional join.
+ */
+function ownerKeyForProposal(proposalId) {
+    return `evolution:cluster:${proposalId}`;
+}
 export class EvolutionManager {
     stateDir;
     config;
     trustElevationTracker = null;
     autonomousEvolution = null;
     autonomyManager = null;
+    // ── TaskFlow Phase 3a dual-write fields ──────────────────────────
+    /** Registry handle (set when TaskFlow is enabled). */
+    taskFlowRegistry = null;
+    /** Per-process controller-instance id (server uuid). */
+    taskFlowControllerInstanceId = null;
+    /**
+     * Set to true by DivergenceChecker when state-divergence is detected.
+     * When true, EvolutionManager continues JSON-only writes and DOES NOT
+     * dual-write to TaskFlow. The signal is reset on next divergence pass that
+     * reports zero divergence. This is a signal-emitted brake, not a hard gate.
+     */
+    taskFlowShadowWritesHalted = false;
+    /** Most recent reason for halting; surfaced in /health and tests. */
+    taskFlowShadowWritesHaltReason = null;
+    /**
+     * Source of the most recent halt. DivergenceChecker uses this to ensure it
+     * only auto-clears halts that DivergenceChecker itself imposed — an operator
+     * or sibling system's manual halt stays in place until that source clears it.
+     */
+    taskFlowShadowWritesHaltSource = null;
+    // ── WikiClaim Evidence Phase 2 fields ───────────────────────────
+    /**
+     * SemanticMemory handle. When wired, EvolutionManager creates a `pattern`
+     * MemoryEntity for each new proposal (the cluster) and populates evidence
+     * rows linking the cluster to its constituent feedback IDs / commit SHAs /
+     * session IDs. JSON state remains the source of truth for proposals; the
+     * cluster entity is a parallel surface for inverse-traceability queries.
+     *
+     * See docs/specs/OPENCLAW-IMPORT-WIKICLAIM-EVIDENCE-SPEC.md § Producers.
+     */
+    semanticMemory = null;
     constructor(config) {
         this.config = config;
         this.stateDir = config.stateDir;
     }
+    // ── TaskFlow wiring ─────────────────────────────────────────────
+    /**
+     * Wire a TaskFlow registry for Phase 3a dual-write. After this call:
+     *   - addProposal / updateProposalStatus shadow-write to TaskFlow
+     *   - migrateExistingToTaskFlow() backfills any in-flight clusters
+     *   - DivergenceChecker can call setShadowWritesHalted() to brake writes
+     *
+     * Idempotent — safe to call multiple times; subsequent calls overwrite.
+     */
+    setTaskFlowRegistry(registry, controllerInstanceId) {
+        this.taskFlowRegistry = registry;
+        this.taskFlowControllerInstanceId = controllerInstanceId;
+    }
+    getTaskFlowRegistry() {
+        return this.taskFlowRegistry;
+    }
+    /**
+     * Halt or resume TaskFlow shadow-writes. Called by DivergenceChecker on a
+     * mismatch. Signal-vs-authority compliance: this is a signal-consumed brake,
+     * not a brittle blocker — the divergence checker decides, EvolutionManager
+     * obeys. JSON writes continue regardless (read-authoritative TaskFlow is
+     * shadow-mode in Phase 3a).
+     *
+     * The optional `source` parameter tags the halt with its origin so an
+     * automatic clearer (DivergenceChecker) only cancels halts it caused.
+     * Defaults to `'manual'` when omitted.
+     */
+    setShadowWritesHalted(halted, reason, source) {
+        this.taskFlowShadowWritesHalted = halted;
+        this.taskFlowShadowWritesHaltReason = halted ? (reason ?? 'divergence-detected') : null;
+        this.taskFlowShadowWritesHaltSource = halted ? (source ?? 'manual') : null;
+    }
+    isShadowWritesHalted() {
+        return {
+            halted: this.taskFlowShadowWritesHalted,
+            reason: this.taskFlowShadowWritesHaltReason,
+            source: this.taskFlowShadowWritesHaltSource,
+        };
+    }
+    // ── WikiClaim Evidence Phase 2 wiring ───────────────────────────
+    /**
+     * Wire a SemanticMemory handle for cluster evidence emission. After this
+     * call, `addProposal()` creates a `pattern` MemoryEntity for the cluster
+     * with evidence linking to the proposal source, and `addClusterEvidence()`
+     * appends evidence atomically as more inputs join the cluster.
+     *
+     * Idempotent — safe to call multiple times; subsequent calls overwrite.
+     * Producers emit signals (evidence rows); they do NOT gate proposal flow.
+     *
+     * See docs/specs/OPENCLAW-IMPORT-WIKICLAIM-EVIDENCE-SPEC.md § Producers.
+     */
+    setSemanticMemory(memory) {
+        this.semanticMemory = memory;
+    }
+    getSemanticMemory() {
+        return this.semanticMemory;
+    }
+    /**
+     * Append evidence to the cluster MemoryEntity for an existing proposal.
+     * Used when more feedback / commits / sessions join an open cluster.
+     * No-op when SemanticMemory is not wired or when the proposal has no
+     * cluster entity yet (legacy proposals from before Phase 2).
+     *
+     * Allowed kinds (per spec § Producers): `feedback`, `pattern-entity`,
+     * `supersedes-evidence`. Mismatches throw `EvidencePolicyError`.
+     */
+    addClusterEvidence(proposalId, evidence) {
+        if (!this.semanticMemory)
+            return;
+        const state = this.loadEvolution();
+        const proposal = state.proposals.find((p) => p.id === proposalId);
+        if (!proposal || !proposal.entityId)
+            return;
+        // Throws EvidencePolicyError on producer/kind mismatch — surfaced to caller
+        // by design so a buggy emit-site isn't silently dropped.
+        this.semanticMemory.addEvidence(proposal.entityId, evidence, 'EvolutionManager');
+    }
+    /**
+     * Build the initial evidence array for a newly-created cluster from the
+     * proposal's `source` field. Recognized patterns (mirrors backfill rules
+     * in spec § Migration of Existing MemoryEntity Records, restricted to
+     * EvolutionManager's allowlist):
+     *
+     *   - `feedback:<id>` → kind: 'feedback', sourceId: '<id>'
+     *
+     * Unrecognized sources produce no evidence (cluster is created with
+     * `evidence: []`). Subsequent `addClusterEvidence()` calls populate as
+     * inputs join. Empty evidence arrays are valid — `rememberWithEvidence`
+     * with `evidence: []` is functionally equivalent to `remember()` plus the
+     * consolidated JSONL action.
+     */
+    buildInitialClusterEvidence(p) {
+        const now = this.now();
+        const evidence = [];
+        const feedbackMatch = p.source.match(/^feedback:(.+)$/);
+        if (feedbackMatch) {
+            evidence.push({
+                kind: 'feedback',
+                sourceId: feedbackMatch[1],
+                weight: 1.0,
+                confidence: 0.8,
+                // privacyTier omitted — inherits cluster's privacyScope ('shared-project')
+                note: p.title.slice(0, 200),
+                updatedAt: now,
+            });
+        }
+        return evidence;
+    }
+    /**
+     * Best-effort cluster-entity creation for a new proposal. Never throws to
+     * the caller — JSON state remains the source of truth for proposals.
+     * `EvidencePolicyError` (e.g., cap exceeded, kind mismatch) is logged and
+     * swallowed; subsequent `addClusterEvidence()` calls can recover.
+     *
+     * Returns the entity id when created, or null when memory is not wired or
+     * the cluster entity cannot be created.
+     */
+    createClusterEntity(p) {
+        if (!this.semanticMemory)
+            return null;
+        try {
+            const evidence = this.buildInitialClusterEvidence(p);
+            const id = this.semanticMemory.rememberWithEvidence({
+                type: 'pattern',
+                name: p.title.slice(0, 200),
+                content: p.description,
+                confidence: p.impact === 'high' ? 0.85 : p.impact === 'low' ? 0.6 : 0.75,
+                lastVerified: p.proposedAt,
+                source: `evolution:${p.id}`,
+                tags: ['cluster', 'evolution', p.type, ...(p.tags ?? [])],
+                privacyScope: 'shared-project',
+            }, evidence, 'EvolutionManager');
+            return id;
+        }
+        catch (err) {
+            console.warn(`[EvolutionManager] cluster-entity create failed for ${p.id}:`, err instanceof Error ? err.message : err);
+            return null;
+        }
+    }
+    taskFlowPrincipal() {
+        if (!this.taskFlowRegistry || !this.taskFlowControllerInstanceId)
+            return null;
+        return {
+            scope: 'controller',
+            controllerId: EVOLUTION_TASKFLOW_CONTROLLER_ID,
+            controllerInstanceId: this.taskFlowControllerInstanceId,
+        };
+    }
+    /**
+     * Best-effort dual-write to TaskFlow. NEVER throws to the caller — JSON
+     * remains the local durability path; TaskFlow is shadow until Phase 3b
+     * cutover. All TaskFlow errors land in console and the divergence checker
+     * picks up via state comparison.
+     */
+    async dualWriteCreate(p) {
+        if (!this.taskFlowRegistry || this.taskFlowShadowWritesHalted)
+            return;
+        const principal = this.taskFlowPrincipal();
+        if (!principal || principal.scope !== 'controller')
+            return;
+        try {
+            await this.taskFlowRegistry.createFlow({
+                controllerId: EVOLUTION_TASKFLOW_CONTROLLER_ID,
+                controllerInstanceId: principal.controllerInstanceId,
+                ownerKey: ownerKeyForProposal(p.id),
+                idempotencyKey: `evolution-cluster-create-${p.id}`,
+                goal: p.title.slice(0, 1024),
+                currentStep: 'proposed',
+                stateJson: { proposalId: p.id, type: p.type, source: p.source },
+            });
+        }
+        catch (err) {
+            // Swallow — JSON is the source of truth in Phase 3a.
+            this.logTaskFlowError('createFlow', p.id, err);
+        }
+    }
+    async dualWriteTransition(proposalId, action) {
+        if (!this.taskFlowRegistry || this.taskFlowShadowWritesHalted)
+            return;
+        if (action.kind === 'noop' || action.kind === 'create')
+            return;
+        const principal = this.taskFlowPrincipal();
+        if (!principal)
+            return;
+        const registry = this.taskFlowRegistry;
+        try {
+            // Look up current flow by owner key + deterministic idempotency key.
+            const existing = registry.findByIdempotency(EVOLUTION_TASKFLOW_CONTROLLER_ID, ownerKeyForProposal(proposalId), `evolution-cluster-create-${proposalId}`);
+            if (!existing) {
+                // No backfill yet — proposal exists in JSON but not TaskFlow. Skip
+                // the transition silently; the next migrate-existing pass will pick
+                // it up at its terminal state.
+                return;
+            }
+            const flow = registry.getFlow(existing.flowId, { bypassCache: true });
+            if (!flow)
+                return;
+            const expectedRevision = flow.revision;
+            switch (action.kind) {
+                case 'start': {
+                    if (flow.status !== 'queued' && flow.status !== 'running')
+                        return;
+                    await registry.startStep({
+                        flowId: flow.flowId,
+                        expectedRevision,
+                        principal,
+                        currentStep: action.step,
+                    });
+                    break;
+                }
+                case 'finish': {
+                    // Need flow in `running` state for finishFlow. If currently queued,
+                    // promote first.
+                    let rev = expectedRevision;
+                    let cur = flow;
+                    if (cur.status === 'queued') {
+                        const r = await registry.startStep({
+                            flowId: flow.flowId,
+                            expectedRevision: rev,
+                            principal,
+                            currentStep: 'implementing',
+                        });
+                        cur = r.flow;
+                        rev = r.flow.revision;
+                    }
+                    if (cur.status !== 'running')
+                        return;
+                    await registry.finishFlow({
+                        flowId: flow.flowId,
+                        expectedRevision: rev,
+                        principal,
+                    });
+                    break;
+                }
+                case 'fail': {
+                    let rev = expectedRevision;
+                    let cur = flow;
+                    // failFlow requires running|waiting. Promote queued → running.
+                    if (cur.status === 'queued') {
+                        const r = await registry.startStep({
+                            flowId: flow.flowId,
+                            expectedRevision: rev,
+                            principal,
+                            currentStep: 'reject-transition',
+                        });
+                        cur = r.flow;
+                        rev = r.flow.revision;
+                    }
+                    if (cur.status !== 'running' && cur.status !== 'waiting')
+                        return;
+                    await registry.failFlow({
+                        flowId: flow.flowId,
+                        expectedRevision: rev,
+                        principal,
+                        failureReason: action.reason,
+                    });
+                    break;
+                }
+                case 'cancel': {
+                    if (flow.status === 'succeeded' || flow.status === 'failed' ||
+                        flow.status === 'cancelled' || flow.status === 'lost')
+                        return;
+                    // Two-phase: request, then cancel.
+                    const r = await registry.requestFlowCancel({
+                        flowId: flow.flowId,
+                        expectedRevision,
+                        requesterOrigin: { kind: 'system', id: 'EvolutionManager' },
+                    });
+                    await registry.cancelFlow({
+                        flowId: flow.flowId,
+                        expectedRevision: r.flow.revision,
+                        principal,
+                    });
+                    break;
+                }
+            }
+        }
+        catch (err) {
+            this.logTaskFlowError(`transition:${action.kind}`, proposalId, err);
+        }
+    }
+    logTaskFlowError(op, proposalId, err) {
+        if (err instanceof TaskFlowError) {
+            // OCC conflicts and invalid-transition errors are expected during
+            // races and concurrent updates — log at debug level only.
+            console.warn(`[EvolutionManager] taskflow ${op} for ${proposalId} skipped: ${err.code} (${err.message})`);
+        }
+        else {
+            console.warn(`[EvolutionManager] taskflow ${op} for ${proposalId} unexpected error:`, err instanceof Error ? err.message : err);
+        }
+    }
+    /**
+     * Backfill all in-flight proposals (status !== 'implemented' && !== 'rejected')
+     * into TaskFlow. Idempotent via `findIdempotent` on
+     * `evolution:cluster:<proposalId>` ownerKey. Safe to call multiple times.
+     *
+     * Returns counts: created, alreadyExisted, advanced (transitioned beyond
+     * 'queued' to match the proposal's current status).
+     */
+    async migrateExistingToTaskFlow() {
+        if (!this.taskFlowRegistry) {
+            return { created: 0, alreadyExisted: 0, advanced: 0, skipped: 0 };
+        }
+        const principal = this.taskFlowPrincipal();
+        if (!principal || principal.scope !== 'controller') {
+            return { created: 0, alreadyExisted: 0, advanced: 0, skipped: 0 };
+        }
+        const registry = this.taskFlowRegistry;
+        const state = this.loadEvolution();
+        let created = 0;
+        let alreadyExisted = 0;
+        let advanced = 0;
+        let skipped = 0;
+        for (const p of state.proposals) {
+            try {
+                const existing = registry.findByIdempotency(EVOLUTION_TASKFLOW_CONTROLLER_ID, ownerKeyForProposal(p.id), `evolution-cluster-create-${p.id}`);
+                let flow;
+                if (existing) {
+                    alreadyExisted++;
+                    flow = existing;
+                }
+                else {
+                    const r = await registry.createFlow({
+                        controllerId: EVOLUTION_TASKFLOW_CONTROLLER_ID,
+                        controllerInstanceId: principal.controllerInstanceId,
+                        ownerKey: ownerKeyForProposal(p.id),
+                        idempotencyKey: `evolution-cluster-create-${p.id}`,
+                        goal: p.title.slice(0, 1024),
+                        currentStep: 'proposed',
+                        stateJson: { proposalId: p.id, type: p.type, source: p.source },
+                    });
+                    if (r.created)
+                        created++;
+                    else
+                        alreadyExisted++;
+                    flow = r.flow;
+                }
+                // Catch the flow up to the proposal's current status.
+                const advancedThis = await this.catchUpFlowToStatus(flow, p, principal);
+                if (advancedThis)
+                    advanced++;
+            }
+            catch (err) {
+                skipped++;
+                this.logTaskFlowError('migrate', p.id, err);
+            }
+        }
+        return { created, alreadyExisted, advanced, skipped };
+    }
+    /** Catch a flow record up to the proposal's current status. */
+    async catchUpFlowToStatus(flow, p, principal) {
+        if (!this.taskFlowRegistry)
+            return false;
+        const registry = this.taskFlowRegistry;
+        let cur = registry.getFlow(flow.flowId, { bypassCache: true }) ?? flow;
+        let mutated = false;
+        const target = p.status;
+        // Already terminal — leave alone.
+        if (cur.status === 'succeeded' || cur.status === 'failed' ||
+            cur.status === 'cancelled' || cur.status === 'lost') {
+            return false;
+        }
+        // Promote queued → running if target is running/terminal-success/failed.
+        if (cur.status === 'queued' && (target === 'approved' || target === 'in_progress' ||
+            target === 'implemented' || target === 'rejected' || target === 'deferred')) {
+            const step = target === 'in_progress' ? 'in_progress'
+                : target === 'approved' ? 'approved' : 'catch-up';
+            const r = await registry.startStep({
+                flowId: cur.flowId,
+                expectedRevision: cur.revision,
+                principal,
+                currentStep: step,
+            });
+            cur = r.flow;
+            mutated = true;
+        }
+        // Finalize.
+        if (target === 'implemented' && cur.status === 'running') {
+            await registry.finishFlow({
+                flowId: cur.flowId,
+                expectedRevision: cur.revision,
+                principal,
+            });
+            mutated = true;
+        }
+        else if (target === 'rejected' &&
+            (cur.status === 'running' || cur.status === 'waiting')) {
+            await registry.failFlow({
+                flowId: cur.flowId,
+                expectedRevision: cur.revision,
+                principal,
+                failureReason: p.resolution ?? 'rejected',
+            });
+            mutated = true;
+        }
+        else if (target === 'deferred' &&
+            (cur.status === 'running' || cur.status === 'waiting' || cur.status === 'queued')) {
+            const r = await registry.requestFlowCancel({
+                flowId: cur.flowId,
+                expectedRevision: cur.revision,
+                requesterOrigin: { kind: 'system', id: 'EvolutionManager' },
+            });
+            await registry.cancelFlow({
+                flowId: cur.flowId,
+                expectedRevision: r.flow.revision,
+                principal,
+            });
+            mutated = true;
+        }
+        return mutated;
+    }
     /**
      * Wire adaptive autonomy modules for runtime integration.
      * - TrustElevationTracker: receives proposal approval/rejection events
@@ -141,8 +622,17 @@ export class EvolutionManager {
             proposedAt: this.now(),
             tags: opts.tags,
         };
+        // WikiClaim Phase 2: create cluster MemoryEntity and capture entityId BEFORE
+        // saving JSON, so the persisted proposal carries its cluster reference.
+        // Failure to create the cluster entity is logged and swallowed inside
+        // createClusterEntity — the proposal still ships with `entityId: undefined`.
+        const entityId = this.createClusterEntity(proposal);
+        if (entityId)
+            proposal.entityId = entityId;
         state.proposals.push(proposal);
         this.saveEvolution(state);
+        // TaskFlow Phase 3a shadow-write — best-effort, JSON remains source of truth.
+        void this.dualWriteCreate(proposal);
         return proposal;
     }
     updateProposalStatus(id, status, resolution) {
@@ -150,6 +640,7 @@ export class EvolutionManager {
         const proposal = state.proposals.find(p => p.id === id);
         if (!proposal)
             return false;
+        const prevStatus = proposal.status;
         proposal.status = status;
         if (resolution)
             proposal.resolution = resolution;
@@ -161,6 +652,9 @@ export class EvolutionManager {
             const decision = status === 'approved' ? 'approved' : 'rejected';
             this.trustElevationTracker.recordProposalDecision(proposal, decision, false);
         }
+        // TaskFlow Phase 3a shadow-write — best-effort, JSON remains source of truth.
+        const action = statusToFlowAction(prevStatus, status, resolution);
+        void this.dualWriteTransition(id, action);
         return true;
     }
     /**