@sapiom/harness 0.8.9 → 0.10.0

package/dist/server/index.js

@@ -8,7 +8,7 @@
  */
 import { createServer as createHttpServer, } from "node:http";
 import { readFileSync } from "node:fs";
-import { dirname, join, sep } from "node:path";
+import { dirname, join, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import express from "express";
 import { WebSocketServer } from "ws";
@@ -21,6 +21,7 @@ import { TaskManager } from "../core/task-manager.js";
 import { createClaudeCodeAdapter } from "../core/adapters/claude-code.js";
 import { createCodexAdapter } from "../core/adapters/codex.js";
 import { WorkflowRegistry, createWorkflowsRouter, } from "../core/workflow-registry.js";
+import { AgentProjectScanBudget } from "../core/agent-project-discovery.js";
 import { DEFAULT_MACROS } from "../core/macros.js";
 import { createEventStore } from "../core/collector/store.js";
 import { createClaudeTranscriptEnricher, createSessionRecordReader, } from "../core/session-record.js";
@@ -36,7 +37,7 @@ import { findRolloutFile, tailCodexRollout, } from "../core/collector/codex-tail
 import { getOrCreateMachineId } from "../cli/machine-id.js";
 import { loadSettings, pruneDeadRecentDirs } from "../cli/settings.js";
 import { generateClaudeSettings } from "../core/inject/claude-settings.js";
-import { generateMcpConfig } from "../core/inject/mcp-config.js";
+import { generateMcpConfig, } from "../core/inject/mcp-config.js";
 import { generateSystemPromptFile } from "../core/inject/system-prompt.js";
 import { generateSkillsPlugin } from "../core/inject/skills-plugin.js";
 import { removeGeneratedSessionDir, sweepGeneratedDirs, } from "../core/inject/retention.js";
@@ -50,12 +51,17 @@ import { prepareHarnessContextForResume, writeHarnessContext, writeHarnessContex
 import { ensureCanvasTemplate } from "../core/canvas-template.js";
 import { renderCanvasForSession } from "../core/canvas-render.js";
 import { invalidateExtractionCache } from "../core/canvas-cache.js";
+import { CachedAgentRelationshipProvider, HarnessRegistryInventoryProvider, LocalWorkspaceScopeCatalog, SourceAgentRelationshipProvider, StaticSystemGraphBuilder, } from "../core/system-graph.js";
+import { canonicalGraphPath, dirtyGraphSourceRoots, graphSourceRootsWithinScope, isWithinGraphPath, } from "../core/system-graph-inventory.js";
+import { SystemGraphStore } from "../core/system-graph-store.js";
+import { SystemGraphWatcherManager } from "../core/system-graph-watcher.js";
 import { sweepNdjson } from "../core/collector/store-retention.js";
 import { createDefinitionSlugResolver, resolveAgentsBaseUrl, } from "../core/definition-slug-resolver.js";
-import { resolveManifestName } from "../core/definition-name.js";
+import { inspectManifestName, resolveManifestName, } from "../core/definition-name.js";
 import { createBootTokenMiddleware } from "./auth.js";
 import { createApiKeyProvider } from "../core/api-key-provider.js";
 import { createRestRouter } from "./rest.js";
+import { createSystemGraphRouter } from "./system-graph.js";
 import { createStaticRouter } from "./static.js";
 import { createTerminalWebSocketHandler } from "./terminal-ws.js";
 import { createEventsWebSocketHandler } from "./events-ws.js";
@@ -63,13 +69,16 @@ import { attachWebSocketRouters } from "./ws-router.js";
 import { createIngestRouter, processIngest, } from "./ingest.js";
 import { createCanvasRouter } from "./canvas.js";
 import { createCanvasRenderRouter } from "./canvas-render.js";
+import { createWorkflowGraphRouter } from "./workflow-graph.js";
+import { createStudioRailRouter } from "./studio-rail.js";
+import { createAgentMoveRouter, moveTargetDirs, remapSessions, } from "./agent-move.js";
 import { createMacrosRouter } from "./macros.js";
 import { createFsRouter } from "./fs.js";
 import { createRunsRouter } from "./runs.js";
 import { createTemplatesRouter } from "./templates.js";
 import { createAccountRouter } from "./account.js";
 import { createActionsRouter } from "./actions.js";
-import { createAuthRouter, createMutableAuthState, } from "./auth-routes.js";
+import { createAuthRouter, createMutableAuthState } from "./auth-routes.js";
 // resolveAgentsBaseUrl is imported above from definition-slug-resolver.js
 // (an identical helper); the runs router reuses it for its agents base URL.
 /**
@@ -102,6 +111,19 @@ const NDJSON_RETENTION_SWEEP_MS = 6 * 60 * 60 * 1000; // every 6 hours
 function packageRoot() {
     return join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 }
+/**
+ * One line per scan, on stderr with the rest of the harness's operational log.
+ * Deliberately unconditional rather than debug-gated: a scan happens a handful
+ * of times per session, and the one question this log answers ("what put that
+ * there?") is only ever asked after the fact.
+ */
+function logAgentScan(reason, root, found, budget) {
+    const truncated = budget.truncated
+        ? `, TRUNCATED at depth ${String(budget.truncatedAtDepth)} (envelope ${budget.envelopeDepth})`
+        : "";
+    console.error(`[harness] agent scan (${reason}) ${root}: ${found} agent(s), ` +
+        `${budget.visited} dirs${truncated}`);
+}
 /** Whether two workflow lists are equivalent for the rail's purposes —
  *  compares the fields the SPA actually renders/keys on, order-insensitive, so
  *  a rescan that turned up nothing new doesn't trigger a needless broadcast. */
@@ -164,7 +186,11 @@ function createDefaultBuildLaunchOpts(apiKey, generatedRoot, rehydration, sapiom
         // dim text loses contrast. Only when the theme is known: an unthemed path
         // (server-side auto-create, a legacy session resumed before this existed)
         // omits it and Claude keeps its default 256-color rendering, exactly as before.
-        const claudeTheme = req.theme === "light" ? "light-ansi" : req.theme === "dark" ? "dark-ansi" : undefined;
+        const claudeTheme = req.theme === "light"
+            ? "light-ansi"
+            : req.theme === "dark"
+                ? "dark-ansi"
+                : undefined;
         const [settings, mcpConfigFile, systemPromptFile, pluginDir] = await Promise.all([
             generateClaudeSettings({
                 harnessSessionId,
@@ -191,7 +217,9 @@ function createDefaultBuildLaunchOpts(apiKey, generatedRoot, rehydration, sapiom
             ...(pluginDir ? { pluginDir } : {}),
             // Set on BOTH channels: the post-ready path hasn't delivered yet, but a
             // brief exists and will, and this is the flag that tells it to.
-            ...(brief !== null && rehydrateFrom ? { rehydratedFrom: rehydrateFrom } : {}),
+            ...(brief !== null && rehydrateFrom
+                ? { rehydratedFrom: rehydrateFrom }
+                : {}),
         };
     };
 }
@@ -439,7 +467,8 @@ export const startServer = async (options) => {
             return null;
         try {
             const rows = await adapter.listPastSessions(record.cwd);
-            return rows.find((row) => row.agentSessionId === record.agentSessionId)?.gitBranch ?? null;
+            return (rows.find((row) => row.agentSessionId === record.agentSessionId)
+                ?.gitBranch ?? null);
         }
         catch {
             return null;
@@ -472,7 +501,11 @@ export const startServer = async (options) => {
                 title: prior?.title ?? null,
                 gitBranch: await priorGitBranch(record),
                 workflow: workflow
-                    ? { name: workflow.name, path: workflow.path, definitionId: workflow.definitionId }
+                    ? {
+                        name: workflow.name,
+                        path: workflow.path,
+                        definitionId: workflow.definitionId,
+                    }
                     : null,
             };
         },
@@ -500,6 +533,43 @@ export const startServer = async (options) => {
         ensureCanvasTemplate,
     });
     await sessionManager.init();
+    const workspaceScopeCatalog = new LocalWorkspaceScopeCatalog(async () => [
+        ...(await loadSettings(statePaths.settings)).recentDirs,
+        ...sessionManager.list().map((session) => session.cwd),
+    ]);
+    const systemGraphRelationships = new CachedAgentRelationshipProvider(new SourceAgentRelationshipProvider());
+    const systemGraphStore = new SystemGraphStore(new StaticSystemGraphBuilder(new HarnessRegistryInventoryProvider({
+        listWorkflows: () => workflowsCache,
+        inspectManifestName,
+    }), systemGraphRelationships), {
+        onChange: ({ workspaceKey, revision, state }) => {
+            bus.publish({
+                type: "system-graph.changed",
+                workspaceKey,
+                revision,
+                state,
+            });
+        },
+    });
+    const activeSystemGraphScopes = new Map();
+    const refreshSystemGraphScopesForRoot = (changedRoot, excludedWorkspaceKey) => {
+        const canonicalChangedRoot = canonicalGraphPath(changedRoot);
+        for (const scope of activeSystemGraphScopes.values()) {
+            if (scope.workspaceKey === excludedWorkspaceKey)
+                continue;
+            if (!systemGraphStore.peek(scope.workspaceKey))
+                continue;
+            const scopeRoot = canonicalGraphPath(scope.root);
+            if (!isWithinGraphPath(scopeRoot, canonicalChangedRoot) &&
+                !isWithinGraphPath(canonicalChangedRoot, scopeRoot)) {
+                continue;
+            }
+            systemGraphStore.requestRefresh({
+                workspaceKey: scope.workspaceKey,
+                root: scopeRoot,
+            });
+        }
+    };
     const sessionSweepTimer = setInterval(() => sessionManager.sweepDeadSessions(), SESSION_LIVENESS_SWEEP_MS);
     sessionSweepTimer.unref?.();
     // Portable continue, second channel: for a harness whose adapter never puts
@@ -518,7 +588,8 @@ export const startServer = async (options) => {
         const rehydratedFrom = session.rehydratedFrom;
         if (!session.ready || !rehydratedFrom)
             return;
-        if (systemPromptDeliveryFor(adapters[session.harness]) !== "post-ready-injection")
+        if (systemPromptDeliveryFor(adapters[session.harness]) !==
+            "post-ready-injection")
             return;
         if (briefsDelivered.has(session.id))
             return;
@@ -571,7 +642,9 @@ export const startServer = async (options) => {
         readRecord: (harnessSessionId) => sessionRecordReader.read(harnessSessionId),
         getSession: (harnessSessionId) => {
             const session = sessionManager.get(harnessSessionId);
-            return session ? { harness: session.harness, cwd: session.cwd } : undefined;
+            return session
+                ? { harness: session.harness, cwd: session.cwd }
+                : undefined;
         },
         runTask: (req) => taskManager.run(req),
     });
@@ -606,9 +679,15 @@ export const startServer = async (options) => {
             return;
         const before = workflowsCache;
         await workflowRegistry.prune();
-        await workflowRegistry.scan(session.cwd);
+        const rescanBudget = new AgentProjectScanBudget();
+        const rescanFound = await workflowRegistry.scan(session.cwd, rescanBudget);
+        logAgentScan("workspace-change", session.cwd, rescanFound.length, rescanBudget);
         const after = await workflowRegistry.list();
         workflowsCache = after;
+        const inventoryChanged = !workflowListsEqual(before, after);
+        if (inventoryChanged) {
+            await refreshSystemGraphScopesForRoot(session.cwd);
+        }
         // A removed project must not leave a live session permanently bound to a
         // path that the registry can no longer resolve. Clear only stale bindings;
         // the triggering session may immediately auto-bind to another candidate
@@ -646,7 +725,7 @@ export const startServer = async (options) => {
                 });
             }
         }
-        if (workflowListsEqual(before, after))
+        if (!inventoryChanged)
             return;
         await Promise.all(sessionManager.list().map((s) => writeSessionContext(s)));
         bus.publish({ type: "workflows.changed" });
@@ -773,17 +852,92 @@ export const startServer = async (options) => {
     // merged registry) — callers use this to decide whether THIS scan turned
     // up something new worth an unprompted canvas render, without conflating
     // it with unrelated workflows some earlier scan already found elsewhere.
-    const scanWorkflowsAndBroadcast = async (root) => {
-        const found = await workflowRegistry.scan(root);
-        workflowsCache = await workflowRegistry.list();
+    //
+    // Every call names WHY it is scanning. The registry is the thing the user
+    // asked "how am I finding my agents?" about, and until this line existed the
+    // only honest answer available after the fact was "something scanned
+    // something". One install ended up with 88 agents across twelve repositories
+    // nobody had opened, and reconstructing which root did it took a filesystem
+    // archaeology session. Now every scan says its root, its reason, what it
+    // found, what it cost, and what it declined to enter.
+    const scanWorkflowsAndBroadcast = async (root, reason, options = {}) => {
+        const before = workflowsCache;
+        const budget = new AgentProjectScanBudget();
+        const found = await workflowRegistry.scan(root, budget);
+        logAgentScan(reason, root, found.length, budget);
+        const after = await workflowRegistry.list();
+        workflowsCache = after;
+        const changed = !workflowListsEqual(before, after);
         // Rewrite every open session's context file before broadcasting — a
         // listener reacting to workflows.changed (the SPA, or an agent that
         // happens to re-read the file right then) must never see the
         // notification before the file it describes is actually updated.
-        await Promise.all(sessionManager.list().map((session) => writeSessionContext(session)));
-        bus.publish({ type: "workflows.changed" });
+        if (changed) {
+            await Promise.all(sessionManager.list().map((session) => writeSessionContext(session)));
+        }
+        if (options.refreshGraphs)
+            await options.refreshGraphs();
+        else if (changed)
+            refreshSystemGraphScopesForRoot(root);
+        if (changed)
+            bus.publish({ type: "workflows.changed" });
         return found;
     };
+    const refreshSystemGraphInventory = async (scope) => {
+        const canonicalScope = {
+            workspaceKey: scope.workspaceKey,
+            root: canonicalGraphPath(scope.root),
+        };
+        let refreshPromise = null;
+        const refreshGraph = () => {
+            refreshSystemGraphScopesForRoot(canonicalScope.root, canonicalScope.workspaceKey);
+            refreshPromise = systemGraphStore.refresh(canonicalScope);
+            return refreshPromise.then(() => undefined);
+        };
+        await scanWorkflowsAndBroadcast(canonicalScope.root, "graph-refresh", {
+            refreshGraphs: refreshGraph,
+        });
+        return await (refreshPromise ?? systemGraphStore.refresh(canonicalScope));
+    };
+    const workflowRootsForGraphScope = (scope) => graphSourceRootsWithinScope(scope.root, workflowsCache.map((workflow) => workflow.path));
+    const systemGraphWatcher = new SystemGraphWatcherManager({
+        listSourceRoots: workflowRootsForGraphScope,
+        onSourceChange: (scope, sourcePaths) => {
+            const canonicalScope = {
+                workspaceKey: scope.workspaceKey,
+                root: canonicalGraphPath(scope.root),
+            };
+            const dirtyRoots = dirtyGraphSourceRoots(canonicalScope.root, workflowsCache.map((workflow) => workflow.path), sourcePaths);
+            if (dirtyRoots.length === 0)
+                return;
+            for (const workflowRoot of dirtyRoots) {
+                systemGraphRelationships.invalidateSource(workflowRoot);
+            }
+            systemGraphStore.requestRefresh(canonicalScope);
+        },
+        onInventoryChange: async (scope) => {
+            try {
+                await refreshSystemGraphInventory(scope);
+            }
+            catch (err) {
+                systemGraphStore.reportRefreshFailure(scope);
+                console.error("[harness] workspace graph inventory refresh failed");
+                throw err;
+            }
+        },
+    });
+    const listWorkspaceScopesAndRetain = async () => {
+        const scopes = await workspaceScopeCatalog.list();
+        const retained = new Set(scopes.map((scope) => scope.workspaceKey));
+        systemGraphWatcher.retain(retained);
+        systemGraphStore.retain(retained);
+        for (const workspaceKey of activeSystemGraphScopes.keys()) {
+            if (!retained.has(workspaceKey)) {
+                activeSystemGraphScopes.delete(workspaceKey);
+            }
+        }
+        return scopes;
+    };
     /** Enrich only the bound workflow before a Canvas render. Canvas extraction
      *  needs the registry snapshot to resolve the binding, but its cloud badge
      *  needs the same mutable build projection exposed by /api/state. Limiting
@@ -838,7 +992,7 @@ export const startServer = async (options) => {
         const outcome = await renderCanvasForSession(session, await canvasWorkflowsForSession(session), { surfaceErrorOnMissingDeps: true });
         reactToRenderOutcome(session, outcome);
     };
-    const initialWorkflowScan = scanWorkflowsAndBroadcast(launchDir).catch((err) => {
+    const initialWorkflowScan = scanWorkflowsAndBroadcast(launchDir, "boot").catch((err) => {
         console.error("[harness] initial agent scan failed:", err);
         return [];
     });
@@ -941,13 +1095,14 @@ export const startServer = async (options) => {
             }
             : null,
         listWorkflows: () => workflowRegistry.list().then(enrichWorkflows),
+        listWorkspaceScopes: listWorkspaceScopesAndRetain,
         listMacros: () => DEFAULT_MACROS,
         findWorkflow: (workflowPath) => workflowsCache.find((w) => w.path === workflowPath) ?? null,
         writeWorkspaceContext: writeSessionContext,
         renderCanvas,
         onTelemetryOptInChange: (optIn) => batcher.setTelemetryOptIn(optIn),
         onSessionCreated: (cwd, harnessSessionId) => {
-            scanWorkflowsAndBroadcast(cwd)
+            scanWorkflowsAndBroadcast(cwd, "session-create")
                 .then((found) => {
                 // Only auto-render when THIS session's own directory turned up a
                 // workflow — an unrelated project scanned earlier elsewhere in
@@ -982,6 +1137,23 @@ export const startServer = async (options) => {
         },
         settingsPath: statePaths.settings,
     }));
+    app.use("/api", createSystemGraphRouter({
+        scopeResolver: workspaceScopeCatalog,
+        store: systemGraphStore,
+        onScopeAccess: (scope) => {
+            activeSystemGraphScopes.set(scope.workspaceKey, scope);
+            return systemGraphWatcher.start(scope);
+        },
+        onScopeRefresh: async (scope) => {
+            try {
+                return await refreshSystemGraphInventory(scope);
+            }
+            catch {
+                console.error("[harness] workspace graph manual refresh failed");
+                return systemGraphStore.reportRefreshFailure(scope);
+            }
+        },
+    }));
     app.use("/api", createCanvasRenderRouter({
         getSession: (harnessSessionId) => sessionManager.get(harnessSessionId),
         listWorkflows: canvasWorkflowsForSession,
@@ -1000,8 +1172,23 @@ export const startServer = async (options) => {
     // as WorkflowRegistryLike so this wrapper needs no unsafe cast.
     const enrichedWorkflowRegistry = {
         list: () => workflowRegistry.list().then(enrichWorkflows),
-        scan: (root) => workflowRegistry.scan(root),
-        connectPath: (inputPath) => workflowRegistry.connectPath(inputPath),
+        scan: (root) => scanWorkflowsAndBroadcast(root, "requested"),
+        connectPath: async (inputPath) => {
+            const workflow = await workflowRegistry.connectPath(inputPath);
+            workflowsCache = await workflowRegistry.list();
+            await Promise.all(sessionManager.list().map((session) => writeSessionContext(session)));
+            await refreshSystemGraphScopesForRoot(workflow.path);
+            bus.publish({ type: "workflows.changed" });
+            return workflow;
+        },
+        scanWithBoundaries: async (root) => {
+            const budget = new AgentProjectScanBudget();
+            const found = await workflowRegistry.scan(root, budget);
+            logAgentScan("requested", root, found.length, budget);
+            // The boundaries the walk stopped at travel with the result so the rail
+            // can explain an empty project instead of misdescribing one.
+            return { found, repositoryBoundaries: budget.repositoryBoundaries };
+        },
     };
     app.use(createRunsRouter({
         // Pass the provider (not a static key) so the live-status path can refresh
@@ -1044,7 +1231,69 @@ export const startServer = async (options) => {
         // which the watcher's directory-set diff cannot see — rescan that project
         // so the Draft→Deployed chip and the deploy-gated actions update.
         onLinked: async (workflow) => {
-            await scanWorkflowsAndBroadcast(workflow.path);
+            await scanWorkflowsAndBroadcast(workflow.path, "agent-linked");
+        },
+    }));
+    // IA-01: the session-free, workflow-keyed canvas route. Same derivation the
+    // session-bound render uses, keyed by the agent's absolute path instead of a
+    // session id — so a board can be read for an agent that has never hosted a
+    // session. Resolution goes through the same live cache actions.ts uses, so
+    // only registered agents are ever read from disk.
+    app.use(createWorkflowGraphRouter({
+        resolveWorkflow: (agentPath) => workflowsCache.find((w) => resolve(w.path) === agentPath) ?? null,
+    }));
+    // SAP-2929: the Group axis's stored arrangement, one `.sapiom/studio-rail.json`
+    // per project root, plus the launch edges it seeds from. Writable roots are
+    // exactly the roots the rail can SHOW — recentDirs, the configured project
+    // root, and live session cwds — so the route cannot be aimed anywhere the
+    // studio has not already been pointed.
+    app.use(createStudioRailRouter({
+        listKnownRoots: async () => {
+            const stored = await loadSettings(statePaths.settings);
+            return [
+                ...stored.recentDirs,
+                ...(stored.projectRoot ? [stored.projectRoot] : []),
+                ...sessionManager.list().map((session) => session.cwd),
+            ];
+        },
+        listWorkflows: () => workflowsCache,
+    }));
+    // SAP-2930: the Project axis's drag, performed on disk. Its own guards live
+    // in the module — a planner is not a permission system, so the route stats
+    // the destination itself. Only a REGISTERED agent may be moved, resolved
+    // through the same live cache the canvas and action routes use, and it may
+    // only be moved INTO a directory the rail can show: the roots studio-rail
+    // already treats as writable, plus the branching directories the Project
+    // axis draws between a root and an agent. Both sides of the move are
+    // therefore server-authored paths, not strings off the request.
+    app.use(createAgentMoveRouter({
+        resolveAgent: (agentPath) => workflowsCache.find((w) => resolve(w.path) === agentPath) ?? null,
+        listMoveTargetDirs: async () => {
+            const stored = await loadSettings(statePaths.settings);
+            return moveTargetDirs([
+                ...stored.recentDirs,
+                ...(stored.projectRoot ? [stored.projectRoot] : []),
+                ...sessionManager.list().map((session) => session.cwd),
+            ], workflowsCache.map((w) => w.path));
+        },
+        // Everything under the moved directory travelled with it, so every live
+        // session whose cwd sat inside follows — a session left pointing at a
+        // directory that no longer exists is the whole reason this remap exists.
+        // Then prune the path that is gone and rescan the destination, which
+        // broadcasts `workflows.changed`: the rail re-derives the tree from the
+        // NEW path rather than from a stale registry row.
+        onMoved: async (from, to) => {
+            remapSessions(sessionManager.list(), from, to);
+            await workflowRegistry.prune();
+            await scanWorkflowsAndBroadcast(dirname(to), "agent-moved", {
+                refreshGraphs: async () => {
+                    // A cross-project move changes two containment inventories. Refresh
+                    // every active parent/nested graph touching either side so neither
+                    // keeps a ghost node or misses the arrival.
+                    refreshSystemGraphScopesForRoot(dirname(from));
+                    refreshSystemGraphScopesForRoot(dirname(to));
+                },
+            });
         },
     }));
     app.use(createWorkflowsRouter(enrichedWorkflowRegistry), createFsRouter(), createMacrosRouter({
@@ -1320,6 +1569,10 @@ export const startServer = async (options) => {
             clearInterval(ndjsonRetentionTimer);
             canvasWatcher.stopAll();
             workspaceWatcher.stopAll();
+            systemGraphWatcher.stopAll();
+            activeSystemGraphScopes.clear();
+            systemGraphRelationships.clear();
+            systemGraphStore.clear();
             installWatcher.stopAll();
             for (const tailer of codexTailers.values())
                 tailer.stop();