graphlin 0.1.3 → 0.2.1

package/runtime/pipeline.mjs

@@ -1,6 +1,5 @@
 import { createHash, randomUUID } from 'node:crypto';
 import { realpathSync } from 'node:fs';
-import { readdir } from 'node:fs/promises';
 import path from 'node:path';
 import {
   createPolicy, normalizeHostEvent, metadataEvent, EvidenceStore,
@@ -8,6 +7,9 @@ import {
   applyPatch, projectGraph,
 } from './core/index.mjs';
 import { safeLabel, safeText, excluded } from './core/privacy.mjs';
+import { createPlatform } from './platform.mjs';
+import { createArchitectureController } from './architecture/controller.mjs';
+import { analyzeArchitecture, ARCHITECTURE_NAMESPACE } from './architecture/analysis.mjs';
 
 const MAX_ACTIVITY = 200;
 const MAX_HOOK_EVENTS = 200;
@@ -23,11 +25,6 @@ const CLASSIFICATION_QUEUE_TTL_MS = 120_000;
 const DEADLINE_MS = 2000;
 const PENDING_LEASE_MS = 60_000;
 const TERMINAL = new Set(['succeeded', 'failed', 'interrupted', 'unresolved']);
-const SOURCE_EXT = /\.(?:[cm]?[jt]sx?|py|go|rs|java|kt|rb|php|cs|swift|sql|ya?ml|json|toml|tf)$/i;
-const SKIP_DIR = new Set([
-  '.git', '.graphlin', '.graphlin-data', 'node_modules', 'dist', 'build',
-  'coverage', '.next', '.cache', '.venv', 'venv', 'vendor', 'research',
-]);
 const FIXED_CLASSIFIER = new Set([
   'ready', 'metadata_only', 'missing_key', 'paused', 'unavailable', 'timeout', 'demo',
 ]);
@@ -97,8 +94,8 @@ function restoreGraph(input, { stale = true } = {}) {
  */
 export function createPipeline({
   projectRoot, policy: policyOptions, decisionService, onChange = () => {},
-  onDiagnostic = () => {}, restoredState, mode = 'live', clock = Date.now,
-  classificationDeadlineMs = DEADLINE_MS,
+  onDiagnostic = () => {}, restoredState, restoredModel, mode = 'live', clock = Date.now,
+  classificationDeadlineMs = DEADLINE_MS, missingKey = false,
 } = {}) {
   if (!Number.isSafeInteger(classificationDeadlineMs) ||
       classificationDeadlineMs < DEADLINE_MS || classificationDeadlineMs > 10_000) {
@@ -107,13 +104,37 @@ export function createPipeline({
   const root = realpathSync(projectRoot);
   const inputRoot = path.resolve(projectRoot);
   const projectId = opaque(root);
+  const modelProjectId = createHash('sha256').update(root).digest('hex');
   const policy = createPolicy(policyOptions ?? {});
-  const evidence = new EvidenceStore({ projectRoot, policy });
+  // Restored claims stay stale until recaptured, and their versions must never
+  // collide with a new daemon's counters. No saved content is trusted here.
+  let generationFloor = 0;
+  const retainGeneration = value => {
+    if (Number.isSafeInteger(value) && value > generationFloor && value < Number.MAX_SAFE_INTEGER) generationFloor = value;
+  };
+  const retainRefs = rows => {
+    for (const row of Array.isArray(rows) ? rows.slice(0, 40000) : []) {
+      for (const ref of Array.isArray(row?.sourceRefs) ? row.sourceRefs.slice(0, 16) : []) retainGeneration(ref.generation);
+    }
+  };
+  if (restoredModel?.projectId === modelProjectId) {
+    const artifacts = restoredModel.coverage?.artifacts;
+    for (const artifact of Array.isArray(artifacts) ? artifacts.slice(0, 10000) : []) retainGeneration(artifact?.generation);
+    for (const rows of [restoredModel.entities, restoredModel.relations, restoredModel.interpretations]) retainRefs(rows);
+  }
+  if (restoredState?.projectId === projectId) {
+    const states = Array.isArray(restoredState.sessionStates) ? restoredState.sessionStates.slice(0, MAX_SESSIONS) : [];
+    for (const session of [restoredState, ...states]) {
+      retainRefs(session?.graph?.nodes); retainRefs(session?.graph?.edges);
+    }
+  }
+  const evidence = new EvidenceStore({ projectRoot, policy, maxTrackedPaths: 10000, generationFloor });
   const sessions = new Map();
   const dedup = new Map();
   const sessionStarts = new Map();
   const hookEvents = [];
   const knownArtifacts = new Map();
+  const lineageWork = new Set();
   const messageVersions = new Map();
   const deferredWork = new Map();
   const classificationQueue = [];
@@ -130,16 +151,144 @@ export function createPipeline({
   let pending = 0;
   let classifier = mode === 'demo' ? 'demo' : policy.transmitSource ? 'ready' : 'metadata_only';
   let serial = Promise.resolve();
-  let lastDiscoveryAt = -Infinity;
-  let discoveredPaths = [];
   let reconciliationTask = null;
   let resumeScheduled = false;
+  let lineageId = null;
+  let architecture;
+  let architectureEpoch = 0;
+  const architectureGuards = new WeakMap();
 
   const serialized = (fn) => {
     const operation = serial.then(fn);
     serial = operation.catch(() => {});
     return operation;
   };
+  const platform = createPlatform({
+    projectRoot: root, projectId: modelProjectId,
+    policy, restoredState: restoredModel, now: clock,
+    accept: operation => serialized(operation),
+    revalidate: refs => serialized(async () => {
+      if (closed) return false;
+      registerArtifacts(await evidence.reconcile({ refs }));
+      return evidence.isCurrent(refs);
+    }),
+    onChange: () => { architecture?.wake(); notify(); },
+  });
+  architecture = createArchitectureController({
+    snapshot: () => platform.snapshot(),
+    capture: captureArchitecture,
+    commit: commitArchitecture,
+    analyze: async input => {
+      const guard = {
+        epoch: architectureEpoch, signal: input.signal, policyVersion: policy.version,
+        lineageId: input.model.coverage.lineage?.id ?? modelProjectId,
+      };
+      const result = await analyzeArchitecture({ ...input, service: decisionService, policy });
+      architectureGuards.set(result, guard);
+      return result;
+    },
+    available: architectureUnavailable,
+    ready: () => {
+      const { queued, active } = platform.stats();
+      return queued === 0 && active === 0;
+    },
+    onChange: notify,
+    onDiagnostic: result => trace(null, 'classification', {
+      status: result.status === 'complete' ? 'accepted' : result.status === 'partial' ? 'partial' : 'unavailable',
+      reason: result.status === 'complete' ? 'ok' : result.status === 'partial' ? 'unknown' : 'decision_failure',
+      diagnostics: { calls: result.providerRequests },
+    }),
+    now: clock,
+  });
+
+  function architectureUnavailable() {
+    if (closed) return 'closed';
+    if (!policy.transmitSource) return 'source_consent_required';
+    if (missingKey) return 'missing_key';
+    if (paused) return 'paused';
+    if (mode === 'demo') return 'demo';
+    if (typeof decisionService?.analyze !== 'function' || typeof decisionService?.evaluate !== 'function') {
+      return 'unsupported_service';
+    }
+    return null;
+  }
+
+  async function captureArchitecture(ids) {
+    const artifacts = await serialized(async () => {
+      if (architectureUnavailable()) return [];
+      const refs = ids.slice(0, 64).filter(id => knownArtifacts.has(id)).map(artifactId => ({ artifactId }));
+      const captures = await evidence.reconcile({ refs });
+      registerArtifacts(captures);
+      return captures;
+    });
+    // Parser acceptance uses the same serial queue; waiting inside it deadlocks.
+    await platform.whenIdle();
+    return artifacts;
+  }
+
+  function commitArchitecture(result, context) {
+    return serialized(async () => {
+      const guard = architectureGuards.get(result);
+      architectureGuards.delete(result);
+      const current = () => guard && guard.signal === context.signal && !context.signal.aborted &&
+        guard.epoch === architectureEpoch && guard.policyVersion === policy.version &&
+        guard.lineageId === (lineageId ?? platform.snapshot().coverage.lineage?.id ?? modelProjectId) &&
+        !architectureUnavailable();
+      if (!current() || result.sourceRefs.length > 256 || context.artifacts.length > 64) return false;
+      const expected = new Map();
+      const sameVersion = (a, b) => a && b && a.hash === b.hash &&
+        a.generation === b.generation && a.status === b.status;
+      for (const value of [...context.artifacts, ...result.sourceRefs.map(ref => ({ ...ref, status: 'present' }))]) {
+        if (!knownArtifacts.has(value.artifactId) ||
+            expected.has(value.artifactId) && !sameVersion(expected.get(value.artifactId), value)) return false;
+        expected.set(value.artifactId, value);
+      }
+      if (expected.size > 256) return false;
+      const observed = new Map();
+      const refs = [...expected.keys()].map(artifactId => ({ artifactId }));
+      // Re-read all selected versions, including absence, and any extra
+      // membership support. Retain metadata only across these bounded reads.
+      for (let offset = 0; offset < refs.length; offset += 32) {
+        const captures = await evidence.reconcile({ refs: refs.slice(offset, offset + 32) });
+        registerArtifacts(captures);
+        for (const artifact of captures) observed.set(artifact.id, {
+          hash: artifact.hash, generation: artifact.generation, status: artifact.status,
+          exists: artifact.exists, complete: artifact.complete,
+        });
+        if (!current()) return false;
+      }
+      if ([...expected].some(([id, value]) => !sameVersion(value, observed.get(id)))) return false;
+      const missing = new Set(result.coverage.missingArtifactIds ?? []);
+      const selected = new Set(context.artifacts.map(value => value.artifactId));
+      for (const id of missing) {
+        const value = observed.get(id);
+        if (!selected.has(id) || value?.status !== 'missing' || value.hash !== null ||
+            value.exists !== false || value.complete !== true) return false;
+      }
+      const withdrawn = new Set(result.coverage.withdrawnEntityIds ?? []);
+      const model = platform.snapshot();
+      const entities = new Map(model.entities.map(value => [value.id, value]));
+      if ([...withdrawn].some(id => !missing.has(entities.get(id)?.artifactId))) return false;
+      const affected = result.affectedEntityIds.filter(id => !withdrawn.has(id));
+      let omitted = 0;
+      if (affected.length || result.interpretations.length) {
+        const replacement = platform.model.replaceInterpretations(ARCHITECTURE_NAMESPACE, result.interpretations, {
+          affectedEntityIds: affected, sourceRefs: result.sourceRefs,
+        });
+        if (!replacement.accepted) return false;
+        omitted = Math.max(0, result.interpretations.length - replacement.retained);
+      }
+      if (missing.size) {
+        // Present-source guards cannot prove deletion. The serialized reread
+        // above authorizes this clear, even when deleted anchors were evicted.
+        const cleared = platform.model.replaceInterpretations(ARCHITECTURE_NAMESPACE, [], {
+          affectedEntityIds: [...withdrawn], artifactIds: [...missing],
+        });
+        if (!cleared.accepted) return false;
+      }
+      return omitted ? { accepted: true, omitted } : true;
+    });
+  }
 
   function artifactMetadata(artifact) {
     const relative = artifact.relativePath;
@@ -217,17 +366,24 @@ export function createPipeline({
     const session = freshSession(id, `Session ${sessions.size + 1}`);
     sessions.set(id, session);
     selectedSession ??= id;
+    platform.setSessions([...sessions.values()].map(item => ({ id: item.id, label: item.label })));
     return session;
   }
 
   function evictSession(id) {
     sessions.delete(id);
+    syncSessions();
     deferredWork.delete(id);
     for (const key of completedClassifications.keys()) {
       if (key.startsWith(`${id}:`)) completedClassifications.delete(key);
     }
   }
 
+  function syncSessions() {
+    platform.setSessions([...sessions.values()].map(({ id, label, host, status, startedAt, endedAt }) =>
+      ({ id, label, host, status, startedAt, endedAt })));
+  }
+
   if (restoredState?.projectId === projectId) {
     const saved = Array.isArray(restoredState.sessionStates)
       ? [restoredState, ...restoredState.sessionStates.filter(s => s.id !== restoredState.sessionId)].slice(-MAX_SESSIONS)
@@ -237,6 +393,7 @@ export function createPipeline({
       if (typeof id !== 'string' || !/^[a-zA-Z0-9_-]{1,100}$/.test(id)) continue;
       const session = ensureSession(id);
       session.graph = restoreGraph(entry.graph);
+      platform.observeLegacy(session.graph, { sessionId: session.id });
       // Activity is reconstructed through the metadata allowlist, never trusted verbatim.
       session.activity = (Array.isArray(entry.activity) ? entry.activity : [])
         .slice(-MAX_ACTIVITY).map(item => {
@@ -329,6 +486,7 @@ export function createPipeline({
   function recordPatch(session, patch) {
     if (!patch || !patch.operations?.length) return;
     session.graph = applyPatch(session.graph, patch);
+    platform.observeLegacy(session.graph, { sessionId: session.id, fresh: true });
     session.history.push({
       revision: session.graph.revision, at: new Date(clock()).toISOString(),
       graph: structuredClone(session.graph),
@@ -356,7 +514,7 @@ export function createPipeline({
     }
   }
 
-  function registerArtifacts(artifacts) {
+  function registerArtifacts(artifacts, event) {
     const changed = [];
     for (const artifact of artifacts) {
       const previous = knownArtifacts.get(artifact.id);
@@ -390,12 +548,15 @@ export function createPipeline({
         recordPatch(session, invalidateArtifacts(session.graph, changed));
       }
     }
+    platform.observeArtifacts(artifacts, event);
+    if (changed.length) architecture.observe(changed);
     return changed;
   }
 
   function addActivity(session, event) {
     const state = activityState(event);
     const row = { ...metadataEvent(event), label: activityLabel(event), state };
+    platform.recordActivity(row);
     const existing = event.toolCallId
       ? session.activity.findIndex(item => item.toolCallId === event.toolCallId &&
           item.agentId === event.agentId && item.kind?.startsWith('tool.'))
@@ -422,6 +583,7 @@ export function createPipeline({
           sequence: ++sequence, incomplete: true,
         });
         session.activity[index] = { ...event, label: activityLabel(event), state: 'unresolved' };
+        platform.recordActivity(event);
         changed = true;
       }
     }
@@ -429,31 +591,7 @@ export function createPipeline({
   }
 
   async function discoverPaths() {
-    // Bounded name discovery lets a shell-created file enter the next observation.
-    // Core still authorizes each resolved file before reading any content.
-    if (clock() - lastDiscoveryAt < 1000) return discoveredPaths;
-    lastDiscoveryAt = clock();
-    const found = [];
-    const dirs = [{ dir: root, depth: 0 }];
-    let visited = 0;
-    while (dirs.length && found.length < 64 && visited++ < 100) {
-      const { dir, depth } = dirs.shift();
-      let entries;
-      try { entries = await readdir(dir, { withFileTypes: true }); } catch { continue; }
-      entries.sort((a, b) => a.name.localeCompare(b.name));
-      for (const entry of entries) {
-        if (entry.isSymbolicLink() || entry.name.startsWith('.')) continue;
-        if (entry.isDirectory() && depth < 5 && !SKIP_DIR.has(entry.name)) {
-          dirs.push({ dir: path.join(dir, entry.name), depth: depth + 1 });
-        } else if (entry.isFile() && SOURCE_EXT.test(entry.name) &&
-            !/(?:package-lock|pnpm-lock|yarn\.lock)/.test(entry.name)) {
-          found.push(path.join(dir, entry.name));
-          if (found.length >= 64) break;
-        }
-      }
-    }
-    discoveredPaths = found;
-    return found;
+    return platform.discover({ limit: 64 });
   }
 
   function canonicalNamedPaths(paths, raw) {
@@ -555,7 +693,15 @@ export function createPipeline({
     if (closed || paused || !deferredWork.size) return;
     const work = [...deferredWork];
     deferredWork.clear();
-    const artifacts = await evidence.reconcile();
+    const needed = new Set(work.flatMap(([, entry]) => [...entry.artifacts]));
+    for (const [sessionId] of work) {
+      for (const edge of sessions.get(sessionId)?.graph.edges ?? []) {
+        if (edge.sourceRefs.some(ref => needed.has(ref.artifactId))) {
+          edge.sourceRefs.forEach(ref => needed.add(ref.artifactId));
+        }
+      }
+    }
+    const artifacts = await evidence.reconcile({ refs: [...needed].map(artifactId => ({ artifactId })) });
     registerArtifacts(artifacts);
     for (const [sessionId, entry] of work) {
       const session = sessions.get(sessionId);
@@ -602,7 +748,7 @@ export function createPipeline({
   const sourceIdentity = job => job.candidates.some(candidate => candidate.sourceRef?.type === 'message')
     ? null : sourceVersions(job.candidates).map(ref => ref.artifactId).sort().join(',');
   const classificationKey = job => `${job.event.sessionId}:${opaque(JSON.stringify([
-    policy.version,
+    policy.version, job.lineageId,
     { kind: job.event.kind, toolCategory: job.event.toolCategory,
       outcome: job.event.outcome, incomplete: job.event.incomplete !== false },
     job.candidates.map(candidate => candidate.digest),
@@ -650,7 +796,7 @@ export function createPipeline({
     const session = sessions.get(event.sessionId);
     if (!session) { skip('session_evicted'); return; }
     if (paused) { deferClassification(event, candidates); skip('paused_deferred'); return; }
-    const job = { event, candidates, sourceEventId, session, enqueuedAt: clock(),
+    const job = { event, candidates, sourceEventId, session, lineageId, enqueuedAt: clock(),
       needsRefresh: activeClassifications.size >= MAX_CLASSIFICATIONS };
     const covered = coverageReason(job);
     if (covered) { skipJob(job, covered); return; }
@@ -783,6 +929,7 @@ export function createPipeline({
         diagnostics: result?.diagnostics ?? {} });
       await serialized(async () => {
         if (closed || !sessions.has(event.sessionId)) { skip(closed ? 'pipeline_closed' : 'session_evicted'); return; }
+        if (job.lineageId !== lineageId) { skip('source_changed_during_classification'); return; }
         if (paused) { deferClassification(event, candidates); skip('paused_deferred'); notify(); return; }
         if (clock() >= deadlineAt) { classifier = 'timeout'; dropped++; skip('deadline_before_apply'); notify(); return; }
         if (result.status === 'timeout') classifier = 'timeout';
@@ -796,7 +943,7 @@ export function createPipeline({
             bundle?.policyVersion === policy.version && Array.isArray(bundle.candidates)) {
           // Reobserve the worktree before accepting remote answers; a tool could
           // have edited these files while either Jev request was in flight.
-          registerArtifacts(await evidence.reconcile());
+          registerArtifacts(await evidence.reconcile({ refs: sourceVersions(candidates) }));
           if (closed || sessions.get(event.sessionId) !== session) { skip(closed ? 'pipeline_closed' : 'session_evicted'); return; }
           if (paused) { deferClassification(event, candidates); skip('paused_deferred'); notify(); return; }
           const artifactRefs = sourceVersions(candidates);
@@ -825,6 +972,10 @@ export function createPipeline({
             const patch = compileDecision(before, { event, decision: result, policy,
               onDiagnostic: entry => admission.push(entry) });
             recordPatch(session, patch);
+            // The legacy canvas cap is not a semantic admission cap. Compile
+            // this bounded decision independently so the model can retain it.
+            const independent = compileDecision(emptyGraph(), { event, decision: result, policy });
+            if (independent) platform.observeLegacy(applyPatch(emptyGraph(), independent), { sessionId: session.id, fresh: true });
             remember();
             const after = session.graph;
             const counts = { revisionBefore: before.revision, revisionAfter: after.revision };
@@ -923,6 +1074,11 @@ export function createPipeline({
           if (dedup.size > MAX_DEDUP) dedup.delete(dedup.keys().next().value);
         }
         const session = ensureSession(event.sessionId);
+        session.host = host;
+        session.startedAt ??= event.at;
+        session.status = event.kind === 'session.ended' ? 'ended' : 'active';
+        if (event.kind === 'session.ended') session.endedAt = event.at;
+        syncSessions();
         if (followSession) selectedSession = session.id;
         observeMessage(event, prepared.publicText);
         addActivity(session, event);
@@ -935,20 +1091,22 @@ export function createPipeline({
           const named = Array.isArray(prepared.paths) ? canonicalNamedPaths(prepared.paths, raw) : [];
           const discover = event.kind === 'session.started' ||
             ['tool.succeeded', 'tool.failed'].includes(event.kind);
-          const paths = [...new Set([...named, ...(discover ? await discoverPaths() : [])])];
+          const sessionPaths = event.kind === 'session.started'
+            ? [...knownArtifacts.values()].filter(item => item.status === 'present').slice(0, 64).map(item => item.path) : [];
+          const paths = [...new Set([...named, ...sessionPaths, ...(discover ? await discoverPaths() : [])])];
           // EvidenceStore bounds each capture to 32 paths. Process the bounded
           // discovery list in chunks rather than silently losing its tail.
           for (let index = 0; index < paths.length; index += 32) {
             artifacts.push(...await evidence.capture(paths.slice(index, index + 32)));
           }
-          const changed = registerArtifacts(artifacts);
+          const changed = registerArtifacts(artifacts, metadataEvent(event));
           trace(event, 'capture', { status: 'observed', reason: 'artifacts_observed',
             artifacts: artifacts.map(artifact => ({ ...artifactMetadata(artifact),
               reason: changed.some(item => item.id === artifact.id) ? 'artifact_changed' : 'artifact_unchanged' })) });
           // A tool completion also reconciles prior support, including deletions
           // omitted from the tool's returned file list.
           if (event.kind.startsWith('tool.') && event.kind !== 'tool.requested') {
-            registerArtifacts(await evidence.reconcile());
+            registerArtifacts(await evidence.reconcile({ limit: 32 }));
           }
           namedSet = new Set(named.map(file => path.resolve(root, file)));
           // Global observation history is not a session's classification
@@ -998,10 +1156,26 @@ export function createPipeline({
     reconciliationTask = serialized(async () => {
       if (closed) return;
       const expired = expirePending();
-      const artifacts = await evidence.capture(await discoverPaths());
-      const known = await evidence.reconcile();
-      const changed = registerArtifacts([...artifacts, ...known]);
-      if (!changed.length) { if (expired) notify(); return; }
+      const beforeModel = platform.model.stats();
+      const paths = await discoverPaths();
+      const artifacts = [];
+      for (let index = 0; index < paths.length; index += 32) {
+        artifacts.push(...await evidence.capture(paths.slice(index, index + 32)));
+      }
+      const known = await evidence.reconcile({ limit: 32 });
+      const observed = [...artifacts, ...known];
+      const changed = registerArtifacts(observed);
+      // Parser revalidation may have already refreshed a file's generation.
+      // Retain canceled classifier work until this dispatch path sees it.
+      for (const artifact of observed) {
+        if (!lineageWork.delete(artifact.id)) continue;
+        if (!changed.some(item => item.id === artifact.id)) changed.push(artifact);
+      }
+      if (!changed.length) {
+        const afterModel = platform.model.stats();
+        if (expired || beforeModel.revision !== afterModel.revision || beforeModel.sequence !== afterModel.sequence) notify();
+        return;
+      }
       const session = selectedSession ? sessions.get(selectedSession) : null;
       if (session) {
         const event = observationEvent(session.id);
@@ -1021,6 +1195,7 @@ export function createPipeline({
   function setPaused(value) {
     const wasPaused = paused;
     paused = Boolean(value);
+    if (wasPaused !== paused) architectureEpoch++;
     if (!wasPaused && paused) {
       for (const job of classificationQueue.splice(0)) {
         deferClassification(job.event, job.candidates);
@@ -1033,10 +1208,30 @@ export function createPipeline({
       serialized(flushDeferred).catch(() => { dropped++; })
         .finally(() => { resumeScheduled = false; });
     }
+    architecture.wake();
     notify();
     return getState();
   }
 
+  function observeLineage(lineage) {
+    return serialized(() => {
+      if (closed || lineage.id === lineageId) return;
+      platform.observeLineage(lineage);
+      lineageId = lineage.id;
+      architectureEpoch++;
+      architecture.invalidate();
+      completedClassifications.clear();
+      for (const job of activeClassifications) job.controller.abort();
+      for (const job of classificationQueue.splice(0)) {
+        pending--;
+        skipJob(job, 'source_changed_during_classification');
+      }
+      for (const artifactId of knownArtifacts.keys()) lineageWork.add(artifactId);
+      registerArtifacts(evidence.setLineage(lineage.id));
+      notify();
+    });
+  }
+
   function selectSession(id) {
     if (!sessions.has(id)) return false;
     selectedSession = id;
@@ -1045,18 +1240,26 @@ export function createPipeline({
   }
 
   async function whenIdle() {
-    await serial;
-    while (tasks.size || classificationQueue.length) {
-      pumpClassifications();
-      await Promise.allSettled([...tasks]);
+    do {
       await serial;
-    }
-    await serial;
+      await platform.whenIdle();
+      await serial;
+      while (tasks.size || classificationQueue.length) {
+        pumpClassifications();
+        await Promise.allSettled([...tasks]);
+        await serial;
+      }
+      await platform.whenIdle();
+      await architecture.whenIdle();
+      await serial;
+    } while (tasks.size || classificationQueue.length || platform.stats().queued || platform.stats().active);
   }
 
   async function close() {
     if (closed) return;
     closed = true;
+    architectureEpoch++;
+    const architectureClosed = architecture.close();
     deferredWork.clear();
     for (const job of classificationQueue.splice(0)) {
       skipJob(job, 'pipeline_closed');
@@ -1064,8 +1267,17 @@ export function createPipeline({
     }
     for (const job of activeClassifications) job.controller.abort();
     decisionService?.close?.();
+    await platform.close();
+    await architectureClosed;
     await whenIdle();
   }
 
-  return { ingest, getState, reconcile, setPaused, selectSession, whenIdle, close };
+  return {
+    ingest, getState, reconcile, observeLineage, setPaused, selectSession, whenIdle, close,
+    getArchitectureStatus: () => architecture.status(),
+    discoverArchitecture: () => architecture.request(),
+    getModelState: options => platform.snapshot(options),
+    createCheckpoint: options => platform.checkpoint(options),
+    model: platform.model,
+  };
 }