@shapeshift-labs/frontier-swarm 0.2.0 → 0.4.0

package/README.md

@@ -36,7 +36,7 @@ The published Frontier package family is generated from one shared package catal
 - [`@shapeshift-labs/frontier-workflow`](https://www.npmjs.com/package/@shapeshift-labs/frontier-workflow): Serializable durable workflow/process manifests for Frontier apps, including steps, waits, approvals, timers, retries, expected patches, compensation, records, timelines, and registry graph output.
 - [`@shapeshift-labs/frontier-worker`](https://www.npmjs.com/package/@shapeshift-labs/frontier-worker): Serializable worker and edge task descriptors for Frontier apps, including queues, idempotency keys, retry and timeout policy, declared reads/writes/effects, snapshots, patch outputs, produced assets, execution records, logs, trace links, proof hashes, dedupe indexes, and registry graph output.
 - [`@shapeshift-labs/frontier-queue`](https://www.npmjs.com/package/@shapeshift-labs/frontier-queue): Serializable durable queue state, leases, retries, dedupe keys, patch-carrying jobs, dead-letter records, replay evidence, and queue inspection for Frontier apps.
-- [`@shapeshift-labs/frontier-swarm-codex`](https://www.npmjs.com/package/@shapeshift-labs/frontier-swarm-codex): Node Codex CLI adapter for Frontier swarm plans, including prompt rendering, worktree and snapshot workspaces, Codex argument construction, JSONL capture, verification commands, and result artifacts.
+- [`@shapeshift-labs/frontier-swarm-codex`](https://www.npmjs.com/package/@shapeshift-labs/frontier-swarm-codex): Node Codex CLI adapter for Frontier swarm plans, including prompt rendering, worktree and snapshot workspaces, Codex argument compatibility, JSONL capture, verification commands, pid-backed stop, collect/apply workflows, merge indexes, queue overlays, merge bundles, and result artifacts.
 - [`@shapeshift-labs/frontier-kv`](https://www.npmjs.com/package/@shapeshift-labs/frontier-kv): Serializable in-memory key/value state for Frontier apps, including TTL, versioned compare-and-set, batched patch mutations, scans, watchers, snapshots, JSONL event evidence, and replay verification.
 - [`@shapeshift-labs/frontier-kv-locks`](https://www.npmjs.com/package/@shapeshift-labs/frontier-kv-locks): Lease-style lock records on top of Frontier KV, including acquire, renew, release, fencing tokens, expiration, owner evidence, and replayable lock events.
 - [`@shapeshift-labs/frontier-kv-rate-limit`](https://www.npmjs.com/package/@shapeshift-labs/frontier-kv-rate-limit): Patch-native rate limit buckets for Frontier KV, including fixed windows, sliding windows, token buckets, deterministic refill, consume evidence, and reset records.
@@ -211,10 +211,13 @@ Large swarms need a control plane, not just a flat worker loop. `frontier-swarm`
 import {
   checkSwarmBudget,
   createSwarmArtifactIndex,
+  createSwarmEventStream,
   createSwarmLeases,
   createSwarmMergePlan,
+  createSwarmQueueSnapshot,
   createSwarmReviewPlan,
-  createSwarmSchedule
+  createSwarmSchedule,
+  routeSwarmEventToMailboxes
 } from '@shapeshift-labs/frontier-swarm';
 
 const schedule = createSwarmSchedule({
@@ -237,10 +240,27 @@ The scale APIs are runtime-neutral and serializable:
 - dependency DAGs are compiled into `plan.graph`,
 - `createSwarmSchedule` returns ready, blocked, running, completed, and failed jobs under lane/compute/contention limits,
 - `createSwarmLeases` gives workers expiring leases with fencing tokens,
+- lane/task `capabilities` and `resourceRequirements` can reserve browser work with lower concurrency, port pools, profile directories, and explicit capability checks,
+- `createSwarmQueueSnapshot` and run checkpoints give durable queue/run-store adapters a stable serialization shape,
+- `createSwarmEventStream` and `routeSwarmEventToMailboxes` route global, lane, task, and worker events into coordinator-facing JSONL streams,
 - `checkSwarmBudget` records token/cost/time/retry budget decisions,
 - `createSwarmArtifactIndex` groups evidence, timelines, logs, and produced files,
 - `createSwarmReviewPlan` samples or requires reviewer assignments,
 - `createSwarmMergePlan` blocks jobs with failed checks, required reviews, ownership violations, or conflicting changed paths,
+- job results include merge-readiness classification: `discovery-only`, `patch-candidate`, `verified-patch`, `rejected`, or `blocked`,
+- `ownershipRegions` allow hot files to be split into semantic regions such as `content.docs.*` or `adminSettings.quota.*`; merge conflict detection compares explicit changed regions when both sides report them and falls back to path conflicts when either side omits regions,
+- `createSwarmMergeBundle` builds a compact worker `merge.json` shape with touched owned files, patch path, evidence, verification, queue items satisfied, risk, and disposition,
+- `createSwarmQueueOverlay` and `deriveSwarmQueueStatus` keep central queue files immutable while deriving status from worker result overlays,
+- `createSwarmMergeIndex` records stale/patch status and region-aware conflicts so coordinators can review ready bundles before reading every worker directory,
+- `checkSwarmRegionOwnership` makes semantic region ownership enforceable instead of only advisory,
+- `createSwarmHotspotReport` highlights repeatedly touched files and suggests module/region splits for merge throughput,
+- `createSwarmReviewerLanePlan` turns risky/conflicting merge bundles into reviewer-lane tasks,
+- `createSwarmRunStoreShards` describes sharded event/result/checkpoint paths for large run stores,
+- `createSwarmMergeAdmission` limits ready merges by count, touched paths/regions, and risk budget,
+- `createSwarmPatchStackPlan` clusters compatible bundles into candidate patch stacks by lane, path, region, disposition, and risk so reviewers can evaluate batches instead of individual worker directories,
+- `createSwarmContextPack` gives workers compact task context: relevant files, API maps, known failures, focused/oracle commands, expected evidence, exclusions, evidence schema, playbooks, and explicit dead ends to avoid,
+- `createSwarmOracleCorpus` indexes deterministic reference artifacts such as traces, snapshots, classifications, expected outputs, or fixtures without assuming a project domain,
+- `createSwarmLanePlaybook` turns successful prior bundles into persistent lane-specific guidance with commands, hot paths, evidence patterns, and avoid-investigating notes,
 - `decomposeSwarmFeature` creates an initial task queue for feature work across lanes.
 
 ## Hierarchical Compute
@@ -260,11 +280,22 @@ That lets a parent swarm route implementation jobs to a deep model while evidenc
 - `defineSwarmManifest`, `createSwarmManifest`
 - `defineSwarmTasks`
 - `compileSwarm`, `validateSwarmManifest`
+- `createSwarmTaskSelection`
 - `createSwarmPlan`, `createSwarmRun`
 - `createSwarmSchedule`, `createSwarmLeases`
+- `createSwarmQueueSnapshot`, `createSwarmRunCheckpoint`
+- `createSwarmQueueOverlay`, `deriveSwarmQueueStatus`
+- `createSwarmEventStream`, `createSwarmMailbox`, `routeSwarmEventToMailboxes`
 - `checkSwarmBudget`
 - `createSwarmArtifactIndex`
-- `createSwarmReviewPlan`, `createSwarmMergePlan`
+- `createSwarmReviewPlan`, `createSwarmReviewerLanePlan`, `createSwarmMergePlan`
+- `createSwarmMergeBundle`
+- `createSwarmMergeIndex`, `createSwarmMergeAdmission`
+- `createSwarmHotspotReport`, `createSwarmRunStoreShards`
+- `createSwarmPatchStackPlan`
+- `createSwarmContextPack`, `createSwarmOracleCorpus`, `createSwarmLanePlaybook`
+- `classifySwarmMergeReadiness`, `classifySwarmMergeDisposition`
+- `resolveSwarmChangedRegions`, `checkSwarmRegionOwnership`
 - `decomposeSwarmFeature`
 - `recordSwarmEvent`, `completeSwarmJob`
 - `resolveSwarmCompute`
@@ -280,7 +311,7 @@ Run the package-local benchmark:
 npm run bench
 ```
 
-The benchmark writes `benchmarks/results/frontier-swarm-package-bench-latest.json` when run from the monorepo. These are Frontier-only package measurements for plan creation, manifest validation, hierarchical compute resolution, ownership checks, run creation, JSONL, and proof hashing.
+The benchmark writes `benchmarks/results/frontier-swarm-package-bench-latest.json` when run from the monorepo. These are Frontier-only package measurements for plan creation, manifest validation, hierarchical compute resolution, ownership checks, scheduling/leases, queue snapshots, queue overlays, merge bundles, merge indexes, merge admission, hotspot reports, context packs, oracle corpora, lane playbooks, patch stack plans, event routing, run checkpoints, JSONL, and proof hashing.
 
 ## Source Repository
 

package/benchmarks/package-bench.mjs

@@ -4,13 +4,28 @@ import { performance } from 'node:perf_hooks';
 import { fileURLToPath } from 'node:url';
 import {
   checkSwarmOwnership,
+  createSwarmContextPack,
+  createSwarmHotspotReport,
+  createSwarmLanePlaybook,
+  createSwarmMergeAdmission,
   createSwarmManifest,
+  createSwarmEventStream,
+  createSwarmLeases,
+  createSwarmMergeBundle,
+  createSwarmMergeIndex,
+  createSwarmOracleCorpus,
+  createSwarmPatchStackPlan,
   createSwarmPlan,
   createSwarmProof,
+  createSwarmQueueOverlay,
+  createSwarmQueueSnapshot,
   createSwarmRun,
+  createSwarmRunCheckpoint,
+  createSwarmSchedule,
   decodeSwarmJsonl,
   defineSwarmTasks,
   encodeSwarmJsonl,
+  routeSwarmEventToMailboxes,
   resolveSwarmCompute,
   validateSwarmManifest
 } from '../dist/index.js';
@@ -31,6 +46,11 @@ let plan = createSwarmPlan(manifest, tasks, { limit: 64 });
 let run = createSwarmRun({ plan });
 let jsonl = encodeSwarmJsonl([plan, run]);
 let cursor = 0;
+let schedule = createSwarmSchedule({ plan, maxReadyJobs: 128 });
+let leases = createSwarmLeases({ schedule, workerId: 'bench-worker', now: 1000, leaseMs: 60000, count: 16 });
+let eventStream = createSwarmEventStream({ runId: 'bench', root: 'agent-runs/bench/streams', lanes: manifest.lanes });
+let bundles = makeBundles(plan, 32);
+let mergeIndex = createSwarmMergeIndex({ bundles });
 
 const rows = [
   measure('create-plan-' + taskCount, 8, () => {
@@ -49,7 +69,70 @@ const rows = [
     return jsonl.length;
   }),
   measure('jsonl-decode', 16, () => decodeSwarmJsonl(jsonl).length),
-  measure('proof', 16, () => createSwarmProof(plan).hash.length)
+  measure('proof', 16, () => createSwarmProof(plan).hash.length),
+  measure('schedule-lease-' + taskCount, 8, () => {
+    schedule = createSwarmSchedule({ plan, maxReadyJobs: 128, maxComputeConcurrency: { fast: 64, deep: 32 } });
+    leases = createSwarmLeases({ schedule, workerId: 'bench-worker', now: 1000 + cursor++, leaseMs: 60000, count: 16 });
+    return schedule.ready.length + leases.length;
+  }),
+  measure('queue-snapshot-' + taskCount, 8, () => {
+    const snapshot = createSwarmQueueSnapshot({ plan, run, leases, generatedAt: 2000 + cursor++ });
+    return snapshot.summary.jobCount + snapshot.summary.leaseCount;
+  }),
+  measure('run-checkpoint-' + taskCount, 16, () => createSwarmRunCheckpoint({ run, sequence: cursor++ }).hash.length),
+  measure('merge-bundle-' + taskCount, 32, () => createSwarmMergeBundle({
+    job: plan.jobs[cursor % plan.jobs.length],
+    result: {
+      jobId: plan.jobs[cursor++ % plan.jobs.length].id,
+      status: 'completed',
+      changedPaths: ['src/runtime/file.ts'],
+      verification: [{ status: 0 }]
+    },
+    patchPath: 'agent-runs/bench/changes.patch'
+  }).id.length),
+  measure('queue-overlay-' + taskCount, 16, () => createSwarmQueueOverlay({ bundles, generatedAt: 3000 + cursor++ }).summary.entryCount),
+  measure('merge-index-' + taskCount, 8, () => {
+    bundles = makeBundles(plan, 32);
+    mergeIndex = createSwarmMergeIndex({ bundles, generatedAt: 4000 + cursor++ });
+    return mergeIndex.summary.entryCount + mergeIndex.summary.conflictCount;
+  }),
+  measure('merge-admission-' + taskCount, 16, () => createSwarmMergeAdmission({ index: mergeIndex, maxReady: 8, maxChangedPaths: 16 }).summary.admittedCount),
+  measure('hotspot-report-' + taskCount, 16, () => createSwarmHotspotReport({ bundles, threshold: 3 }).summary.recommendationCount),
+  measure('context-pack-' + taskCount, 32, () => createSwarmContextPack({
+    job: plan.jobs[cursor % plan.jobs.length],
+    files: ['src/runtime/file.ts', 'test/runtime-smoke.mjs'],
+    apiMap: {
+      runtime: ['createRuntime', 'stepRuntime'],
+      tests: ['runtime smoke gate']
+    },
+    knownFailures: ['shared renderer gate is noisy on old snapshots'],
+    oracleCommands: [{ name: 'focused-gate', command: 'npm', args: ['test'], required: true }],
+    evidenceSchema: { type: 'object', required: ['ok', 'commands'] },
+    avoidInvestigating: ['unrelated route snapshots'],
+    playbookIds: ['runtime-playbook']
+  }).files.length),
+  measure('oracle-corpus-' + taskCount, 32, () => createSwarmOracleCorpus({
+    artifacts: [
+      { id: 'trace-runtime', path: 'oracles/runtime-trace.jsonl', kind: 'trace', tags: ['runtime', 'reference'], hash: 'fnv1a32:trace' },
+      { id: 'snapshot-routing', path: 'oracles/routing-snapshot.json', kind: 'snapshot', tags: ['routing', 'reference'] }
+    ]
+  }).summary.artifactCount),
+  measure('lane-playbook-' + taskCount, 16, () => createSwarmLanePlaybook({
+    lane: 'runtime',
+    successfulBundles: bundles,
+    notes: ['prefer narrow patches with focused evidence'],
+    commands: [{ name: 'runtime-smoke', command: 'npm', args: ['test'], required: true }],
+    avoidInvestigating: ['generated fixtures unless task owns them'],
+    evidencePatterns: ['evidence.json', 'commands.md']
+  }).successfulJobIds.length),
+  measure('patch-stack-plan-' + taskCount, 16, () => createSwarmPatchStackPlan({
+    index: mergeIndex,
+    maxStackSize: 8
+  }).summary.stackCount),
+  measure('event-route-' + taskCount, 64, () => {
+    eventStream = createSwarmEventStream({ runId: 'bench', root: 'agent-runs/bench/streams', lanes: manifest.lanes });
+    return routeSwarmEventToMailboxes(eventStream, { type: 'agent.evidence', jobId: plan.jobs[cursor++ % plan.jobs.length].id, lane: 'runtime' }).length;
+  })
 ];
 
 const report = {
@@ -112,6 +195,25 @@ function makeTasks(count) {
   return tasks;
 }
 
+function makeBundles(plan, count) {
+  const bundles = [];
+  for (let i = 0; i < Math.min(count, plan.jobs.length); i += 1) {
+    const job = plan.jobs[i];
+    bundles.push(createSwarmMergeBundle({
+      job,
+      result: {
+        jobId: job.id,
+        status: 'verified',
+        changedPaths: [job.task.targetRefs[0] ?? `src/runtime/file-${i}.ts`],
+        changedRegions: i % 2 === 0 ? [`region.${i}`] : [],
+        verification: [{ status: 0 }]
+      },
+      patchPath: `agent-runs/bench/${job.id}/changes.patch`
+    }));
+  }
+  return bundles;
+}
+
 function measure(fixture, operationsPerRound, fn) {
   const samples = [];
   let checksum = 0;