@swarmclawai/swarmclaw 1.8.11 → 1.8.13

package/src/lib/server/operations/operation-pulse.ts

@@ -1,11 +1,13 @@
 import { listPendingApprovals } from '@/lib/server/approvals'
 import { getConnectorReadiness } from '@/lib/connectors/connector-readiness'
 import { loadConnectors } from '@/lib/server/connectors/connector-repository'
+import { listOpenClawGatewayProfiles } from '@/lib/server/gateways/gateway-profile-service'
 import { listMissions } from '@/lib/server/missions/mission-repository'
 import { listUnifiedRuns } from '@/lib/server/runs/unified-run-queries'
 import type {
   ApprovalRequest,
   Connector,
+  GatewayProfile,
   Mission,
   OperationPulse,
   OperationPulseAction,
@@ -26,6 +28,7 @@ const SEVERITY_RANK: Record<OperationPulseSeverity, number> = {
 }
 
 const ACTIVE_MISSION_STATUSES = new Set<Mission['status']>(['running', 'paused'])
+const GATEWAY_TOPOLOGY_STALE_MS = 30 * 60_000
 
 export function normalizeOperationPulseRange(value: string | null | undefined): OperationPulseRange {
   return value === '7d' ? '7d' : '24h'
@@ -68,6 +71,67 @@ function addAction(actions: OperationPulseAction[], action: OperationPulseAction
   actions.push(action)
 }
 
+function gatewayPendingPairings(gateway: GatewayProfile): number {
+  return (gateway.stats?.pendingNodePairings || 0) + (gateway.stats?.pendingDevicePairings || 0)
+}
+
+function gatewayAttentionReason(gateway: GatewayProfile, now: number): {
+  severity: OperationPulseSeverity
+  summary: string
+  evidence: string[]
+} | null {
+  const pendingPairings = gatewayPendingPairings(gateway)
+  const errorCount = gateway.stats?.lastTopologyErrorCount || 0
+  const checkedAt = gateway.stats?.lastTopologyCheckedAt || gateway.lastCheckedAt || null
+  const staleTopology = !checkedAt || now - checkedAt > GATEWAY_TOPOLOGY_STALE_MS
+  const evidence = [
+    `status:${gateway.status}`,
+    `${gateway.stats?.connectedNodeCount || 0}/${gateway.stats?.nodeCount || 0} nodes`,
+  ]
+
+  if (gateway.status === 'offline') {
+    return {
+      severity: 'high',
+      summary: `${gateway.name} is offline${gateway.lastError ? `: ${gateway.lastError}` : '.'}`,
+      evidence,
+    }
+  }
+
+  if (gateway.status === 'degraded') {
+    return {
+      severity: 'high',
+      summary: `${gateway.name} is degraded${gateway.lastError ? `: ${gateway.lastError}` : '.'}`,
+      evidence,
+    }
+  }
+
+  if (errorCount > 0) {
+    return {
+      severity: 'medium',
+      summary: `${gateway.name} topology refresh reported ${errorCount} error${errorCount === 1 ? '' : 's'}.`,
+      evidence: [...evidence, gateway.stats?.lastTopologyError || 'topology error'].filter(Boolean),
+    }
+  }
+
+  if (pendingPairings > 0) {
+    return {
+      severity: 'medium',
+      summary: `${gateway.name} has ${pendingPairings} pending OpenClaw pairing request${pendingPairings === 1 ? '' : 's'}.`,
+      evidence: [...evidence, `${pendingPairings} pending pairings`],
+    }
+  }
+
+  if (staleTopology) {
+    return {
+      severity: 'medium',
+      summary: `${gateway.name} topology has not been refreshed in the last 30 minutes.`,
+      evidence,
+    }
+  }
+
+  return null
+}
+
 function sortActions(actions: OperationPulseAction[]): OperationPulseAction[] {
   return [...actions]
     .sort((left, right) => {
@@ -85,6 +149,7 @@ export function buildOperationPulse(input: {
   runs: SessionRunRecord[]
   approvals: ApprovalRequest[]
   connectors: Connector[]
+  gateways?: GatewayProfile[]
 }): OperationPulse {
   const windowStart = input.now - RANGE_MS[input.range]
   const windowRuns = input.runs.filter((run) => run.status === 'running' || run.status === 'queued' || isWithinWindow(runActivityAt(run), windowStart))
@@ -94,6 +159,9 @@ export function buildOperationPulse(input: {
   const pendingApprovals = input.approvals.filter((approval) => approval.status === 'pending')
   const connectorReadiness = input.connectors.map((connector) => ({ connector, readiness: getConnectorReadiness(connector) }))
   const connectorAttention = connectorReadiness.filter((item) => item.readiness.state !== 'healthy')
+  const gatewayAttention = (input.gateways || [])
+    .map((gateway) => ({ gateway, reason: gatewayAttentionReason(gateway, input.now) }))
+    .filter((item): item is { gateway: GatewayProfile; reason: NonNullable<ReturnType<typeof gatewayAttentionReason>> } => Boolean(item.reason))
   const budgetWarnings = input.missions
     .map((mission) => ({ mission, pressure: budgetPressure(mission, input.now) }))
     .filter((item) => item.pressure)
@@ -141,6 +209,19 @@ export function buildOperationPulse(input: {
     })
   }
 
+  for (const item of gatewayAttention.slice(0, 5)) {
+    addAction(actions, {
+      id: `gateway:${item.gateway.id}`,
+      kind: 'gateway',
+      severity: item.reason.severity,
+      title: 'Review OpenClaw gateway',
+      summary: item.reason.summary,
+      href: '/providers',
+      evidence: item.reason.evidence,
+      createdAt: item.gateway.stats?.lastTopologyCheckedAt || item.gateway.lastCheckedAt || item.gateway.updatedAt || item.gateway.createdAt,
+    })
+  }
+
   for (const item of budgetWarnings.slice(0, 5)) {
     if (!item.pressure) continue
     addAction(actions, {
@@ -178,6 +259,7 @@ export function buildOperationPulse(input: {
       failedRuns: failedRuns.length,
       pendingApprovals: pendingApprovals.length,
       connectorAttention: connectorAttention.length,
+      gatewayAttention: gatewayAttention.length,
       budgetWarnings: budgetWarnings.length,
     },
     actions: sortActions(actions),
@@ -193,5 +275,6 @@ export function getOperationPulse(range: OperationPulseRange): OperationPulse {
     runs: listUnifiedRuns({ limit: 500 }),
     approvals: listPendingApprovals(),
     connectors: Object.values(loadConnectors()),
+    gateways: listOpenClawGatewayProfiles(),
   })
 }

package/src/lib/server/runtime/queue/core.ts

@@ -124,6 +124,14 @@ function applyTaskPolicyDefaults(task: BoardTask): void {
   if (task.deadLetteredAt === undefined) task.deadLetteredAt = null
 }
 
+function formatRetryError(reason: string): string {
+  return `Retry scheduled after failure: ${reason}`.slice(0, 500)
+}
+
+function isRepeatedRetryFailure(previousError: unknown, reason: string): boolean {
+  return typeof previousError === 'string' && previousError === formatRetryError(reason)
+}
+
 export interface TaskResumeState {
   claudeSessionId: string | null
   codexThreadId: string | null
@@ -971,9 +979,10 @@ function scheduleRetryOrDeadLetter(task: BoardTask, reason: string): 'retry' | '
   }
   applyTaskPolicyDefaults(task)
   const now = Date.now()
+  const repeatedRetryFailure = isRepeatedRetryFailure(task.error, reason)
   task.attempts = (task.attempts || 0) + 1
 
-  if ((task.attempts || 0) < (task.maxAttempts || 1)) {
+  if ((task.attempts || 0) < (task.maxAttempts || 1) && !repeatedRetryFailure) {
     const delayMs = jitteredBackoff((task.retryBackoffSec || 30) * 1000, Math.max(0, (task.attempts || 1) - 1), 6 * 3600_000)
     task.status = 'queued'
     task.retryScheduledAt = now + delayMs
@@ -982,7 +991,7 @@ function scheduleRetryOrDeadLetter(task: BoardTask, reason: string): 'retry' | '
     // recovery loop burns CPU re-queueing a task that can never run.
     task.checkoutRunId = null
     task.updatedAt = now
-    task.error = `Retry scheduled after failure: ${reason}`.slice(0, 500)
+    task.error = formatRetryError(reason)
     if (!task.comments) task.comments = []
     task.comments.push({
       id: genId(),

package/src/lib/server/runtime/queue-retry-policy.test.ts

@@ -0,0 +1,98 @@
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { spawnSync } from 'node:child_process'
+import { test } from 'node:test'
+
+const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../../..')
+
+function runWithTempDataDir<T extends Record<string, unknown>>(script: string): T {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-queue-retry-policy-'))
+  try {
+    const result = spawnSync(process.execPath, ['--import', 'tsx', '--input-type=module', '--eval', script], {
+      cwd: repoRoot,
+      env: {
+        ...process.env,
+        DATA_DIR: path.join(tempDir, 'data'),
+        WORKSPACE_DIR: path.join(tempDir, 'workspace'),
+        SWARMCLAW_BUILD_MODE: '1',
+      },
+      encoding: 'utf-8',
+      timeout: 15000,
+    })
+    assert.equal(result.status, 0, result.stderr || result.stdout || 'subprocess failed')
+    const lines = (result.stdout || '')
+      .trim()
+      .split('\n')
+      .map((line) => line.trim())
+      .filter(Boolean)
+    const jsonLine = [...lines].reverse().find((line) => line.startsWith('{'))
+    return JSON.parse(jsonLine || '{}') as T
+  } finally {
+    fs.rmSync(tempDir, { recursive: true, force: true })
+  }
+}
+
+test('recoverStalledRunningTasks dead-letters repeated retry failures instead of repeating identical work', () => {
+  const output = runWithTempDataDir<{
+    result: { recovered: number; deadLettered: number }
+    status: string | null
+    attempts: number | null
+    queued: string[]
+    retryScheduledAt: number | null
+    deadLetteredAt: number | null
+    error: string | null
+  }>(`
+    const storageMod = await import('@/lib/server/storage')
+    const queueMod = await import('@/lib/server/runtime/queue')
+    const storage = storageMod.default || storageMod
+    const queue = queueMod.default || queueMod
+
+    const now = Date.now()
+    const reason = 'Detected stalled run after 5m without progress'
+    storage.saveSettings({
+      ...storage.loadSettings(),
+      taskStallTimeoutMin: 5,
+      taskRetryBackoffSec: 30,
+    })
+    storage.saveTasks({
+      repeat: {
+        id: 'repeat',
+        title: 'Repeated structural failure',
+        description: 'A task that already retried the same failure reason',
+        status: 'running',
+        agentId: 'agent-a',
+        startedAt: now - 600_000,
+        updatedAt: now - 600_000,
+        createdAt: now - 700_000,
+        maxAttempts: 3,
+        attempts: 1,
+        checkoutRunId: 'repeat-run-id',
+        error: 'Retry scheduled after failure: ' + reason,
+      },
+    })
+    storage.saveQueue([])
+
+    const result = queue.recoverStalledRunningTasks()
+    const task = storage.loadTasks().repeat
+    console.log(JSON.stringify({
+      result,
+      status: task?.status ?? null,
+      attempts: task?.attempts ?? null,
+      queued: storage.loadQueue(),
+      retryScheduledAt: task?.retryScheduledAt ?? null,
+      deadLetteredAt: task?.deadLetteredAt ?? null,
+      error: task?.error ?? null,
+    }))
+  `)
+
+  assert.equal(output.result.recovered, 0)
+  assert.equal(output.result.deadLettered, 1)
+  assert.equal(output.status, 'failed')
+  assert.equal(output.attempts, 2)
+  assert.deepEqual(output.queued, [])
+  assert.equal(output.retryScheduledAt, null)
+  assert.equal(typeof output.deadLetteredAt, 'number')
+  assert.match(output.error || '', /Dead-lettered after 2\/3 attempts/)
+})

package/src/lib/server/session-tools/context.ts

@@ -94,6 +94,8 @@ export interface ToolBuildContext {
   fileAccessPolicy?: { allowedPaths?: string[]; blockedPaths?: string[] } | null
   /** Agent's sandbox config — passed to shell for session-scoped container execution */
   sandboxConfig?: NonNullable<Agent['sandboxConfig']> | null
+  /** Loaded agent record for tool builders that need per-agent runtime settings */
+  agentRecord?: Agent | null
   /** Agent's filesystem scope — 'machine' allows file access outside the workspace */
   filesystemScope?: 'workspace' | 'machine'
 }

package/src/lib/server/session-tools/execute.test.ts

@@ -1,7 +1,40 @@
-import { describe, it } from 'node:test'
 import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { spawnSync } from 'node:child_process'
+import { describe, it, test } from 'node:test'
 import { redactSecrets, buildCredentialEnv } from './credential-env'
 
+const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../../..')
+
+function runWithTempDataDir<T extends Record<string, unknown>>(script: string): T {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-execute-tool-'))
+  try {
+    const result = spawnSync(process.execPath, ['--import', 'tsx', '--input-type=module', '--eval', script], {
+      cwd: repoRoot,
+      env: {
+        ...process.env,
+        DATA_DIR: path.join(tempDir, 'data'),
+        WORKSPACE_DIR: path.join(tempDir, 'workspace'),
+        SWARMCLAW_BUILD_MODE: '1',
+      },
+      encoding: 'utf-8',
+      timeout: 15000,
+    })
+    assert.equal(result.status, 0, result.stderr || result.stdout || 'subprocess failed')
+    const lines = (result.stdout || '')
+      .trim()
+      .split('\n')
+      .map((line) => line.trim())
+      .filter(Boolean)
+    const jsonLine = [...lines].reverse().find((line) => line.startsWith('{'))
+    return JSON.parse(jsonLine || '{}') as T
+  } finally {
+    fs.rmSync(tempDir, { recursive: true, force: true })
+  }
+}
+
 describe('credential-env', () => {
   describe('redactSecrets', () => {
     it('redacts secret values from text', () => {
@@ -56,3 +89,62 @@ describe('credential-env', () => {
     })
   })
 })
+
+test('buildExecuteTools uses the current agent executeConfig backend', () => {
+  const output = runWithTempDataDir<{
+    hostOutput: string
+    sandboxOutput: string
+  }>(`
+    const storageMod = await import('@/lib/server/storage')
+    const executeModImport = await import('./src/lib/server/session-tools/execute.ts')
+    const executeMod = executeModImport.default || executeModImport
+    const storage = storageMod.default || storageMod
+    const now = Date.now()
+
+    storage.saveAgents({
+      'agent-host': {
+        id: 'agent-host',
+        name: 'Host Agent',
+        provider: 'openai',
+        model: 'gpt-test',
+        executeConfig: { backend: 'host', network: { enabled: true }, timeout: 30 },
+        createdAt: now,
+        updatedAt: now,
+      },
+      'agent-sandbox': {
+        id: 'agent-sandbox',
+        name: 'Sandbox Agent',
+        provider: 'openai',
+        model: 'gpt-test',
+        createdAt: now,
+        updatedAt: now,
+      },
+    })
+
+    const makeContext = (agentId) => ({
+      cwd: process.cwd(),
+      ctx: { agentId, sessionId: 'session-' + agentId },
+      hasExtension: (name) => name === 'execute',
+      hasTool: (name) => name === 'execute',
+      cleanupFns: [],
+      commandTimeoutMs: 30000,
+      claudeTimeoutMs: 30000,
+      cliProcessTimeoutMs: 30000,
+      persistDelegateResumeId: () => {},
+      readStoredDelegateResumeId: () => null,
+      resolveCurrentSession: () => ({ id: 'session-' + agentId, agentId }),
+      activeExtensions: ['execute'],
+      agentRecord: storage.loadAgent(agentId),
+    })
+
+    const hostTool = executeMod.buildExecuteTools(makeContext('agent-host'))[0]
+    const sandboxTool = executeMod.buildExecuteTools(makeContext('agent-sandbox'))[0]
+    const hostOutput = await hostTool.invoke({ code: 'printf host-ok', persistent: true })
+    const sandboxOutput = await sandboxTool.invoke({ code: 'printf sandbox-no', persistent: true })
+
+    console.log(JSON.stringify({ hostOutput, sandboxOutput }))
+  `)
+
+  assert.equal(output.hostOutput, 'host-ok')
+  assert.match(output.sandboxOutput, /requires `executeConfig\.backend = "host"`/)
+})

package/src/lib/server/session-tools/execute.ts

@@ -26,6 +26,7 @@ import {
   normalizeAgentExecuteConfig,
   type AgentExecuteConfig,
 } from '@/lib/agent-execute-defaults'
+import { loadAgent } from '../storage'
 
 const TAG = 'execute'
 
@@ -310,9 +311,9 @@ registerNativeCapability('execute', ExecuteExtension)
 export function buildExecuteTools(bctx: ToolBuildContext) {
   if (!bctx.hasExtension('execute')) return []
 
-  // Resolve execute config from the agent
-  const session = bctx.resolveCurrentSession?.()
-  const agent = session?.agent as (Agent & { executeConfig?: ExecuteConfig }) | undefined
+  const agentId = typeof bctx.ctx?.agentId === 'string' ? bctx.ctx.agentId.trim() : ''
+  const agent = (bctx.agentRecord as (Agent & { executeConfig?: ExecuteConfig }) | null | undefined)
+    ?? (agentId ? (loadAgent(agentId) as (Agent & { executeConfig?: ExecuteConfig }) | null) : null)
   const executeConfig = normalizeAgentExecuteConfig(agent?.executeConfig)
 
   return [

package/src/lib/server/session-tools/index.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod'
 import { tool, type StructuredToolInterface } from '@langchain/core/tools'
-import type { Session } from '@/types'
+import type { Agent, Session } from '@/types'
 import { dedup, errorMessage } from '@/lib/shared-utils'
 import { loadSettings, loadSession, loadAgent, loadMcpServers, patchAgent, patchSession } from '../storage'
 import { loadRuntimeSettings } from '@/lib/server/runtime/runtime-settings'
@@ -221,6 +221,7 @@ export async function buildSessionTools(cwd: string, enabledExtensions: string[]
       activeExtensions,
       fileAccessPolicy: effectiveFileAccessPolicy,
       sandboxConfig: agentRecord?.sandboxConfig ?? null,
+      agentRecord: agentRecord as Agent | null,
       filesystemScope,
     }
 

package/src/lib/server/tasks/task-validation.test.ts

@@ -49,6 +49,39 @@ test('validateTaskCompletion still enforces stricter minimum for implementation
   assert.ok(validation.reasons.some((reason) => reason.includes('Result summary is too short')))
 })
 
+test('validateTaskCompletion does not auto-apply implementation quality gates to scheduled tasks', () => {
+  const validation = validateTaskCompletion({
+    title: '[Sched] Daily wiki hygiene (run #1)',
+    description: 'Run scripts/wiki-hygiene.mjs and post the digest to Slack.',
+    result: 'Ran node scripts/wiki-hygiene.mjs and posted the digest to Slack with exit code 0.',
+    sourceType: 'schedule',
+    sourceScheduleId: 'schedule-wiki',
+    error: null,
+  } as Partial<BoardTask>)
+
+  assert.equal(validation.ok, true)
+})
+
+test('validateTaskCompletion still enforces explicit quality gates on scheduled tasks', () => {
+  const validation = validateTaskCompletion({
+    title: '[Sched] Daily wiki hygiene (run #1)',
+    description: 'Run scripts/wiki-hygiene.mjs and post the digest to Slack.',
+    result: 'Ran node scripts/wiki-hygiene.mjs and posted the digest to Slack with exit code 0.',
+    sourceType: 'schedule',
+    sourceScheduleId: 'schedule-wiki',
+    qualityGate: {
+      enabled: true,
+      minResultChars: 20,
+      minEvidenceItems: 2,
+      requireVerification: true,
+    },
+    error: null,
+  } as Partial<BoardTask>)
+
+  assert.equal(validation.ok, false)
+  assert.ok(validation.reasons.some((reason) => reason.includes('verification evidence is required')))
+})
+
 test('validateTaskCompletion fails implementation task with unfinished next-step language', () => {
   const validation = validateTaskCompletion({
     title: 'Build weather dashboard',

package/src/lib/server/tasks/task-validation.ts

@@ -85,7 +85,8 @@ export function validateTaskCompletion(
   const report = options.report || null
   const hasExplicitQualityGate = !!task.qualityGate && typeof task.qualityGate === 'object'
   const qualityGate = normalizeTaskQualityGate(task.qualityGate || null, options.settings || null)
-  const implementationTask = IMPLEMENTATION_HINT.test(title) || IMPLEMENTATION_HINT.test(description)
+  const scheduleTask = task.sourceType === 'schedule' || typeof task.sourceScheduleId === 'string'
+  const implementationTask = !scheduleTask && (IMPLEMENTATION_HINT.test(title) || IMPLEMENTATION_HINT.test(description))
 
   if (error) reasons.push('Task has a non-empty error field.')
   if (/^untitled task$/i.test(title) && !description) {

package/src/types/misc.ts

@@ -670,6 +670,11 @@ export interface OpenClawGatewayStats {
   pairedDeviceCount?: number
   pendingDevicePairings?: number
   externalRuntimeCount?: number
+  sessionCount?: number
+  presenceCount?: number
+  lastTopologyCheckedAt?: number
+  lastTopologyErrorCount?: number
+  lastTopologyError?: string | null
 }
 
 export interface OpenClawDeploymentConfig {
@@ -773,6 +778,59 @@ export interface OpenClawPairingSnapshot {
   paired?: OpenClawPairedDevice[]
 }
 
+export interface OpenClawGatewayRpcError {
+  method: string
+  message: string
+}
+
+export interface OpenClawGatewaySession {
+  id: string
+  key?: string | null
+  title?: string | null
+  channel?: string | null
+  sender?: string | null
+  updatedAt?: number | null
+  status?: string | null
+}
+
+export interface OpenClawGatewayPresenceEntry {
+  id: string
+  label?: string | null
+  mode?: string | null
+  deviceId?: string | null
+  host?: string | null
+  status?: string | null
+  updatedAt?: number | null
+}
+
+export interface OpenClawGatewayTopologyStats extends OpenClawGatewayStats {
+  pendingPairingCount: number
+  hasErrors: boolean
+}
+
+export interface OpenClawGatewayTopology {
+  profile: GatewayProfile
+  connected: boolean
+  refreshedAt: number
+  stats: OpenClawGatewayTopologyStats
+  nodes: OpenClawNode[]
+  nodePairings: OpenClawNodePairRequest[]
+  devicePairings: OpenClawDevicePairRequest[]
+  pairedDevices: OpenClawPairedDevice[]
+  sessions: OpenClawGatewaySession[]
+  presence: OpenClawGatewayPresenceEntry[]
+  errors: OpenClawGatewayRpcError[]
+}
+
+export interface OpenClawGatewayFleetTopology {
+  generatedAt: number
+  gateways: OpenClawGatewayTopology[]
+  totals: OpenClawGatewayTopologyStats & {
+    gatewayCount: number
+    connectedGatewayCount: number
+  }
+}
+
 // --- ClawHub ---
 
 export interface ClawHubSkill {

package/src/types/operations.ts

@@ -7,6 +7,7 @@ export type OperationPulseActionKind =
   | 'run'
   | 'approval'
   | 'connector'
+  | 'gateway'
   | 'budget'
   | 'quality'
 
@@ -16,6 +17,7 @@ export interface OperationPulseKpis {
   failedRuns: number
   pendingApprovals: number
   connectorAttention: number
+  gatewayAttention: number
   budgetWarnings: number
 }