@swarmclawai/swarmclaw 1.9.38 → 1.9.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/package.json +2 -2
- package/src/lib/server/autonomy/supervisor-reflection.test.ts +10 -1
- package/src/lib/server/connectors/outbox.ts +22 -2
- package/src/lib/server/runtime/queue/core.ts +160 -18
- package/src/lib/server/runtime/queue/orphan-recovery.test.ts +49 -0
- package/src/lib/server/runtime/queue/orphan-recovery.ts +32 -0
- package/src/lib/server/runtime/scheduled-run-preflight.test.ts +73 -0
- package/src/lib/server/runtime/scheduled-run-preflight.ts +83 -0
- package/src/lib/server/schedules/schedule-lifecycle.test.ts +44 -0
- package/src/lib/server/schedules/schedule-lifecycle.ts +27 -0
- package/src/lib/server/storage-normalization.ts +13 -0
- package/src/lib/server/tasks/task-followups.test.ts +124 -41
- package/src/lib/server/tasks/task-followups.ts +28 -3
- package/src/lib/server/tasks/task-lifecycle.test.ts +25 -0
- package/src/lib/server/tasks/task-lifecycle.ts +6 -0
- package/src/lib/server/tasks/task-result.test.ts +25 -1
- package/src/lib/server/tasks/task-result.ts +22 -0
- package/src/lib/server/workspace-paths.test.ts +72 -0
- package/src/lib/server/workspace-paths.ts +60 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { getProvider } from '@/lib/providers'
|
|
2
|
+
import { resolveCredentialSecret } from '@/lib/server/credentials/credential-service'
|
|
3
|
+
import { resolveProviderCredentialId } from '@/lib/server/provider-endpoint'
|
|
4
|
+
|
|
5
|
+
export interface ProviderCredentialPreflightInput {
|
|
6
|
+
provider?: string | null
|
|
7
|
+
ollamaMode?: string | null
|
|
8
|
+
credentialId?: string | null
|
|
9
|
+
fallbackCredentialIds?: readonly string[] | null
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface ProviderCredentialPreflightDeps {
|
|
13
|
+
getProvider: (id: string) => { requiresApiKey?: boolean } | null
|
|
14
|
+
resolveProviderCredentialId: (input: {
|
|
15
|
+
provider?: string | null
|
|
16
|
+
ollamaMode?: string | null
|
|
17
|
+
credentialId?: string | null
|
|
18
|
+
}) => string | null
|
|
19
|
+
resolveCredentialSecret: (credentialId: string | null | undefined) => string | null
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export type ProviderCredentialPreflightResult =
|
|
23
|
+
| { ok: true }
|
|
24
|
+
| { ok: false; error: string }
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Fail-fast credential check for scheduled runs. Catches the "schedule fires
|
|
28
|
+
* deep into execution and dies on a 401" case before the run starts. No
|
|
29
|
+
* network calls: it only verifies that at least one credential with a
|
|
30
|
+
* decryptable secret exists for a provider that requires an API key.
|
|
31
|
+
*
|
|
32
|
+
* Deliberately permissive: it passes whenever ANY candidate credential
|
|
33
|
+
* resolves, so it can only block runs that are guaranteed to fail.
|
|
34
|
+
*/
|
|
35
|
+
export function preflightProviderCredential(
|
|
36
|
+
input: ProviderCredentialPreflightInput,
|
|
37
|
+
deps?: Partial<ProviderCredentialPreflightDeps>,
|
|
38
|
+
): ProviderCredentialPreflightResult {
|
|
39
|
+
const provider = typeof input.provider === 'string' ? input.provider.trim() : ''
|
|
40
|
+
if (!provider) return { ok: true }
|
|
41
|
+
|
|
42
|
+
const resolved: ProviderCredentialPreflightDeps = {
|
|
43
|
+
getProvider,
|
|
44
|
+
resolveProviderCredentialId,
|
|
45
|
+
resolveCredentialSecret,
|
|
46
|
+
...deps,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
let providerConfig: { requiresApiKey?: boolean } | null = null
|
|
50
|
+
try {
|
|
51
|
+
providerConfig = resolved.getProvider(provider)
|
|
52
|
+
} catch {
|
|
53
|
+
return { ok: true }
|
|
54
|
+
}
|
|
55
|
+
// Unknown/custom providers and key-optional providers (ollama, CLI, gateway
|
|
56
|
+
// routes) are exempt; execution resolves those credentials differently.
|
|
57
|
+
if (providerConfig?.requiresApiKey !== true) return { ok: true }
|
|
58
|
+
|
|
59
|
+
const candidateIds = [
|
|
60
|
+
resolved.resolveProviderCredentialId({
|
|
61
|
+
provider,
|
|
62
|
+
ollamaMode: input.ollamaMode ?? null,
|
|
63
|
+
credentialId: input.credentialId ?? null,
|
|
64
|
+
}),
|
|
65
|
+
...(input.fallbackCredentialIds || []),
|
|
66
|
+
// Last resort: any credential stored for this provider
|
|
67
|
+
input.credentialId
|
|
68
|
+
? resolved.resolveProviderCredentialId({ provider, ollamaMode: input.ollamaMode ?? null, credentialId: null })
|
|
69
|
+
: null,
|
|
70
|
+
]
|
|
71
|
+
const seen = new Set<string>()
|
|
72
|
+
for (const candidateId of candidateIds) {
|
|
73
|
+
const id = typeof candidateId === 'string' ? candidateId.trim() : ''
|
|
74
|
+
if (!id || seen.has(id)) continue
|
|
75
|
+
seen.add(id)
|
|
76
|
+
if (resolved.resolveCredentialSecret(id)) return { ok: true }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
ok: false,
|
|
81
|
+
error: `Provider authentication preflight failed: no API credential configured for provider "${provider}". Add a key in Settings > Providers (or assign one to the agent), then re-run the schedule.`,
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -177,4 +177,48 @@ describe('schedule lifecycle helpers', () => {
|
|
|
177
177
|
assert.equal(Boolean(schedules['sched-archived']), false)
|
|
178
178
|
assert.equal(Boolean(schedules['sched-live']), true)
|
|
179
179
|
})
|
|
180
|
+
|
|
181
|
+
it('applyScheduleRunOutcome records a successful run and clears the previous error', () => {
|
|
182
|
+
const schedule = makeSchedule({
|
|
183
|
+
lastDeliveryStatus: 'error',
|
|
184
|
+
lastDeliveryError: 'previous failure',
|
|
185
|
+
}) as unknown as import('@/types').Schedule
|
|
186
|
+
|
|
187
|
+
const changed = lifecycle.applyScheduleRunOutcome(schedule, { status: 'completed', error: null }, 500)
|
|
188
|
+
|
|
189
|
+
assert.equal(changed, true)
|
|
190
|
+
assert.equal(schedule.lastDeliveryStatus, 'ok')
|
|
191
|
+
assert.equal(schedule.lastDeliveryError, null)
|
|
192
|
+
assert.equal(schedule.lastDeliveredAt, 500)
|
|
193
|
+
assert.equal(schedule.updatedAt, 500)
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('applyScheduleRunOutcome records a failed run with a truncated error', () => {
|
|
197
|
+
const schedule = makeSchedule() as unknown as import('@/types').Schedule
|
|
198
|
+
const longError = 'x'.repeat(600)
|
|
199
|
+
|
|
200
|
+
const changed = lifecycle.applyScheduleRunOutcome(schedule, { status: 'failed', error: longError }, 700)
|
|
201
|
+
|
|
202
|
+
assert.equal(changed, true)
|
|
203
|
+
assert.equal(schedule.lastDeliveryStatus, 'error')
|
|
204
|
+
assert.equal(schedule.lastDeliveryError?.length, 500)
|
|
205
|
+
assert.equal(schedule.lastDeliveredAt, 700)
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
it('applyScheduleRunOutcome uses a fallback message when the task has no error', () => {
|
|
209
|
+
const schedule = makeSchedule() as unknown as import('@/types').Schedule
|
|
210
|
+
|
|
211
|
+
lifecycle.applyScheduleRunOutcome(schedule, { status: 'failed', error: null }, 800)
|
|
212
|
+
|
|
213
|
+
assert.match(schedule.lastDeliveryError || '', /Scheduled run failed without a recorded error/)
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
it('applyScheduleRunOutcome ignores non-terminal task statuses', () => {
|
|
217
|
+
const schedule = makeSchedule() as unknown as import('@/types').Schedule
|
|
218
|
+
|
|
219
|
+
const changed = lifecycle.applyScheduleRunOutcome(schedule, { status: 'queued', error: null }, 900)
|
|
220
|
+
|
|
221
|
+
assert.equal(changed, false)
|
|
222
|
+
assert.equal(schedule.lastDeliveryStatus, undefined)
|
|
223
|
+
})
|
|
180
224
|
})
|
|
@@ -84,6 +84,33 @@ function disableSessionHeartbeatLocally(
|
|
|
84
84
|
return true
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
/**
|
|
88
|
+
* Records the outcome of a scheduled run on the schedule itself so failures
|
|
89
|
+
* are visible without digging through the linked task. Returns true when the
|
|
90
|
+
* schedule was updated and needs to be persisted.
|
|
91
|
+
*/
|
|
92
|
+
export function applyScheduleRunOutcome(
|
|
93
|
+
schedule: Schedule,
|
|
94
|
+
task: Pick<BoardTask, 'status' | 'error'>,
|
|
95
|
+
now: number,
|
|
96
|
+
): boolean {
|
|
97
|
+
if (task.status === 'completed') {
|
|
98
|
+
schedule.lastDeliveryStatus = 'ok'
|
|
99
|
+
schedule.lastDeliveryError = null
|
|
100
|
+
schedule.lastDeliveredAt = now
|
|
101
|
+
schedule.updatedAt = now
|
|
102
|
+
return true
|
|
103
|
+
}
|
|
104
|
+
if (task.status === 'failed') {
|
|
105
|
+
schedule.lastDeliveryStatus = 'error'
|
|
106
|
+
schedule.lastDeliveryError = (task.error || 'Scheduled run failed without a recorded error.').slice(0, 500)
|
|
107
|
+
schedule.lastDeliveredAt = now
|
|
108
|
+
schedule.updatedAt = now
|
|
109
|
+
return true
|
|
110
|
+
}
|
|
111
|
+
return false
|
|
112
|
+
}
|
|
113
|
+
|
|
87
114
|
function markTaskCancelled(task: BoardTask, reason: string, now: number): void {
|
|
88
115
|
task.status = 'cancelled'
|
|
89
116
|
task.retryScheduledAt = null
|
|
@@ -171,6 +171,19 @@ function normalizeStoredScheduleRecord(value: unknown, loadItem: CollectionItemL
|
|
|
171
171
|
if (archivedAt != null) schedule.archivedAt = archivedAt
|
|
172
172
|
else delete schedule.archivedAt
|
|
173
173
|
|
|
174
|
+
const lastDeliveryStatus = typeof schedule.lastDeliveryStatus === 'string'
|
|
175
|
+
? schedule.lastDeliveryStatus.trim().toLowerCase()
|
|
176
|
+
: ''
|
|
177
|
+
if (lastDeliveryStatus === 'ok' || lastDeliveryStatus === 'error') {
|
|
178
|
+
schedule.lastDeliveryStatus = lastDeliveryStatus
|
|
179
|
+
} else {
|
|
180
|
+
delete schedule.lastDeliveryStatus
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const lastDeliveredAt = normalizeStoredScheduleTimestamp(schedule.lastDeliveredAt)
|
|
184
|
+
if (lastDeliveredAt != null) schedule.lastDeliveredAt = lastDeliveredAt
|
|
185
|
+
else delete schedule.lastDeliveredAt
|
|
186
|
+
|
|
174
187
|
const archivedFromStatus = typeof schedule.archivedFromStatus === 'string'
|
|
175
188
|
? schedule.archivedFromStatus.trim().toLowerCase()
|
|
176
189
|
: ''
|
|
@@ -596,7 +596,8 @@ describe('task-followups', () => {
|
|
|
596
596
|
})
|
|
597
597
|
|
|
598
598
|
describe('taskAlreadyDeliveredToConnectorTarget', () => {
|
|
599
|
-
it('returns true when the task session already delivered to the same connector target', () => {
|
|
599
|
+
it('returns true when the task session already delivered to the same connector target', async () => {
|
|
600
|
+
const { appendMessage } = await import('@/lib/server/messages/message-repository')
|
|
600
601
|
const task = {
|
|
601
602
|
id: 'task-delivered',
|
|
602
603
|
title: 'Delivered task',
|
|
@@ -607,27 +608,29 @@ describe('task-followups', () => {
|
|
|
607
608
|
createdAt: Date.now(),
|
|
608
609
|
updatedAt: Date.now(),
|
|
609
610
|
}
|
|
611
|
+
// Delivery evidence is read through the repo-backed message reader, so
|
|
612
|
+
// the message must be seeded via the repository, not inline on the session.
|
|
613
|
+
appendMessage('task-session', {
|
|
614
|
+
role: 'assistant',
|
|
615
|
+
text: 'Sent it.',
|
|
616
|
+
time: Date.now(),
|
|
617
|
+
toolEvents: [
|
|
618
|
+
{
|
|
619
|
+
name: 'connector_message_tool',
|
|
620
|
+
input: '{}',
|
|
621
|
+
output: JSON.stringify({
|
|
622
|
+
status: 'voice_sent',
|
|
623
|
+
connectorId: 'conn-wa',
|
|
624
|
+
to: '447700900111@s.whatsapp.net',
|
|
625
|
+
messageId: 'msg-1',
|
|
626
|
+
}),
|
|
627
|
+
},
|
|
628
|
+
],
|
|
629
|
+
} as import('@/types').Message)
|
|
610
630
|
const sessions = {
|
|
611
631
|
'task-session': {
|
|
612
632
|
id: 'task-session',
|
|
613
|
-
messages: [
|
|
614
|
-
{
|
|
615
|
-
role: 'assistant',
|
|
616
|
-
text: 'Sent it.',
|
|
617
|
-
toolEvents: [
|
|
618
|
-
{
|
|
619
|
-
name: 'connector_message_tool',
|
|
620
|
-
input: '{}',
|
|
621
|
-
output: JSON.stringify({
|
|
622
|
-
status: 'voice_sent',
|
|
623
|
-
connectorId: 'conn-wa',
|
|
624
|
-
to: '447700900111@s.whatsapp.net',
|
|
625
|
-
messageId: 'msg-1',
|
|
626
|
-
}),
|
|
627
|
-
},
|
|
628
|
-
],
|
|
629
|
-
},
|
|
630
|
-
],
|
|
633
|
+
messages: [],
|
|
631
634
|
},
|
|
632
635
|
}
|
|
633
636
|
const connectors = {
|
|
@@ -657,38 +660,39 @@ describe('task-followups', () => {
|
|
|
657
660
|
assert.equal(delivered, true)
|
|
658
661
|
})
|
|
659
662
|
|
|
660
|
-
it('returns false when connector delivery was to a different target', () => {
|
|
663
|
+
it('returns false when connector delivery was to a different target', async () => {
|
|
664
|
+
const { appendMessage } = await import('@/lib/server/messages/message-repository')
|
|
661
665
|
const task = {
|
|
662
666
|
id: 'task-other-target',
|
|
663
667
|
title: 'Other target',
|
|
664
668
|
description: '',
|
|
665
669
|
agentId: 'agent-1',
|
|
666
|
-
sessionId: 'task-session',
|
|
670
|
+
sessionId: 'task-session-other',
|
|
667
671
|
status: 'completed' as const,
|
|
668
672
|
createdAt: Date.now(),
|
|
669
673
|
updatedAt: Date.now(),
|
|
670
674
|
}
|
|
675
|
+
appendMessage('task-session-other', {
|
|
676
|
+
role: 'assistant',
|
|
677
|
+
text: 'Sent it.',
|
|
678
|
+
time: Date.now(),
|
|
679
|
+
toolEvents: [
|
|
680
|
+
{
|
|
681
|
+
name: 'connector_message_tool',
|
|
682
|
+
input: '{}',
|
|
683
|
+
output: JSON.stringify({
|
|
684
|
+
status: 'sent',
|
|
685
|
+
connectorId: 'conn-wa',
|
|
686
|
+
to: '447700900222@s.whatsapp.net',
|
|
687
|
+
messageId: 'msg-2',
|
|
688
|
+
}),
|
|
689
|
+
},
|
|
690
|
+
],
|
|
691
|
+
} as import('@/types').Message)
|
|
671
692
|
const sessions = {
|
|
672
|
-
'task-session': {
|
|
673
|
-
id: 'task-session',
|
|
674
|
-
messages: [
|
|
675
|
-
{
|
|
676
|
-
role: 'assistant',
|
|
677
|
-
text: 'Sent it.',
|
|
678
|
-
toolEvents: [
|
|
679
|
-
{
|
|
680
|
-
name: 'connector_message_tool',
|
|
681
|
-
input: '{}',
|
|
682
|
-
output: JSON.stringify({
|
|
683
|
-
status: 'sent',
|
|
684
|
-
connectorId: 'conn-wa',
|
|
685
|
-
to: '447700900222@s.whatsapp.net',
|
|
686
|
-
messageId: 'msg-2',
|
|
687
|
-
}),
|
|
688
|
-
},
|
|
689
|
-
],
|
|
690
|
-
},
|
|
691
|
-
],
|
|
693
|
+
'task-session-other': {
|
|
694
|
+
id: 'task-session-other',
|
|
695
|
+
messages: [],
|
|
692
696
|
},
|
|
693
697
|
}
|
|
694
698
|
const connectors = {
|
|
@@ -719,6 +723,85 @@ describe('task-followups', () => {
|
|
|
719
723
|
})
|
|
720
724
|
})
|
|
721
725
|
|
|
726
|
+
// ---- buildTaskFollowupDedupeKey ----
|
|
727
|
+
|
|
728
|
+
describe('buildTaskFollowupDedupeKey', () => {
|
|
729
|
+
it('is scoped to the task, run number, and target', () => {
|
|
730
|
+
const key = mod.buildTaskFollowupDedupeKey(
|
|
731
|
+
{ id: 'task-1', runNumber: 7, attempts: 2 },
|
|
732
|
+
{ connectorId: 'conn-1', channelId: 'ch-1', threadId: 'th-1' },
|
|
733
|
+
)
|
|
734
|
+
assert.equal(key, 'task-followup:task-1:run7:conn-1|ch-1|th-1')
|
|
735
|
+
})
|
|
736
|
+
|
|
737
|
+
it('changes across runs so scheduled reruns can deliver again', () => {
|
|
738
|
+
const target = { connectorId: 'conn-1', channelId: 'ch-1' }
|
|
739
|
+
const run1 = mod.buildTaskFollowupDedupeKey({ id: 'task-1', runNumber: 1 }, target)
|
|
740
|
+
const run2 = mod.buildTaskFollowupDedupeKey({ id: 'task-1', runNumber: 2 }, target)
|
|
741
|
+
assert.notEqual(run1, run2)
|
|
742
|
+
})
|
|
743
|
+
|
|
744
|
+
it('falls back to attempts, then zero, when runNumber is missing', () => {
|
|
745
|
+
const target = { connectorId: 'conn-1', channelId: 'ch-1' }
|
|
746
|
+
assert.equal(
|
|
747
|
+
mod.buildTaskFollowupDedupeKey({ id: 'task-1', attempts: 3 }, target),
|
|
748
|
+
'task-followup:task-1:run3:conn-1|ch-1|',
|
|
749
|
+
)
|
|
750
|
+
assert.equal(
|
|
751
|
+
mod.buildTaskFollowupDedupeKey({ id: 'task-1' }, target),
|
|
752
|
+
'task-followup:task-1:run0:conn-1|ch-1|',
|
|
753
|
+
)
|
|
754
|
+
})
|
|
755
|
+
})
|
|
756
|
+
|
|
757
|
+
// ---- outbox-backed follow-up delivery ----
|
|
758
|
+
|
|
759
|
+
describe('outbox-backed follow-up delivery', () => {
|
|
760
|
+
it('stores taskId/scheduleId on the outbox entry and keeps them through normalization', async () => {
|
|
761
|
+
const outbox = await import('@/lib/server/connectors/outbox')
|
|
762
|
+
const dedupeKey = 'task-followup:task-evidence:run1:conn-1|ch-1|'
|
|
763
|
+
outbox.enqueueConnectorOutbox({
|
|
764
|
+
connectorId: 'conn-1',
|
|
765
|
+
channelId: 'ch-1',
|
|
766
|
+
text: 'Task completed: evidence',
|
|
767
|
+
taskId: 'task-evidence',
|
|
768
|
+
scheduleId: 'sched-evidence',
|
|
769
|
+
dedupeKey,
|
|
770
|
+
sendAt: Date.now() + 60_000,
|
|
771
|
+
})
|
|
772
|
+
|
|
773
|
+
const entry = outbox.findPendingConnectorOutboxByDedupe(dedupeKey)
|
|
774
|
+
assert.ok(entry)
|
|
775
|
+
assert.equal(entry!.taskId, 'task-evidence')
|
|
776
|
+
assert.equal(entry!.scheduleId, 'sched-evidence')
|
|
777
|
+
assert.equal(entry!.connectorId, 'conn-1')
|
|
778
|
+
assert.equal(entry!.status, 'pending')
|
|
779
|
+
})
|
|
780
|
+
|
|
781
|
+
it('hasSentConnectorOutboxForDedupe reports successful terminal deliveries', async () => {
|
|
782
|
+
const outbox = await import('@/lib/server/connectors/outbox')
|
|
783
|
+
const storage = await import('@/lib/server/storage')
|
|
784
|
+
const dedupeKey = 'task-followup:task-sent:run1:conn-1|ch-1|'
|
|
785
|
+
const now = Date.now()
|
|
786
|
+
storage.upsertConnectorOutboxItem('sent-entry-1', {
|
|
787
|
+
id: 'sent-entry-1',
|
|
788
|
+
channelId: 'ch-1',
|
|
789
|
+
text: 'done',
|
|
790
|
+
status: 'sent',
|
|
791
|
+
sendAt: now,
|
|
792
|
+
createdAt: now,
|
|
793
|
+
updatedAt: now,
|
|
794
|
+
attemptCount: 1,
|
|
795
|
+
maxAttempts: 6,
|
|
796
|
+
dedupeKey,
|
|
797
|
+
deliveredAt: now,
|
|
798
|
+
})
|
|
799
|
+
|
|
800
|
+
assert.equal(outbox.hasSentConnectorOutboxForDedupe(dedupeKey), true)
|
|
801
|
+
assert.equal(outbox.hasSentConnectorOutboxForDedupe('task-followup:other:run1:c|c|'), false)
|
|
802
|
+
})
|
|
803
|
+
})
|
|
804
|
+
|
|
722
805
|
// ---- isSendableAttachment ----
|
|
723
806
|
|
|
724
807
|
describe('isSendableAttachment', () => {
|
|
@@ -4,6 +4,7 @@ import type { BoardTask, Connector, MessageToolEvent } from '@/types'
|
|
|
4
4
|
import { normalizeWhatsappTarget } from '@/lib/server/connectors/response-media'
|
|
5
5
|
import { isDirectConnectorSession } from '@/lib/server/connectors/session-kind'
|
|
6
6
|
import { loadConnectors } from '@/lib/server/connectors/connector-repository'
|
|
7
|
+
import { enqueueConnectorOutbox, hasSentConnectorOutboxForDedupe } from '@/lib/server/connectors/outbox'
|
|
7
8
|
import { WORKSPACE_DIR } from '@/lib/server/data-dir'
|
|
8
9
|
import { loadSessions } from '@/lib/server/sessions/session-repository'
|
|
9
10
|
import { UPLOAD_DIR } from '@/lib/server/upload-path'
|
|
@@ -434,6 +435,23 @@ export function taskAlreadyDeliveredToConnectorTarget(params: {
|
|
|
434
435
|
return false
|
|
435
436
|
}
|
|
436
437
|
|
|
438
|
+
/**
|
|
439
|
+
* Run-scoped dedupe key for a task follow-up delivery. Scheduled reruns reuse
|
|
440
|
+
* the same linked task id, so the run number keeps each run's follow-up
|
|
441
|
+
* distinct while suppressing duplicate sends within one run.
|
|
442
|
+
*/
|
|
443
|
+
export function buildTaskFollowupDedupeKey(
|
|
444
|
+
task: Pick<BoardTask, 'id' | 'runNumber' | 'attempts'>,
|
|
445
|
+
target: { connectorId: string; channelId: string; threadId?: string | null },
|
|
446
|
+
): string {
|
|
447
|
+
const run = typeof task.runNumber === 'number'
|
|
448
|
+
? task.runNumber
|
|
449
|
+
: typeof task.attempts === 'number'
|
|
450
|
+
? task.attempts
|
|
451
|
+
: 0
|
|
452
|
+
return `task-followup:${task.id}:run${run}:${target.connectorId}|${target.channelId}|${target.threadId || ''}`
|
|
453
|
+
}
|
|
454
|
+
|
|
437
455
|
export async function notifyConnectorTaskFollowups(params: {
|
|
438
456
|
task: BoardTask
|
|
439
457
|
statusLabel: string
|
|
@@ -446,7 +464,6 @@ export async function notifyConnectorTaskFollowups(params: {
|
|
|
446
464
|
|
|
447
465
|
const connectors = loadConnectors()
|
|
448
466
|
const running = (await import('@/lib/server/connectors/manager')).listRunningConnectors()
|
|
449
|
-
const manager = await import('@/lib/server/connectors/manager')
|
|
450
467
|
const sessions = loadSessions()
|
|
451
468
|
const targets = collectTaskConnectorFollowupTargets({
|
|
452
469
|
task,
|
|
@@ -499,12 +516,20 @@ export async function notifyConnectorTaskFollowups(params: {
|
|
|
499
516
|
const outboundMessage = `${message}${preferredChannelNote}`
|
|
500
517
|
|
|
501
518
|
const resolvedMediaPath = mediaPath || maybeResolveUploadMediaPathFromUrl(imageUrl)
|
|
519
|
+
// Deliver through the connector outbox so every schedule/task follow-up
|
|
520
|
+
// leaves a durable delivery record (pending/sent/failed with lastError)
|
|
521
|
+
// and gets retried with backoff instead of one fire-and-forget attempt.
|
|
522
|
+
const dedupeKey = buildTaskFollowupDedupeKey(task, target)
|
|
523
|
+
if (hasSentConnectorOutboxForDedupe(dedupeKey)) continue
|
|
502
524
|
try {
|
|
503
|
-
|
|
525
|
+
enqueueConnectorOutbox({
|
|
504
526
|
connectorId: target.connectorId,
|
|
505
527
|
channelId: target.channelId,
|
|
506
528
|
threadId: target.threadId || undefined,
|
|
507
529
|
text: outboundMessage,
|
|
530
|
+
taskId: task.id,
|
|
531
|
+
scheduleId: typeof task.sourceScheduleId === 'string' ? task.sourceScheduleId : null,
|
|
532
|
+
dedupeKey,
|
|
508
533
|
...(resolvedMediaPath
|
|
509
534
|
? {
|
|
510
535
|
mediaPath: resolvedMediaPath,
|
|
@@ -515,7 +540,7 @@ export async function notifyConnectorTaskFollowups(params: {
|
|
|
515
540
|
})
|
|
516
541
|
} catch (err: unknown) {
|
|
517
542
|
const errMsg = errorMessage(err)
|
|
518
|
-
log.warn(TAG, `Failed task follow-up
|
|
543
|
+
log.warn(TAG, `Failed to queue task follow-up for connector ${target.connectorId}: ${errMsg}`)
|
|
519
544
|
}
|
|
520
545
|
}
|
|
521
546
|
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
|
+
import path from 'node:path'
|
|
2
3
|
import { describe, it } from 'node:test'
|
|
3
4
|
|
|
4
5
|
import type { BoardTask, Schedule } from '@/types'
|
|
5
6
|
|
|
7
|
+
import { WORKSPACE_DIR } from '@/lib/server/data-dir'
|
|
8
|
+
|
|
6
9
|
import {
|
|
7
10
|
buildBoardTask,
|
|
8
11
|
didTaskValidationChange,
|
|
@@ -109,6 +112,28 @@ describe('task lifecycle helpers', () => {
|
|
|
109
112
|
assert.equal(task.runNumber, 2)
|
|
110
113
|
})
|
|
111
114
|
|
|
115
|
+
it('resetTaskForRerun remaps a legacy workspace cwd onto the current root', () => {
|
|
116
|
+
const task = makeTask({
|
|
117
|
+
status: 'failed',
|
|
118
|
+
cwd: '/root/.swarmclaw/workspace/tasks/task-1',
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
resetTaskForRerun(task, { title: 'Rerun', now: 99 })
|
|
122
|
+
|
|
123
|
+
assert.equal(task.cwd, path.join(WORKSPACE_DIR, 'tasks', 'task-1'))
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
it('resetTaskForRerun preserves an intentional custom cwd', () => {
|
|
127
|
+
const task = makeTask({
|
|
128
|
+
status: 'failed',
|
|
129
|
+
cwd: '/home/me/code/myrepo',
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
resetTaskForRerun(task, { title: 'Rerun', now: 99 })
|
|
133
|
+
|
|
134
|
+
assert.equal(task.cwd, '/home/me/code/myrepo')
|
|
135
|
+
})
|
|
136
|
+
|
|
112
137
|
it('prepareScheduledTaskRun creates a schedule-backed task when no reusable task exists', () => {
|
|
113
138
|
const schedule = makeSchedule({
|
|
114
139
|
createdInSessionId: 'session-1',
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
type TaskCompletionValidation,
|
|
9
9
|
} from '@/lib/server/tasks/task-validation'
|
|
10
10
|
import { syncTaskExecutionPolicyState } from '@/lib/server/tasks/task-execution-policy'
|
|
11
|
+
import { normalizeLegacyWorkspacePath } from '@/lib/server/workspace-paths'
|
|
11
12
|
import { createMission, startMission } from '@/lib/server/missions/mission-service'
|
|
12
13
|
import { getMission } from '@/lib/server/missions/mission-repository'
|
|
13
14
|
import { loadSessions } from '@/lib/server/storage'
|
|
@@ -74,6 +75,11 @@ export function resetTaskForRerun(task: BoardTask, options: ResetTaskForRerunOpt
|
|
|
74
75
|
task.retryScheduledAt = null
|
|
75
76
|
task.deadLetteredAt = null
|
|
76
77
|
task.validation = null
|
|
78
|
+
// Remap cwds persisted under a previous workspace root so reruns don't keep
|
|
79
|
+
// executing in the pre-migration directory. Custom cwds pass through unchanged.
|
|
80
|
+
if (typeof task.cwd === 'string' && task.cwd.trim()) {
|
|
81
|
+
task.cwd = normalizeLegacyWorkspacePath(task.cwd, { taskId: task.id })
|
|
82
|
+
}
|
|
77
83
|
task.executionPolicyState = syncTaskExecutionPolicyState(task.executionPolicy || null, null, options.now)
|
|
78
84
|
if (options.runNumber !== undefined) stats.runNumber = options.runNumber
|
|
79
85
|
return task
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, it } from 'node:test'
|
|
2
2
|
import assert from 'node:assert/strict'
|
|
3
|
-
import { extractTaskResult } from '@/lib/server/tasks/task-result'
|
|
3
|
+
import { classifyEmptyRunOutcome, extractTaskResult } from '@/lib/server/tasks/task-result'
|
|
4
4
|
|
|
5
5
|
describe('extractTaskResult', () => {
|
|
6
6
|
it('limits artifact extraction to messages from the current run window', () => {
|
|
@@ -38,3 +38,27 @@ describe('extractTaskResult', () => {
|
|
|
38
38
|
assert.deepEqual(result.artifacts.map((a) => a.url), ['/api/uploads/dated.png'])
|
|
39
39
|
})
|
|
40
40
|
})
|
|
41
|
+
|
|
42
|
+
describe('classifyEmptyRunOutcome', () => {
|
|
43
|
+
it('flags a run with no text, no tool calls, and no error', () => {
|
|
44
|
+
const reason = classifyEmptyRunOutcome({ text: '', error: null, toolEvents: [] })
|
|
45
|
+
assert.match(reason || '', /Run produced no output/)
|
|
46
|
+
assert.match(reason || '', /provider credential, model name, and endpoint/)
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
it('returns null when the run produced text', () => {
|
|
50
|
+
assert.equal(classifyEmptyRunOutcome({ text: 'done', error: null, toolEvents: [] }), null)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
it('returns null when the run made tool calls', () => {
|
|
54
|
+
assert.equal(classifyEmptyRunOutcome({ text: '', error: null, toolEvents: [{ name: 'bash' }] }), null)
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
it('returns null when the run reported an error', () => {
|
|
58
|
+
assert.equal(classifyEmptyRunOutcome({ text: '', error: 'boom', toolEvents: [] }), null)
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('treats whitespace-only text as empty', () => {
|
|
62
|
+
assert.match(classifyEmptyRunOutcome({ text: ' \n ', error: ' ', toolEvents: null }) || '', /Run produced no output/)
|
|
63
|
+
})
|
|
64
|
+
})
|
|
@@ -53,6 +53,28 @@ interface ExtractTaskResultOptions {
|
|
|
53
53
|
sinceTime?: number | null
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
export interface RunOutputSnapshot {
|
|
57
|
+
text?: string | null
|
|
58
|
+
error?: string | null
|
|
59
|
+
toolEvents?: readonly unknown[] | null
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export const EMPTY_RUN_OUTCOME_MESSAGE =
|
|
63
|
+
'Run produced no output: the model returned no text, made no tool calls, and reported no error. '
|
|
64
|
+
+ "This usually means the provider returned an empty response. Verify the agent's provider credential, model name, and endpoint."
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Detects the "silent empty run" case (no text, no tool calls, no error) and
|
|
68
|
+
* returns an actionable failure reason for it. Returns null when the run
|
|
69
|
+
* produced any signal at all.
|
|
70
|
+
*/
|
|
71
|
+
export function classifyEmptyRunOutcome(run: RunOutputSnapshot): string | null {
|
|
72
|
+
if (typeof run.error === 'string' && run.error.trim()) return null
|
|
73
|
+
if (typeof run.text === 'string' && run.text.trim()) return null
|
|
74
|
+
if (Array.isArray(run.toolEvents) && run.toolEvents.length > 0) return null
|
|
75
|
+
return EMPTY_RUN_OUTCOME_MESSAGE
|
|
76
|
+
}
|
|
77
|
+
|
|
56
78
|
// ---------------------------------------------------------------------------
|
|
57
79
|
// Core extraction
|
|
58
80
|
// ---------------------------------------------------------------------------
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import test from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import os from 'node:os'
|
|
5
|
+
|
|
6
|
+
import { normalizeLegacyWorkspacePath } from './workspace-paths'
|
|
7
|
+
|
|
8
|
+
const ROOT = '/app/data/workspace'
|
|
9
|
+
|
|
10
|
+
test('remaps a legacy .swarmclaw workspace task path onto the current root', () => {
|
|
11
|
+
assert.equal(
|
|
12
|
+
normalizeLegacyWorkspacePath('/root/.swarmclaw/workspace/tasks/a974c8b0', { workspaceRoot: ROOT }),
|
|
13
|
+
path.join(ROOT, 'tasks', 'a974c8b0'),
|
|
14
|
+
)
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
test('remaps the home-dir default workspace root', () => {
|
|
18
|
+
const legacy = path.join(os.homedir(), '.swarmclaw', 'workspace', 'projects', 'site')
|
|
19
|
+
assert.equal(
|
|
20
|
+
normalizeLegacyWorkspacePath(legacy, { workspaceRoot: ROOT }),
|
|
21
|
+
path.join(ROOT, 'projects', 'site'),
|
|
22
|
+
)
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
test('remaps a non-.swarmclaw legacy root when the tail matches tasks/<taskId>', () => {
|
|
26
|
+
assert.equal(
|
|
27
|
+
normalizeLegacyWorkspacePath('/old/home/workspace/tasks/abc123', { workspaceRoot: ROOT, taskId: 'abc123' }),
|
|
28
|
+
path.join(ROOT, 'tasks', 'abc123'),
|
|
29
|
+
)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
test('remaps subdirectories under a matching tasks/<taskId> tail', () => {
|
|
33
|
+
assert.equal(
|
|
34
|
+
normalizeLegacyWorkspacePath('/old/home/workspace/tasks/abc123/output', { workspaceRoot: ROOT, taskId: 'abc123' }),
|
|
35
|
+
path.join(ROOT, 'tasks', 'abc123', 'output'),
|
|
36
|
+
)
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
test('does not remap when the taskId does not match the tail', () => {
|
|
40
|
+
const input = '/old/home/workspace/tasks/othertask'
|
|
41
|
+
assert.equal(normalizeLegacyWorkspacePath(input, { workspaceRoot: ROOT, taskId: 'abc123' }), input)
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
test('does not remap intentional custom cwds', () => {
|
|
45
|
+
const input = '/home/me/code/myrepo'
|
|
46
|
+
assert.equal(normalizeLegacyWorkspacePath(input, { workspaceRoot: ROOT, taskId: 'abc123' }), input)
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
test('does not remap an unrelated path that merely contains a workspace segment', () => {
|
|
50
|
+
const input = '/home/me/code/workspace/notes'
|
|
51
|
+
assert.equal(normalizeLegacyWorkspacePath(input, { workspaceRoot: ROOT }), input)
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
test('leaves paths already under the current root unchanged', () => {
|
|
55
|
+
const input = path.join(ROOT, 'tasks', 'abc123')
|
|
56
|
+
assert.equal(normalizeLegacyWorkspacePath(input, { workspaceRoot: ROOT, taskId: 'abc123' }), input)
|
|
57
|
+
assert.equal(normalizeLegacyWorkspacePath(ROOT, { workspaceRoot: ROOT }), ROOT)
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
test('leaves relative and empty inputs unchanged', () => {
|
|
61
|
+
assert.equal(normalizeLegacyWorkspacePath('relative/dir', { workspaceRoot: ROOT }), 'relative/dir')
|
|
62
|
+
assert.equal(normalizeLegacyWorkspacePath('', { workspaceRoot: ROOT }), '')
|
|
63
|
+
assert.equal(normalizeLegacyWorkspacePath(null, { workspaceRoot: ROOT }), '')
|
|
64
|
+
assert.equal(normalizeLegacyWorkspacePath(undefined, { workspaceRoot: ROOT }), '')
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
test('remaps a bare legacy root with no tail to the current root', () => {
|
|
68
|
+
assert.equal(
|
|
69
|
+
normalizeLegacyWorkspacePath('/root/.swarmclaw/workspace', { workspaceRoot: ROOT }),
|
|
70
|
+
ROOT,
|
|
71
|
+
)
|
|
72
|
+
})
|