@lota-sdk/core 0.1.19 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/infrastructure/schema/09_queue_job.surql +38 -0
- package/infrastructure/schema/10_autonomous_job.surql +44 -0
- package/package.json +2 -2
- package/src/ai-gateway/ai-gateway.ts +55 -16
- package/src/create-runtime.ts +10 -1
- package/src/db/base.service.ts +6 -1
- package/src/db/tables.ts +4 -0
- package/src/queues/autonomous-job.queue.ts +134 -0
- package/src/queues/document-processor.queue.ts +13 -2
- package/src/queues/index.ts +1 -0
- package/src/queues/memory-consolidation.queue.ts +22 -3
- package/src/queues/queue-factory.ts +33 -4
- package/src/runtime/runtime-worker-registry.ts +3 -0
- package/src/services/autonomous-job.service.ts +692 -0
- package/src/services/index.ts +2 -0
- package/src/services/plan-deadline.service.ts +6 -4
- package/src/services/queue-job.service.ts +356 -0
- package/src/services/workstream-message.service.ts +25 -14
- package/src/services/workstream-turn-preparation.service.ts +18 -5
- package/src/services/workstream-turn.ts +11 -3
- package/src/workers/worker-utils.ts +35 -7
|
@@ -0,0 +1,692 @@
|
|
|
1
|
+
import {
|
|
2
|
+
AutonomousJobRunSchema,
|
|
3
|
+
AutonomousJobRunStatusSchema,
|
|
4
|
+
AutonomousJobScheduleSchema,
|
|
5
|
+
AutonomousJobSchema,
|
|
6
|
+
AutonomousJobStatusSchema,
|
|
7
|
+
CreateAutonomousJobInputSchema,
|
|
8
|
+
QueueJobErrorSchema,
|
|
9
|
+
UpdateAutonomousJobInputSchema,
|
|
10
|
+
recordIdSchema,
|
|
11
|
+
} from '@lota-sdk/shared'
|
|
12
|
+
import type {
|
|
13
|
+
AutonomousJob,
|
|
14
|
+
AutonomousJobRun,
|
|
15
|
+
AutonomousJobRunStatus,
|
|
16
|
+
AutonomousJobSchedule,
|
|
17
|
+
AutonomousJobStatus,
|
|
18
|
+
ChatMessage,
|
|
19
|
+
CreateAutonomousJobInput,
|
|
20
|
+
QueueJobError,
|
|
21
|
+
UpdateAutonomousJobInput,
|
|
22
|
+
} from '@lota-sdk/shared'
|
|
23
|
+
import type { Job } from 'bullmq'
|
|
24
|
+
import { CronExpressionParser } from 'cron-parser'
|
|
25
|
+
import { RecordId } from 'surrealdb'
|
|
26
|
+
import { z } from 'zod'
|
|
27
|
+
|
|
28
|
+
import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
29
|
+
import type { RecordIdInput } from '../db/record-id'
|
|
30
|
+
import { databaseService } from '../db/service'
|
|
31
|
+
import { TABLES } from '../db/tables'
|
|
32
|
+
import type { AutonomousJobQueuePayload } from '../queues/autonomous-job.queue'
|
|
33
|
+
import { extractMessageText } from '../runtime/workstream-chat-helpers'
|
|
34
|
+
import { toIsoDateTimeString, toOptionalIsoDateTimeString } from '../utils/date-time'
|
|
35
|
+
import { compactRecord, compactWhitespace, stringifyUnknown, truncateText } from '../utils/string'
|
|
36
|
+
import { executionPlanService } from './execution-plan.service'
|
|
37
|
+
import { getNotificationService } from './notification.service'
|
|
38
|
+
import { queueJobService } from './queue-job.service'
|
|
39
|
+
import { runWorkstreamTurnInBackground } from './workstream-turn'
|
|
40
|
+
import { workstreamService } from './workstream.service'
|
|
41
|
+
|
|
42
|
+
const AUTONOMOUS_JOB_QUEUE_NAME = 'autonomous-job'
|
|
43
|
+
|
|
44
|
+
function encodeBullmqId(raw: string): string {
|
|
45
|
+
return Buffer.from(raw).toString('base64url')
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function buildAutonomousAtJobId(autonomousJobId: string): string {
|
|
49
|
+
return `autonomous-at-${encodeBullmqId(autonomousJobId)}`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function buildAutonomousManualJobId(autonomousJobId: string): string {
|
|
53
|
+
return `autonomous-manual-${encodeBullmqId(autonomousJobId)}-${Date.now()}`
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const AutonomousJobRowSchema = z.object({
|
|
57
|
+
id: recordIdSchema,
|
|
58
|
+
organizationId: recordIdSchema,
|
|
59
|
+
ownerUserId: recordIdSchema,
|
|
60
|
+
ownerUserName: z.string().optional(),
|
|
61
|
+
workstreamId: recordIdSchema,
|
|
62
|
+
agentId: z.string(),
|
|
63
|
+
title: z.string(),
|
|
64
|
+
prompt: z.string(),
|
|
65
|
+
schedule: AutonomousJobScheduleSchema,
|
|
66
|
+
status: AutonomousJobStatusSchema,
|
|
67
|
+
autoPauseThreshold: z.number().int().positive(),
|
|
68
|
+
consecutiveErrorCount: z.number().int().nonnegative(),
|
|
69
|
+
lastRunStatus: AutonomousJobRunStatusSchema.optional(),
|
|
70
|
+
lastRunAt: z.coerce.date().optional(),
|
|
71
|
+
nextRunAt: z.coerce.date().optional(),
|
|
72
|
+
linkedPlanSpecId: recordIdSchema.optional(),
|
|
73
|
+
linkedPlanRunId: recordIdSchema.optional(),
|
|
74
|
+
lastError: QueueJobErrorSchema.optional(),
|
|
75
|
+
createdAt: z.coerce.date(),
|
|
76
|
+
updatedAt: z.coerce.date(),
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
const AutonomousJobRunRowSchema = z.object({
|
|
80
|
+
id: recordIdSchema,
|
|
81
|
+
autonomousJobId: recordIdSchema,
|
|
82
|
+
workstreamId: recordIdSchema,
|
|
83
|
+
queueJobId: recordIdSchema.optional(),
|
|
84
|
+
status: AutonomousJobRunStatusSchema,
|
|
85
|
+
inputMessageId: z.string().optional(),
|
|
86
|
+
assistantMessageIds: z.array(z.string()).default([]),
|
|
87
|
+
summary: z.string().optional(),
|
|
88
|
+
error: QueueJobErrorSchema.optional(),
|
|
89
|
+
linkedPlanSpecId: recordIdSchema.optional(),
|
|
90
|
+
linkedPlanRunId: recordIdSchema.optional(),
|
|
91
|
+
startedAt: z.coerce.date().optional(),
|
|
92
|
+
completedAt: z.coerce.date().optional(),
|
|
93
|
+
createdAt: z.coerce.date(),
|
|
94
|
+
updatedAt: z.coerce.date(),
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
type AutonomousJobRow = z.infer<typeof AutonomousJobRowSchema>
|
|
98
|
+
type AutonomousJobRunRow = z.infer<typeof AutonomousJobRunRowSchema>
|
|
99
|
+
|
|
100
|
+
function toQueueJobError(error: unknown): QueueJobError {
|
|
101
|
+
if (error instanceof Error) {
|
|
102
|
+
return QueueJobErrorSchema.parse(
|
|
103
|
+
compactRecord({
|
|
104
|
+
name: error.name || undefined,
|
|
105
|
+
message: truncateText(error.message || 'Unknown error', 5_000),
|
|
106
|
+
stack: typeof error.stack === 'string' ? truncateText(error.stack, 20_000) : undefined,
|
|
107
|
+
}),
|
|
108
|
+
)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return QueueJobErrorSchema.parse({ message: truncateText(stringifyUnknown(error) ?? 'Unknown error', 5_000) })
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
class AutonomousJobService {
|
|
115
|
+
private toPublicJob(row: AutonomousJobRow): AutonomousJob {
|
|
116
|
+
return AutonomousJobSchema.parse({
|
|
117
|
+
id: recordIdToString(row.id, TABLES.AUTONOMOUS_JOB),
|
|
118
|
+
organizationId: recordIdToString(row.organizationId, TABLES.ORGANIZATION),
|
|
119
|
+
ownerUserId: recordIdToString(row.ownerUserId, TABLES.USER),
|
|
120
|
+
ownerUserName: row.ownerUserName,
|
|
121
|
+
workstreamId: recordIdToString(row.workstreamId, TABLES.WORKSTREAM),
|
|
122
|
+
agentId: row.agentId,
|
|
123
|
+
title: row.title,
|
|
124
|
+
prompt: row.prompt,
|
|
125
|
+
schedule: row.schedule,
|
|
126
|
+
status: row.status,
|
|
127
|
+
autoPauseThreshold: row.autoPauseThreshold,
|
|
128
|
+
consecutiveErrorCount: row.consecutiveErrorCount,
|
|
129
|
+
lastRunStatus: row.lastRunStatus,
|
|
130
|
+
lastRunAt: toOptionalIsoDateTimeString(row.lastRunAt),
|
|
131
|
+
nextRunAt: toOptionalIsoDateTimeString(row.nextRunAt),
|
|
132
|
+
linkedPlanSpecId: row.linkedPlanSpecId ? recordIdToString(row.linkedPlanSpecId, TABLES.PLAN_SPEC) : undefined,
|
|
133
|
+
linkedPlanRunId: row.linkedPlanRunId ? recordIdToString(row.linkedPlanRunId, TABLES.PLAN_RUN) : undefined,
|
|
134
|
+
lastError: row.lastError,
|
|
135
|
+
createdAt: toIsoDateTimeString(row.createdAt),
|
|
136
|
+
updatedAt: toIsoDateTimeString(row.updatedAt),
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private toPublicRun(row: AutonomousJobRunRow): AutonomousJobRun {
|
|
141
|
+
return AutonomousJobRunSchema.parse({
|
|
142
|
+
id: recordIdToString(row.id, TABLES.AUTONOMOUS_JOB_RUN),
|
|
143
|
+
autonomousJobId: recordIdToString(row.autonomousJobId, TABLES.AUTONOMOUS_JOB),
|
|
144
|
+
workstreamId: recordIdToString(row.workstreamId, TABLES.WORKSTREAM),
|
|
145
|
+
queueJobId: row.queueJobId ? recordIdToString(row.queueJobId, TABLES.QUEUE_JOB) : undefined,
|
|
146
|
+
status: row.status,
|
|
147
|
+
inputMessageId: row.inputMessageId,
|
|
148
|
+
assistantMessageIds: row.assistantMessageIds,
|
|
149
|
+
summary: row.summary,
|
|
150
|
+
error: row.error,
|
|
151
|
+
linkedPlanSpecId: row.linkedPlanSpecId ? recordIdToString(row.linkedPlanSpecId, TABLES.PLAN_SPEC) : undefined,
|
|
152
|
+
linkedPlanRunId: row.linkedPlanRunId ? recordIdToString(row.linkedPlanRunId, TABLES.PLAN_RUN) : undefined,
|
|
153
|
+
startedAt: toOptionalIsoDateTimeString(row.startedAt),
|
|
154
|
+
completedAt: toOptionalIsoDateTimeString(row.completedAt),
|
|
155
|
+
createdAt: toIsoDateTimeString(row.createdAt),
|
|
156
|
+
updatedAt: toIsoDateTimeString(row.updatedAt),
|
|
157
|
+
})
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
computeNextRunAt(schedule: AutonomousJobSchedule, baseTime: Date = new Date()): Date | null {
|
|
161
|
+
switch (schedule.kind) {
|
|
162
|
+
case 'cron': {
|
|
163
|
+
try {
|
|
164
|
+
const expr = CronExpressionParser.parse(schedule.cron, { currentDate: baseTime })
|
|
165
|
+
return expr.next().toDate()
|
|
166
|
+
} catch {
|
|
167
|
+
return null
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
case 'every':
|
|
171
|
+
return new Date(baseTime.getTime() + schedule.intervalMs)
|
|
172
|
+
case 'at': {
|
|
173
|
+
const at = new Date(schedule.at)
|
|
174
|
+
return Number.isNaN(at.getTime()) ? null : at
|
|
175
|
+
}
|
|
176
|
+
default:
|
|
177
|
+
return null
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
private async maybeNotify(
|
|
182
|
+
kind: 'notify',
|
|
183
|
+
params: {
|
|
184
|
+
organizationId: string
|
|
185
|
+
workstreamId: string
|
|
186
|
+
title: string
|
|
187
|
+
body: string
|
|
188
|
+
severity: 'info' | 'warning'
|
|
189
|
+
metadata?: Record<string, unknown>
|
|
190
|
+
},
|
|
191
|
+
): Promise<void> {
|
|
192
|
+
try {
|
|
193
|
+
const service = getNotificationService()
|
|
194
|
+
await service[kind](params)
|
|
195
|
+
} catch (error) {
|
|
196
|
+
if (error instanceof Error && error.message === 'Notification service is not configured.') {
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
throw error
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
private buildSyntheticUserMessage(prompt: string): ChatMessage {
|
|
204
|
+
return {
|
|
205
|
+
id: Bun.randomUUIDv7(),
|
|
206
|
+
role: 'user',
|
|
207
|
+
parts: [{ type: 'text', text: prompt }],
|
|
208
|
+
metadata: { createdAt: Date.now() },
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private async createRunRow(params: {
|
|
213
|
+
autonomousJobId: RecordIdInput
|
|
214
|
+
workstreamId: RecordIdInput
|
|
215
|
+
queueJobId?: RecordIdInput
|
|
216
|
+
status?: AutonomousJobRunStatus
|
|
217
|
+
}): Promise<AutonomousJobRunRow> {
|
|
218
|
+
const runId = new RecordId(TABLES.AUTONOMOUS_JOB_RUN, Bun.randomUUIDv7())
|
|
219
|
+
return databaseService.createWithId(
|
|
220
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
221
|
+
runId,
|
|
222
|
+
compactRecord({
|
|
223
|
+
autonomousJobId: ensureRecordId(params.autonomousJobId, TABLES.AUTONOMOUS_JOB),
|
|
224
|
+
workstreamId: ensureRecordId(params.workstreamId, TABLES.WORKSTREAM),
|
|
225
|
+
queueJobId: params.queueJobId ? ensureRecordId(params.queueJobId, TABLES.QUEUE_JOB) : undefined,
|
|
226
|
+
status: params.status ?? 'queued',
|
|
227
|
+
assistantMessageIds: [],
|
|
228
|
+
}),
|
|
229
|
+
AutonomousJobRunRowSchema,
|
|
230
|
+
)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
private async getRow(jobId: RecordIdInput): Promise<AutonomousJobRow> {
|
|
234
|
+
await databaseService.connect()
|
|
235
|
+
const row = await databaseService.findOne(
|
|
236
|
+
TABLES.AUTONOMOUS_JOB,
|
|
237
|
+
{ id: ensureRecordId(jobId, TABLES.AUTONOMOUS_JOB) },
|
|
238
|
+
AutonomousJobRowSchema,
|
|
239
|
+
)
|
|
240
|
+
if (!row) {
|
|
241
|
+
throw new Error(`Autonomous job not found: ${recordIdToString(jobId, TABLES.AUTONOMOUS_JOB)}`)
|
|
242
|
+
}
|
|
243
|
+
return row
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
private async getRunRow(runId: RecordIdInput): Promise<AutonomousJobRunRow> {
|
|
247
|
+
await databaseService.connect()
|
|
248
|
+
const row = await databaseService.findOne(
|
|
249
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
250
|
+
{ id: ensureRecordId(runId, TABLES.AUTONOMOUS_JOB_RUN) },
|
|
251
|
+
AutonomousJobRunRowSchema,
|
|
252
|
+
)
|
|
253
|
+
if (!row) {
|
|
254
|
+
throw new Error(`Autonomous job run not found: ${recordIdToString(runId, TABLES.AUTONOMOUS_JOB_RUN)}`)
|
|
255
|
+
}
|
|
256
|
+
return row
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private async unscheduleRow(row: AutonomousJobRow): Promise<void> {
|
|
260
|
+
if (row.schedule.kind === 'at') {
|
|
261
|
+
const { removeAutonomousAtJob } = await import('../queues/autonomous-job.queue')
|
|
262
|
+
await removeAutonomousAtJob(recordIdToString(row.id, TABLES.AUTONOMOUS_JOB))
|
|
263
|
+
return
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const { removeAutonomousJobScheduler } = await import('../queues/autonomous-job.queue')
|
|
267
|
+
await removeAutonomousJobScheduler(recordIdToString(row.id, TABLES.AUTONOMOUS_JOB))
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
private async findRecoverableRunRow(autonomousJobId: RecordIdInput): Promise<AutonomousJobRunRow | null> {
|
|
271
|
+
const rows = await databaseService.queryMany(
|
|
272
|
+
{
|
|
273
|
+
query: `SELECT * FROM ${TABLES.AUTONOMOUS_JOB_RUN}
|
|
274
|
+
WHERE autonomousJobId = $autonomousJobId
|
|
275
|
+
AND status IN $statuses
|
|
276
|
+
ORDER BY createdAt DESC
|
|
277
|
+
LIMIT 1`,
|
|
278
|
+
bindings: {
|
|
279
|
+
autonomousJobId: ensureRecordId(autonomousJobId, TABLES.AUTONOMOUS_JOB),
|
|
280
|
+
statuses: ['queued', 'running'],
|
|
281
|
+
},
|
|
282
|
+
},
|
|
283
|
+
AutonomousJobRunRowSchema,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
return rows[0] ?? null
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private async scheduleRow(
|
|
290
|
+
row: AutonomousJobRow,
|
|
291
|
+
options: { runImmediate?: boolean; referenceTime?: Date; reusePendingAtRun?: boolean } = {},
|
|
292
|
+
): Promise<void> {
|
|
293
|
+
const jobId = recordIdToString(row.id, TABLES.AUTONOMOUS_JOB)
|
|
294
|
+
const referenceTime = options.referenceTime ?? new Date()
|
|
295
|
+
const nextRunAt =
|
|
296
|
+
row.schedule.kind === 'at'
|
|
297
|
+
? (row.nextRunAt ?? this.computeNextRunAt(row.schedule, referenceTime))
|
|
298
|
+
: this.computeNextRunAt(row.schedule, referenceTime)
|
|
299
|
+
|
|
300
|
+
if (row.schedule.kind === 'at') {
|
|
301
|
+
const queuedRun = options.reusePendingAtRun
|
|
302
|
+
? ((await this.findRecoverableRunRow(row.id)) ??
|
|
303
|
+
(await this.createRunRow({ autonomousJobId: row.id, workstreamId: row.workstreamId })))
|
|
304
|
+
: await this.createRunRow({ autonomousJobId: row.id, workstreamId: row.workstreamId })
|
|
305
|
+
const { enqueueAutonomousJobRun } = await import('../queues/autonomous-job.queue')
|
|
306
|
+
const enqueueResult = await enqueueAutonomousJobRun({
|
|
307
|
+
payload: {
|
|
308
|
+
autonomousJobId: jobId,
|
|
309
|
+
autonomousJobRunId: recordIdToString(queuedRun.id, TABLES.AUTONOMOUS_JOB_RUN),
|
|
310
|
+
trigger: 'scheduled',
|
|
311
|
+
},
|
|
312
|
+
delayMs: Math.max(0, (nextRunAt ?? referenceTime).getTime() - referenceTime.getTime()),
|
|
313
|
+
jobId: buildAutonomousAtJobId(jobId),
|
|
314
|
+
})
|
|
315
|
+
|
|
316
|
+
await databaseService.update(
|
|
317
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
318
|
+
queuedRun.id,
|
|
319
|
+
{ queueJobId: ensureRecordId(enqueueResult.queueJobId, TABLES.QUEUE_JOB) },
|
|
320
|
+
AutonomousJobRunRowSchema,
|
|
321
|
+
)
|
|
322
|
+
} else {
|
|
323
|
+
const { upsertAutonomousJobScheduler } = await import('../queues/autonomous-job.queue')
|
|
324
|
+
await upsertAutonomousJobScheduler({ autonomousJobId: jobId, schedule: row.schedule })
|
|
325
|
+
|
|
326
|
+
if (options.runImmediate ?? row.schedule.immediately) {
|
|
327
|
+
await this.runNow(jobId)
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
await databaseService.update(
|
|
332
|
+
TABLES.AUTONOMOUS_JOB,
|
|
333
|
+
row.id,
|
|
334
|
+
{ nextRunAt: nextRunAt ?? undefined },
|
|
335
|
+
AutonomousJobRowSchema,
|
|
336
|
+
)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
async create(input: CreateAutonomousJobInput): Promise<AutonomousJob> {
|
|
340
|
+
await databaseService.connect()
|
|
341
|
+
const parsed = CreateAutonomousJobInputSchema.parse(input)
|
|
342
|
+
const organizationId = ensureRecordId(parsed.organizationId, TABLES.ORGANIZATION)
|
|
343
|
+
const ownerUserId = ensureRecordId(parsed.ownerUserId, TABLES.USER)
|
|
344
|
+
const workstream = await workstreamService.createWorkstream(ownerUserId, organizationId, {
|
|
345
|
+
mode: 'group',
|
|
346
|
+
core: false,
|
|
347
|
+
title: parsed.workstreamTitle ?? parsed.title,
|
|
348
|
+
})
|
|
349
|
+
const jobId = new RecordId(TABLES.AUTONOMOUS_JOB, Bun.randomUUIDv7())
|
|
350
|
+
const nextRunAt = this.computeNextRunAt(parsed.schedule)
|
|
351
|
+
const created = await databaseService.createWithId(
|
|
352
|
+
TABLES.AUTONOMOUS_JOB,
|
|
353
|
+
jobId,
|
|
354
|
+
compactRecord({
|
|
355
|
+
organizationId,
|
|
356
|
+
ownerUserId,
|
|
357
|
+
ownerUserName: parsed.ownerUserName,
|
|
358
|
+
workstreamId: ensureRecordId(workstream.id, TABLES.WORKSTREAM),
|
|
359
|
+
agentId: parsed.agentId,
|
|
360
|
+
title: parsed.title,
|
|
361
|
+
prompt: parsed.prompt,
|
|
362
|
+
schedule: parsed.schedule,
|
|
363
|
+
status: 'active',
|
|
364
|
+
autoPauseThreshold: parsed.autoPauseThreshold,
|
|
365
|
+
consecutiveErrorCount: 0,
|
|
366
|
+
nextRunAt: nextRunAt ?? undefined,
|
|
367
|
+
}),
|
|
368
|
+
AutonomousJobRowSchema,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
await this.scheduleRow(created)
|
|
372
|
+
return this.toPublicJob(await this.getRow(created.id))
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
async recoverActiveJobs(now = new Date()): Promise<void> {
|
|
376
|
+
await databaseService.connect()
|
|
377
|
+
const activeRows = await databaseService.queryMany(
|
|
378
|
+
{
|
|
379
|
+
query: `SELECT * FROM ${TABLES.AUTONOMOUS_JOB} WHERE status = $status ORDER BY createdAt ASC`,
|
|
380
|
+
bindings: { status: 'active' },
|
|
381
|
+
},
|
|
382
|
+
AutonomousJobRowSchema,
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
for (const row of activeRows) {
|
|
386
|
+
await this.scheduleRow(row, { runImmediate: false, referenceTime: now, reusePendingAtRun: true })
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
async get(jobId: RecordIdInput): Promise<AutonomousJob> {
|
|
391
|
+
return this.toPublicJob(await this.getRow(jobId))
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
async list(params: {
|
|
395
|
+
organizationId: RecordIdInput
|
|
396
|
+
ownerUserId?: RecordIdInput
|
|
397
|
+
status?: AutonomousJobStatus
|
|
398
|
+
}): Promise<AutonomousJob[]> {
|
|
399
|
+
await databaseService.connect()
|
|
400
|
+
const rows = await databaseService.findMany(
|
|
401
|
+
TABLES.AUTONOMOUS_JOB,
|
|
402
|
+
compactRecord({
|
|
403
|
+
organizationId: ensureRecordId(params.organizationId, TABLES.ORGANIZATION),
|
|
404
|
+
ownerUserId: params.ownerUserId ? ensureRecordId(params.ownerUserId, TABLES.USER) : undefined,
|
|
405
|
+
status: params.status,
|
|
406
|
+
}),
|
|
407
|
+
AutonomousJobRowSchema,
|
|
408
|
+
{ orderBy: 'createdAt', orderDir: 'DESC' },
|
|
409
|
+
)
|
|
410
|
+
return rows.map((row) => this.toPublicJob(row))
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
async update(jobId: RecordIdInput, input: UpdateAutonomousJobInput): Promise<AutonomousJob> {
|
|
414
|
+
const parsed = UpdateAutonomousJobInputSchema.parse(input)
|
|
415
|
+
const existing = await this.getRow(jobId)
|
|
416
|
+
await this.unscheduleRow(existing)
|
|
417
|
+
|
|
418
|
+
if (parsed.title && compactWhitespace(parsed.title) !== compactWhitespace(existing.title)) {
|
|
419
|
+
await workstreamService.updateTitle(existing.workstreamId, parsed.title)
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
const nextRunAt = this.computeNextRunAt(parsed.schedule ?? existing.schedule)
|
|
423
|
+
const updated = await databaseService.update(
|
|
424
|
+
TABLES.AUTONOMOUS_JOB,
|
|
425
|
+
existing.id,
|
|
426
|
+
compactRecord({
|
|
427
|
+
title: parsed.title,
|
|
428
|
+
prompt: parsed.prompt,
|
|
429
|
+
schedule: parsed.schedule,
|
|
430
|
+
autoPauseThreshold: parsed.autoPauseThreshold,
|
|
431
|
+
nextRunAt: existing.status === 'active' ? (nextRunAt ?? undefined) : undefined,
|
|
432
|
+
}),
|
|
433
|
+
AutonomousJobRowSchema,
|
|
434
|
+
)
|
|
435
|
+
if (!updated) {
|
|
436
|
+
throw new Error(`Failed to update autonomous job: ${recordIdToString(existing.id, TABLES.AUTONOMOUS_JOB)}`)
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
if (updated.status === 'active') {
|
|
440
|
+
await this.scheduleRow(updated)
|
|
441
|
+
}
|
|
442
|
+
return this.toPublicJob(await this.getRow(updated.id))
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
async pause(jobId: RecordIdInput): Promise<AutonomousJob> {
|
|
446
|
+
const row = await this.getRow(jobId)
|
|
447
|
+
await this.unscheduleRow(row)
|
|
448
|
+
await databaseService.update(
|
|
449
|
+
TABLES.AUTONOMOUS_JOB,
|
|
450
|
+
row.id,
|
|
451
|
+
{ status: 'paused', nextRunAt: undefined },
|
|
452
|
+
AutonomousJobRowSchema,
|
|
453
|
+
)
|
|
454
|
+
return this.toPublicJob(await this.getRow(row.id))
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
async resume(jobId: RecordIdInput): Promise<AutonomousJob> {
|
|
458
|
+
const row = await this.getRow(jobId)
|
|
459
|
+
const nextRunAt = this.computeNextRunAt(row.schedule)
|
|
460
|
+
const resumed = await databaseService.update(
|
|
461
|
+
TABLES.AUTONOMOUS_JOB,
|
|
462
|
+
row.id,
|
|
463
|
+
{ status: 'active', nextRunAt: nextRunAt ?? undefined },
|
|
464
|
+
AutonomousJobRowSchema,
|
|
465
|
+
)
|
|
466
|
+
if (!resumed) {
|
|
467
|
+
throw new Error(`Failed to resume autonomous job: ${recordIdToString(row.id, TABLES.AUTONOMOUS_JOB)}`)
|
|
468
|
+
}
|
|
469
|
+
await this.scheduleRow(resumed)
|
|
470
|
+
return this.toPublicJob(await this.getRow(resumed.id))
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
async runNow(jobId: RecordIdInput): Promise<AutonomousJobRun> {
|
|
474
|
+
const row = await this.getRow(jobId)
|
|
475
|
+
const queuedRun = await this.createRunRow({ autonomousJobId: row.id, workstreamId: row.workstreamId })
|
|
476
|
+
const { enqueueAutonomousJobRun } = await import('../queues/autonomous-job.queue')
|
|
477
|
+
const enqueueResult = await enqueueAutonomousJobRun({
|
|
478
|
+
payload: {
|
|
479
|
+
autonomousJobId: recordIdToString(row.id, TABLES.AUTONOMOUS_JOB),
|
|
480
|
+
autonomousJobRunId: recordIdToString(queuedRun.id, TABLES.AUTONOMOUS_JOB_RUN),
|
|
481
|
+
trigger: 'manual',
|
|
482
|
+
},
|
|
483
|
+
jobId: buildAutonomousManualJobId(recordIdToString(row.id, TABLES.AUTONOMOUS_JOB)),
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
await databaseService.update(
|
|
487
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
488
|
+
queuedRun.id,
|
|
489
|
+
{ queueJobId: ensureRecordId(enqueueResult.queueJobId, TABLES.QUEUE_JOB) },
|
|
490
|
+
AutonomousJobRunRowSchema,
|
|
491
|
+
)
|
|
492
|
+
|
|
493
|
+
return this.toPublicRun(await this.getRunRow(queuedRun.id))
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
async cancel(jobId: RecordIdInput): Promise<AutonomousJob> {
|
|
497
|
+
const row = await this.getRow(jobId)
|
|
498
|
+
await this.unscheduleRow(row)
|
|
499
|
+
await databaseService.update(
|
|
500
|
+
TABLES.AUTONOMOUS_JOB,
|
|
501
|
+
row.id,
|
|
502
|
+
{ status: 'cancelled', nextRunAt: undefined },
|
|
503
|
+
AutonomousJobRowSchema,
|
|
504
|
+
)
|
|
505
|
+
return this.toPublicJob(await this.getRow(row.id))
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async delete(jobId: RecordIdInput): Promise<AutonomousJob> {
|
|
509
|
+
const row = await this.getRow(jobId)
|
|
510
|
+
const cancelled = await this.cancel(row.id)
|
|
511
|
+
await workstreamService.updateStatus(row.workstreamId, 'archived')
|
|
512
|
+
return cancelled
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
async listRuns(jobId: RecordIdInput): Promise<AutonomousJobRun[]> {
|
|
516
|
+
await databaseService.connect()
|
|
517
|
+
const rows = await databaseService.findMany(
|
|
518
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
519
|
+
{ autonomousJobId: ensureRecordId(jobId, TABLES.AUTONOMOUS_JOB) },
|
|
520
|
+
AutonomousJobRunRowSchema,
|
|
521
|
+
{ orderBy: 'createdAt', orderDir: 'DESC' },
|
|
522
|
+
)
|
|
523
|
+
return rows.map((row) => this.toPublicRun(row))
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
async executeQueuedRun(job: Job<AutonomousJobQueuePayload>): Promise<{ status: string; summary?: string }> {
|
|
527
|
+
await databaseService.connect()
|
|
528
|
+
|
|
529
|
+
const autonomousJobRow = await this.getRow(job.data.autonomousJobId)
|
|
530
|
+
const queueJobId = queueJobService.getQueueJobId(AUTONOMOUS_JOB_QUEUE_NAME, String(job.id))
|
|
531
|
+
let runRow =
|
|
532
|
+
job.data.autonomousJobRunId !== undefined
|
|
533
|
+
? await this.getRunRow(job.data.autonomousJobRunId)
|
|
534
|
+
: await this.createRunRow({
|
|
535
|
+
autonomousJobId: autonomousJobRow.id,
|
|
536
|
+
workstreamId: autonomousJobRow.workstreamId,
|
|
537
|
+
queueJobId,
|
|
538
|
+
status: 'queued',
|
|
539
|
+
})
|
|
540
|
+
|
|
541
|
+
const activeStatus = autonomousJobRow.status
|
|
542
|
+
if (activeStatus !== 'active' && job.data.trigger === 'scheduled') {
|
|
543
|
+
return { status: 'skipped' }
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
const startedAt = new Date()
|
|
547
|
+
runRow =
|
|
548
|
+
(await databaseService.update(
|
|
549
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
550
|
+
runRow.id,
|
|
551
|
+
{ queueJobId: ensureRecordId(queueJobId, TABLES.QUEUE_JOB), status: 'running', startedAt },
|
|
552
|
+
AutonomousJobRunRowSchema,
|
|
553
|
+
)) ?? runRow
|
|
554
|
+
|
|
555
|
+
try {
|
|
556
|
+
const workstream = await workstreamService.getWorkstream(autonomousJobRow.workstreamId)
|
|
557
|
+
const inputMessage = this.buildSyntheticUserMessage(autonomousJobRow.prompt)
|
|
558
|
+
const turnResult = await runWorkstreamTurnInBackground({
|
|
559
|
+
workstream,
|
|
560
|
+
workstreamRef: ensureRecordId(autonomousJobRow.workstreamId, TABLES.WORKSTREAM),
|
|
561
|
+
orgRef: ensureRecordId(autonomousJobRow.organizationId, TABLES.ORGANIZATION),
|
|
562
|
+
userRef: ensureRecordId(autonomousJobRow.ownerUserId, TABLES.USER),
|
|
563
|
+
userName: autonomousJobRow.ownerUserName,
|
|
564
|
+
agentIdOverride: autonomousJobRow.agentId,
|
|
565
|
+
inputMessage,
|
|
566
|
+
})
|
|
567
|
+
const activePlan = await executionPlanService.getActivePlanForWorkstream(autonomousJobRow.workstreamId)
|
|
568
|
+
const runStatus: AutonomousJobRunStatus = activePlan?.status === 'awaiting-human' ? 'awaiting-human' : 'completed'
|
|
569
|
+
const summary = truncateText(
|
|
570
|
+
turnResult.assistantMessages
|
|
571
|
+
.map((message) => extractMessageText(message))
|
|
572
|
+
.filter(Boolean)
|
|
573
|
+
.join('\n\n')
|
|
574
|
+
.trim() || `${autonomousJobRow.title} completed.`,
|
|
575
|
+
2_000,
|
|
576
|
+
)
|
|
577
|
+
const completedAt = new Date()
|
|
578
|
+
|
|
579
|
+
await databaseService.update(
|
|
580
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
581
|
+
runRow.id,
|
|
582
|
+
compactRecord({
|
|
583
|
+
status: runStatus,
|
|
584
|
+
inputMessageId: turnResult.inputMessageId,
|
|
585
|
+
assistantMessageIds: turnResult.assistantMessages.map((message) => message.id),
|
|
586
|
+
summary,
|
|
587
|
+
linkedPlanSpecId: activePlan?.specId ? ensureRecordId(activePlan.specId, TABLES.PLAN_SPEC) : undefined,
|
|
588
|
+
linkedPlanRunId: activePlan?.runId ? ensureRecordId(activePlan.runId, TABLES.PLAN_RUN) : undefined,
|
|
589
|
+
completedAt,
|
|
590
|
+
}),
|
|
591
|
+
AutonomousJobRunRowSchema,
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
const nextRunAt =
|
|
595
|
+
job.data.trigger === 'scheduled' && autonomousJobRow.schedule.kind !== 'at'
|
|
596
|
+
? this.computeNextRunAt(autonomousJobRow.schedule, completedAt)
|
|
597
|
+
: autonomousJobRow.nextRunAt
|
|
598
|
+
const nextStatus =
|
|
599
|
+
autonomousJobRow.schedule.kind === 'at' && job.data.trigger === 'scheduled'
|
|
600
|
+
? 'completed'
|
|
601
|
+
: autonomousJobRow.status
|
|
602
|
+
|
|
603
|
+
await databaseService.update(
|
|
604
|
+
TABLES.AUTONOMOUS_JOB,
|
|
605
|
+
autonomousJobRow.id,
|
|
606
|
+
compactRecord({
|
|
607
|
+
status: nextStatus,
|
|
608
|
+
consecutiveErrorCount: 0,
|
|
609
|
+
lastRunStatus: runStatus,
|
|
610
|
+
lastRunAt: completedAt,
|
|
611
|
+
nextRunAt: nextStatus === 'active' ? (nextRunAt ?? undefined) : undefined,
|
|
612
|
+
linkedPlanSpecId: activePlan?.specId ? ensureRecordId(activePlan.specId, TABLES.PLAN_SPEC) : undefined,
|
|
613
|
+
linkedPlanRunId: activePlan?.runId ? ensureRecordId(activePlan.runId, TABLES.PLAN_RUN) : undefined,
|
|
614
|
+
lastError: undefined,
|
|
615
|
+
}),
|
|
616
|
+
AutonomousJobRowSchema,
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
await this.maybeNotify('notify', {
|
|
620
|
+
organizationId: recordIdToString(autonomousJobRow.organizationId, TABLES.ORGANIZATION),
|
|
621
|
+
workstreamId: recordIdToString(autonomousJobRow.workstreamId, TABLES.WORKSTREAM),
|
|
622
|
+
severity: 'info',
|
|
623
|
+
title: `${autonomousJobRow.title} completed`,
|
|
624
|
+
body: summary,
|
|
625
|
+
metadata: {
|
|
626
|
+
autonomousJobId: job.data.autonomousJobId,
|
|
627
|
+
runId: recordIdToString(runRow.id, TABLES.AUTONOMOUS_JOB_RUN),
|
|
628
|
+
},
|
|
629
|
+
})
|
|
630
|
+
|
|
631
|
+
return { status: runStatus, summary }
|
|
632
|
+
} catch (error) {
|
|
633
|
+
const normalizedError = toQueueJobError(error)
|
|
634
|
+
const completedAt = new Date()
|
|
635
|
+
const nextConsecutiveErrorCount = autonomousJobRow.consecutiveErrorCount + 1
|
|
636
|
+
const autoPause = nextConsecutiveErrorCount >= autonomousJobRow.autoPauseThreshold
|
|
637
|
+
const terminalOneShot = autonomousJobRow.schedule.kind === 'at' && job.data.trigger === 'scheduled'
|
|
638
|
+
const nextStatus: AutonomousJobStatus = terminalOneShot
|
|
639
|
+
? 'failed'
|
|
640
|
+
: autoPause
|
|
641
|
+
? 'paused'
|
|
642
|
+
: autonomousJobRow.status
|
|
643
|
+
|
|
644
|
+
await databaseService.update(
|
|
645
|
+
TABLES.AUTONOMOUS_JOB_RUN,
|
|
646
|
+
runRow.id,
|
|
647
|
+
{ status: 'failed', error: normalizedError, completedAt },
|
|
648
|
+
AutonomousJobRunRowSchema,
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
if (autoPause || terminalOneShot) {
|
|
652
|
+
await this.unscheduleRow(autonomousJobRow)
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
await databaseService.update(
|
|
656
|
+
TABLES.AUTONOMOUS_JOB,
|
|
657
|
+
autonomousJobRow.id,
|
|
658
|
+
compactRecord({
|
|
659
|
+
status: nextStatus,
|
|
660
|
+
consecutiveErrorCount: nextConsecutiveErrorCount,
|
|
661
|
+
lastRunStatus: 'failed',
|
|
662
|
+
lastRunAt: completedAt,
|
|
663
|
+
nextRunAt:
|
|
664
|
+
nextStatus === 'active' && autonomousJobRow.schedule.kind !== 'at'
|
|
665
|
+
? (this.computeNextRunAt(autonomousJobRow.schedule, completedAt) ?? undefined)
|
|
666
|
+
: undefined,
|
|
667
|
+
lastError: normalizedError,
|
|
668
|
+
}),
|
|
669
|
+
AutonomousJobRowSchema,
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
await this.maybeNotify('notify', {
|
|
673
|
+
organizationId: recordIdToString(autonomousJobRow.organizationId, TABLES.ORGANIZATION),
|
|
674
|
+
workstreamId: recordIdToString(autonomousJobRow.workstreamId, TABLES.WORKSTREAM),
|
|
675
|
+
severity: 'warning',
|
|
676
|
+
title: autoPause
|
|
677
|
+
? `${autonomousJobRow.title} paused after repeated failures`
|
|
678
|
+
: `${autonomousJobRow.title} failed`,
|
|
679
|
+
body: normalizedError.message,
|
|
680
|
+
metadata: {
|
|
681
|
+
autonomousJobId: job.data.autonomousJobId,
|
|
682
|
+
runId: recordIdToString(runRow.id, TABLES.AUTONOMOUS_JOB_RUN),
|
|
683
|
+
autoPaused: autoPause,
|
|
684
|
+
},
|
|
685
|
+
})
|
|
686
|
+
|
|
687
|
+
throw error
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
export const autonomousJobService = new AutonomousJobService()
|
package/src/services/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ export * from './adaptive-playbook.service'
|
|
|
2
2
|
export * from './agent-executor.service'
|
|
3
3
|
export * from './artifact-provenance.service'
|
|
4
4
|
export * from './attachment.service'
|
|
5
|
+
export * from './autonomous-job.service'
|
|
5
6
|
export * from './context-enrichment.service'
|
|
6
7
|
export * from './coordination-registry.service'
|
|
7
8
|
export * from './document-chunk.service'
|
|
@@ -25,6 +26,7 @@ export * from './plan-scheduler.service'
|
|
|
25
26
|
export * from './plan-template.service'
|
|
26
27
|
export * from './playbook-registry.service'
|
|
27
28
|
export * from './quality-metrics.service'
|
|
29
|
+
export * from './queue-job.service'
|
|
28
30
|
export * from './monitoring-window.service'
|
|
29
31
|
export * from './plugin-executor.service'
|
|
30
32
|
export * from './recent-activity-title.service'
|