bopodev-api 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/app.ts +11 -1
- package/src/http.ts +39 -0
- package/src/lib/comment-recipients.ts +105 -0
- package/src/lib/hiring-delegate.ts +7 -6
- package/src/lib/instance-paths.ts +11 -0
- package/src/realtime/attention.ts +47 -0
- package/src/realtime/governance.ts +11 -3
- package/src/realtime/heartbeat-runs.ts +33 -11
- package/src/realtime/hub.ts +34 -2
- package/src/realtime/office-space.ts +17 -1
- package/src/routes/agents.ts +81 -12
- package/src/routes/attention.ts +112 -0
- package/src/routes/companies.ts +13 -5
- package/src/routes/goals.ts +10 -2
- package/src/routes/governance.ts +85 -2
- package/src/routes/heartbeats.ts +81 -43
- package/src/routes/issues.ts +293 -62
- package/src/routes/observability.ts +219 -10
- package/src/routes/projects.ts +7 -2
- package/src/scripts/onboard-seed.ts +8 -7
- package/src/server.ts +3 -1
- package/src/services/attention-service.ts +412 -0
- package/src/services/budget-service.ts +99 -2
- package/src/services/comment-recipient-dispatch-service.ts +158 -0
- package/src/services/governance-service.ts +237 -14
- package/src/services/heartbeat-queue-service.ts +318 -0
- package/src/services/heartbeat-service.ts +2341 -278
- package/src/services/memory-file-service.ts +510 -35
- package/src/services/plugin-runtime.ts +33 -1
- package/src/services/template-apply-service.ts +37 -2
- package/src/worker/scheduler.ts +46 -8
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import { and, eq } from "drizzle-orm";
|
|
2
2
|
import { mkdir } from "node:fs/promises";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
AGENT_ROLE_LABELS,
|
|
6
|
+
AgentCreateRequestSchema,
|
|
7
|
+
AgentRoleKeySchema,
|
|
8
|
+
TemplateManifestDefault,
|
|
9
|
+
TemplateManifestSchema
|
|
10
|
+
} from "bopodev-contracts";
|
|
5
11
|
import type { BopoDb } from "bopodev-db";
|
|
6
12
|
import {
|
|
7
13
|
approvalRequests,
|
|
14
|
+
agents,
|
|
15
|
+
appendAuditEvent,
|
|
8
16
|
createAgent,
|
|
9
17
|
createGoal,
|
|
10
18
|
createIssue,
|
|
@@ -31,7 +39,6 @@ import {
|
|
|
31
39
|
import { resolveOpencodeRuntimeModel } from "../lib/opencode-model";
|
|
32
40
|
import {
|
|
33
41
|
normalizeCompanyWorkspacePath,
|
|
34
|
-
resolveAgentFallbackWorkspacePath,
|
|
35
42
|
resolveProjectWorkspacePath
|
|
36
43
|
} from "../lib/instance-paths";
|
|
37
44
|
import { assertRuntimeCwdForCompany, hasText, resolveDefaultRuntimeCwdForCompany } from "../lib/workspace-policy";
|
|
@@ -107,6 +114,32 @@ const applyTemplatePayloadSchema = z.object({
|
|
|
107
114
|
templateVersion: z.string().min(1),
|
|
108
115
|
variables: z.record(z.string(), z.unknown()).default({})
|
|
109
116
|
});
|
|
117
|
+
const overrideBudgetPayloadSchema = z
|
|
118
|
+
.object({
|
|
119
|
+
agentId: z.string().min(1).optional(),
|
|
120
|
+
projectId: z.string().min(1).optional(),
|
|
121
|
+
reason: z.string().optional(),
|
|
122
|
+
additionalBudgetUsd: z.number().positive().optional(),
|
|
123
|
+
revisedMonthlyBudgetUsd: z.number().positive().optional()
|
|
124
|
+
})
|
|
125
|
+
.refine(
|
|
126
|
+
(value) => Boolean(value.agentId) || Boolean(value.projectId),
|
|
127
|
+
"Budget override payload requires agentId or projectId."
|
|
128
|
+
)
|
|
129
|
+
.refine(
|
|
130
|
+
(value) => !(value.agentId && value.projectId),
|
|
131
|
+
"Budget override payload must target either agentId or projectId, not both."
|
|
132
|
+
)
|
|
133
|
+
.refine(
|
|
134
|
+
(value) =>
|
|
135
|
+
(typeof value.additionalBudgetUsd === "number" && value.additionalBudgetUsd > 0) ||
|
|
136
|
+
(typeof value.revisedMonthlyBudgetUsd === "number" && value.revisedMonthlyBudgetUsd > 0),
|
|
137
|
+
"Budget override payload requires additionalBudgetUsd or revisedMonthlyBudgetUsd."
|
|
138
|
+
);
|
|
139
|
+
const pauseOrTerminateAgentPayloadSchema = z.object({
|
|
140
|
+
agentId: z.string().min(1),
|
|
141
|
+
reason: z.string().optional()
|
|
142
|
+
});
|
|
110
143
|
const AGENT_STARTUP_PROJECT_NAME = "Agent Onboarding";
|
|
111
144
|
const AGENT_STARTUP_TASK_MARKER = "[bopodev:onboarding:agent-startup:v1]";
|
|
112
145
|
|
|
@@ -153,7 +186,7 @@ export async function resolveApproval(
|
|
|
153
186
|
let execution:
|
|
154
187
|
| {
|
|
155
188
|
applied: boolean;
|
|
156
|
-
entityType?: "agent" | "goal" | "memory" | "template";
|
|
189
|
+
entityType?: "agent" | "goal" | "project" | "memory" | "template";
|
|
157
190
|
entityId?: string;
|
|
158
191
|
entity?: Record<string, unknown>;
|
|
159
192
|
}
|
|
@@ -255,7 +288,8 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
|
|
|
255
288
|
const existingAgents = await listAgents(db, companyId);
|
|
256
289
|
const duplicate = existingAgents.find(
|
|
257
290
|
(agent) =>
|
|
258
|
-
agent.
|
|
291
|
+
((parsed.data.roleKey && agent.roleKey === parsed.data.roleKey) ||
|
|
292
|
+
(!parsed.data.roleKey && parsed.data.role && agent.role === parsed.data.role)) &&
|
|
259
293
|
(agent.managerAgentId ?? null) === (parsed.data.managerAgentId ?? null) &&
|
|
260
294
|
agent.status !== "terminated"
|
|
261
295
|
);
|
|
@@ -271,7 +305,9 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
|
|
|
271
305
|
const agent = await createAgent(db, {
|
|
272
306
|
companyId,
|
|
273
307
|
managerAgentId: parsed.data.managerAgentId,
|
|
274
|
-
role: parsed.data.role,
|
|
308
|
+
role: resolveAgentRoleText(parsed.data.role, parsed.data.roleKey, parsed.data.title),
|
|
309
|
+
roleKey: normalizeRoleKey(parsed.data.roleKey),
|
|
310
|
+
title: normalizeTitle(parsed.data.title),
|
|
275
311
|
name: parsed.data.name,
|
|
276
312
|
providerType: parsed.data.providerType,
|
|
277
313
|
heartbeatCron: parsed.data.heartbeatCron,
|
|
@@ -281,7 +317,13 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
|
|
|
281
317
|
initialState: runtimeConfigToStateBlobPatch(runtimeConfig)
|
|
282
318
|
});
|
|
283
319
|
const startupProjectId = await ensureAgentStartupProject(db, companyId);
|
|
284
|
-
await ensureAgentStartupIssue(
|
|
320
|
+
await ensureAgentStartupIssue(
|
|
321
|
+
db,
|
|
322
|
+
companyId,
|
|
323
|
+
startupProjectId,
|
|
324
|
+
agent.id,
|
|
325
|
+
resolveAgentDisplayTitle(agent.title, agent.roleKey, agent.role)
|
|
326
|
+
);
|
|
285
327
|
|
|
286
328
|
return {
|
|
287
329
|
applied: true,
|
|
@@ -347,8 +389,144 @@ async function applyApprovalAction(db: BopoDb, companyId: string, action: string
|
|
|
347
389
|
};
|
|
348
390
|
}
|
|
349
391
|
|
|
350
|
-
if (action === "
|
|
351
|
-
|
|
392
|
+
if (action === "override_budget") {
|
|
393
|
+
const parsed = overrideBudgetPayloadSchema.safeParse(payload);
|
|
394
|
+
if (!parsed.success) {
|
|
395
|
+
throw new GovernanceError("Approval payload for budget override is invalid.");
|
|
396
|
+
}
|
|
397
|
+
if (parsed.data.agentId) {
|
|
398
|
+
const [agent] = await db
|
|
399
|
+
.select({
|
|
400
|
+
id: agents.id,
|
|
401
|
+
monthlyBudgetUsd: agents.monthlyBudgetUsd,
|
|
402
|
+
usedBudgetUsd: agents.usedBudgetUsd
|
|
403
|
+
})
|
|
404
|
+
.from(agents)
|
|
405
|
+
.where(and(eq(agents.companyId, companyId), eq(agents.id, parsed.data.agentId)))
|
|
406
|
+
.limit(1);
|
|
407
|
+
if (!agent) {
|
|
408
|
+
throw new GovernanceError("Agent not found for budget override request.");
|
|
409
|
+
}
|
|
410
|
+
const currentMonthlyBudget = Number(agent.monthlyBudgetUsd);
|
|
411
|
+
const currentUsedBudget = Number(agent.usedBudgetUsd);
|
|
412
|
+
const nextMonthlyBudget =
|
|
413
|
+
typeof parsed.data.revisedMonthlyBudgetUsd === "number"
|
|
414
|
+
? parsed.data.revisedMonthlyBudgetUsd
|
|
415
|
+
: currentMonthlyBudget + (parsed.data.additionalBudgetUsd ?? 0);
|
|
416
|
+
if (!Number.isFinite(nextMonthlyBudget) || nextMonthlyBudget <= 0) {
|
|
417
|
+
throw new GovernanceError("Budget override must resolve to a positive monthly budget.");
|
|
418
|
+
}
|
|
419
|
+
if (nextMonthlyBudget <= currentUsedBudget) {
|
|
420
|
+
throw new GovernanceError("Budget override must exceed current used budget.");
|
|
421
|
+
}
|
|
422
|
+
await db
|
|
423
|
+
.update(agents)
|
|
424
|
+
.set({
|
|
425
|
+
monthlyBudgetUsd: nextMonthlyBudget.toFixed(4),
|
|
426
|
+
updatedAt: new Date()
|
|
427
|
+
})
|
|
428
|
+
.where(and(eq(agents.companyId, companyId), eq(agents.id, parsed.data.agentId)));
|
|
429
|
+
return {
|
|
430
|
+
applied: true,
|
|
431
|
+
entityType: "agent" as const,
|
|
432
|
+
entityId: parsed.data.agentId,
|
|
433
|
+
entity: {
|
|
434
|
+
agentId: parsed.data.agentId,
|
|
435
|
+
previousMonthlyBudgetUsd: currentMonthlyBudget,
|
|
436
|
+
monthlyBudgetUsd: nextMonthlyBudget,
|
|
437
|
+
usedBudgetUsd: currentUsedBudget,
|
|
438
|
+
reason: parsed.data.reason ?? null
|
|
439
|
+
}
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
const [project] = await db
|
|
443
|
+
.select({
|
|
444
|
+
id: projects.id,
|
|
445
|
+
monthlyBudgetUsd: projects.monthlyBudgetUsd,
|
|
446
|
+
usedBudgetUsd: projects.usedBudgetUsd
|
|
447
|
+
})
|
|
448
|
+
.from(projects)
|
|
449
|
+
.where(and(eq(projects.companyId, companyId), eq(projects.id, parsed.data.projectId!)))
|
|
450
|
+
.limit(1);
|
|
451
|
+
if (!project) {
|
|
452
|
+
throw new GovernanceError("Project not found for budget override request.");
|
|
453
|
+
}
|
|
454
|
+
const currentMonthlyBudget = Number(project.monthlyBudgetUsd);
|
|
455
|
+
const currentUsedBudget = Number(project.usedBudgetUsd);
|
|
456
|
+
const nextMonthlyBudget =
|
|
457
|
+
typeof parsed.data.revisedMonthlyBudgetUsd === "number"
|
|
458
|
+
? parsed.data.revisedMonthlyBudgetUsd
|
|
459
|
+
: currentMonthlyBudget + (parsed.data.additionalBudgetUsd ?? 0);
|
|
460
|
+
if (!Number.isFinite(nextMonthlyBudget) || nextMonthlyBudget <= 0) {
|
|
461
|
+
throw new GovernanceError("Budget override must resolve to a positive monthly budget.");
|
|
462
|
+
}
|
|
463
|
+
if (nextMonthlyBudget <= currentUsedBudget) {
|
|
464
|
+
throw new GovernanceError("Budget override must exceed current used budget.");
|
|
465
|
+
}
|
|
466
|
+
await db
|
|
467
|
+
.update(projects)
|
|
468
|
+
.set({
|
|
469
|
+
monthlyBudgetUsd: nextMonthlyBudget.toFixed(4),
|
|
470
|
+
updatedAt: new Date()
|
|
471
|
+
})
|
|
472
|
+
.where(and(eq(projects.companyId, companyId), eq(projects.id, parsed.data.projectId!)));
|
|
473
|
+
await appendAuditEvent(db, {
|
|
474
|
+
companyId,
|
|
475
|
+
actorType: "system",
|
|
476
|
+
eventType: "project_budget.override_applied",
|
|
477
|
+
entityType: "project",
|
|
478
|
+
entityId: parsed.data.projectId!,
|
|
479
|
+
payload: {
|
|
480
|
+
previousMonthlyBudgetUsd: currentMonthlyBudget,
|
|
481
|
+
monthlyBudgetUsd: nextMonthlyBudget,
|
|
482
|
+
usedBudgetUsd: currentUsedBudget,
|
|
483
|
+
reason: parsed.data.reason ?? null
|
|
484
|
+
}
|
|
485
|
+
});
|
|
486
|
+
return {
|
|
487
|
+
applied: true,
|
|
488
|
+
entityType: "project" as const,
|
|
489
|
+
entityId: parsed.data.projectId!,
|
|
490
|
+
entity: {
|
|
491
|
+
projectId: parsed.data.projectId!,
|
|
492
|
+
previousMonthlyBudgetUsd: currentMonthlyBudget,
|
|
493
|
+
monthlyBudgetUsd: nextMonthlyBudget,
|
|
494
|
+
usedBudgetUsd: currentUsedBudget,
|
|
495
|
+
reason: parsed.data.reason ?? null
|
|
496
|
+
}
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
if (action === "pause_agent" || action === "terminate_agent") {
|
|
501
|
+
const parsed = pauseOrTerminateAgentPayloadSchema.safeParse(payload);
|
|
502
|
+
if (!parsed.success) {
|
|
503
|
+
throw new GovernanceError(`Approval payload for ${action} is invalid.`);
|
|
504
|
+
}
|
|
505
|
+
const nextStatus = action === "pause_agent" ? "paused" : "terminated";
|
|
506
|
+
const [updated] = await db
|
|
507
|
+
.update(agents)
|
|
508
|
+
.set({
|
|
509
|
+
status: nextStatus,
|
|
510
|
+
updatedAt: new Date()
|
|
511
|
+
})
|
|
512
|
+
.where(and(eq(agents.companyId, companyId), eq(agents.id, parsed.data.agentId)))
|
|
513
|
+
.returning({
|
|
514
|
+
id: agents.id,
|
|
515
|
+
status: agents.status
|
|
516
|
+
});
|
|
517
|
+
if (!updated) {
|
|
518
|
+
throw new GovernanceError("Agent not found for lifecycle governance action.");
|
|
519
|
+
}
|
|
520
|
+
return {
|
|
521
|
+
applied: true,
|
|
522
|
+
entityType: "agent" as const,
|
|
523
|
+
entityId: updated.id,
|
|
524
|
+
entity: {
|
|
525
|
+
id: updated.id,
|
|
526
|
+
status: updated.status,
|
|
527
|
+
reason: parsed.data.reason ?? null
|
|
528
|
+
}
|
|
529
|
+
};
|
|
352
530
|
}
|
|
353
531
|
|
|
354
532
|
if (action === "promote_memory_fact") {
|
|
@@ -516,9 +694,9 @@ async function ensureAgentStartupIssue(
|
|
|
516
694
|
companyId: string,
|
|
517
695
|
projectId: string,
|
|
518
696
|
agentId: string,
|
|
519
|
-
|
|
697
|
+
roleTitle: string
|
|
520
698
|
) {
|
|
521
|
-
const title = `Set up ${
|
|
699
|
+
const title = `Set up ${roleTitle} operating files`;
|
|
522
700
|
const body = buildAgentStartupTaskBody(companyId, agentId);
|
|
523
701
|
const existingIssues = await listIssues(db, companyId);
|
|
524
702
|
const existing = existingIssues.find(
|
|
@@ -545,15 +723,60 @@ async function ensureAgentStartupIssue(
|
|
|
545
723
|
return created.id;
|
|
546
724
|
}
|
|
547
725
|
|
|
726
|
+
function normalizeRoleKey(input: string | null | undefined) {
|
|
727
|
+
const normalized = input?.trim().toLowerCase();
|
|
728
|
+
if (!normalized) {
|
|
729
|
+
return null;
|
|
730
|
+
}
|
|
731
|
+
return AgentRoleKeySchema.safeParse(normalized).success ? normalized : null;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
function normalizeTitle(input: string | null | undefined) {
|
|
735
|
+
const normalized = input?.trim();
|
|
736
|
+
return normalized ? normalized : null;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
function resolveAgentRoleText(
|
|
740
|
+
legacyRole: string | undefined,
|
|
741
|
+
roleKeyInput: string | undefined,
|
|
742
|
+
titleInput: string | null | undefined
|
|
743
|
+
) {
|
|
744
|
+
const normalizedLegacy = legacyRole?.trim();
|
|
745
|
+
if (normalizedLegacy) {
|
|
746
|
+
return normalizedLegacy;
|
|
747
|
+
}
|
|
748
|
+
const normalizedTitle = normalizeTitle(titleInput);
|
|
749
|
+
if (normalizedTitle) {
|
|
750
|
+
return normalizedTitle;
|
|
751
|
+
}
|
|
752
|
+
const roleKey = normalizeRoleKey(roleKeyInput);
|
|
753
|
+
if (roleKey) {
|
|
754
|
+
return AGENT_ROLE_LABELS[roleKey as keyof typeof AGENT_ROLE_LABELS];
|
|
755
|
+
}
|
|
756
|
+
return AGENT_ROLE_LABELS.general;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
function resolveAgentDisplayTitle(title: string | null | undefined, roleKeyInput: string | null | undefined, role: string) {
|
|
760
|
+
const normalizedTitle = normalizeTitle(title);
|
|
761
|
+
if (normalizedTitle) {
|
|
762
|
+
return normalizedTitle;
|
|
763
|
+
}
|
|
764
|
+
const roleKey = normalizeRoleKey(roleKeyInput);
|
|
765
|
+
if (roleKey) {
|
|
766
|
+
return AGENT_ROLE_LABELS[roleKey as keyof typeof AGENT_ROLE_LABELS];
|
|
767
|
+
}
|
|
768
|
+
return role;
|
|
769
|
+
}
|
|
770
|
+
|
|
548
771
|
function buildAgentStartupTaskBody(companyId: string, agentId: string) {
|
|
549
|
-
const
|
|
550
|
-
const agentOperatingFolder = `${
|
|
772
|
+
const companyScopedAgentRoot = `workspace/${companyId}/agents/${agentId}`;
|
|
773
|
+
const agentOperatingFolder = `${companyScopedAgentRoot}/operating`;
|
|
551
774
|
return [
|
|
552
775
|
AGENT_STARTUP_TASK_MARKER,
|
|
553
776
|
"",
|
|
554
777
|
`Create your operating baseline before starting feature delivery work.`,
|
|
555
778
|
"",
|
|
556
|
-
`1. Create your operating folder at \`${agentOperatingFolder}
|
|
779
|
+
`1. Create your operating folder at \`${agentOperatingFolder}/\`.`,
|
|
557
780
|
"2. Author these files with your own responsibilities and working style:",
|
|
558
781
|
` - \`${agentOperatingFolder}/AGENTS.md\``,
|
|
559
782
|
` - \`${agentOperatingFolder}/HEARTBEAT.md\``,
|
|
@@ -563,7 +786,7 @@ function buildAgentStartupTaskBody(companyId: string, agentId: string) {
|
|
|
563
786
|
"4. Post an issue comment summarizing completed setup artifacts.",
|
|
564
787
|
"",
|
|
565
788
|
"Safety checks:",
|
|
566
|
-
|
|
789
|
+
`- Keep operating files inside \`workspace/${companyId}/agents/${agentId}/\` only.`,
|
|
567
790
|
"- Do not overwrite another agent's operating folder.",
|
|
568
791
|
"- Keep content original to your role and scope."
|
|
569
792
|
].join("\n");
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import { and, eq } from "drizzle-orm";
|
|
2
|
+
import {
|
|
3
|
+
cancelHeartbeatJob,
|
|
4
|
+
getHeartbeatRun,
|
|
5
|
+
claimNextHeartbeatJob,
|
|
6
|
+
enqueueHeartbeatJob,
|
|
7
|
+
issueComments,
|
|
8
|
+
markHeartbeatJobCompleted,
|
|
9
|
+
markHeartbeatJobDeadLetter,
|
|
10
|
+
markHeartbeatJobRetry,
|
|
11
|
+
updateIssueCommentRecipients,
|
|
12
|
+
type BopoDb
|
|
13
|
+
} from "bopodev-db";
|
|
14
|
+
import { parseIssueCommentRecipients } from "../lib/comment-recipients";
|
|
15
|
+
import type { RealtimeHub } from "../realtime/hub";
|
|
16
|
+
import { runHeartbeatForAgent } from "./heartbeat-service";
|
|
17
|
+
|
|
18
|
+
type QueueJobPayload = {
|
|
19
|
+
sourceRunId?: string;
|
|
20
|
+
wakeContext?: {
|
|
21
|
+
reason?: string | null;
|
|
22
|
+
commentId?: string | null;
|
|
23
|
+
issueIds?: string[];
|
|
24
|
+
};
|
|
25
|
+
commentDispatch?: {
|
|
26
|
+
commentId: string;
|
|
27
|
+
issueId: string;
|
|
28
|
+
recipientId: string;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const activeCompanyQueueWorkers = new Set<string>();
|
|
33
|
+
|
|
34
|
+
export async function enqueueHeartbeatQueueJob(
|
|
35
|
+
db: BopoDb,
|
|
36
|
+
input: {
|
|
37
|
+
companyId: string;
|
|
38
|
+
agentId: string;
|
|
39
|
+
jobType: "manual" | "scheduler" | "resume" | "redo" | "comment_dispatch";
|
|
40
|
+
payload?: QueueJobPayload;
|
|
41
|
+
priority?: number;
|
|
42
|
+
idempotencyKey?: string | null;
|
|
43
|
+
maxAttempts?: number;
|
|
44
|
+
availableAt?: Date;
|
|
45
|
+
}
|
|
46
|
+
) {
|
|
47
|
+
return enqueueHeartbeatJob(db, {
|
|
48
|
+
companyId: input.companyId,
|
|
49
|
+
agentId: input.agentId,
|
|
50
|
+
jobType: input.jobType,
|
|
51
|
+
payload: input.payload ?? {},
|
|
52
|
+
priority: input.priority ?? 100,
|
|
53
|
+
idempotencyKey: input.idempotencyKey ?? null,
|
|
54
|
+
maxAttempts: input.maxAttempts ?? 10,
|
|
55
|
+
availableAt: input.availableAt
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function triggerHeartbeatQueueWorker(
|
|
60
|
+
db: BopoDb,
|
|
61
|
+
companyId: string,
|
|
62
|
+
options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
|
|
63
|
+
) {
|
|
64
|
+
if (activeCompanyQueueWorkers.has(companyId)) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
activeCompanyQueueWorkers.add(companyId);
|
|
68
|
+
queueMicrotask(() => {
|
|
69
|
+
void runHeartbeatQueueSweep(db, companyId, options)
|
|
70
|
+
.catch((error) => {
|
|
71
|
+
// eslint-disable-next-line no-console
|
|
72
|
+
console.error("[heartbeat-queue] worker run failed", error);
|
|
73
|
+
})
|
|
74
|
+
.finally(() => {
|
|
75
|
+
activeCompanyQueueWorkers.delete(companyId);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export async function runHeartbeatQueueSweep(
|
|
81
|
+
db: BopoDb,
|
|
82
|
+
companyId: string,
|
|
83
|
+
options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
|
|
84
|
+
) {
|
|
85
|
+
const maxJobs = Math.max(1, Math.min(options?.maxJobsPerSweep ?? 50, 500));
|
|
86
|
+
let processed = 0;
|
|
87
|
+
while (processed < maxJobs) {
|
|
88
|
+
const job = await claimNextHeartbeatJob(db, companyId);
|
|
89
|
+
if (!job) {
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
processed += 1;
|
|
93
|
+
try {
|
|
94
|
+
await processHeartbeatQueueJob(db, {
|
|
95
|
+
companyId,
|
|
96
|
+
job: {
|
|
97
|
+
id: job.id,
|
|
98
|
+
agentId: job.agentId,
|
|
99
|
+
jobType: job.jobType as "manual" | "scheduler" | "resume" | "redo" | "comment_dispatch",
|
|
100
|
+
attemptCount: job.attemptCount,
|
|
101
|
+
maxAttempts: job.maxAttempts,
|
|
102
|
+
payload: (job.payload ?? {}) as QueueJobPayload
|
|
103
|
+
},
|
|
104
|
+
requestId: options?.requestId,
|
|
105
|
+
realtimeHub: options?.realtimeHub
|
|
106
|
+
});
|
|
107
|
+
} catch (error) {
|
|
108
|
+
await handleQueueRetryOrDeadLetter(db, {
|
|
109
|
+
companyId,
|
|
110
|
+
jobId: job.id,
|
|
111
|
+
attemptCount: job.attemptCount,
|
|
112
|
+
maxAttempts: job.maxAttempts,
|
|
113
|
+
error: `Queue processing failed: ${String((error as Error)?.message ?? error)}`
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return { processed };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function processHeartbeatQueueJob(
|
|
121
|
+
db: BopoDb,
|
|
122
|
+
input: {
|
|
123
|
+
companyId: string;
|
|
124
|
+
job: {
|
|
125
|
+
id: string;
|
|
126
|
+
agentId: string;
|
|
127
|
+
jobType: "manual" | "scheduler" | "resume" | "redo" | "comment_dispatch";
|
|
128
|
+
attemptCount: number;
|
|
129
|
+
maxAttempts: number;
|
|
130
|
+
payload: QueueJobPayload;
|
|
131
|
+
};
|
|
132
|
+
requestId?: string;
|
|
133
|
+
realtimeHub?: RealtimeHub;
|
|
134
|
+
}
|
|
135
|
+
) {
|
|
136
|
+
let runId: string | null = null;
|
|
137
|
+
try {
|
|
138
|
+
runId = await runHeartbeatForAgent(db, input.companyId, input.job.agentId, {
|
|
139
|
+
trigger: input.job.jobType === "manual" || input.job.jobType === "resume" || input.job.jobType === "redo" ? "manual" : "scheduler",
|
|
140
|
+
requestId: input.requestId,
|
|
141
|
+
realtimeHub: input.realtimeHub,
|
|
142
|
+
...(input.job.jobType === "resume" || input.job.jobType === "redo" ? { mode: input.job.jobType } : {}),
|
|
143
|
+
...(input.job.payload.sourceRunId ? { sourceRunId: input.job.payload.sourceRunId } : {}),
|
|
144
|
+
...(input.job.payload.wakeContext ? { wakeContext: input.job.payload.wakeContext } : {})
|
|
145
|
+
});
|
|
146
|
+
} catch (error) {
|
|
147
|
+
await handleQueueRetryOrDeadLetter(db, {
|
|
148
|
+
companyId: input.companyId,
|
|
149
|
+
jobId: input.job.id,
|
|
150
|
+
attemptCount: input.job.attemptCount,
|
|
151
|
+
maxAttempts: input.job.maxAttempts,
|
|
152
|
+
error: `Heartbeat executor failed: ${String((error as Error)?.message ?? error)}`
|
|
153
|
+
});
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (!runId) {
|
|
158
|
+
await handleQueueRetryOrDeadLetter(db, {
|
|
159
|
+
companyId: input.companyId,
|
|
160
|
+
jobId: input.job.id,
|
|
161
|
+
attemptCount: input.job.attemptCount,
|
|
162
|
+
maxAttempts: input.job.maxAttempts,
|
|
163
|
+
error: "Heartbeat execution returned no run id."
|
|
164
|
+
});
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const run = await getHeartbeatRun(db, input.companyId, runId);
|
|
169
|
+
if (!run) {
|
|
170
|
+
await handleQueueRetryOrDeadLetter(db, {
|
|
171
|
+
companyId: input.companyId,
|
|
172
|
+
jobId: input.job.id,
|
|
173
|
+
attemptCount: input.job.attemptCount,
|
|
174
|
+
maxAttempts: input.job.maxAttempts,
|
|
175
|
+
heartbeatRunId: runId,
|
|
176
|
+
error: "Heartbeat run record was not found after execution."
|
|
177
|
+
});
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (run.status === "skipped" && String(run.message ?? "").toLowerCase().includes("already in progress")) {
|
|
182
|
+
await markHeartbeatJobRetry(db, {
|
|
183
|
+
companyId: input.companyId,
|
|
184
|
+
id: input.job.id,
|
|
185
|
+
heartbeatRunId: runId,
|
|
186
|
+
retryAt: new Date(Date.now() + resolveRetryDelayMs(input.job.attemptCount)),
|
|
187
|
+
error: "Agent busy, retry queued."
|
|
188
|
+
});
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (run.status === "skipped") {
|
|
193
|
+
if (isProjectBudgetHardStopMessage(run.message)) {
|
|
194
|
+
await cancelHeartbeatJob(db, {
|
|
195
|
+
companyId: input.companyId,
|
|
196
|
+
id: input.job.id
|
|
197
|
+
});
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
await handleQueueRetryOrDeadLetter(db, {
|
|
201
|
+
companyId: input.companyId,
|
|
202
|
+
jobId: input.job.id,
|
|
203
|
+
attemptCount: input.job.attemptCount,
|
|
204
|
+
maxAttempts: input.job.maxAttempts,
|
|
205
|
+
heartbeatRunId: runId,
|
|
206
|
+
error: run.message ?? "Heartbeat skipped."
|
|
207
|
+
});
|
|
208
|
+
await markCommentRecipientDeliveryIfNeeded(db, input.companyId, input.job.payload.commentDispatch, {
|
|
209
|
+
deliveryStatus: "failed",
|
|
210
|
+
dispatchedRunId: null,
|
|
211
|
+
dispatchedAt: null
|
|
212
|
+
});
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
await markHeartbeatJobCompleted(db, {
|
|
217
|
+
companyId: input.companyId,
|
|
218
|
+
id: input.job.id,
|
|
219
|
+
heartbeatRunId: runId
|
|
220
|
+
});
|
|
221
|
+
await markCommentRecipientDeliveryIfNeeded(db, input.companyId, input.job.payload.commentDispatch, {
|
|
222
|
+
deliveryStatus: "dispatched",
|
|
223
|
+
dispatchedRunId: runId,
|
|
224
|
+
dispatchedAt: new Date().toISOString()
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async function handleQueueRetryOrDeadLetter(
|
|
229
|
+
db: BopoDb,
|
|
230
|
+
input: {
|
|
231
|
+
companyId: string;
|
|
232
|
+
jobId: string;
|
|
233
|
+
attemptCount: number;
|
|
234
|
+
maxAttempts: number;
|
|
235
|
+
error: string;
|
|
236
|
+
heartbeatRunId?: string;
|
|
237
|
+
}
|
|
238
|
+
) {
|
|
239
|
+
if (input.attemptCount >= input.maxAttempts) {
|
|
240
|
+
await markHeartbeatJobDeadLetter(db, {
|
|
241
|
+
companyId: input.companyId,
|
|
242
|
+
id: input.jobId,
|
|
243
|
+
heartbeatRunId: input.heartbeatRunId ?? null,
|
|
244
|
+
error: input.error
|
|
245
|
+
});
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
await markHeartbeatJobRetry(db, {
|
|
249
|
+
companyId: input.companyId,
|
|
250
|
+
id: input.jobId,
|
|
251
|
+
heartbeatRunId: input.heartbeatRunId ?? null,
|
|
252
|
+
retryAt: new Date(Date.now() + resolveRetryDelayMs(input.attemptCount)),
|
|
253
|
+
error: input.error
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function resolveRetryDelayMs(attemptCount: number) {
|
|
258
|
+
const baseDelayMs = Number(process.env.BOPO_HEARTBEAT_QUEUE_RETRY_BASE_MS ?? 1000);
|
|
259
|
+
const cappedAttempt = Math.max(0, Math.min(attemptCount, 8));
|
|
260
|
+
return Math.max(500, baseDelayMs) * 2 ** cappedAttempt;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function isProjectBudgetHardStopMessage(message: string | null | undefined) {
|
|
264
|
+
const normalized = String(message ?? "")
|
|
265
|
+
.trim()
|
|
266
|
+
.toLowerCase();
|
|
267
|
+
return normalized.includes("project budget hard-stop");
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async function markCommentRecipientDeliveryIfNeeded(
|
|
271
|
+
db: BopoDb,
|
|
272
|
+
companyId: string,
|
|
273
|
+
dispatch: QueueJobPayload["commentDispatch"] | undefined,
|
|
274
|
+
input: {
|
|
275
|
+
deliveryStatus: "dispatched" | "failed";
|
|
276
|
+
dispatchedRunId: string | null;
|
|
277
|
+
dispatchedAt: string | null;
|
|
278
|
+
}
|
|
279
|
+
) {
|
|
280
|
+
if (!dispatch) {
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
const [comment] = await db
|
|
284
|
+
.select({ recipientsJson: issueComments.recipientsJson })
|
|
285
|
+
.from(issueComments)
|
|
286
|
+
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.id, dispatch.commentId)))
|
|
287
|
+
.limit(1);
|
|
288
|
+
if (!comment) {
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
const recipients = parseIssueCommentRecipients(comment.recipientsJson);
|
|
292
|
+
let changed = false;
|
|
293
|
+
const updatedRecipients = recipients.map((recipient) => {
|
|
294
|
+
if (recipient.recipientType !== "agent" || recipient.recipientId !== dispatch.recipientId) {
|
|
295
|
+
return recipient;
|
|
296
|
+
}
|
|
297
|
+
if (recipient.deliveryStatus !== "pending") {
|
|
298
|
+
return recipient;
|
|
299
|
+
}
|
|
300
|
+
changed = true;
|
|
301
|
+
return {
|
|
302
|
+
...recipient,
|
|
303
|
+
deliveryStatus: input.deliveryStatus,
|
|
304
|
+
dispatchedRunId: input.dispatchedRunId,
|
|
305
|
+
dispatchedAt: input.dispatchedAt
|
|
306
|
+
};
|
|
307
|
+
});
|
|
308
|
+
if (!changed) {
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
await updateIssueCommentRecipients(db, {
|
|
312
|
+
companyId,
|
|
313
|
+
issueId: dispatch.issueId,
|
|
314
|
+
id: dispatch.commentId,
|
|
315
|
+
recipients: updatedRecipients
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
|