bopodev-api 0.1.27 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/app.ts +1 -2
- package/src/lib/drainable-work.ts +36 -0
- package/src/lib/workspace-policy.ts +1 -2
- package/src/realtime/office-space.ts +3 -1
- package/src/routes/heartbeats.ts +1 -2
- package/src/routes/issues.ts +4 -1
- package/src/server.ts +57 -39
- package/src/services/attention-service.ts +4 -1
- package/src/services/budget-service.ts +1 -2
- package/src/services/comment-recipient-dispatch-service.ts +39 -2
- package/src/services/governance-service.ts +2 -1
- package/src/services/heartbeat-queue-service.ts +34 -3
- package/src/services/heartbeat-service.ts +38 -6
- package/src/worker/scheduler.ts +20 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bopodev-api",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.28",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"files": [
|
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
"nanoid": "^5.1.5",
|
|
18
18
|
"ws": "^8.19.0",
|
|
19
19
|
"zod": "^4.1.5",
|
|
20
|
-
"bopodev-
|
|
21
|
-
"bopodev-
|
|
22
|
-
"bopodev-
|
|
20
|
+
"bopodev-agent-sdk": "0.1.28",
|
|
21
|
+
"bopodev-db": "0.1.28",
|
|
22
|
+
"bopodev-contracts": "0.1.28"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
25
|
"@types/cors": "^2.8.19",
|
package/src/app.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import cors from "cors";
|
|
2
2
|
import express from "express";
|
|
3
3
|
import type { NextFunction, Request, Response } from "express";
|
|
4
|
-
import { sql } from "
|
|
5
|
-
import { RepositoryValidationError } from "bopodev-db";
|
|
4
|
+
import { RepositoryValidationError, sql } from "bopodev-db";
|
|
6
5
|
import { nanoid } from "nanoid";
|
|
7
6
|
import type { AppContext } from "./context";
|
|
8
7
|
import { createAgentsRouter } from "./routes/agents";
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export type DrainableWorkTracker = {
|
|
2
|
+
beginShutdown: () => void;
|
|
3
|
+
isShuttingDown: () => boolean;
|
|
4
|
+
track: <T>(promise: Promise<T>) => Promise<T>;
|
|
5
|
+
drain: () => Promise<void>;
|
|
6
|
+
resetForTests: () => void;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export function createDrainableWorkTracker(): DrainableWorkTracker {
|
|
10
|
+
let shuttingDown = false;
|
|
11
|
+
const pending = new Set<Promise<unknown>>();
|
|
12
|
+
|
|
13
|
+
return {
|
|
14
|
+
beginShutdown() {
|
|
15
|
+
shuttingDown = true;
|
|
16
|
+
},
|
|
17
|
+
isShuttingDown() {
|
|
18
|
+
return shuttingDown;
|
|
19
|
+
},
|
|
20
|
+
track<T>(promise: Promise<T>) {
|
|
21
|
+
let tracked: Promise<T>;
|
|
22
|
+
tracked = promise.finally(() => {
|
|
23
|
+
pending.delete(tracked);
|
|
24
|
+
});
|
|
25
|
+
pending.add(tracked);
|
|
26
|
+
return tracked;
|
|
27
|
+
},
|
|
28
|
+
async drain() {
|
|
29
|
+
await Promise.allSettled([...pending]);
|
|
30
|
+
},
|
|
31
|
+
resetForTests() {
|
|
32
|
+
shuttingDown = false;
|
|
33
|
+
pending.clear();
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { and, eq, inArray } from "drizzle-orm";
|
|
2
1
|
import type { BopoDb } from "bopodev-db";
|
|
3
|
-
import { projectWorkspaces, projects } from "bopodev-db";
|
|
2
|
+
import { and, eq, inArray, projectWorkspaces, projects } from "bopodev-db";
|
|
4
3
|
import {
|
|
5
4
|
assertPathInsideCompanyWorkspaceRoot,
|
|
6
5
|
isInsidePath,
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { and, desc, eq } from "drizzle-orm";
|
|
2
1
|
import type { OfficeOccupant, RealtimeEventEnvelope, RealtimeMessage } from "bopodev-contracts";
|
|
3
2
|
import { AGENT_ROLE_LABELS, AgentRoleKeySchema } from "bopodev-contracts";
|
|
4
3
|
import {
|
|
4
|
+
and,
|
|
5
5
|
agents,
|
|
6
6
|
approvalRequests,
|
|
7
|
+
desc,
|
|
8
|
+
eq,
|
|
7
9
|
getApprovalRequest,
|
|
8
10
|
heartbeatRuns,
|
|
9
11
|
issues,
|
package/src/routes/heartbeats.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Router } from "express";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
-
import { and, eq } from "
|
|
4
|
-
import { agents, heartbeatRuns, listHeartbeatQueueJobs } from "bopodev-db";
|
|
3
|
+
import { agents, and, eq, heartbeatRuns, listHeartbeatQueueJobs } from "bopodev-db";
|
|
5
4
|
import type { AppContext } from "../context";
|
|
6
5
|
import { sendError, sendOk } from "../http";
|
|
7
6
|
import { requireCompanyScope } from "../middleware/company-scope";
|
package/src/routes/issues.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Router } from "express";
|
|
2
2
|
import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
3
3
|
import { basename, extname, join, resolve } from "node:path";
|
|
4
|
-
import { and, desc, eq, inArray } from "drizzle-orm";
|
|
5
4
|
import multer from "multer";
|
|
6
5
|
import { z } from "zod";
|
|
7
6
|
import { IssueDetailSchema, IssueSchema } from "bopodev-contracts";
|
|
@@ -9,15 +8,19 @@ import {
|
|
|
9
8
|
addIssueAttachment,
|
|
10
9
|
addIssueComment,
|
|
11
10
|
agents,
|
|
11
|
+
and,
|
|
12
12
|
appendActivity,
|
|
13
13
|
appendAuditEvent,
|
|
14
14
|
createIssue,
|
|
15
15
|
deleteIssueAttachment,
|
|
16
16
|
deleteIssueComment,
|
|
17
17
|
deleteIssue,
|
|
18
|
+
desc,
|
|
19
|
+
eq,
|
|
18
20
|
getIssue,
|
|
19
21
|
heartbeatRuns,
|
|
20
22
|
getIssueAttachment,
|
|
23
|
+
inArray,
|
|
21
24
|
issues,
|
|
22
25
|
listIssueAttachments,
|
|
23
26
|
listIssueActivity,
|
package/src/server.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { createServer } from "node:http";
|
|
2
2
|
import { dirname, resolve } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { sql } from "drizzle-orm";
|
|
5
4
|
import { config as loadDotenv } from "dotenv";
|
|
6
|
-
import { bootstrapDatabase, listCompanies, resolveDefaultDbPath } from "bopodev-db";
|
|
5
|
+
import { bootstrapDatabase, listCompanies, resolveDefaultDbPath, sql } from "bopodev-db";
|
|
7
6
|
import { checkRuntimeCommandHealth } from "bopodev-agent-sdk";
|
|
8
7
|
import type { RuntimeCommandHealth } from "bopodev-agent-sdk";
|
|
9
8
|
import { createApp } from "./app";
|
|
@@ -21,6 +20,8 @@ import {
|
|
|
21
20
|
} from "./security/deployment-mode";
|
|
22
21
|
import { ensureBuiltinPluginsRegistered } from "./services/plugin-runtime";
|
|
23
22
|
import { ensureBuiltinTemplatesRegistered } from "./services/template-catalog";
|
|
23
|
+
import { beginIssueCommentDispatchShutdown, waitForIssueCommentDispatchDrain } from "./services/comment-recipient-dispatch-service";
|
|
24
|
+
import { beginHeartbeatQueueShutdown, waitForHeartbeatQueueDrain } from "./services/heartbeat-queue-service";
|
|
24
25
|
import { createHeartbeatScheduler } from "./worker/scheduler";
|
|
25
26
|
|
|
26
27
|
loadApiEnv();
|
|
@@ -32,6 +33,7 @@ async function main() {
|
|
|
32
33
|
const publicBaseUrl = resolvePublicBaseUrl();
|
|
33
34
|
validateDeploymentConfiguration(deploymentMode, allowedOrigins, allowedHostnames, publicBaseUrl);
|
|
34
35
|
const dbPath = normalizeOptionalDbPath(process.env.BOPO_DB_PATH);
|
|
36
|
+
const usingExternalDatabase = Boolean(process.env.DATABASE_URL?.trim());
|
|
35
37
|
const port = Number(process.env.PORT ?? 4020);
|
|
36
38
|
const effectiveDbPath = dbPath ?? resolveDefaultDbPath();
|
|
37
39
|
let db: Awaited<ReturnType<typeof bootstrapDatabase>>["db"];
|
|
@@ -41,17 +43,20 @@ async function main() {
|
|
|
41
43
|
db = boot.db;
|
|
42
44
|
dbClient = boot.client;
|
|
43
45
|
} catch (error) {
|
|
44
|
-
if (
|
|
46
|
+
if (isProbablyDatabaseStartupError(error)) {
|
|
45
47
|
// eslint-disable-next-line no-console
|
|
46
|
-
console.error(
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
48
|
+
console.error("[startup] Database bootstrap failed before the API could start.");
|
|
49
|
+
if (usingExternalDatabase) {
|
|
50
|
+
// eslint-disable-next-line no-console
|
|
51
|
+
console.error("[startup] Check DATABASE_URL connectivity, permissions, and migration state.");
|
|
52
|
+
} else {
|
|
53
|
+
// eslint-disable-next-line no-console
|
|
54
|
+
console.error(`[startup] Embedded Postgres data path: ${effectiveDbPath}`);
|
|
55
|
+
// eslint-disable-next-line no-console
|
|
56
|
+
console.error(
|
|
57
|
+
"[startup] Recovery: stop all API/node processes using this DB, back up the path above, delete the directory if it is corrupted, then restart. Or set BOPO_DB_PATH to a fresh path."
|
|
58
|
+
);
|
|
59
|
+
}
|
|
55
60
|
}
|
|
56
61
|
throw error;
|
|
57
62
|
}
|
|
@@ -141,9 +146,9 @@ async function main() {
|
|
|
141
146
|
|
|
142
147
|
const defaultCompanyId = process.env.BOPO_DEFAULT_COMPANY_ID;
|
|
143
148
|
const schedulerCompanyId = await resolveSchedulerCompanyId(db, defaultCompanyId ?? null);
|
|
144
|
-
let
|
|
149
|
+
let scheduler: ReturnType<typeof createHeartbeatScheduler> | undefined;
|
|
145
150
|
if (schedulerCompanyId && shouldStartScheduler()) {
|
|
146
|
-
|
|
151
|
+
scheduler = createHeartbeatScheduler(db, schedulerCompanyId, realtimeHub);
|
|
147
152
|
} else if (schedulerCompanyId) {
|
|
148
153
|
// eslint-disable-next-line no-console
|
|
149
154
|
console.log("[startup] Scheduler disabled for this instance (BOPO_SCHEDULER_ROLE is follower/off).");
|
|
@@ -151,38 +156,50 @@ async function main() {
|
|
|
151
156
|
|
|
152
157
|
let shutdownInFlight: Promise<void> | null = null;
|
|
153
158
|
function shutdown(signal: string) {
|
|
159
|
+
const shutdownTimeoutMs = Number(process.env.BOPO_SHUTDOWN_TIMEOUT_MS ?? 15_000);
|
|
160
|
+
const forcedExit = setTimeout(() => {
|
|
161
|
+
// eslint-disable-next-line no-console
|
|
162
|
+
console.error(`[shutdown] timed out after ${shutdownTimeoutMs}ms; forcing exit.`);
|
|
163
|
+
process.exit(process.exitCode ?? 1);
|
|
164
|
+
}, shutdownTimeoutMs);
|
|
165
|
+
forcedExit.unref();
|
|
154
166
|
shutdownInFlight ??= (async () => {
|
|
155
167
|
// eslint-disable-next-line no-console
|
|
156
|
-
console.log(`[shutdown] ${signal} —
|
|
157
|
-
|
|
168
|
+
console.log(`[shutdown] ${signal} — draining HTTP/background work before closing the embedded database…`);
|
|
169
|
+
beginHeartbeatQueueShutdown();
|
|
170
|
+
beginIssueCommentDispatchShutdown();
|
|
171
|
+
await Promise.allSettled([
|
|
172
|
+
scheduler?.stop() ?? Promise.resolve(),
|
|
173
|
+
new Promise<void>((resolve, reject) => {
|
|
174
|
+
server.close((err) => {
|
|
175
|
+
if (err) {
|
|
176
|
+
reject(err);
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
resolve();
|
|
180
|
+
});
|
|
181
|
+
})
|
|
182
|
+
]);
|
|
183
|
+
await Promise.allSettled([waitForHeartbeatQueueDrain(), waitForIssueCommentDispatchDrain()]);
|
|
158
184
|
try {
|
|
159
185
|
await realtimeHub.close();
|
|
160
186
|
} catch (closeError) {
|
|
161
187
|
// eslint-disable-next-line no-console
|
|
162
188
|
console.error("[shutdown] realtime hub close error", closeError);
|
|
163
189
|
}
|
|
164
|
-
await new Promise<void>((resolve, reject) => {
|
|
165
|
-
server.close((err) => {
|
|
166
|
-
if (err) {
|
|
167
|
-
reject(err);
|
|
168
|
-
return;
|
|
169
|
-
}
|
|
170
|
-
resolve();
|
|
171
|
-
});
|
|
172
|
-
});
|
|
173
190
|
try {
|
|
174
|
-
await
|
|
191
|
+
await closeDatabaseClient(dbClient);
|
|
175
192
|
} catch (closeDbError) {
|
|
176
193
|
// eslint-disable-next-line no-console
|
|
177
|
-
console.error("[shutdown]
|
|
194
|
+
console.error("[shutdown] database close error", closeDbError);
|
|
178
195
|
}
|
|
179
196
|
// eslint-disable-next-line no-console
|
|
180
197
|
console.log("[shutdown] clean exit");
|
|
181
|
-
process.
|
|
198
|
+
process.exitCode = 0;
|
|
182
199
|
})().catch((error) => {
|
|
183
200
|
// eslint-disable-next-line no-console
|
|
184
201
|
console.error("[shutdown] failed", error);
|
|
185
|
-
process.
|
|
202
|
+
process.exitCode = 1;
|
|
186
203
|
});
|
|
187
204
|
return shutdownInFlight;
|
|
188
205
|
}
|
|
@@ -193,7 +210,7 @@ async function main() {
|
|
|
193
210
|
|
|
194
211
|
void main();
|
|
195
212
|
|
|
196
|
-
async function
|
|
213
|
+
async function closeDatabaseClient(client: unknown) {
|
|
197
214
|
if (!client || typeof client !== "object") {
|
|
198
215
|
return;
|
|
199
216
|
}
|
|
@@ -211,7 +228,7 @@ async function hasCodexAgentsConfigured(db: Awaited<ReturnType<typeof bootstrapD
|
|
|
211
228
|
WHERE provider_type = 'codex'
|
|
212
229
|
LIMIT 1
|
|
213
230
|
`);
|
|
214
|
-
return
|
|
231
|
+
return result.length > 0;
|
|
215
232
|
}
|
|
216
233
|
|
|
217
234
|
async function hasOpenCodeAgentsConfigured(db: Awaited<ReturnType<typeof bootstrapDatabase>>["db"]) {
|
|
@@ -221,7 +238,7 @@ async function hasOpenCodeAgentsConfigured(db: Awaited<ReturnType<typeof bootstr
|
|
|
221
238
|
WHERE provider_type = 'opencode'
|
|
222
239
|
LIMIT 1
|
|
223
240
|
`);
|
|
224
|
-
return
|
|
241
|
+
return result.length > 0;
|
|
225
242
|
}
|
|
226
243
|
|
|
227
244
|
async function resolveSchedulerCompanyId(
|
|
@@ -235,7 +252,7 @@ async function resolveSchedulerCompanyId(
|
|
|
235
252
|
WHERE id = ${configuredCompanyId}
|
|
236
253
|
LIMIT 1
|
|
237
254
|
`);
|
|
238
|
-
if (
|
|
255
|
+
if (configured.length > 0) {
|
|
239
256
|
return configuredCompanyId;
|
|
240
257
|
}
|
|
241
258
|
// eslint-disable-next-line no-console
|
|
@@ -248,7 +265,7 @@ async function resolveSchedulerCompanyId(
|
|
|
248
265
|
ORDER BY created_at ASC
|
|
249
266
|
LIMIT 1
|
|
250
267
|
`);
|
|
251
|
-
const id = fallback
|
|
268
|
+
const id = fallback[0]?.id;
|
|
252
269
|
return typeof id === "string" && id.length > 0 ? id : null;
|
|
253
270
|
}
|
|
254
271
|
|
|
@@ -337,14 +354,15 @@ function normalizeOptionalDbPath(value: string | undefined) {
|
|
|
337
354
|
return normalized && normalized.length > 0 ? normalized : undefined;
|
|
338
355
|
}
|
|
339
356
|
|
|
340
|
-
function
|
|
357
|
+
function isProbablyDatabaseStartupError(error: unknown): boolean {
|
|
341
358
|
const message = error instanceof Error ? error.message : String(error);
|
|
342
359
|
const cause = error instanceof Error ? error.cause : undefined;
|
|
343
360
|
const causeMessage = cause instanceof Error ? cause.message : String(cause ?? "");
|
|
344
361
|
return (
|
|
345
|
-
message.includes("
|
|
346
|
-
|
|
347
|
-
message.includes("
|
|
348
|
-
causeMessage.includes("
|
|
362
|
+
message.includes("database") ||
|
|
363
|
+
message.includes("postgres") ||
|
|
364
|
+
message.includes("migration") ||
|
|
365
|
+
causeMessage.includes("postgres") ||
|
|
366
|
+
causeMessage.includes("connection")
|
|
349
367
|
);
|
|
350
368
|
}
|
|
@@ -1,21 +1,38 @@
|
|
|
1
|
-
import { and, desc, eq, inArray, like } from "drizzle-orm";
|
|
2
1
|
import {
|
|
2
|
+
and,
|
|
3
3
|
agents,
|
|
4
|
+
desc,
|
|
5
|
+
eq,
|
|
6
|
+
inArray,
|
|
4
7
|
issueComments,
|
|
8
|
+
like,
|
|
5
9
|
updateIssueCommentRecipients,
|
|
6
10
|
type BopoDb
|
|
7
11
|
} from "bopodev-db";
|
|
12
|
+
import { createDrainableWorkTracker } from "../lib/drainable-work";
|
|
8
13
|
import { parseIssueCommentRecipients, type PersistedCommentRecipient } from "../lib/comment-recipients";
|
|
9
14
|
import type { RealtimeHub } from "../realtime/hub";
|
|
10
15
|
import { enqueueHeartbeatQueueJob, triggerHeartbeatQueueWorker } from "./heartbeat-queue-service";
|
|
11
16
|
|
|
12
17
|
const COMMENT_DISPATCH_SWEEP_LIMIT = 100;
|
|
13
18
|
const activeCompanyDispatchRuns = new Set<string>();
|
|
19
|
+
const commentDispatchTracker = createDrainableWorkTracker();
|
|
14
20
|
|
|
15
21
|
export async function runIssueCommentDispatchSweep(
|
|
16
22
|
db: BopoDb,
|
|
17
23
|
companyId: string,
|
|
18
24
|
options?: { requestId?: string; realtimeHub?: RealtimeHub; limit?: number }
|
|
25
|
+
) {
|
|
26
|
+
if (commentDispatchTracker.isShuttingDown()) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
return commentDispatchTracker.track(runIssueCommentDispatchSweepInternal(db, companyId, options));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function runIssueCommentDispatchSweepInternal(
|
|
33
|
+
db: BopoDb,
|
|
34
|
+
companyId: string,
|
|
35
|
+
options?: { requestId?: string; realtimeHub?: RealtimeHub; limit?: number }
|
|
19
36
|
) {
|
|
20
37
|
const rows = await db
|
|
21
38
|
.select({
|
|
@@ -34,6 +51,9 @@ export async function runIssueCommentDispatchSweep(
|
|
|
34
51
|
.limit(options?.limit ?? COMMENT_DISPATCH_SWEEP_LIMIT);
|
|
35
52
|
|
|
36
53
|
for (const row of rows) {
|
|
54
|
+
if (commentDispatchTracker.isShuttingDown()) {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
37
57
|
const recipients = parseIssueCommentRecipients(row.recipientsJson);
|
|
38
58
|
if (!recipients.some((recipient) => recipient.deliveryStatus === "pending")) {
|
|
39
59
|
continue;
|
|
@@ -60,11 +80,15 @@ export function triggerIssueCommentDispatchWorker(
|
|
|
60
80
|
companyId: string,
|
|
61
81
|
options?: { requestId?: string; realtimeHub?: RealtimeHub; limit?: number }
|
|
62
82
|
) {
|
|
63
|
-
if (activeCompanyDispatchRuns.has(companyId)) {
|
|
83
|
+
if (commentDispatchTracker.isShuttingDown() || activeCompanyDispatchRuns.has(companyId)) {
|
|
64
84
|
return;
|
|
65
85
|
}
|
|
66
86
|
activeCompanyDispatchRuns.add(companyId);
|
|
67
87
|
queueMicrotask(() => {
|
|
88
|
+
if (commentDispatchTracker.isShuttingDown()) {
|
|
89
|
+
activeCompanyDispatchRuns.delete(companyId);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
68
92
|
void runIssueCommentDispatchSweep(db, companyId, options)
|
|
69
93
|
.catch((error) => {
|
|
70
94
|
// eslint-disable-next-line no-console
|
|
@@ -156,3 +180,16 @@ async function dispatchCommentRecipients(
|
|
|
156
180
|
return dispatchedRecipients;
|
|
157
181
|
}
|
|
158
182
|
|
|
183
|
+
export function beginIssueCommentDispatchShutdown() {
|
|
184
|
+
commentDispatchTracker.beginShutdown();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export async function waitForIssueCommentDispatchDrain() {
|
|
188
|
+
await commentDispatchTracker.drain();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export function resetIssueCommentDispatchShutdownForTests() {
|
|
192
|
+
activeCompanyDispatchRuns.clear();
|
|
193
|
+
commentDispatchTracker.resetForTests();
|
|
194
|
+
}
|
|
195
|
+
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { and, eq } from "drizzle-orm";
|
|
2
1
|
import { mkdir } from "node:fs/promises";
|
|
3
2
|
import { z } from "zod";
|
|
4
3
|
import {
|
|
@@ -10,6 +9,7 @@ import {
|
|
|
10
9
|
} from "bopodev-contracts";
|
|
11
10
|
import type { BopoDb } from "bopodev-db";
|
|
12
11
|
import {
|
|
12
|
+
and,
|
|
13
13
|
approvalRequests,
|
|
14
14
|
agents,
|
|
15
15
|
appendAuditEvent,
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
listProjectWorkspaces,
|
|
27
27
|
listProjects,
|
|
28
28
|
projects,
|
|
29
|
+
eq,
|
|
29
30
|
updateProjectWorkspace,
|
|
30
31
|
updatePluginConfig
|
|
31
32
|
} from "bopodev-db";
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { and, eq } from "drizzle-orm";
|
|
2
1
|
import {
|
|
2
|
+
and,
|
|
3
3
|
cancelHeartbeatJob,
|
|
4
|
+
eq,
|
|
4
5
|
getHeartbeatRun,
|
|
5
6
|
claimNextHeartbeatJob,
|
|
6
7
|
enqueueHeartbeatJob,
|
|
@@ -11,6 +12,7 @@ import {
|
|
|
11
12
|
updateIssueCommentRecipients,
|
|
12
13
|
type BopoDb
|
|
13
14
|
} from "bopodev-db";
|
|
15
|
+
import { createDrainableWorkTracker } from "../lib/drainable-work";
|
|
14
16
|
import { parseIssueCommentRecipients } from "../lib/comment-recipients";
|
|
15
17
|
import type { RealtimeHub } from "../realtime/hub";
|
|
16
18
|
import { runHeartbeatForAgent } from "./heartbeat-service";
|
|
@@ -30,6 +32,7 @@ type QueueJobPayload = {
|
|
|
30
32
|
};
|
|
31
33
|
|
|
32
34
|
const activeCompanyQueueWorkers = new Set<string>();
|
|
35
|
+
const queueWorkTracker = createDrainableWorkTracker();
|
|
33
36
|
|
|
34
37
|
export async function enqueueHeartbeatQueueJob(
|
|
35
38
|
db: BopoDb,
|
|
@@ -61,11 +64,15 @@ export function triggerHeartbeatQueueWorker(
|
|
|
61
64
|
companyId: string,
|
|
62
65
|
options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
|
|
63
66
|
) {
|
|
64
|
-
if (activeCompanyQueueWorkers.has(companyId)) {
|
|
67
|
+
if (queueWorkTracker.isShuttingDown() || activeCompanyQueueWorkers.has(companyId)) {
|
|
65
68
|
return;
|
|
66
69
|
}
|
|
67
70
|
activeCompanyQueueWorkers.add(companyId);
|
|
68
71
|
queueMicrotask(() => {
|
|
72
|
+
if (queueWorkTracker.isShuttingDown()) {
|
|
73
|
+
activeCompanyQueueWorkers.delete(companyId);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
69
76
|
void runHeartbeatQueueSweep(db, companyId, options)
|
|
70
77
|
.catch((error) => {
|
|
71
78
|
// eslint-disable-next-line no-console
|
|
@@ -81,10 +88,21 @@ export async function runHeartbeatQueueSweep(
|
|
|
81
88
|
db: BopoDb,
|
|
82
89
|
companyId: string,
|
|
83
90
|
options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
|
|
91
|
+
) {
|
|
92
|
+
if (queueWorkTracker.isShuttingDown()) {
|
|
93
|
+
return { processed: 0 };
|
|
94
|
+
}
|
|
95
|
+
return queueWorkTracker.track(runHeartbeatQueueSweepInternal(db, companyId, options));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function runHeartbeatQueueSweepInternal(
|
|
99
|
+
db: BopoDb,
|
|
100
|
+
companyId: string,
|
|
101
|
+
options?: { requestId?: string; realtimeHub?: RealtimeHub; maxJobsPerSweep?: number }
|
|
84
102
|
) {
|
|
85
103
|
const maxJobs = Math.max(1, Math.min(options?.maxJobsPerSweep ?? 50, 500));
|
|
86
104
|
let processed = 0;
|
|
87
|
-
while (processed < maxJobs) {
|
|
105
|
+
while (processed < maxJobs && !queueWorkTracker.isShuttingDown()) {
|
|
88
106
|
const job = await claimNextHeartbeatJob(db, companyId);
|
|
89
107
|
if (!job) {
|
|
90
108
|
break;
|
|
@@ -117,6 +135,19 @@ export async function runHeartbeatQueueSweep(
|
|
|
117
135
|
return { processed };
|
|
118
136
|
}
|
|
119
137
|
|
|
138
|
+
export function beginHeartbeatQueueShutdown() {
|
|
139
|
+
queueWorkTracker.beginShutdown();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export async function waitForHeartbeatQueueDrain() {
|
|
143
|
+
await queueWorkTracker.drain();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function resetHeartbeatQueueShutdownForTests() {
|
|
147
|
+
activeCompanyQueueWorkers.clear();
|
|
148
|
+
queueWorkTracker.resetForTests();
|
|
149
|
+
}
|
|
150
|
+
|
|
120
151
|
async function processHeartbeatQueueJob(
|
|
121
152
|
db: BopoDb,
|
|
122
153
|
input: {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { mkdir, stat } from "node:fs/promises";
|
|
2
2
|
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
3
|
-
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
|
4
3
|
import { nanoid } from "nanoid";
|
|
5
4
|
import { resolveAdapter } from "bopodev-agent-sdk";
|
|
6
5
|
import type { AdapterExecutionResult, AgentState, HeartbeatContext } from "bopodev-agent-sdk";
|
|
@@ -19,18 +18,24 @@ import {
|
|
|
19
18
|
import type { BopoDb } from "bopodev-db";
|
|
20
19
|
import {
|
|
21
20
|
addIssueComment,
|
|
21
|
+
and,
|
|
22
22
|
approvalRequests,
|
|
23
23
|
agents,
|
|
24
24
|
appendActivity,
|
|
25
25
|
appendHeartbeatRunMessages,
|
|
26
|
+
auditEvents,
|
|
26
27
|
companies,
|
|
27
28
|
createApprovalRequest,
|
|
29
|
+
desc,
|
|
30
|
+
eq,
|
|
28
31
|
goals,
|
|
29
32
|
heartbeatRuns,
|
|
33
|
+
inArray,
|
|
30
34
|
issueComments,
|
|
31
35
|
issueAttachments,
|
|
32
36
|
issues,
|
|
33
|
-
projects
|
|
37
|
+
projects,
|
|
38
|
+
sql
|
|
34
39
|
} from "bopodev-db";
|
|
35
40
|
import { appendAuditEvent, appendCost } from "bopodev-db";
|
|
36
41
|
import { parseRuntimeConfigFromAgentRow } from "../lib/agent-config";
|
|
@@ -49,7 +54,7 @@ import {
|
|
|
49
54
|
resolveAgentFallbackWorkspace
|
|
50
55
|
} from "../lib/workspace-policy";
|
|
51
56
|
import type { RealtimeHub } from "../realtime/hub";
|
|
52
|
-
import { createHeartbeatRunsRealtimeEvent } from "../realtime/heartbeat-runs";
|
|
57
|
+
import { createHeartbeatRunsRealtimeEvent, loadHeartbeatRunsRealtimeSnapshot } from "../realtime/heartbeat-runs";
|
|
53
58
|
import { publishAttentionSnapshot } from "../realtime/attention";
|
|
54
59
|
import { publishOfficeOccupantForAgent } from "../realtime/office-space";
|
|
55
60
|
import { appendProjectBudgetUsage, checkAgentBudget, checkProjectBudget } from "./budget-service";
|
|
@@ -158,7 +163,7 @@ export async function claimIssuesForAgent(
|
|
|
158
163
|
RETURNING i.id, i.project_id, i.parent_issue_id, i.title, i.body, i.status, i.priority, i.labels_json, i.tags_json;
|
|
159
164
|
`);
|
|
160
165
|
|
|
161
|
-
return
|
|
166
|
+
return result as unknown as Array<{
|
|
162
167
|
id: string;
|
|
163
168
|
project_id: string;
|
|
164
169
|
parent_issue_id: string | null;
|
|
@@ -713,6 +718,8 @@ export async function runHeartbeatForAgent(
|
|
|
713
718
|
|
|
714
719
|
let issueIds: string[] = [];
|
|
715
720
|
let claimedIssueIds: string[] = [];
|
|
721
|
+
/** After transcript flush: remove DB row + audit noise for idle heartbeats with no issues. */
|
|
722
|
+
let discardIdleNoWorkRunAfterFlush = false;
|
|
716
723
|
let executionWorkItemsForBudget: Array<{ issueId: string; projectId: string }> = [];
|
|
717
724
|
let state: AgentState & {
|
|
718
725
|
runtime?: {
|
|
@@ -1716,6 +1723,11 @@ export async function runHeartbeatForAgent(
|
|
|
1716
1723
|
}
|
|
1717
1724
|
}
|
|
1718
1725
|
});
|
|
1726
|
+
discardIdleNoWorkRunAfterFlush =
|
|
1727
|
+
issueIds.length === 0 &&
|
|
1728
|
+
!isCommentOrderWake &&
|
|
1729
|
+
(terminalPresentation.completionReason === "no_assigned_work" ||
|
|
1730
|
+
(isIdleNoWork && heartbeatIdlePolicy === "skip_adapter" && persistedRunStatus === "completed"));
|
|
1719
1731
|
} catch (error) {
|
|
1720
1732
|
const classified = classifyHeartbeatError(error);
|
|
1721
1733
|
executionSummary =
|
|
@@ -1951,6 +1963,17 @@ export async function runHeartbeatForAgent(
|
|
|
1951
1963
|
}
|
|
1952
1964
|
} finally {
|
|
1953
1965
|
await transcriptWriteQueue;
|
|
1966
|
+
if (discardIdleNoWorkRunAfterFlush) {
|
|
1967
|
+
try {
|
|
1968
|
+
await purgeIdleNoWorkHeartbeatRun(db, companyId, runId);
|
|
1969
|
+
if (options?.realtimeHub) {
|
|
1970
|
+
options.realtimeHub.publish(await loadHeartbeatRunsRealtimeSnapshot(db, companyId));
|
|
1971
|
+
}
|
|
1972
|
+
} catch (purgeError) {
|
|
1973
|
+
// eslint-disable-next-line no-console
|
|
1974
|
+
console.error("[heartbeat] failed to purge idle no-work run", runId, purgeError);
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1954
1977
|
unregisterActiveHeartbeatRun(runId);
|
|
1955
1978
|
try {
|
|
1956
1979
|
await releaseClaimedIssues(db, companyId, claimedIssueIds);
|
|
@@ -1984,6 +2007,15 @@ export async function runHeartbeatForAgent(
|
|
|
1984
2007
|
return runId;
|
|
1985
2008
|
}
|
|
1986
2009
|
|
|
2010
|
+
async function purgeIdleNoWorkHeartbeatRun(db: BopoDb, companyId: string, runId: string) {
|
|
2011
|
+
await db
|
|
2012
|
+
.delete(auditEvents)
|
|
2013
|
+
.where(
|
|
2014
|
+
and(eq(auditEvents.companyId, companyId), eq(auditEvents.entityType, "heartbeat_run"), eq(auditEvents.entityId, runId))
|
|
2015
|
+
);
|
|
2016
|
+
await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
|
|
2017
|
+
}
|
|
2018
|
+
|
|
1987
2019
|
async function insertStartedRunAtomic(
|
|
1988
2020
|
db: BopoDb,
|
|
1989
2021
|
input: { id: string; companyId: string; agentId: string; message: string }
|
|
@@ -1994,7 +2026,7 @@ async function insertStartedRunAtomic(
|
|
|
1994
2026
|
ON CONFLICT DO NOTHING
|
|
1995
2027
|
RETURNING id
|
|
1996
2028
|
`);
|
|
1997
|
-
return
|
|
2029
|
+
return result.length > 0;
|
|
1998
2030
|
}
|
|
1999
2031
|
|
|
2000
2032
|
async function recoverStaleHeartbeatRuns(
|
|
@@ -2138,7 +2170,7 @@ async function listLatestRunByAgent(db: BopoDb, companyId: string) {
|
|
|
2138
2170
|
GROUP BY agent_id
|
|
2139
2171
|
`);
|
|
2140
2172
|
const latestRunByAgent = new Map<string, Date>();
|
|
2141
|
-
for (const row of result
|
|
2173
|
+
for (const row of result as Array<Record<string, unknown>>) {
|
|
2142
2174
|
const agentId = typeof row.agent_id === "string" ? row.agent_id : null;
|
|
2143
2175
|
if (!agentId) {
|
|
2144
2176
|
continue;
|
package/src/worker/scheduler.ts
CHANGED
|
@@ -4,6 +4,10 @@ import { runHeartbeatSweep } from "../services/heartbeat-service";
|
|
|
4
4
|
import { runHeartbeatQueueSweep } from "../services/heartbeat-queue-service";
|
|
5
5
|
import { runIssueCommentDispatchSweep } from "../services/comment-recipient-dispatch-service";
|
|
6
6
|
|
|
7
|
+
export type HeartbeatSchedulerHandle = {
|
|
8
|
+
stop: () => Promise<void>;
|
|
9
|
+
};
|
|
10
|
+
|
|
7
11
|
export function createHeartbeatScheduler(db: BopoDb, companyId: string, realtimeHub?: RealtimeHub) {
|
|
8
12
|
const heartbeatIntervalMs = Number(process.env.BOPO_HEARTBEAT_SWEEP_MS ?? 60_000);
|
|
9
13
|
const queueIntervalMs = Number(process.env.BOPO_HEARTBEAT_QUEUE_SWEEP_MS ?? 2_000);
|
|
@@ -11,18 +15,22 @@ export function createHeartbeatScheduler(db: BopoDb, companyId: string, realtime
|
|
|
11
15
|
let heartbeatRunning = false;
|
|
12
16
|
let queueRunning = false;
|
|
13
17
|
let commentDispatchRunning = false;
|
|
18
|
+
let heartbeatPromise: Promise<unknown> | null = null;
|
|
19
|
+
let queuePromise: Promise<unknown> | null = null;
|
|
20
|
+
let commentDispatchPromise: Promise<unknown> | null = null;
|
|
14
21
|
const heartbeatTimer = setInterval(() => {
|
|
15
22
|
if (heartbeatRunning) {
|
|
16
23
|
return;
|
|
17
24
|
}
|
|
18
25
|
heartbeatRunning = true;
|
|
19
|
-
|
|
26
|
+
heartbeatPromise = runHeartbeatSweep(db, companyId, { realtimeHub })
|
|
20
27
|
.catch((error) => {
|
|
21
28
|
// eslint-disable-next-line no-console
|
|
22
29
|
console.error("[scheduler] heartbeat sweep failed", error);
|
|
23
30
|
})
|
|
24
31
|
.finally(() => {
|
|
25
32
|
heartbeatRunning = false;
|
|
33
|
+
heartbeatPromise = null;
|
|
26
34
|
});
|
|
27
35
|
}, heartbeatIntervalMs);
|
|
28
36
|
const queueTimer = setInterval(() => {
|
|
@@ -30,13 +38,14 @@ export function createHeartbeatScheduler(db: BopoDb, companyId: string, realtime
|
|
|
30
38
|
return;
|
|
31
39
|
}
|
|
32
40
|
queueRunning = true;
|
|
33
|
-
|
|
41
|
+
queuePromise = runHeartbeatQueueSweep(db, companyId, { realtimeHub })
|
|
34
42
|
.catch((error) => {
|
|
35
43
|
// eslint-disable-next-line no-console
|
|
36
44
|
console.error("[scheduler] queue sweep failed", error);
|
|
37
45
|
})
|
|
38
46
|
.finally(() => {
|
|
39
47
|
queueRunning = false;
|
|
48
|
+
queuePromise = null;
|
|
40
49
|
});
|
|
41
50
|
}, queueIntervalMs);
|
|
42
51
|
const commentDispatchTimer = setInterval(() => {
|
|
@@ -44,18 +53,25 @@ export function createHeartbeatScheduler(db: BopoDb, companyId: string, realtime
|
|
|
44
53
|
return;
|
|
45
54
|
}
|
|
46
55
|
commentDispatchRunning = true;
|
|
47
|
-
|
|
56
|
+
commentDispatchPromise = runIssueCommentDispatchSweep(db, companyId, { realtimeHub })
|
|
48
57
|
.catch((error) => {
|
|
49
58
|
// eslint-disable-next-line no-console
|
|
50
59
|
console.error("[scheduler] comment dispatch sweep failed", error);
|
|
51
60
|
})
|
|
52
61
|
.finally(() => {
|
|
53
62
|
commentDispatchRunning = false;
|
|
63
|
+
commentDispatchPromise = null;
|
|
54
64
|
});
|
|
55
65
|
}, commentDispatchIntervalMs);
|
|
56
|
-
|
|
66
|
+
const stop = async () => {
|
|
57
67
|
clearInterval(heartbeatTimer);
|
|
58
68
|
clearInterval(queueTimer);
|
|
59
69
|
clearInterval(commentDispatchTimer);
|
|
70
|
+
await Promise.allSettled(
|
|
71
|
+
[heartbeatPromise, queuePromise, commentDispatchPromise].filter(
|
|
72
|
+
(promise): promise is Promise<unknown> => promise !== null
|
|
73
|
+
)
|
|
74
|
+
);
|
|
60
75
|
};
|
|
76
|
+
return { stop } satisfies HeartbeatSchedulerHandle;
|
|
61
77
|
}
|