@robodev-ai/runtime 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/cron.test.ts +19 -0
- package/src/cron.ts +86 -0
- package/src/index.ts +30 -1
- package/src/invoke.ts +5 -18
- package/src/job-queue.test.ts +68 -0
- package/src/job-queue.ts +58 -0
- package/src/local-jobs.test.ts +448 -0
- package/src/local-jobs.ts +353 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@robodev-ai/runtime",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Robodev project runtime — deploy-file classification, esbuild compile, module loading, schema push, route matching, OpenAPI, handler invocation, and Robodev Auth. Shared by hosted Starbase and `robodev dev`.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"pg": "^8.16.3",
|
|
33
33
|
"zod": "^3.25.76",
|
|
34
34
|
"zod-to-json-schema": "^3.24.6",
|
|
35
|
-
"@robodev-ai/sdk": "0.
|
|
35
|
+
"@robodev-ai/sdk": "0.11.0"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@types/busboy": "^1.5.4",
|
package/src/cron.test.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import { nextRunAfter, parseCron } from "./cron.js";
|
|
4
|
+
|
|
5
|
+
test("* * * * * is the next minute", () => {
|
|
6
|
+
const from = new Date(Date.UTC(2026, 0, 1, 12, 0, 30));
|
|
7
|
+
assert.deepEqual(nextRunAfter("* * * * *", from), new Date(Date.UTC(2026, 0, 1, 12, 1, 0)));
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
test("0 0 * * * is the next midnight", () => {
|
|
11
|
+
const from = new Date(Date.UTC(2026, 0, 1, 12, 30, 0));
|
|
12
|
+
assert.deepEqual(nextRunAfter("0 0 * * *", from), new Date(Date.UTC(2026, 0, 2, 0, 0, 0)));
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("rejects 6-field and invalid expressions", () => {
|
|
16
|
+
assert.throws(() => parseCron("* * * * * *"), /exactly 5 fields/);
|
|
17
|
+
assert.throws(() => parseCron("not-cron"), /exactly 5 fields/);
|
|
18
|
+
assert.throws(() => parseCron("60 * * * *"), /invalid cron field/);
|
|
19
|
+
});
|
package/src/cron.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
const FIELD_RANGES = [
|
|
2
|
+
{ min: 0, max: 59 },
|
|
3
|
+
{ min: 0, max: 23 },
|
|
4
|
+
{ min: 1, max: 31 },
|
|
5
|
+
{ min: 1, max: 12 },
|
|
6
|
+
{ min: 0, max: 7 },
|
|
7
|
+
] as const;
|
|
8
|
+
|
|
9
|
+
export type CronFields = [Set<number>, Set<number>, Set<number>, Set<number>, Set<number>];
|
|
10
|
+
|
|
11
|
+
function expandField(field: string, min: number, max: number): Set<number> {
|
|
12
|
+
const values = new Set<number>();
|
|
13
|
+
for (const part of field.split(",")) {
|
|
14
|
+
const [rangePart, stepPart] = part.split("/");
|
|
15
|
+
const step = stepPart ? Number(stepPart) : 1;
|
|
16
|
+
if (!Number.isInteger(step) || step < 1) {
|
|
17
|
+
throw new Error("invalid cron field");
|
|
18
|
+
}
|
|
19
|
+
let start = min;
|
|
20
|
+
let end = max;
|
|
21
|
+
if (rangePart && rangePart !== "*") {
|
|
22
|
+
if (rangePart.includes("-")) {
|
|
23
|
+
const [fromRaw, toRaw] = rangePart.split("-");
|
|
24
|
+
start = Number(fromRaw);
|
|
25
|
+
end = Number(toRaw);
|
|
26
|
+
} else {
|
|
27
|
+
start = Number(rangePart);
|
|
28
|
+
end = start;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (
|
|
32
|
+
!Number.isInteger(start) ||
|
|
33
|
+
!Number.isInteger(end) ||
|
|
34
|
+
start < min ||
|
|
35
|
+
end > max ||
|
|
36
|
+
start > end
|
|
37
|
+
) {
|
|
38
|
+
throw new Error("invalid cron field");
|
|
39
|
+
}
|
|
40
|
+
for (let value = start; value <= end; value += step) {
|
|
41
|
+
values.add(value);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (max === 7 && values.has(7)) values.add(0);
|
|
45
|
+
return values;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function parseCron(expr: string): CronFields {
|
|
49
|
+
const fields = expr.trim().split(/\s+/);
|
|
50
|
+
if (fields.length !== 5) {
|
|
51
|
+
throw new Error("cron must have exactly 5 fields");
|
|
52
|
+
}
|
|
53
|
+
return FIELD_RANGES.map((range, index) =>
|
|
54
|
+
expandField(fields[index] ?? "", range.min, range.max),
|
|
55
|
+
) as CronFields;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function matches(date: Date, fields: CronFields): boolean {
|
|
59
|
+
const [minutes, hours, days, months, weekdays] = fields;
|
|
60
|
+
const dow = date.getUTCDay();
|
|
61
|
+
return (
|
|
62
|
+
minutes.has(date.getUTCMinutes()) &&
|
|
63
|
+
hours.has(date.getUTCHours()) &&
|
|
64
|
+
days.has(date.getUTCDate()) &&
|
|
65
|
+
months.has(date.getUTCMonth() + 1) &&
|
|
66
|
+
(weekdays.has(dow) || (dow === 0 && weekdays.has(7)))
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function nextRunAfter(expr: string, from: Date): Date {
|
|
71
|
+
const fields = parseCron(expr);
|
|
72
|
+
const cursor = new Date(from.getTime());
|
|
73
|
+
cursor.setUTCSeconds(0, 0);
|
|
74
|
+
cursor.setUTCMinutes(cursor.getUTCMinutes() + 1);
|
|
75
|
+
const limit = cursor.getTime() + 366 * 24 * 60 * 60 * 1000;
|
|
76
|
+
while (cursor.getTime() <= limit) {
|
|
77
|
+
if (matches(cursor, fields)) return new Date(cursor);
|
|
78
|
+
cursor.setUTCMinutes(cursor.getUTCMinutes() + 1);
|
|
79
|
+
}
|
|
80
|
+
throw new Error("could not find next cron run");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function assertFiveFieldCron(expr: string): string {
|
|
84
|
+
parseCron(expr);
|
|
85
|
+
return expr.trim();
|
|
86
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `@robodev-ai/runtime` — the project runtime shared by hosted Starbase deploys and the
|
|
3
3
|
* offline `robodev dev` loop: deploy-file classification, the esbuild compile, module
|
|
4
|
-
* loading, schema push, route matching, OpenAPI, handler invocation,
|
|
4
|
+
* loading, schema push, route matching, OpenAPI, handler invocation, Robodev Auth,
|
|
5
|
+
* and the offline `robodev dev` jobs engine.
|
|
5
6
|
*/
|
|
6
7
|
|
|
7
8
|
export {
|
|
@@ -193,3 +194,31 @@ export {
|
|
|
193
194
|
type LocalAuthRequest,
|
|
194
195
|
type LocalAuthResult,
|
|
195
196
|
} from "./local-auth.js";
|
|
197
|
+
|
|
198
|
+
export { assertFiveFieldCron, nextRunAfter, parseCron, type CronFields } from "./cron.js";
|
|
199
|
+
|
|
200
|
+
export {
|
|
201
|
+
JOB_CONCURRENCY,
|
|
202
|
+
JOB_MAX_ATTEMPTS,
|
|
203
|
+
JOB_PAYLOAD_MAX_BYTES,
|
|
204
|
+
assertJobCrons,
|
|
205
|
+
assertKnownJob,
|
|
206
|
+
jobBackoffMs,
|
|
207
|
+
nextJobFailureState,
|
|
208
|
+
serializeJobPayload,
|
|
209
|
+
type JobCronDefinition,
|
|
210
|
+
} from "./job-queue.js";
|
|
211
|
+
|
|
212
|
+
export {
|
|
213
|
+
LOCAL_JOB_CRON_MS,
|
|
214
|
+
LOCAL_JOB_WORKER_MS,
|
|
215
|
+
createLocalJobsEngine,
|
|
216
|
+
ensureJobsSchema,
|
|
217
|
+
type JobsQueryable,
|
|
218
|
+
type LocalCronRow,
|
|
219
|
+
type LocalJobRow,
|
|
220
|
+
type LocalJobStatus,
|
|
221
|
+
type LocalJobsEngine,
|
|
222
|
+
type LocalJobsEngineOptions,
|
|
223
|
+
type LocalJobsGeneration,
|
|
224
|
+
} from "./local-jobs.js";
|
package/src/invoke.ts
CHANGED
|
@@ -56,9 +56,6 @@ export type RunRouteInput = {
|
|
|
56
56
|
resolveUser: (authorization?: string) => Promise<AuthUser | null>;
|
|
57
57
|
/** Built after auth passes, right before the handler runs. */
|
|
58
58
|
clients: () => Promise<RouteClients> | RouteClients;
|
|
59
|
-
/** True when the request carries a valid schedule signature. */
|
|
60
|
-
verifySchedule?: (headers: Record<string, string | string[] | undefined>) => boolean;
|
|
61
|
-
scheduleId?: string;
|
|
62
59
|
/** Lets the host translate its own error types before the generic 500. */
|
|
63
60
|
mapError?: (error: unknown) => MappedError | null;
|
|
64
61
|
};
|
|
@@ -145,22 +142,13 @@ export async function runRoute(input: RunRouteInput): Promise<RouteOutcome> {
|
|
|
145
142
|
body = parsed.data;
|
|
146
143
|
}
|
|
147
144
|
|
|
148
|
-
const scheduleValid = input.verifySchedule ? input.verifySchedule(input.headers) : false;
|
|
149
|
-
const schedule = scheduleValid
|
|
150
|
-
? { id: input.scheduleId ?? "schedule" }
|
|
151
|
-
: input.scheduleId
|
|
152
|
-
? { id: input.scheduleId }
|
|
153
|
-
: null;
|
|
154
|
-
|
|
155
145
|
let user: AuthUser | null = null;
|
|
156
146
|
const auth = route.def.auth ?? false;
|
|
157
|
-
if (auth === "
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
});
|
|
163
|
-
}
|
|
147
|
+
if (typeof auth === "string" && auth !== "required" && auth !== "optional") {
|
|
148
|
+
return invalid(401, {
|
|
149
|
+
error: "unauthorized",
|
|
150
|
+
message: "legacy schedule auth is no longer supported",
|
|
151
|
+
});
|
|
164
152
|
} else if (typeof auth === "function") {
|
|
165
153
|
try {
|
|
166
154
|
user = await auth({ headers: input.headers });
|
|
@@ -202,7 +190,6 @@ export async function runRoute(input: RunRouteInput): Promise<RouteOutcome> {
|
|
|
202
190
|
push: clients.push,
|
|
203
191
|
jobs: clients.jobs,
|
|
204
192
|
sockets: clients.sockets,
|
|
205
|
-
schedule,
|
|
206
193
|
};
|
|
207
194
|
const result = await withHandlerTimeout(
|
|
208
195
|
() => Promise.resolve(route.def.handler(context)),
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import {
|
|
4
|
+
assertJobCrons,
|
|
5
|
+
assertKnownJob,
|
|
6
|
+
JOB_MAX_ATTEMPTS,
|
|
7
|
+
JOB_PAYLOAD_MAX_BYTES,
|
|
8
|
+
jobBackoffMs,
|
|
9
|
+
nextJobFailureState,
|
|
10
|
+
serializeJobPayload,
|
|
11
|
+
} from "./job-queue.js";
|
|
12
|
+
|
|
13
|
+
function job(name: string, cron?: string, file = `jobs/${name}.ts`) {
|
|
14
|
+
return { name, file, def: cron === undefined ? {} : { cron } };
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
test("enqueue row is queued with serialized payload", () => {
|
|
18
|
+
assertKnownJob("resize", ["resize", "email"]);
|
|
19
|
+
const payloadJson = serializeJobPayload({ key: "avatar.png" });
|
|
20
|
+
assert.equal(JSON.parse(payloadJson).key, "avatar.png");
|
|
21
|
+
const row = {
|
|
22
|
+
name: "resize",
|
|
23
|
+
payload_json: payloadJson,
|
|
24
|
+
status: "queued" as const,
|
|
25
|
+
attempts: 0,
|
|
26
|
+
};
|
|
27
|
+
assert.equal(row.status, "queued");
|
|
28
|
+
assert.equal(row.attempts, 0);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("unknown name throws", () => {
|
|
32
|
+
assert.throws(() => assertKnownJob("missing", ["resize"]), /Unknown job name: missing/);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("oversize payload throws", () => {
|
|
36
|
+
const oversized = "x".repeat(JOB_PAYLOAD_MAX_BYTES);
|
|
37
|
+
assert.throws(() => serializeJobPayload(oversized), /job payload must be at most 256KB/);
|
|
38
|
+
assert.equal(serializeJobPayload(undefined), "null");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("backoff then dead after 5 attempts", () => {
|
|
42
|
+
assert.equal(jobBackoffMs(1), 1000);
|
|
43
|
+
assert.equal(jobBackoffMs(2), 4000);
|
|
44
|
+
let state = nextJobFailureState(1);
|
|
45
|
+
for (let attempts = 2; attempts < JOB_MAX_ATTEMPTS; attempts++) {
|
|
46
|
+
assert.equal(state.status, "queued");
|
|
47
|
+
assert.ok(state.runAt);
|
|
48
|
+
state = nextJobFailureState(attempts);
|
|
49
|
+
}
|
|
50
|
+
state = nextJobFailureState(JOB_MAX_ATTEMPTS);
|
|
51
|
+
assert.equal(state.status, "dead");
|
|
52
|
+
assert.equal(state.runAt, null);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("invalid cron fails with hosted error shape", () => {
|
|
56
|
+
assert.throws(
|
|
57
|
+
() => assertJobCrons([job("digest", "not-a-cron")]),
|
|
58
|
+
(error: unknown) => {
|
|
59
|
+
const err = error as { statusCode?: number; code?: string; path?: string };
|
|
60
|
+
return (
|
|
61
|
+
err.statusCode === 400 && err.code === "invalid_job_cron" && err.path === "jobs/digest.ts"
|
|
62
|
+
);
|
|
63
|
+
},
|
|
64
|
+
);
|
|
65
|
+
assert.throws(() => assertJobCrons([job("digest", "")]));
|
|
66
|
+
assert.throws(() => assertJobCrons([job("digest", " ")]));
|
|
67
|
+
assert.doesNotThrow(() => assertJobCrons([job("digest"), job("hourly", "0 * * * *")]));
|
|
68
|
+
});
|
package/src/job-queue.ts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { assertFiveFieldCron } from "./cron.js";
|
|
2
|
+
|
|
3
|
+
export const JOB_MAX_ATTEMPTS = 5;
|
|
4
|
+
export const JOB_PAYLOAD_MAX_BYTES = 256 * 1024;
|
|
5
|
+
export const JOB_CONCURRENCY = 4;
|
|
6
|
+
|
|
7
|
+
export type JobCronDefinition = {
|
|
8
|
+
name: string;
|
|
9
|
+
file?: string;
|
|
10
|
+
def: { cron?: string };
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export function jobBackoffMs(attempt: number): number {
|
|
14
|
+
return Math.min(1000 * 4 ** Math.max(attempt - 1, 0), 300_000);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function nextJobFailureState(attempts: number): {
|
|
18
|
+
status: "queued" | "dead";
|
|
19
|
+
runAt: Date | null;
|
|
20
|
+
} {
|
|
21
|
+
if (attempts >= JOB_MAX_ATTEMPTS) {
|
|
22
|
+
return { status: "dead", runAt: null };
|
|
23
|
+
}
|
|
24
|
+
return { status: "queued", runAt: new Date(Date.now() + jobBackoffMs(attempts)) };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function serializeJobPayload(payload: unknown): string {
|
|
28
|
+
const json = JSON.stringify(payload ?? null);
|
|
29
|
+
if (Buffer.byteLength(json, "utf8") > JOB_PAYLOAD_MAX_BYTES) {
|
|
30
|
+
throw new Error("job payload must be at most 256KB");
|
|
31
|
+
}
|
|
32
|
+
return json;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function assertKnownJob(name: string, known: Iterable<string>): void {
|
|
36
|
+
const names = new Set(known);
|
|
37
|
+
if (!names.has(name)) {
|
|
38
|
+
throw new Error(`Unknown job name: ${name}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function assertJobCrons(jobs: readonly JobCronDefinition[]): void {
|
|
43
|
+
for (const job of jobs) {
|
|
44
|
+
if (job.def.cron === undefined) continue;
|
|
45
|
+
try {
|
|
46
|
+
if (typeof job.def.cron !== "string" || job.def.cron.trim() === "") {
|
|
47
|
+
throw new Error("cron must be a non-empty 5-field expression");
|
|
48
|
+
}
|
|
49
|
+
assertFiveFieldCron(job.def.cron);
|
|
50
|
+
} catch (error) {
|
|
51
|
+
throw Object.assign(new Error(error instanceof Error ? error.message : "invalid cron"), {
|
|
52
|
+
statusCode: 400,
|
|
53
|
+
code: "invalid_job_cron",
|
|
54
|
+
path: job.file ?? `jobs/${job.name}.ts`,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import type { JobDefinition, JobHandlerContext, RobodevDb } from "@robodev-ai/sdk";
|
|
4
|
+
import { JOB_KIND } from "@robodev-ai/sdk";
|
|
5
|
+
import { nextRunAfter } from "./cron.js";
|
|
6
|
+
import type { RouteClients } from "./invoke.js";
|
|
7
|
+
import { JOB_MAX_ATTEMPTS, JOB_PAYLOAD_MAX_BYTES } from "./job-queue.js";
|
|
8
|
+
import {
|
|
9
|
+
createLocalJobsEngine,
|
|
10
|
+
ensureJobsSchema,
|
|
11
|
+
type JobsQueryable,
|
|
12
|
+
type LocalCronRow,
|
|
13
|
+
type LocalJobRow,
|
|
14
|
+
type LocalJobsGeneration,
|
|
15
|
+
} from "./local-jobs.js";
|
|
16
|
+
|
|
17
|
+
function jobDef(
|
|
18
|
+
handler: (ctx: JobHandlerContext) => Promise<unknown> | unknown,
|
|
19
|
+
extras: { timeoutMs?: number; cron?: string } = {},
|
|
20
|
+
): JobDefinition {
|
|
21
|
+
return { _kind: JOB_KIND, handler, ...extras };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function unavailable(feature: string): never {
|
|
25
|
+
throw new Error(`ctx.${feature} needs hosted Starbase. Run robodev deploy to use it.`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function mockClients(): RouteClients {
|
|
29
|
+
return {
|
|
30
|
+
email: {
|
|
31
|
+
async send() {
|
|
32
|
+
return { id: "local_mail", status: "sent" };
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
llm: { complete: () => unavailable("llm"), stream: () => unavailable("llm") },
|
|
36
|
+
agent: {
|
|
37
|
+
createSession: () => unavailable("agent"),
|
|
38
|
+
start: () => unavailable("agent"),
|
|
39
|
+
events: () => unavailable("agent"),
|
|
40
|
+
subscribe: () => unavailable("agent"),
|
|
41
|
+
destroy: () => unavailable("agent"),
|
|
42
|
+
},
|
|
43
|
+
storage: {
|
|
44
|
+
upload: () => unavailable("storage"),
|
|
45
|
+
get: () => unavailable("storage"),
|
|
46
|
+
getUrl: () => unavailable("storage"),
|
|
47
|
+
delete: () => unavailable("storage"),
|
|
48
|
+
list: () => unavailable("storage"),
|
|
49
|
+
},
|
|
50
|
+
push: { send: () => unavailable("push") },
|
|
51
|
+
jobs: { enqueue: async () => ({ id: "nested" }) },
|
|
52
|
+
sockets: {
|
|
53
|
+
send: () => unavailable("sockets"),
|
|
54
|
+
broadcast: () => unavailable("sockets"),
|
|
55
|
+
},
|
|
56
|
+
env: {},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function memoryJobs() {
|
|
61
|
+
const jobs: LocalJobRow[] = [];
|
|
62
|
+
const crons: LocalCronRow[] = [];
|
|
63
|
+
const now = () => new Date();
|
|
64
|
+
|
|
65
|
+
const query: JobsQueryable["query"] = async (text, values = []) => {
|
|
66
|
+
const sql = text.replace(/\s+/g, " ").trim();
|
|
67
|
+
if (/^(CREATE SCHEMA|CREATE TABLE|CREATE INDEX)/.test(sql)) return { rows: [] };
|
|
68
|
+
|
|
69
|
+
if (sql.includes("INSERT INTO robodev_jobs.jobs")) {
|
|
70
|
+
const created = now();
|
|
71
|
+
jobs.push({
|
|
72
|
+
id: String(values[0]),
|
|
73
|
+
name: String(values[1]),
|
|
74
|
+
payload_json: String(values[2]),
|
|
75
|
+
status: "queued",
|
|
76
|
+
attempts: 0,
|
|
77
|
+
max_attempts: Number(values[3]),
|
|
78
|
+
run_at: values[4] as Date,
|
|
79
|
+
last_error: null,
|
|
80
|
+
created_at: created,
|
|
81
|
+
updated_at: created,
|
|
82
|
+
});
|
|
83
|
+
return { rows: [] };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (sql.includes("FOR UPDATE SKIP LOCKED")) {
|
|
87
|
+
const due = jobs
|
|
88
|
+
.filter((row) => row.status === "queued" && row.run_at.getTime() <= Date.now())
|
|
89
|
+
.sort((a, b) => a.run_at.getTime() - b.run_at.getTime())[0];
|
|
90
|
+
if (!due) return { rows: [] };
|
|
91
|
+
due.status = "running";
|
|
92
|
+
due.updated_at = now();
|
|
93
|
+
return { rows: [{ ...due }] };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (sql.includes("SET status = 'succeeded'")) {
|
|
97
|
+
const row = jobs.find((entry) => entry.id === values[0]);
|
|
98
|
+
if (row) {
|
|
99
|
+
row.status = "succeeded";
|
|
100
|
+
row.last_error = null;
|
|
101
|
+
row.updated_at = now();
|
|
102
|
+
}
|
|
103
|
+
return { rows: [] };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (sql.includes("run_at = COALESCE")) {
|
|
107
|
+
const row = jobs.find((entry) => entry.id === values[0]);
|
|
108
|
+
if (row) {
|
|
109
|
+
row.status = values[1] as LocalJobRow["status"];
|
|
110
|
+
row.attempts = Number(values[2]);
|
|
111
|
+
row.last_error = (values[3] as string | null) ?? null;
|
|
112
|
+
if (values[4]) row.run_at = values[4] as Date;
|
|
113
|
+
row.updated_at = now();
|
|
114
|
+
}
|
|
115
|
+
return { rows: [] };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (sql.includes("WHERE status = 'running'") && sql.includes("SET status = 'queued'")) {
|
|
119
|
+
for (const row of jobs) {
|
|
120
|
+
if (row.status === "running") {
|
|
121
|
+
row.status = "queued";
|
|
122
|
+
row.updated_at = now();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return { rows: [] };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (sql.includes("INSERT INTO robodev_jobs.crons")) {
|
|
129
|
+
crons.push({
|
|
130
|
+
job_name: String(values[0]),
|
|
131
|
+
cron: String(values[1]),
|
|
132
|
+
next_run_at: values[2] as Date,
|
|
133
|
+
});
|
|
134
|
+
return { rows: [] };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (sql.includes("FROM robodev_jobs.crons") && sql.includes("WHERE next_run_at <=")) {
|
|
138
|
+
const at = values[0] as Date;
|
|
139
|
+
return {
|
|
140
|
+
rows: crons
|
|
141
|
+
.filter((row) => row.next_run_at.getTime() <= at.getTime())
|
|
142
|
+
.sort((a, b) => a.next_run_at.getTime() - b.next_run_at.getTime())
|
|
143
|
+
.slice(0, 50),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (sql.includes("FROM robodev_jobs.crons") && sql.includes("SELECT job_name")) {
|
|
148
|
+
return { rows: [...crons] };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (sql.includes("UPDATE robodev_jobs.crons") && sql.includes("SET cron")) {
|
|
152
|
+
const row = crons.find((entry) => entry.job_name === values[0]);
|
|
153
|
+
if (row) {
|
|
154
|
+
row.cron = String(values[1]);
|
|
155
|
+
row.next_run_at = values[2] as Date;
|
|
156
|
+
}
|
|
157
|
+
return { rows: [] };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (sql.includes("UPDATE robodev_jobs.crons") && sql.includes("SET next_run_at")) {
|
|
161
|
+
const row = crons.find((entry) => entry.job_name === values[0]);
|
|
162
|
+
if (row) row.next_run_at = values[1] as Date;
|
|
163
|
+
return { rows: [] };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (sql.includes("DELETE FROM robodev_jobs.crons")) {
|
|
167
|
+
const keep = new Set(values[0] as string[]);
|
|
168
|
+
for (let i = crons.length - 1; i >= 0; i--) {
|
|
169
|
+
if (!keep.has(crons[i]!.job_name)) crons.splice(i, 1);
|
|
170
|
+
}
|
|
171
|
+
return { rows: [] };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
throw new Error(`unexpected query: ${sql}`);
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
return { jobs, crons, query };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function waitUntil(pred: () => boolean, ms = 1000): Promise<void> {
|
|
181
|
+
const start = Date.now();
|
|
182
|
+
while (!pred()) {
|
|
183
|
+
if (Date.now() - start > ms) throw new Error("timed out waiting");
|
|
184
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function engineFor(
|
|
189
|
+
store: ReturnType<typeof memoryJobs>,
|
|
190
|
+
jobs: LocalJobsGeneration["jobs"],
|
|
191
|
+
logs: string[] = [],
|
|
192
|
+
) {
|
|
193
|
+
const generation: LocalJobsGeneration = { db: {} as RobodevDb, jobs };
|
|
194
|
+
return createLocalJobsEngine({
|
|
195
|
+
query: store.query,
|
|
196
|
+
getGeneration: () => generation,
|
|
197
|
+
log: (line) => logs.push(line),
|
|
198
|
+
clients: mockClients,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
test("ensureJobsSchema is idempotent SQL", async () => {
|
|
203
|
+
const store = memoryJobs();
|
|
204
|
+
await ensureJobsSchema({ query: store.query });
|
|
205
|
+
await ensureJobsSchema({ query: store.query });
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
test("enqueue known job; unknown and oversize throw without insert", async () => {
|
|
209
|
+
const store = memoryJobs();
|
|
210
|
+
const engine = engineFor(store, [{ name: "digest", def: jobDef(() => undefined) }]);
|
|
211
|
+
const queued = await engine.enqueue({ name: "digest", payload: { ok: true } });
|
|
212
|
+
assert.match(queued.id, /^job_/);
|
|
213
|
+
assert.equal(store.jobs.length, 1);
|
|
214
|
+
assert.equal(JSON.parse(store.jobs[0]!.payload_json).ok, true);
|
|
215
|
+
|
|
216
|
+
await assert.rejects(() => engine.enqueue({ name: "missing" }), /Unknown job name: missing/);
|
|
217
|
+
await assert.rejects(
|
|
218
|
+
() => engine.enqueue({ name: "digest", payload: "x".repeat(JOB_PAYLOAD_MAX_BYTES) }),
|
|
219
|
+
/job payload must be at most 256KB/,
|
|
220
|
+
);
|
|
221
|
+
assert.equal(store.jobs.length, 1);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
test("successful handler marks succeeded and logs", async () => {
|
|
225
|
+
const store = memoryJobs();
|
|
226
|
+
const logs: string[] = [];
|
|
227
|
+
const seen: unknown[] = [];
|
|
228
|
+
const engine = engineFor(
|
|
229
|
+
store,
|
|
230
|
+
[{ name: "digest", def: jobDef((ctx) => seen.push(ctx.payload)) }],
|
|
231
|
+
logs,
|
|
232
|
+
);
|
|
233
|
+
await engine.enqueue({ name: "digest", payload: { n: 1 } });
|
|
234
|
+
await engine.tickJobs();
|
|
235
|
+
await waitUntil(() => store.jobs[0]?.status === "succeeded" && logs.length > 0);
|
|
236
|
+
assert.deepEqual(seen, [{ n: 1 }]);
|
|
237
|
+
assert.equal(logs[0], " digest succeeded");
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
test("throw backs off then dead at 5", async () => {
|
|
241
|
+
const store = memoryJobs();
|
|
242
|
+
const logs: string[] = [];
|
|
243
|
+
const engine = engineFor(
|
|
244
|
+
store,
|
|
245
|
+
[
|
|
246
|
+
{
|
|
247
|
+
name: "digest",
|
|
248
|
+
def: jobDef(() => {
|
|
249
|
+
throw new Error("boom");
|
|
250
|
+
}),
|
|
251
|
+
},
|
|
252
|
+
],
|
|
253
|
+
logs,
|
|
254
|
+
);
|
|
255
|
+
await engine.enqueue({ name: "digest" });
|
|
256
|
+
for (let attempt = 1; attempt <= JOB_MAX_ATTEMPTS; attempt++) {
|
|
257
|
+
store.jobs[0]!.run_at = new Date(0);
|
|
258
|
+
store.jobs[0]!.status = "queued";
|
|
259
|
+
await engine.tickJobs();
|
|
260
|
+
await waitUntil(() => store.jobs[0]!.status !== "running");
|
|
261
|
+
}
|
|
262
|
+
assert.equal(store.jobs[0]?.status, "dead");
|
|
263
|
+
assert.equal(store.jobs[0]?.attempts, 5);
|
|
264
|
+
assert.equal(store.jobs[0]?.last_error, "boom");
|
|
265
|
+
assert.match(logs[0] ?? "", /failed attempt 1\/5: boom/);
|
|
266
|
+
assert.match(logs.at(-1) ?? "", /digest dead: boom/);
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
test("timeout is handler_timeout and retries", async () => {
|
|
270
|
+
const store = memoryJobs();
|
|
271
|
+
const logs: string[] = [];
|
|
272
|
+
const engine = engineFor(
|
|
273
|
+
store,
|
|
274
|
+
[
|
|
275
|
+
{
|
|
276
|
+
name: "slow",
|
|
277
|
+
def: jobDef(() => new Promise((resolve) => setTimeout(resolve, 50)), { timeoutMs: 10 }),
|
|
278
|
+
},
|
|
279
|
+
],
|
|
280
|
+
logs,
|
|
281
|
+
);
|
|
282
|
+
await engine.enqueue({ name: "slow" });
|
|
283
|
+
await engine.tickJobs();
|
|
284
|
+
await waitUntil(() => store.jobs[0]?.last_error === "handler_timeout");
|
|
285
|
+
assert.equal(store.jobs[0]?.status, "queued");
|
|
286
|
+
assert.equal(store.jobs[0]?.last_error, "handler_timeout");
|
|
287
|
+
assert.match(logs[0] ?? "", /failed attempt 1\/5: handler_timeout/);
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
test("reconcile keeps unchanged next_run, reschedules, and deletes", async () => {
|
|
291
|
+
const now = new Date("2026-01-01T00:00:00.000Z");
|
|
292
|
+
const keptNext = new Date("2026-01-01T01:00:00.000Z");
|
|
293
|
+
const store = memoryJobs();
|
|
294
|
+
store.crons.push(
|
|
295
|
+
{ job_name: "hourly", cron: "0 * * * *", next_run_at: keptNext },
|
|
296
|
+
{ job_name: "stale", cron: "0 0 * * *", next_run_at: new Date("2026-01-02T00:00:00.000Z") },
|
|
297
|
+
{ job_name: "daily", cron: "0 0 * * *", next_run_at: new Date("2026-01-02T00:00:00.000Z") },
|
|
298
|
+
);
|
|
299
|
+
const engine = engineFor(store, []);
|
|
300
|
+
await engine.reconcile(
|
|
301
|
+
[
|
|
302
|
+
{ name: "hourly", def: { cron: "0 * * * *" } },
|
|
303
|
+
{ name: "daily", def: { cron: "0 12 * * *" } },
|
|
304
|
+
{ name: "fresh", def: { cron: "*/5 * * * *" } },
|
|
305
|
+
],
|
|
306
|
+
now,
|
|
307
|
+
);
|
|
308
|
+
const byName = Object.fromEntries(store.crons.map((row) => [row.job_name, row]));
|
|
309
|
+
assert.equal(store.crons.length, 3);
|
|
310
|
+
assert.equal(byName.hourly?.next_run_at, keptNext);
|
|
311
|
+
assert.equal(byName.daily?.cron, "0 12 * * *");
|
|
312
|
+
assert.deepEqual(byName.daily?.next_run_at, nextRunAfter("0 12 * * *", now));
|
|
313
|
+
assert.deepEqual(byName.fresh?.next_run_at, nextRunAfter("*/5 * * * *", now));
|
|
314
|
+
assert.equal(byName.stale, undefined);
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
test("removing cron deletes the row", async () => {
|
|
318
|
+
const store = memoryJobs();
|
|
319
|
+
store.crons.push({
|
|
320
|
+
job_name: "hourly",
|
|
321
|
+
cron: "0 * * * *",
|
|
322
|
+
next_run_at: new Date("2026-01-01T01:00:00.000Z"),
|
|
323
|
+
});
|
|
324
|
+
const engine = engineFor(store, []);
|
|
325
|
+
await engine.reconcile([{ name: "hourly", def: {} }]);
|
|
326
|
+
assert.equal(store.crons.length, 0);
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
test("tick enqueues null payload, advances first, and does not roll back", async () => {
|
|
330
|
+
const now = new Date("2026-01-01T00:00:00.000Z");
|
|
331
|
+
const store = memoryJobs();
|
|
332
|
+
store.crons.push({ job_name: "hourly", cron: "0 * * * *", next_run_at: now });
|
|
333
|
+
const events: string[] = [];
|
|
334
|
+
const engine = createLocalJobsEngine({
|
|
335
|
+
query: async (text, values) => {
|
|
336
|
+
if (text.includes("UPDATE robodev_jobs.crons") && text.includes("SET next_run_at")) {
|
|
337
|
+
events.push("advance");
|
|
338
|
+
}
|
|
339
|
+
if (text.includes("INSERT INTO robodev_jobs.jobs")) events.push("enqueue");
|
|
340
|
+
return store.query(text, values);
|
|
341
|
+
},
|
|
342
|
+
getGeneration: () => ({
|
|
343
|
+
db: {} as RobodevDb,
|
|
344
|
+
jobs: [{ name: "hourly", def: jobDef(() => undefined) }],
|
|
345
|
+
}),
|
|
346
|
+
clients: mockClients,
|
|
347
|
+
});
|
|
348
|
+
await engine.tickDue(now);
|
|
349
|
+
assert.deepEqual(events, ["advance", "enqueue"]);
|
|
350
|
+
assert.equal(JSON.parse(store.jobs[0]!.payload_json), null);
|
|
351
|
+
assert.deepEqual(store.crons[0]?.next_run_at, nextRunAfter("0 * * * *", now));
|
|
352
|
+
|
|
353
|
+
const failStore = memoryJobs();
|
|
354
|
+
failStore.crons.push({ job_name: "hourly", cron: "0 * * * *", next_run_at: now });
|
|
355
|
+
const failing = createLocalJobsEngine({
|
|
356
|
+
query: failStore.query,
|
|
357
|
+
getGeneration: () => ({ db: {} as RobodevDb, jobs: [] }),
|
|
358
|
+
clients: mockClients,
|
|
359
|
+
});
|
|
360
|
+
await failing.tickDue(now);
|
|
361
|
+
assert.equal(failStore.jobs.length, 0);
|
|
362
|
+
assert.deepEqual(failStore.crons[0]?.next_run_at, nextRunAfter("0 * * * *", now));
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
test("overlap piles up queued rows", async () => {
|
|
366
|
+
const now = new Date("2026-01-01T00:00:00.000Z");
|
|
367
|
+
const store = memoryJobs();
|
|
368
|
+
store.crons.push({ job_name: "hourly", cron: "0 * * * *", next_run_at: now });
|
|
369
|
+
const engine = engineFor(store, [{ name: "hourly", def: jobDef(() => undefined) }]);
|
|
370
|
+
await engine.tickDue(now);
|
|
371
|
+
store.crons[0]!.next_run_at = now;
|
|
372
|
+
await engine.tickDue(now);
|
|
373
|
+
assert.equal(store.jobs.length, 2);
|
|
374
|
+
assert.equal(
|
|
375
|
+
store.jobs.every((row) => row.status === "queued"),
|
|
376
|
+
true,
|
|
377
|
+
);
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test("missing handler is route_not_found", async () => {
|
|
381
|
+
const store = memoryJobs();
|
|
382
|
+
const logs: string[] = [];
|
|
383
|
+
const engine = engineFor(store, [], logs);
|
|
384
|
+
store.jobs.push({
|
|
385
|
+
id: "job_ghost",
|
|
386
|
+
name: "gone",
|
|
387
|
+
payload_json: "null",
|
|
388
|
+
status: "queued",
|
|
389
|
+
attempts: 0,
|
|
390
|
+
max_attempts: 5,
|
|
391
|
+
run_at: new Date(0),
|
|
392
|
+
last_error: null,
|
|
393
|
+
created_at: new Date(),
|
|
394
|
+
updated_at: new Date(),
|
|
395
|
+
});
|
|
396
|
+
await engine.tickJobs();
|
|
397
|
+
await waitUntil(() => store.jobs[0]?.last_error === "route_not_found");
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
test("hosted-only clients still throw in job handlers; email is real", async () => {
|
|
401
|
+
const store = memoryJobs();
|
|
402
|
+
const seen: string[] = [];
|
|
403
|
+
const engine = engineFor(store, [
|
|
404
|
+
{
|
|
405
|
+
name: "probe",
|
|
406
|
+
def: jobDef(async (ctx) => {
|
|
407
|
+
await ctx.email.send({ to: "a@b.c", subject: "hi" });
|
|
408
|
+
seen.push("email");
|
|
409
|
+
try {
|
|
410
|
+
await ctx.llm.complete({ messages: [{ role: "user", content: "x" }] });
|
|
411
|
+
} catch (error) {
|
|
412
|
+
seen.push((error as Error).message);
|
|
413
|
+
}
|
|
414
|
+
}),
|
|
415
|
+
},
|
|
416
|
+
]);
|
|
417
|
+
await engine.enqueue({ name: "probe" });
|
|
418
|
+
await engine.tickJobs();
|
|
419
|
+
await waitUntil(() => store.jobs[0]?.status === "succeeded");
|
|
420
|
+
assert.equal(seen[0], "email");
|
|
421
|
+
assert.match(seen[1] ?? "", /ctx\.llm needs hosted Starbase/);
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
test("start resets running rows; stop clears timers", async () => {
|
|
425
|
+
const store = memoryJobs();
|
|
426
|
+
store.jobs.push({
|
|
427
|
+
id: "job_stuck",
|
|
428
|
+
name: "digest",
|
|
429
|
+
payload_json: "null",
|
|
430
|
+
status: "running",
|
|
431
|
+
attempts: 1,
|
|
432
|
+
max_attempts: 5,
|
|
433
|
+
run_at: new Date(),
|
|
434
|
+
last_error: null,
|
|
435
|
+
created_at: new Date(),
|
|
436
|
+
updated_at: new Date(),
|
|
437
|
+
});
|
|
438
|
+
const engine = engineFor(store, [{ name: "digest", def: jobDef(() => undefined) }]);
|
|
439
|
+
assert.equal(engine.running, false);
|
|
440
|
+
await engine.start();
|
|
441
|
+
assert.equal(store.jobs[0]?.status, "queued");
|
|
442
|
+
assert.equal(engine.running, true);
|
|
443
|
+
await engine.start();
|
|
444
|
+
assert.equal(engine.running, true);
|
|
445
|
+
await engine.stop();
|
|
446
|
+
assert.equal(engine.running, false);
|
|
447
|
+
await engine.stop();
|
|
448
|
+
});
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import type { JobDefinition, JobEnqueueInput, RobodevDb } from "@robodev-ai/sdk";
|
|
2
|
+
import { assertFiveFieldCron, nextRunAfter } from "./cron.js";
|
|
3
|
+
import { HandlerTimeoutError, clampTimeoutMs, withHandlerTimeout } from "./handler-timeout.js";
|
|
4
|
+
import { id } from "./ids.js";
|
|
5
|
+
import type { RouteClients } from "./invoke.js";
|
|
6
|
+
import {
|
|
7
|
+
JOB_CONCURRENCY,
|
|
8
|
+
JOB_MAX_ATTEMPTS,
|
|
9
|
+
assertKnownJob,
|
|
10
|
+
nextJobFailureState,
|
|
11
|
+
serializeJobPayload,
|
|
12
|
+
type JobCronDefinition,
|
|
13
|
+
} from "./job-queue.js";
|
|
14
|
+
|
|
15
|
+
/** Hosted uses metadata `now()`; local worker/ticker intervals match hosted. */
|
|
16
|
+
export const LOCAL_JOB_WORKER_MS = 1000;
|
|
17
|
+
export const LOCAL_JOB_CRON_MS = 60_000;
|
|
18
|
+
|
|
19
|
+
export type JobsQueryable = {
|
|
20
|
+
query: <T extends Record<string, unknown> = Record<string, unknown>>(
|
|
21
|
+
sql: string,
|
|
22
|
+
values?: unknown[],
|
|
23
|
+
) => Promise<{ rows: T[] }>;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export type LocalJobStatus = "queued" | "running" | "succeeded" | "dead";
|
|
27
|
+
|
|
28
|
+
export type LocalJobRow = {
|
|
29
|
+
id: string;
|
|
30
|
+
name: string;
|
|
31
|
+
payload_json: string;
|
|
32
|
+
status: LocalJobStatus;
|
|
33
|
+
attempts: number;
|
|
34
|
+
max_attempts: number;
|
|
35
|
+
run_at: Date;
|
|
36
|
+
last_error: string | null;
|
|
37
|
+
created_at: Date;
|
|
38
|
+
updated_at: Date;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export type LocalCronRow = {
|
|
42
|
+
job_name: string;
|
|
43
|
+
cron: string;
|
|
44
|
+
next_run_at: Date;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type LocalJobsGeneration = {
|
|
48
|
+
db: RobodevDb;
|
|
49
|
+
jobs: readonly { name: string; file?: string; def: JobDefinition }[];
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export type LocalJobsEngineOptions = {
|
|
53
|
+
query: JobsQueryable["query"];
|
|
54
|
+
getGeneration: () => LocalJobsGeneration | null;
|
|
55
|
+
log?: (line: string) => void;
|
|
56
|
+
clients: () => RouteClients;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
export type LocalJobsEngine = {
|
|
60
|
+
enqueue: (input: JobEnqueueInput) => Promise<{ id: string }>;
|
|
61
|
+
reconcile: (jobs: readonly JobCronDefinition[], now?: Date) => Promise<void>;
|
|
62
|
+
tickDue: (now?: Date) => Promise<void>;
|
|
63
|
+
tickJobs: () => Promise<void>;
|
|
64
|
+
start: () => Promise<void>;
|
|
65
|
+
stop: () => Promise<void>;
|
|
66
|
+
readonly generation: LocalJobsGeneration | null;
|
|
67
|
+
readonly running: boolean;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
function resolveRunAt(input: JobEnqueueInput): Date {
|
|
71
|
+
if (input.runAt) return input.runAt;
|
|
72
|
+
if (typeof input.delayMs === "number" && Number.isFinite(input.delayMs)) {
|
|
73
|
+
return new Date(Date.now() + Math.max(0, input.delayMs));
|
|
74
|
+
}
|
|
75
|
+
return new Date();
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function desiredCronJobs(jobs: readonly JobCronDefinition[]): { name: string; cron: string }[] {
|
|
79
|
+
return jobs
|
|
80
|
+
.filter((job) => job.def.cron !== undefined)
|
|
81
|
+
.map((job) => ({ name: job.name, cron: assertFiveFieldCron(job.def.cron!) }));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function shortError(message: string): string {
|
|
85
|
+
const oneLine = message.split("\n")[0]?.trim() || "handler_failed";
|
|
86
|
+
return oneLine.length > 160 ? `${oneLine.slice(0, 157)}...` : oneLine;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Creates the reserved `robodev_jobs` schema. Idempotent, not drizzle, not public. */
|
|
90
|
+
export async function ensureJobsSchema(db: JobsQueryable): Promise<void> {
|
|
91
|
+
await db.query(`CREATE SCHEMA IF NOT EXISTS robodev_jobs`);
|
|
92
|
+
await db.query(`
|
|
93
|
+
CREATE TABLE IF NOT EXISTS robodev_jobs.jobs (
|
|
94
|
+
id TEXT PRIMARY KEY,
|
|
95
|
+
name TEXT NOT NULL,
|
|
96
|
+
payload_json TEXT NOT NULL DEFAULT 'null',
|
|
97
|
+
status TEXT NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'dead')),
|
|
98
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
99
|
+
max_attempts INTEGER NOT NULL DEFAULT 5,
|
|
100
|
+
run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
101
|
+
last_error TEXT,
|
|
102
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
103
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
104
|
+
)
|
|
105
|
+
`);
|
|
106
|
+
await db.query(`
|
|
107
|
+
CREATE INDEX IF NOT EXISTS robodev_jobs_queued_run_at
|
|
108
|
+
ON robodev_jobs.jobs (status, run_at)
|
|
109
|
+
WHERE status = 'queued'
|
|
110
|
+
`);
|
|
111
|
+
await db.query(`
|
|
112
|
+
CREATE TABLE IF NOT EXISTS robodev_jobs.crons (
|
|
113
|
+
job_name TEXT PRIMARY KEY,
|
|
114
|
+
cron TEXT NOT NULL,
|
|
115
|
+
next_run_at TIMESTAMPTZ NOT NULL,
|
|
116
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
117
|
+
)
|
|
118
|
+
`);
|
|
119
|
+
await db.query(`
|
|
120
|
+
CREATE INDEX IF NOT EXISTS robodev_jobs_crons_next_run_at
|
|
121
|
+
ON robodev_jobs.crons (next_run_at)
|
|
122
|
+
`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function createLocalJobsEngine(options: LocalJobsEngineOptions): LocalJobsEngine {
|
|
126
|
+
const query = options.query;
|
|
127
|
+
const log = options.log ?? (() => undefined);
|
|
128
|
+
let inFlight = 0;
|
|
129
|
+
const running = new Set<Promise<void>>();
|
|
130
|
+
let workerTimer: ReturnType<typeof setInterval> | undefined;
|
|
131
|
+
let cronTimer: ReturnType<typeof setInterval> | undefined;
|
|
132
|
+
|
|
133
|
+
async function enqueue(input: JobEnqueueInput): Promise<{ id: string }> {
|
|
134
|
+
const generation = options.getGeneration();
|
|
135
|
+
const known = generation?.jobs.map((job) => job.name) ?? [];
|
|
136
|
+
assertKnownJob(input.name, known);
|
|
137
|
+
const payloadJson = serializeJobPayload(input.payload);
|
|
138
|
+
const jobId = id("job");
|
|
139
|
+
await query(
|
|
140
|
+
`INSERT INTO robodev_jobs.jobs
|
|
141
|
+
(id, name, payload_json, status, attempts, max_attempts, run_at)
|
|
142
|
+
VALUES ($1, $2, $3, 'queued', 0, $4, $5)`,
|
|
143
|
+
[jobId, input.name, payloadJson, JOB_MAX_ATTEMPTS, resolveRunAt(input)],
|
|
144
|
+
);
|
|
145
|
+
return { id: jobId };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function reconcile(jobs: readonly JobCronDefinition[], now = new Date()): Promise<void> {
|
|
149
|
+
const desired = desiredCronJobs(jobs);
|
|
150
|
+
const existing = await query<LocalCronRow>(
|
|
151
|
+
`SELECT job_name, cron, next_run_at
|
|
152
|
+
FROM robodev_jobs.crons`,
|
|
153
|
+
);
|
|
154
|
+
const byName = new Map(existing.rows.map((row) => [row.job_name, row]));
|
|
155
|
+
|
|
156
|
+
for (const job of desired) {
|
|
157
|
+
const row = byName.get(job.name);
|
|
158
|
+
if (!row) {
|
|
159
|
+
await query(
|
|
160
|
+
`INSERT INTO robodev_jobs.crons (job_name, cron, next_run_at)
|
|
161
|
+
VALUES ($1, $2, $3)`,
|
|
162
|
+
[job.name, job.cron, nextRunAfter(job.cron, now)],
|
|
163
|
+
);
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (row.cron !== job.cron) {
|
|
167
|
+
await query(
|
|
168
|
+
`UPDATE robodev_jobs.crons
|
|
169
|
+
SET cron = $2, next_run_at = $3, updated_at = now()
|
|
170
|
+
WHERE job_name = $1`,
|
|
171
|
+
[job.name, job.cron, nextRunAfter(job.cron, now)],
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
await query(`DELETE FROM robodev_jobs.crons WHERE NOT (job_name = ANY($1::text[]))`, [
|
|
177
|
+
desired.map((job) => job.name),
|
|
178
|
+
]);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function tickDue(now = new Date()): Promise<void> {
|
|
182
|
+
const due = await query<LocalCronRow>(
|
|
183
|
+
`SELECT job_name, cron, next_run_at
|
|
184
|
+
FROM robodev_jobs.crons
|
|
185
|
+
WHERE next_run_at <= $1
|
|
186
|
+
ORDER BY next_run_at ASC
|
|
187
|
+
LIMIT 50`,
|
|
188
|
+
[now],
|
|
189
|
+
);
|
|
190
|
+
for (const row of due.rows) {
|
|
191
|
+
const nextRunAt = nextRunAfter(row.cron, now);
|
|
192
|
+
await query(
|
|
193
|
+
`UPDATE robodev_jobs.crons
|
|
194
|
+
SET next_run_at = $2, updated_at = now()
|
|
195
|
+
WHERE job_name = $1`,
|
|
196
|
+
[row.job_name, nextRunAt],
|
|
197
|
+
);
|
|
198
|
+
try {
|
|
199
|
+
await enqueue({ name: row.job_name, payload: null });
|
|
200
|
+
} catch {
|
|
201
|
+
// Tick is skipped after next_run_at advanced; do not roll back.
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function claimNextJob(): Promise<LocalJobRow | null> {
|
|
207
|
+
const result = await query<LocalJobRow>(
|
|
208
|
+
`UPDATE robodev_jobs.jobs
|
|
209
|
+
SET status = 'running', updated_at = now()
|
|
210
|
+
WHERE id = (
|
|
211
|
+
SELECT id FROM robodev_jobs.jobs
|
|
212
|
+
WHERE status = 'queued' AND run_at <= now()
|
|
213
|
+
ORDER BY run_at ASC
|
|
214
|
+
FOR UPDATE SKIP LOCKED
|
|
215
|
+
LIMIT 1
|
|
216
|
+
)
|
|
217
|
+
RETURNING id, name, payload_json, status, attempts, max_attempts, run_at,
|
|
218
|
+
last_error, created_at, updated_at`,
|
|
219
|
+
);
|
|
220
|
+
return result.rows[0] ?? null;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function finishJob(row: LocalJobRow, error: string | null): Promise<void> {
|
|
224
|
+
if (!error) {
|
|
225
|
+
await query(
|
|
226
|
+
`UPDATE robodev_jobs.jobs
|
|
227
|
+
SET status = 'succeeded', last_error = NULL, updated_at = now()
|
|
228
|
+
WHERE id = $1`,
|
|
229
|
+
[row.id],
|
|
230
|
+
);
|
|
231
|
+
log(` ${row.name} succeeded`);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const attempts = row.attempts + 1;
|
|
235
|
+
const next = nextJobFailureState(attempts);
|
|
236
|
+
await query(
|
|
237
|
+
`UPDATE robodev_jobs.jobs
|
|
238
|
+
SET status = $2, attempts = $3, last_error = $4, run_at = COALESCE($5, run_at), updated_at = now()
|
|
239
|
+
WHERE id = $1`,
|
|
240
|
+
[row.id, next.status, attempts, error, next.runAt],
|
|
241
|
+
);
|
|
242
|
+
if (next.status === "dead") {
|
|
243
|
+
log(` ${row.name} dead: ${shortError(error)}`);
|
|
244
|
+
} else {
|
|
245
|
+
log(` ${row.name} failed attempt ${attempts}/${JOB_MAX_ATTEMPTS}: ${shortError(error)}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
async function runClaimedJob(
|
|
250
|
+
row: LocalJobRow,
|
|
251
|
+
generation: LocalJobsGeneration | null,
|
|
252
|
+
): Promise<void> {
|
|
253
|
+
const job = generation?.jobs.find((entry) => entry.name === row.name);
|
|
254
|
+
if (!generation || !job) {
|
|
255
|
+
await finishJob(row, "route_not_found");
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
let payload: unknown = null;
|
|
259
|
+
try {
|
|
260
|
+
payload = JSON.parse(row.payload_json) as unknown;
|
|
261
|
+
} catch {
|
|
262
|
+
payload = null;
|
|
263
|
+
}
|
|
264
|
+
const clients = options.clients();
|
|
265
|
+
try {
|
|
266
|
+
await withHandlerTimeout(async () => {
|
|
267
|
+
await (job.def as JobDefinition).handler({
|
|
268
|
+
db: generation.db,
|
|
269
|
+
email: clients.email,
|
|
270
|
+
llm: clients.llm,
|
|
271
|
+
agent: clients.agent,
|
|
272
|
+
env: clients.env,
|
|
273
|
+
push: clients.push,
|
|
274
|
+
jobs: clients.jobs,
|
|
275
|
+
sockets: clients.sockets,
|
|
276
|
+
payload,
|
|
277
|
+
});
|
|
278
|
+
}, clampTimeoutMs(job.def.timeoutMs));
|
|
279
|
+
await finishJob(row, null);
|
|
280
|
+
} catch (error) {
|
|
281
|
+
const message =
|
|
282
|
+
error instanceof HandlerTimeoutError
|
|
283
|
+
? "handler_timeout"
|
|
284
|
+
: error instanceof Error
|
|
285
|
+
? error.message
|
|
286
|
+
: "handler_failed";
|
|
287
|
+
await finishJob(row, message);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function tickJobs(): Promise<void> {
|
|
292
|
+
while (inFlight < JOB_CONCURRENCY) {
|
|
293
|
+
const generation = options.getGeneration();
|
|
294
|
+
const row = await claimNextJob().catch(() => null);
|
|
295
|
+
if (!row) return;
|
|
296
|
+
inFlight += 1;
|
|
297
|
+
const run = runClaimedJob(row, generation);
|
|
298
|
+
running.add(run);
|
|
299
|
+
void run.finally(() => {
|
|
300
|
+
inFlight -= 1;
|
|
301
|
+
running.delete(run);
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
async function requeueRunning(): Promise<void> {
|
|
307
|
+
await query(
|
|
308
|
+
`UPDATE robodev_jobs.jobs
|
|
309
|
+
SET status = 'queued', updated_at = now()
|
|
310
|
+
WHERE status = 'running'`,
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async function start(): Promise<void> {
|
|
315
|
+
if (workerTimer || cronTimer) return;
|
|
316
|
+
await requeueRunning();
|
|
317
|
+
workerTimer = setInterval(() => {
|
|
318
|
+
void tickJobs();
|
|
319
|
+
}, LOCAL_JOB_WORKER_MS);
|
|
320
|
+
workerTimer.unref?.();
|
|
321
|
+
cronTimer = setInterval(() => {
|
|
322
|
+
void tickDue();
|
|
323
|
+
}, LOCAL_JOB_CRON_MS);
|
|
324
|
+
cronTimer.unref?.();
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function stop(): Promise<void> {
|
|
328
|
+
if (workerTimer) {
|
|
329
|
+
clearInterval(workerTimer);
|
|
330
|
+
workerTimer = undefined;
|
|
331
|
+
}
|
|
332
|
+
if (cronTimer) {
|
|
333
|
+
clearInterval(cronTimer);
|
|
334
|
+
cronTimer = undefined;
|
|
335
|
+
}
|
|
336
|
+
await Promise.all([...running]);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return {
|
|
340
|
+
enqueue,
|
|
341
|
+
reconcile,
|
|
342
|
+
tickDue,
|
|
343
|
+
tickJobs,
|
|
344
|
+
start,
|
|
345
|
+
stop,
|
|
346
|
+
get generation() {
|
|
347
|
+
return options.getGeneration();
|
|
348
|
+
},
|
|
349
|
+
get running() {
|
|
350
|
+
return Boolean(workerTimer || cronTimer);
|
|
351
|
+
},
|
|
352
|
+
};
|
|
353
|
+
}
|