@trigger.dev/convex 4.8.2 → 4.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/client/index.d.ts +5 -0
- package/dist/client/index.js +7 -0
- package/dist/component/runs.d.ts +7 -0
- package/dist/component/runs.js +40 -2
- package/dist/component/schema.d.ts +7 -1
- package/dist/component/schema.js +3 -0
- package/dist/component/shared.d.ts +1 -0
- package/dist/component/shared.js +1 -0
- package/dist/component/worker.js +16 -3
- package/package.json +1 -1
- package/src/component/runs.test.ts +162 -15
- package/src/component/runs.ts +52 -1
- package/src/component/schema.ts +3 -0
- package/src/component/shared.ts +1 -0
- package/src/component/worker.ts +15 -3
package/README.md
CHANGED
|
@@ -124,6 +124,7 @@ The ID returned by `trigger.trigger` identifies the **component record**, not th
|
|
|
124
124
|
- `trigger(ctx, taskId, payload, options?)`: enqueue work. Calling from a mutation makes the request atomic with your application's writes.
|
|
125
125
|
- `getRun(ctx, id)`: read one record reactively.
|
|
126
126
|
- `listRuns(ctx, paginationOpts)`: paginate records, newest first, up to 100 per page and a bounded read budget.
|
|
127
|
+
- `refresh(ctx, id)`: schedule an earlier status check for an active, submitted run. Returns `null` after scheduling, not the updated run.
|
|
127
128
|
- `resume(ctx, id)`: resume paused synchronization using the current credentials.
|
|
128
129
|
|
|
129
130
|
Trigger options include tags, delay, queue name, concurrency key, machine, metadata, max attempts, max duration, TTL, and a component-level idempotency key.
|
|
@@ -134,6 +135,35 @@ The constructor accepts `secretKey` (defaults to `TRIGGER_SECRET_KEY`), `baseUrl
|
|
|
134
135
|
|
|
135
136
|
**Authorization belongs in your application's functions.** The component has no tenant or user model. Do not expose arbitrary record IDs or `listRuns` to clients without enforcing ownership. Queries never return stored credentials or request payloads.
|
|
136
137
|
|
|
138
|
+
## Realtime progress and faster result delivery
|
|
139
|
+
|
|
140
|
+
Use [Trigger Realtime](https://trigger.dev/docs/realtime/overview) for responsive browser progress, and Convex queries for durable application state. When Realtime reports that a run is complete, call an ownership-checked application mutation that requests a component refresh:
|
|
141
|
+
|
|
142
|
+
```ts
|
|
143
|
+
// convex/jobs.ts
|
|
144
|
+
export const refresh = mutation({
|
|
145
|
+
args: { jobId: v.id("jobs") },
|
|
146
|
+
handler: async (ctx, { jobId }) => {
|
|
147
|
+
const user = await ctx.auth.getUserIdentity();
|
|
148
|
+
const job = await ctx.db.get(jobId);
|
|
149
|
+
if (!user || !job || job.owner !== user.subject) throw new Error("Unauthorized");
|
|
150
|
+
if (job.runId) await trigger.refresh(ctx, job.runId);
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Pass the component record ID to `refresh`, not the remote Trigger run ID. Your browser subscribes to the latter, available as `run.runId` from `getRun`.
|
|
156
|
+
|
|
157
|
+
A refresh is only a hint: the component retrieves status and output directly from Trigger and uses the same validation, completion callback, and credential cleanup as normal polling. It never accepts client-supplied status or output. Observe `getRun` or your application query for the persisted result; awaiting `refresh` does not wait for that result.
|
|
158
|
+
|
|
159
|
+
- Refresh requests are coalesced per run. Status checks requested this way start at least five seconds after the previous status check.
|
|
160
|
+
- A hint received during a check queues one follow-up instead of interrupting or discarding the in-flight response. If that response completes the run, no follow-up is needed.
|
|
161
|
+
- Pending submissions, paused synchronization, and completed records are no-ops. Refresh never submits a task or resumes paused work; use `resume` after repairing a paused synchronization.
|
|
162
|
+
- Call refresh when Realtime observes completion, including an already-completed run after reconnecting. Do not call it on every metadata update to animate progress.
|
|
163
|
+
- Scheduled polling and its watchdog remain enabled. Convex can still finish the run if the browser closes, the subscription disconnects, or the refresh request fails. No persistent subscription inside a Convex action is required.
|
|
164
|
+
|
|
165
|
+
**Realtime token safety:** authenticate the token request, verify job ownership, and issue a short-lived token scoped to the exact remote run. Keep the root Trigger key server-side. A run-read token can also read the run through REST, including its payload; Realtime `skipColumns` is not a restriction on REST access. Keep worker credentials, storage-upload capabilities, and other secrets out of token-readable payloads, outputs, metadata, and errors. Application mutations must enforce ownership even when component refreshes are coalesced.
|
|
166
|
+
|
|
137
167
|
## Execution and reliability
|
|
138
168
|
|
|
139
169
|
1. The calling mutation writes a component record and schedules submission. If the transaction rolls back, neither survives.
|
package/dist/client/index.d.ts
CHANGED
|
@@ -30,5 +30,10 @@ export declare class Trigger {
|
|
|
30
30
|
isDone: boolean;
|
|
31
31
|
continueCursor: string;
|
|
32
32
|
}>;
|
|
33
|
+
/**
|
|
34
|
+
* Request an earlier status check without accepting client-supplied run data.
|
|
35
|
+
* The check is scheduled, coalesced, and rate-limited; observe getRun for its result.
|
|
36
|
+
*/
|
|
37
|
+
refresh(ctx: MutationCtx, id: string): Promise<null>;
|
|
33
38
|
resume(ctx: MutationCtx, id: string): Promise<null>;
|
|
34
39
|
}
|
package/dist/client/index.js
CHANGED
|
@@ -39,6 +39,13 @@ export class Trigger {
|
|
|
39
39
|
async listRuns(ctx, paginationOpts) {
|
|
40
40
|
return ctx.runQuery(this.component.runs.list, { paginationOpts });
|
|
41
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Request an earlier status check without accepting client-supplied run data.
|
|
44
|
+
* The check is scheduled, coalesced, and rate-limited; observe getRun for its result.
|
|
45
|
+
*/
|
|
46
|
+
async refresh(ctx, id) {
|
|
47
|
+
return ctx.runMutation(this.component.runs.refresh, { id });
|
|
48
|
+
}
|
|
42
49
|
async resume(ctx, id) {
|
|
43
50
|
return ctx.runMutation(this.component.runs.resume, { id, config: this.runtimeConfig() });
|
|
44
51
|
}
|
package/dist/component/runs.d.ts
CHANGED
|
@@ -58,6 +58,10 @@ export declare const list: import("convex/server").RegisteredQuery<"public", {
|
|
|
58
58
|
isDone: boolean;
|
|
59
59
|
continueCursor: string;
|
|
60
60
|
}>>;
|
|
61
|
+
export declare function scheduleRefresh(ctx: GenericMutationCtx<DataModel>, run: RunDoc): Promise<void>;
|
|
62
|
+
export declare const refresh: import("convex/server").RegisteredMutation<"public", {
|
|
63
|
+
id: string;
|
|
64
|
+
}, Promise<null>>;
|
|
61
65
|
export declare const resume: import("convex/server").RegisteredMutation<"public", {
|
|
62
66
|
config: {
|
|
63
67
|
baseUrl: string;
|
|
@@ -72,15 +76,18 @@ export declare const load: import("convex/server").RegisteredQuery<"internal", {
|
|
|
72
76
|
_creationTime: number;
|
|
73
77
|
_id: import("convex/values").GenericId<"runs">;
|
|
74
78
|
callback?: string | undefined;
|
|
79
|
+
checking?: boolean | undefined;
|
|
75
80
|
checks: number;
|
|
76
81
|
error?: any;
|
|
77
82
|
failures: number;
|
|
78
83
|
generation: number;
|
|
79
84
|
idempotencyKey?: string | undefined;
|
|
85
|
+
lastCheckAt?: number | undefined;
|
|
80
86
|
metadata?: any;
|
|
81
87
|
output?: any;
|
|
82
88
|
outputOmitted?: boolean | undefined;
|
|
83
89
|
outputPresignedUrl?: string | undefined;
|
|
90
|
+
refreshRequested?: boolean | undefined;
|
|
84
91
|
runId?: string | undefined;
|
|
85
92
|
status: string;
|
|
86
93
|
syncError?: string | undefined;
|
package/dist/component/runs.js
CHANGED
|
@@ -2,9 +2,9 @@ import { paginationOptsValidator, } from "convex/server";
|
|
|
2
2
|
import { v } from "convex/values";
|
|
3
3
|
import { internal } from "./api.js";
|
|
4
4
|
import { internalMutation, internalQuery, mutation, query } from "./server.js";
|
|
5
|
-
import { MAX_INLINE_BYTES, MAX_PAGE_BYTES, SUBMISSION_WINDOW_MS, vConfig, vRun, vTriggerOptions, } from "./shared.js";
|
|
5
|
+
import { MAX_INLINE_BYTES, MAX_PAGE_BYTES, MIN_REFRESH_INTERVAL_MS, SUBMISSION_WINDOW_MS, vConfig, vRun, vTriggerOptions, } from "./shared.js";
|
|
6
6
|
export function toRun(run) {
|
|
7
|
-
const { _id, _creationTime, generation, checks, failures, callback, ...fields } = run;
|
|
7
|
+
const { _id, _creationTime, generation, checks, failures, callback, checking, refreshRequested, lastCheckAt, ...fields } = run;
|
|
8
8
|
return { id: _id, createdAt: _creationTime, ...fields };
|
|
9
9
|
}
|
|
10
10
|
export const trigger = mutation({
|
|
@@ -83,6 +83,34 @@ export const list = query({
|
|
|
83
83
|
};
|
|
84
84
|
},
|
|
85
85
|
});
|
|
86
|
+
export async function scheduleRefresh(ctx, run) {
|
|
87
|
+
if (!run.refreshRequested)
|
|
88
|
+
return;
|
|
89
|
+
const delay = run.lastCheckAt === undefined
|
|
90
|
+
? 0
|
|
91
|
+
: Math.max(0, run.lastCheckAt + MIN_REFRESH_INTERVAL_MS - Date.now());
|
|
92
|
+
await ctx.scheduler.runAfter(delay, internal.worker.tick, {
|
|
93
|
+
id: run._id,
|
|
94
|
+
generation: run.generation,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
export const refresh = mutation({
|
|
98
|
+
args: { id: v.string() },
|
|
99
|
+
returns: v.null(),
|
|
100
|
+
handler: async (ctx, { id }) => {
|
|
101
|
+
const runId = ctx.db.normalizeId("runs", id);
|
|
102
|
+
const run = runId ? await ctx.db.get("runs", runId) : null;
|
|
103
|
+
if (!run)
|
|
104
|
+
throw new Error("Run not found");
|
|
105
|
+
if (!run.runId || run.syncStatus !== "active" || run.refreshRequested)
|
|
106
|
+
return null;
|
|
107
|
+
await ctx.db.patch(run._id, { refreshRequested: true });
|
|
108
|
+
// Preserve the in-flight observation; its result will schedule any requested follow-up.
|
|
109
|
+
if (!run.checking)
|
|
110
|
+
await scheduleRefresh(ctx, { ...run, refreshRequested: true });
|
|
111
|
+
return null;
|
|
112
|
+
},
|
|
113
|
+
});
|
|
86
114
|
export const resume = mutation({
|
|
87
115
|
args: { id: v.string(), config: vConfig },
|
|
88
116
|
returns: v.null(),
|
|
@@ -110,6 +138,8 @@ export const resume = mutation({
|
|
|
110
138
|
failures: 0,
|
|
111
139
|
checks: 0,
|
|
112
140
|
generation,
|
|
141
|
+
checking: undefined,
|
|
142
|
+
refreshRequested: undefined,
|
|
113
143
|
updatedAt: Date.now(),
|
|
114
144
|
});
|
|
115
145
|
await ctx.scheduler.runAfter(0, internal.worker.tick, { id: run._id, generation });
|
|
@@ -141,6 +171,7 @@ export const recordSubmission = internalMutation({
|
|
|
141
171
|
syncError: undefined,
|
|
142
172
|
failures: 0,
|
|
143
173
|
checks: 0,
|
|
174
|
+
checking: undefined,
|
|
144
175
|
updatedAt: Date.now(),
|
|
145
176
|
});
|
|
146
177
|
return null;
|
|
@@ -168,6 +199,8 @@ export const recordStatus = internalMutation({
|
|
|
168
199
|
syncStatus: completed ? "complete" : "active",
|
|
169
200
|
syncError: undefined,
|
|
170
201
|
failures: 0,
|
|
202
|
+
checking: undefined,
|
|
203
|
+
refreshRequested: completed ? undefined : run.refreshRequested,
|
|
171
204
|
updatedAt: Date.now(),
|
|
172
205
|
};
|
|
173
206
|
await ctx.db.patch("runs", id, updates);
|
|
@@ -184,6 +217,9 @@ export const recordStatus = internalMutation({
|
|
|
184
217
|
if (request)
|
|
185
218
|
await ctx.db.delete("requests", request._id);
|
|
186
219
|
}
|
|
220
|
+
else {
|
|
221
|
+
await scheduleRefresh(ctx, run);
|
|
222
|
+
}
|
|
187
223
|
return null;
|
|
188
224
|
},
|
|
189
225
|
});
|
|
@@ -191,6 +227,8 @@ export async function pause(ctx, run, message) {
|
|
|
191
227
|
await ctx.db.patch("runs", run._id, {
|
|
192
228
|
syncStatus: "paused",
|
|
193
229
|
syncError: message,
|
|
230
|
+
checking: undefined,
|
|
231
|
+
refreshRequested: undefined,
|
|
194
232
|
updatedAt: Date.now(),
|
|
195
233
|
});
|
|
196
234
|
}
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
declare const _default: import("convex/server").SchemaDefinition<{
|
|
2
2
|
runs: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
3
3
|
callback?: string | undefined;
|
|
4
|
+
checking?: boolean | undefined;
|
|
4
5
|
checks: number;
|
|
5
6
|
error?: any;
|
|
6
7
|
failures: number;
|
|
7
8
|
generation: number;
|
|
8
9
|
idempotencyKey?: string | undefined;
|
|
10
|
+
lastCheckAt?: number | undefined;
|
|
9
11
|
metadata?: any;
|
|
10
12
|
output?: any;
|
|
11
13
|
outputOmitted?: boolean | undefined;
|
|
12
14
|
outputPresignedUrl?: string | undefined;
|
|
15
|
+
refreshRequested?: boolean | undefined;
|
|
13
16
|
runId?: string | undefined;
|
|
14
17
|
status: string;
|
|
15
18
|
syncError?: string | undefined;
|
|
@@ -32,8 +35,11 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
32
35
|
generation: import("convex/values").VFloat64<number, "required">;
|
|
33
36
|
checks: import("convex/values").VFloat64<number, "required">;
|
|
34
37
|
failures: import("convex/values").VFloat64<number, "required">;
|
|
38
|
+
checking: import("convex/values").VBoolean<boolean | undefined, "optional">;
|
|
39
|
+
refreshRequested: import("convex/values").VBoolean<boolean | undefined, "optional">;
|
|
40
|
+
lastCheckAt: import("convex/values").VFloat64<number | undefined, "optional">;
|
|
35
41
|
callback: import("convex/values").VString<string | undefined, "optional">;
|
|
36
|
-
}, "required", "callback" | "checks" | "error" | "failures" | "generation" | "idempotencyKey" | "metadata" | "output" | "outputOmitted" | "outputPresignedUrl" | "runId" | "status" | "syncError" | "syncStatus" | "taskId" | "updatedAt" | `error.${string}` | `metadata.${string}` | `output.${string}`>, {
|
|
42
|
+
}, "required", "callback" | "checking" | "checks" | "error" | "failures" | "generation" | "idempotencyKey" | "lastCheckAt" | "metadata" | "output" | "outputOmitted" | "outputPresignedUrl" | "refreshRequested" | "runId" | "status" | "syncError" | "syncStatus" | "taskId" | "updatedAt" | `error.${string}` | `metadata.${string}` | `output.${string}`>, {
|
|
37
43
|
by_key: ["idempotencyKey", "_creationTime"];
|
|
38
44
|
}, {}, {}>;
|
|
39
45
|
requests: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
package/dist/component/schema.js
CHANGED
|
@@ -7,6 +7,9 @@ export default defineSchema({
|
|
|
7
7
|
generation: v.number(),
|
|
8
8
|
checks: v.number(),
|
|
9
9
|
failures: v.number(),
|
|
10
|
+
checking: v.optional(v.boolean()),
|
|
11
|
+
refreshRequested: v.optional(v.boolean()),
|
|
12
|
+
lastCheckAt: v.optional(v.number()),
|
|
10
13
|
callback: v.optional(v.string()),
|
|
11
14
|
}).index("by_key", ["idempotencyKey"]),
|
|
12
15
|
requests: defineTable({
|
|
@@ -82,6 +82,7 @@ export type Run = Infer<typeof vRun>;
|
|
|
82
82
|
export type TriggerOptions = Infer<typeof vTriggerOptions>;
|
|
83
83
|
export type Config = Infer<typeof vConfig>;
|
|
84
84
|
export declare const REQUEST_TIMEOUT_MS = 10000;
|
|
85
|
+
export declare const MIN_REFRESH_INTERVAL_MS = 5000;
|
|
85
86
|
export declare const WATCHDOG_MARGIN_MS = 5000;
|
|
86
87
|
export declare const MAX_SYNC_FAILURES = 8;
|
|
87
88
|
export declare const SUBMISSION_WINDOW_MS: number;
|
package/dist/component/shared.js
CHANGED
|
@@ -36,6 +36,7 @@ export const vRun = v.object({
|
|
|
36
36
|
const FIRST_POLL_MS = 15_000;
|
|
37
37
|
const MAX_POLL_MS = 60_000;
|
|
38
38
|
export const REQUEST_TIMEOUT_MS = 10_000;
|
|
39
|
+
export const MIN_REFRESH_INTERVAL_MS = 5_000;
|
|
39
40
|
export const WATCHDOG_MARGIN_MS = 5_000;
|
|
40
41
|
export const MAX_SYNC_FAILURES = 8;
|
|
41
42
|
export const SUBMISSION_WINDOW_MS = 60 * 60 * 1_000;
|
package/dist/component/worker.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { v } from "convex/values";
|
|
2
2
|
import { internal } from "./api.js";
|
|
3
3
|
import { internalAction, internalMutation } from "./server.js";
|
|
4
|
-
import { pause } from "./runs.js";
|
|
4
|
+
import { pause, scheduleRefresh } from "./runs.js";
|
|
5
5
|
import { ApiError, inlineValue, request } from "./apiClient.js";
|
|
6
6
|
import { IDEMPOTENCY_TTL, MAX_SYNC_FAILURES, REQUEST_TIMEOUT_MS, SUBMISSION_WINDOW_MS, WATCHDOG_MARGIN_MS, pollDelay, } from "./shared.js";
|
|
7
7
|
const jobArgs = { id: v.id("runs"), generation: v.number() };
|
|
@@ -20,7 +20,13 @@ export const tick = internalMutation({
|
|
|
20
20
|
return null;
|
|
21
21
|
}
|
|
22
22
|
const nextGeneration = generation + 1;
|
|
23
|
-
await ctx.db.patch("runs", id, {
|
|
23
|
+
await ctx.db.patch("runs", id, {
|
|
24
|
+
generation: nextGeneration,
|
|
25
|
+
checks: run.checks + 1,
|
|
26
|
+
checking: true,
|
|
27
|
+
refreshRequested: undefined,
|
|
28
|
+
lastCheckAt: run.runId ? Date.now() : run.lastCheckAt,
|
|
29
|
+
});
|
|
24
30
|
// Schedule the watchdog transactionally before starting an action that can be interrupted.
|
|
25
31
|
await ctx.scheduler.runAfter(Math.max(pollDelay(run.checks), REQUEST_TIMEOUT_MS + WATCHDOG_MARGIN_MS), internal.worker.tick, { id, generation: nextGeneration });
|
|
26
32
|
await ctx.scheduler.runAfter(0, internal.worker.step, { id, generation: nextGeneration });
|
|
@@ -97,9 +103,16 @@ export const recordError = internalMutation({
|
|
|
97
103
|
if (!run || run.generation !== generation || run.syncStatus === "complete")
|
|
98
104
|
return null;
|
|
99
105
|
const failures = run.failures + 1;
|
|
100
|
-
await ctx.db.patch("runs", id, {
|
|
106
|
+
await ctx.db.patch("runs", id, {
|
|
107
|
+
failures,
|
|
108
|
+
syncError: message,
|
|
109
|
+
checking: undefined,
|
|
110
|
+
updatedAt: Date.now(),
|
|
111
|
+
});
|
|
101
112
|
if (permanent || failures >= MAX_SYNC_FAILURES)
|
|
102
113
|
await pause(ctx, run, message);
|
|
114
|
+
else
|
|
115
|
+
await scheduleRefresh(ctx, run);
|
|
103
116
|
return null;
|
|
104
117
|
},
|
|
105
118
|
});
|
package/package.json
CHANGED
|
@@ -6,7 +6,7 @@ import { defineSchema, defineTable } from "convex/server";
|
|
|
6
6
|
import { v } from "convex/values";
|
|
7
7
|
import { Trigger, vRun, type ComponentApi } from "../client/index.js";
|
|
8
8
|
import schema from "./schema.js";
|
|
9
|
-
import { MAX_INLINE_BYTES } from "./shared.js";
|
|
9
|
+
import { MAX_INLINE_BYTES, MIN_REFRESH_INTERVAL_MS } from "./shared.js";
|
|
10
10
|
|
|
11
11
|
const modules = {
|
|
12
12
|
"./_generated/component.ts": () => import("./_generated/component.js"),
|
|
@@ -63,6 +63,10 @@ const { createTest, api, internal } = app.defineModules({
|
|
|
63
63
|
return id;
|
|
64
64
|
},
|
|
65
65
|
}),
|
|
66
|
+
refresh: app.mutation({
|
|
67
|
+
args: { id: v.string() },
|
|
68
|
+
handler: async (ctx, { id }) => new Trigger(app.components.trigger).refresh(ctx, id),
|
|
69
|
+
}),
|
|
66
70
|
},
|
|
67
71
|
});
|
|
68
72
|
|
|
@@ -92,6 +96,8 @@ let remoteStatus: string;
|
|
|
92
96
|
let remoteIsCompleted: boolean;
|
|
93
97
|
let largeOutput: boolean;
|
|
94
98
|
let submittedOptions: Record<string, unknown>;
|
|
99
|
+
let holdNextPoll: boolean;
|
|
100
|
+
let releasePoll: (() => void) | undefined;
|
|
95
101
|
|
|
96
102
|
beforeAll(async () => {
|
|
97
103
|
server = createServer(async (req, res) => {
|
|
@@ -125,19 +131,26 @@ beforeAll(async () => {
|
|
|
125
131
|
return;
|
|
126
132
|
}
|
|
127
133
|
const completed = activePolls-- <= 0;
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
?
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
134
|
+
const body = JSON.stringify({
|
|
135
|
+
id: "run_1",
|
|
136
|
+
status: completed ? remoteStatus : "EXECUTING",
|
|
137
|
+
isCompleted: completed && remoteIsCompleted,
|
|
138
|
+
output: completed
|
|
139
|
+
? largeOutput
|
|
140
|
+
? "x".repeat(MAX_INLINE_BYTES + 1)
|
|
141
|
+
: { url: "https://example.com/video.mp4" }
|
|
142
|
+
: undefined,
|
|
143
|
+
metadata: { progress: completed ? 100 : 50 },
|
|
144
|
+
});
|
|
145
|
+
if (holdNextPoll) {
|
|
146
|
+
holdNextPoll = false;
|
|
147
|
+
releasePoll = () => {
|
|
148
|
+
if (!res.writableEnded) res.end(body);
|
|
149
|
+
releasePoll = undefined;
|
|
150
|
+
};
|
|
151
|
+
} else {
|
|
152
|
+
res.end(body);
|
|
153
|
+
}
|
|
141
154
|
}
|
|
142
155
|
});
|
|
143
156
|
server.listen(0, "127.0.0.1");
|
|
@@ -162,8 +175,13 @@ beforeEach(() => {
|
|
|
162
175
|
remoteIsCompleted = true;
|
|
163
176
|
largeOutput = false;
|
|
164
177
|
submittedOptions = {};
|
|
178
|
+
holdNextPoll = false;
|
|
179
|
+
releasePoll = undefined;
|
|
180
|
+
});
|
|
181
|
+
afterEach(() => {
|
|
182
|
+
releasePoll?.();
|
|
183
|
+
vi.useRealTimers();
|
|
165
184
|
});
|
|
166
|
-
afterEach(() => vi.useRealTimers());
|
|
167
185
|
afterAll(
|
|
168
186
|
() =>
|
|
169
187
|
new Promise<void>((resolve, reject) => {
|
|
@@ -344,6 +362,135 @@ test("a failing callback rolls back completion and can be retried without duplic
|
|
|
344
362
|
expect(await t.run((ctx) => ctx.db.query("completions").collect())).toHaveLength(1);
|
|
345
363
|
});
|
|
346
364
|
|
|
365
|
+
async function runScheduled(t: ReturnType<typeof createTest>, advanceMs = 0): Promise<void> {
|
|
366
|
+
// Convex timestamps have sub-millisecond precision, so zero-delay jobs may fall just after now.
|
|
367
|
+
vi.advanceTimersByTime(advanceMs + 1);
|
|
368
|
+
await t.finishInProgressScheduledFunctions();
|
|
369
|
+
vi.advanceTimersByTime(1);
|
|
370
|
+
await t.finishInProgressScheduledFunctions();
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Catches: refresh bursts bypassing the check cooldown or racing the polling callback.
|
|
374
|
+
test("refreshes promptly, bounds repeated checks, and completes once alongside polling", async () => {
|
|
375
|
+
activePolls = 1;
|
|
376
|
+
const t = createTest();
|
|
377
|
+
const id = await t.mutation(api.jobs.request, {});
|
|
378
|
+
await runScheduled(t);
|
|
379
|
+
expect(submissions).toBe(1);
|
|
380
|
+
expect(polls).toBe(0);
|
|
381
|
+
|
|
382
|
+
await Promise.all(Array.from({ length: 3 }, () => t.mutation(api.jobs.refresh, { id })));
|
|
383
|
+
await runScheduled(t);
|
|
384
|
+
expect(polls).toBe(1);
|
|
385
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
386
|
+
status: "EXECUTING",
|
|
387
|
+
metadata: { progress: 50 },
|
|
388
|
+
});
|
|
389
|
+
|
|
390
|
+
await Promise.all(Array.from({ length: 3 }, () => t.mutation(api.jobs.refresh, { id })));
|
|
391
|
+
await runScheduled(t, MIN_REFRESH_INTERVAL_MS / 2);
|
|
392
|
+
expect(polls).toBe(1);
|
|
393
|
+
await runScheduled(t, MIN_REFRESH_INTERVAL_MS / 2);
|
|
394
|
+
expect(polls).toBe(2);
|
|
395
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
396
|
+
status: "COMPLETED",
|
|
397
|
+
syncStatus: "complete",
|
|
398
|
+
output: { url: "https://example.com/video.mp4" },
|
|
399
|
+
});
|
|
400
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
401
|
+
await drain(t);
|
|
402
|
+
expect(polls).toBe(2);
|
|
403
|
+
expect(submissions).toBe(1);
|
|
404
|
+
expect(await t.run((ctx) => ctx.db.query("completions").collect())).toHaveLength(1);
|
|
405
|
+
});
|
|
406
|
+
|
|
407
|
+
// Catches: refresh fencing out an in-flight observation or losing its completion hint.
|
|
408
|
+
test("coalesces hints during an in-flight request and follows it with another check", async () => {
|
|
409
|
+
activePolls = 1;
|
|
410
|
+
const t = createTest();
|
|
411
|
+
const id = await t.mutation(api.jobs.request, {});
|
|
412
|
+
await runScheduled(t);
|
|
413
|
+
holdNextPoll = true;
|
|
414
|
+
const received = once(server, "request");
|
|
415
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
416
|
+
vi.advanceTimersByTime(1);
|
|
417
|
+
await t.finishInProgressScheduledFunctions();
|
|
418
|
+
vi.advanceTimersByTime(1);
|
|
419
|
+
const checking = t.finishInProgressScheduledFunctions();
|
|
420
|
+
await received;
|
|
421
|
+
try {
|
|
422
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
423
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
424
|
+
vi.advanceTimersByTime(MIN_REFRESH_INTERVAL_MS);
|
|
425
|
+
expect(polls).toBe(1);
|
|
426
|
+
releasePoll!();
|
|
427
|
+
await checking;
|
|
428
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
429
|
+
status: "EXECUTING",
|
|
430
|
+
syncStatus: "active",
|
|
431
|
+
metadata: { progress: 50 },
|
|
432
|
+
});
|
|
433
|
+
await runScheduled(t);
|
|
434
|
+
expect(polls).toBe(2);
|
|
435
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
436
|
+
status: "COMPLETED",
|
|
437
|
+
syncStatus: "complete",
|
|
438
|
+
});
|
|
439
|
+
expect(await t.run((ctx) => ctx.db.query("completions").collect())).toHaveLength(1);
|
|
440
|
+
} finally {
|
|
441
|
+
releasePoll?.();
|
|
442
|
+
await checking;
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
// Catches: a failed status request leaving refresh permanently blocked as in-flight.
|
|
447
|
+
test("allows a refresh after a retryable status error", async () => {
|
|
448
|
+
pollFailures = 1;
|
|
449
|
+
const t = createTest();
|
|
450
|
+
const id = await t.mutation(api.jobs.request, {});
|
|
451
|
+
await runScheduled(t);
|
|
452
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
453
|
+
await runScheduled(t);
|
|
454
|
+
expect(polls).toBe(1);
|
|
455
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
456
|
+
syncStatus: "active",
|
|
457
|
+
});
|
|
458
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
459
|
+
await runScheduled(t, MIN_REFRESH_INTERVAL_MS);
|
|
460
|
+
expect(polls).toBe(2);
|
|
461
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
462
|
+
syncStatus: "complete",
|
|
463
|
+
});
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
// Catches: browser hints resubmitting pending work or bypassing a permanent synchronization failure.
|
|
467
|
+
test("does not submit pending work early or resume paused synchronization", async () => {
|
|
468
|
+
pollFailures = 1;
|
|
469
|
+
pollFailureStatus = 401;
|
|
470
|
+
const t = createTest();
|
|
471
|
+
const id = await t.mutation(api.jobs.request, {});
|
|
472
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
473
|
+
expect(submissions).toBe(0);
|
|
474
|
+
await runScheduled(t);
|
|
475
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
476
|
+
await runScheduled(t);
|
|
477
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
478
|
+
syncStatus: "paused",
|
|
479
|
+
});
|
|
480
|
+
await t.mutation(api.jobs.refresh, { id });
|
|
481
|
+
await runScheduled(t, MIN_REFRESH_INTERVAL_MS);
|
|
482
|
+
expect(polls).toBe(1);
|
|
483
|
+
expect(submissions).toBe(1);
|
|
484
|
+
await t.mutation(app.components.trigger.runs.resume, {
|
|
485
|
+
id,
|
|
486
|
+
config: { secretKey: "test-only-key", baseUrl },
|
|
487
|
+
});
|
|
488
|
+
await drain(t);
|
|
489
|
+
expect(await t.query(app.components.trigger.runs.get, { id })).toMatchObject({
|
|
490
|
+
syncStatus: "complete",
|
|
491
|
+
});
|
|
492
|
+
});
|
|
493
|
+
|
|
347
494
|
test("rejects large inputs before scheduling remote work", async () => {
|
|
348
495
|
const t = createTest();
|
|
349
496
|
await expect(
|
package/src/component/runs.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { internalMutation, internalQuery, mutation, query, type DataModel } from
|
|
|
9
9
|
import {
|
|
10
10
|
MAX_INLINE_BYTES,
|
|
11
11
|
MAX_PAGE_BYTES,
|
|
12
|
+
MIN_REFRESH_INTERVAL_MS,
|
|
12
13
|
SUBMISSION_WINDOW_MS,
|
|
13
14
|
vConfig,
|
|
14
15
|
vRun,
|
|
@@ -19,7 +20,18 @@ import {
|
|
|
19
20
|
type RunDoc = DataModel["runs"]["document"];
|
|
20
21
|
|
|
21
22
|
export function toRun(run: RunDoc): Run {
|
|
22
|
-
const {
|
|
23
|
+
const {
|
|
24
|
+
_id,
|
|
25
|
+
_creationTime,
|
|
26
|
+
generation,
|
|
27
|
+
checks,
|
|
28
|
+
failures,
|
|
29
|
+
callback,
|
|
30
|
+
checking,
|
|
31
|
+
refreshRequested,
|
|
32
|
+
lastCheckAt,
|
|
33
|
+
...fields
|
|
34
|
+
} = run;
|
|
23
35
|
return { id: _id, createdAt: _creationTime, ...fields };
|
|
24
36
|
}
|
|
25
37
|
|
|
@@ -110,6 +122,36 @@ export const list = query({
|
|
|
110
122
|
},
|
|
111
123
|
});
|
|
112
124
|
|
|
125
|
+
export async function scheduleRefresh(
|
|
126
|
+
ctx: GenericMutationCtx<DataModel>,
|
|
127
|
+
run: RunDoc
|
|
128
|
+
): Promise<void> {
|
|
129
|
+
if (!run.refreshRequested) return;
|
|
130
|
+
const delay =
|
|
131
|
+
run.lastCheckAt === undefined
|
|
132
|
+
? 0
|
|
133
|
+
: Math.max(0, run.lastCheckAt + MIN_REFRESH_INTERVAL_MS - Date.now());
|
|
134
|
+
await ctx.scheduler.runAfter(delay, internal.worker.tick, {
|
|
135
|
+
id: run._id,
|
|
136
|
+
generation: run.generation,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export const refresh = mutation({
|
|
141
|
+
args: { id: v.string() },
|
|
142
|
+
returns: v.null(),
|
|
143
|
+
handler: async (ctx, { id }): Promise<null> => {
|
|
144
|
+
const runId = ctx.db.normalizeId("runs", id);
|
|
145
|
+
const run = runId ? await ctx.db.get("runs", runId) : null;
|
|
146
|
+
if (!run) throw new Error("Run not found");
|
|
147
|
+
if (!run.runId || run.syncStatus !== "active" || run.refreshRequested) return null;
|
|
148
|
+
await ctx.db.patch(run._id, { refreshRequested: true });
|
|
149
|
+
// Preserve the in-flight observation; its result will schedule any requested follow-up.
|
|
150
|
+
if (!run.checking) await scheduleRefresh(ctx, { ...run, refreshRequested: true });
|
|
151
|
+
return null;
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
|
+
|
|
113
155
|
export const resume = mutation({
|
|
114
156
|
args: { id: v.string(), config: vConfig },
|
|
115
157
|
returns: v.null(),
|
|
@@ -136,6 +178,8 @@ export const resume = mutation({
|
|
|
136
178
|
failures: 0,
|
|
137
179
|
checks: 0,
|
|
138
180
|
generation,
|
|
181
|
+
checking: undefined,
|
|
182
|
+
refreshRequested: undefined,
|
|
139
183
|
updatedAt: Date.now(),
|
|
140
184
|
});
|
|
141
185
|
await ctx.scheduler.runAfter(0, internal.worker.tick, { id: run._id, generation });
|
|
@@ -168,6 +212,7 @@ export const recordSubmission = internalMutation({
|
|
|
168
212
|
syncError: undefined,
|
|
169
213
|
failures: 0,
|
|
170
214
|
checks: 0,
|
|
215
|
+
checking: undefined,
|
|
171
216
|
updatedAt: Date.now(),
|
|
172
217
|
});
|
|
173
218
|
return null;
|
|
@@ -195,6 +240,8 @@ export const recordStatus = internalMutation({
|
|
|
195
240
|
syncStatus: completed ? ("complete" as const) : ("active" as const),
|
|
196
241
|
syncError: undefined,
|
|
197
242
|
failures: 0,
|
|
243
|
+
checking: undefined,
|
|
244
|
+
refreshRequested: completed ? undefined : run.refreshRequested,
|
|
198
245
|
updatedAt: Date.now(),
|
|
199
246
|
};
|
|
200
247
|
await ctx.db.patch("runs", id, updates);
|
|
@@ -209,6 +256,8 @@ export const recordStatus = internalMutation({
|
|
|
209
256
|
.withIndex("by_run", (q) => q.eq("run", id))
|
|
210
257
|
.unique();
|
|
211
258
|
if (request) await ctx.db.delete("requests", request._id);
|
|
259
|
+
} else {
|
|
260
|
+
await scheduleRefresh(ctx, run);
|
|
212
261
|
}
|
|
213
262
|
return null;
|
|
214
263
|
},
|
|
@@ -222,6 +271,8 @@ export async function pause(
|
|
|
222
271
|
await ctx.db.patch("runs", run._id, {
|
|
223
272
|
syncStatus: "paused",
|
|
224
273
|
syncError: message,
|
|
274
|
+
checking: undefined,
|
|
275
|
+
refreshRequested: undefined,
|
|
225
276
|
updatedAt: Date.now(),
|
|
226
277
|
});
|
|
227
278
|
}
|
package/src/component/schema.ts
CHANGED
|
@@ -8,6 +8,9 @@ export default defineSchema({
|
|
|
8
8
|
generation: v.number(),
|
|
9
9
|
checks: v.number(),
|
|
10
10
|
failures: v.number(),
|
|
11
|
+
checking: v.optional(v.boolean()),
|
|
12
|
+
refreshRequested: v.optional(v.boolean()),
|
|
13
|
+
lastCheckAt: v.optional(v.number()),
|
|
11
14
|
callback: v.optional(v.string()),
|
|
12
15
|
}).index("by_key", ["idempotencyKey"]),
|
|
13
16
|
requests: defineTable({
|
package/src/component/shared.ts
CHANGED
|
@@ -49,6 +49,7 @@ export type Config = Infer<typeof vConfig>;
|
|
|
49
49
|
const FIRST_POLL_MS = 15_000;
|
|
50
50
|
const MAX_POLL_MS = 60_000;
|
|
51
51
|
export const REQUEST_TIMEOUT_MS = 10_000;
|
|
52
|
+
export const MIN_REFRESH_INTERVAL_MS = 5_000;
|
|
52
53
|
export const WATCHDOG_MARGIN_MS = 5_000;
|
|
53
54
|
export const MAX_SYNC_FAILURES = 8;
|
|
54
55
|
export const SUBMISSION_WINDOW_MS = 60 * 60 * 1_000;
|
package/src/component/worker.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { v } from "convex/values";
|
|
2
2
|
import { internal } from "./api.js";
|
|
3
3
|
import { internalAction, internalMutation } from "./server.js";
|
|
4
|
-
import { pause } from "./runs.js";
|
|
4
|
+
import { pause, scheduleRefresh } from "./runs.js";
|
|
5
5
|
import { ApiError, inlineValue, request } from "./apiClient.js";
|
|
6
6
|
import {
|
|
7
7
|
IDEMPOTENCY_TTL,
|
|
@@ -35,7 +35,13 @@ export const tick = internalMutation({
|
|
|
35
35
|
return null;
|
|
36
36
|
}
|
|
37
37
|
const nextGeneration = generation + 1;
|
|
38
|
-
await ctx.db.patch("runs", id, {
|
|
38
|
+
await ctx.db.patch("runs", id, {
|
|
39
|
+
generation: nextGeneration,
|
|
40
|
+
checks: run.checks + 1,
|
|
41
|
+
checking: true,
|
|
42
|
+
refreshRequested: undefined,
|
|
43
|
+
lastCheckAt: run.runId ? Date.now() : run.lastCheckAt,
|
|
44
|
+
});
|
|
39
45
|
// Schedule the watchdog transactionally before starting an action that can be interrupted.
|
|
40
46
|
await ctx.scheduler.runAfter(
|
|
41
47
|
Math.max(pollDelay(run.checks), REQUEST_TIMEOUT_MS + WATCHDOG_MARGIN_MS),
|
|
@@ -124,8 +130,14 @@ export const recordError = internalMutation({
|
|
|
124
130
|
const run = await ctx.db.get("runs", id);
|
|
125
131
|
if (!run || run.generation !== generation || run.syncStatus === "complete") return null;
|
|
126
132
|
const failures = run.failures + 1;
|
|
127
|
-
await ctx.db.patch("runs", id, {
|
|
133
|
+
await ctx.db.patch("runs", id, {
|
|
134
|
+
failures,
|
|
135
|
+
syncError: message,
|
|
136
|
+
checking: undefined,
|
|
137
|
+
updatedAt: Date.now(),
|
|
138
|
+
});
|
|
128
139
|
if (permanent || failures >= MAX_SYNC_FAILURES) await pause(ctx, run, message);
|
|
140
|
+
else await scheduleRefresh(ctx, run);
|
|
129
141
|
return null;
|
|
130
142
|
},
|
|
131
143
|
});
|