@pgflow/core 0.0.0-pgflow-installer-45a8ec76-20251211182851 → 0.0.0-pgflow-installer-249c293d-20251211201430

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/CHANGELOG.md +2 -2
  2. package/dist/package.json +1 -5
  3. package/package.json +2 -6
  4. package/dist/migrations/_generated/m_20250429164909.d.ts +0 -3
  5. package/dist/migrations/_generated/m_20250429164909.d.ts.map +0 -1
  6. package/dist/migrations/_generated/m_20250429164909.js +0 -584
  7. package/dist/migrations/_generated/m_20250517072017.d.ts +0 -3
  8. package/dist/migrations/_generated/m_20250517072017.d.ts.map +0 -1
  9. package/dist/migrations/_generated/m_20250517072017.js +0 -106
  10. package/dist/migrations/_generated/m_20250609105135.d.ts +0 -3
  11. package/dist/migrations/_generated/m_20250609105135.d.ts.map +0 -1
  12. package/dist/migrations/_generated/m_20250609105135.js +0 -376
  13. package/dist/migrations/_generated/m_20250610180554.d.ts +0 -3
  14. package/dist/migrations/_generated/m_20250610180554.d.ts.map +0 -1
  15. package/dist/migrations/_generated/m_20250610180554.js +0 -132
  16. package/dist/migrations/_generated/m_20250614124241.d.ts +0 -3
  17. package/dist/migrations/_generated/m_20250614124241.d.ts.map +0 -1
  18. package/dist/migrations/_generated/m_20250614124241.js +0 -506
  19. package/dist/migrations/_generated/m_20250619195327.d.ts +0 -3
  20. package/dist/migrations/_generated/m_20250619195327.d.ts.map +0 -1
  21. package/dist/migrations/_generated/m_20250619195327.js +0 -190
  22. package/dist/migrations/_generated/m_20250627090700.d.ts +0 -3
  23. package/dist/migrations/_generated/m_20250627090700.d.ts.map +0 -1
  24. package/dist/migrations/_generated/m_20250627090700.js +0 -11
  25. package/dist/migrations/_generated/m_20250707210212.d.ts +0 -3
  26. package/dist/migrations/_generated/m_20250707210212.d.ts.map +0 -1
  27. package/dist/migrations/_generated/m_20250707210212.js +0 -108
  28. package/dist/migrations/_generated/m_20250719205006.d.ts +0 -3
  29. package/dist/migrations/_generated/m_20250719205006.d.ts.map +0 -1
  30. package/dist/migrations/_generated/m_20250719205006.js +0 -7
  31. package/dist/migrations/_generated/m_20251006073122.d.ts +0 -3
  32. package/dist/migrations/_generated/m_20251006073122.d.ts.map +0 -1
  33. package/dist/migrations/_generated/m_20251006073122.js +0 -1249
  34. package/dist/migrations/_generated/m_20251103222045.d.ts +0 -3
  35. package/dist/migrations/_generated/m_20251103222045.d.ts.map +0 -1
  36. package/dist/migrations/_generated/m_20251103222045.js +0 -627
  37. package/dist/migrations/_generated/m_20251104080523.d.ts +0 -3
  38. package/dist/migrations/_generated/m_20251104080523.d.ts.map +0 -1
  39. package/dist/migrations/_generated/m_20251104080523.js +0 -98
  40. package/dist/migrations/_generated/m_20251130000000.d.ts +0 -3
  41. package/dist/migrations/_generated/m_20251130000000.d.ts.map +0 -1
  42. package/dist/migrations/_generated/m_20251130000000.js +0 -273
  43. package/dist/migrations/_generated/m_20251209074533.d.ts +0 -3
  44. package/dist/migrations/_generated/m_20251209074533.d.ts.map +0 -1
  45. package/dist/migrations/_generated/m_20251209074533.js +0 -278
  46. package/dist/migrations/index.d.ts +0 -11
  47. package/dist/migrations/index.d.ts.map +0 -1
  48. package/dist/migrations/index.js +0 -39
  49. package/dist/migrations/types.d.ts +0 -12
  50. package/dist/migrations/types.d.ts.map +0 -1
  51. package/dist/migrations/types.js +0 -1
@@ -1,106 +0,0 @@
1
- export const migration = {
2
- timestamp: '20250517072017',
3
- filename: '20250517072017_pgflow_fix_poll_for_tasks_to_use_separate_statement_for_polling.sql',
4
- content: `-- Modify "poll_for_tasks" function
5
- CREATE OR REPLACE FUNCTION "pgflow"."poll_for_tasks" ("queue_name" text, "vt" integer, "qty" integer, "max_poll_seconds" integer DEFAULT 5, "poll_interval_ms" integer DEFAULT 100) RETURNS SETOF "pgflow"."step_task_record" LANGUAGE plpgsql SET "search_path" = '' AS $$
6
- declare
7
- msg_ids bigint[];
8
- begin
9
- -- First statement: Read messages and capture their IDs
10
- -- This gets its own snapshot and can see newly committed messages
11
- select array_agg(msg_id)
12
- into msg_ids
13
- from pgflow.read_with_poll(
14
- queue_name,
15
- vt,
16
- qty,
17
- max_poll_seconds,
18
- poll_interval_ms
19
- );
20
-
21
- -- If no messages were read, return empty set
22
- if msg_ids is null or array_length(msg_ids, 1) is null then
23
- return;
24
- end if;
25
-
26
- -- Second statement: Process tasks with fresh snapshot
27
- -- This can now see step_tasks that were committed during the poll
28
- return query
29
- with tasks as (
30
- select
31
- task.flow_slug,
32
- task.run_id,
33
- task.step_slug,
34
- task.task_index,
35
- task.message_id
36
- from pgflow.step_tasks as task
37
- where task.message_id = any(msg_ids)
38
- and task.status = 'queued'
39
- ),
40
- increment_attempts as (
41
- update pgflow.step_tasks
42
- set attempts_count = attempts_count + 1
43
- from tasks
44
- where step_tasks.message_id = tasks.message_id
45
- and status = 'queued'
46
- ),
47
- runs as (
48
- select
49
- r.run_id,
50
- r.input
51
- from pgflow.runs r
52
- where r.run_id in (select run_id from tasks)
53
- ),
54
- deps as (
55
- select
56
- st.run_id,
57
- st.step_slug,
58
- dep.dep_slug,
59
- dep_task.output as dep_output
60
- from tasks st
61
- join pgflow.deps dep on dep.flow_slug = st.flow_slug and dep.step_slug = st.step_slug
62
- join pgflow.step_tasks dep_task on
63
- dep_task.run_id = st.run_id and
64
- dep_task.step_slug = dep.dep_slug and
65
- dep_task.status = 'completed'
66
- ),
67
- deps_outputs as (
68
- select
69
- d.run_id,
70
- d.step_slug,
71
- jsonb_object_agg(d.dep_slug, d.dep_output) as deps_output
72
- from deps d
73
- group by d.run_id, d.step_slug
74
- ),
75
- timeouts as (
76
- select
77
- task.message_id,
78
- coalesce(step.opt_timeout, flow.opt_timeout) + 2 as vt_delay
79
- from tasks task
80
- join pgflow.flows flow on flow.flow_slug = task.flow_slug
81
- join pgflow.steps step on step.flow_slug = task.flow_slug and step.step_slug = task.step_slug
82
- )
83
- select
84
- st.flow_slug,
85
- st.run_id,
86
- st.step_slug,
87
- jsonb_build_object('run', r.input) ||
88
- coalesce(dep_out.deps_output, '{}'::jsonb) as input,
89
- st.message_id as msg_id
90
- from tasks st
91
- join runs r on st.run_id = r.run_id
92
- left join deps_outputs dep_out on
93
- dep_out.run_id = st.run_id and
94
- dep_out.step_slug = st.step_slug
95
- cross join lateral (
96
- -- TODO: this is slow because it calls set_vt for each row, and set_vt
97
- -- builds dynamic query from string every time it is called
98
- -- implement set_vt_batch(msgs_ids bigint[], vt_delays int[])
99
- select pgmq.set_vt(queue_name, st.message_id,
100
- (select t.vt_delay from timeouts t where t.message_id = st.message_id)
101
- )
102
- ) set_vt;
103
- end;
104
- $$;
105
- `,
106
- };
@@ -1,3 +0,0 @@
1
- import type { Migration } from '../types.js';
2
- export declare const migration: Migration;
3
- //# sourceMappingURL=m_20250609105135.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"m_20250609105135.d.ts","sourceRoot":"","sources":["../../../src/migrations/_generated/m_20250609105135.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,eAAO,MAAM,SAAS,EAAE,SAuXvB,CAAC"}
@@ -1,376 +0,0 @@
1
- export const migration = {
2
- timestamp: '20250609105135',
3
- filename: '20250609105135_pgflow_add_start_tasks_and_started_status.sql',
4
- content: `-- Create index "idx_workers_heartbeat" to table: "workers"
5
- create index "idx_workers_heartbeat" on "pgflow"."workers" ("last_heartbeat_at");
6
- -- Modify "step_tasks" table
7
- alter table "pgflow"."step_tasks" drop constraint "valid_status",
8
- add constraint "valid_status" check (
9
- status = ANY(array['queued'::text, 'started'::text, 'completed'::text, 'failed'::text])
10
- ),
11
- add constraint "completed_at_is_after_started_at" check (
12
- (completed_at is null) or (started_at is null) or (completed_at >= started_at)
13
- ),
14
- add constraint "failed_at_is_after_started_at" check (
15
- (failed_at is null) or (started_at is null) or (failed_at >= started_at)
16
- ),
17
- add constraint "started_at_is_after_queued_at" check ((started_at is null) or (started_at >= queued_at)),
18
- add column "started_at" timestamptz null,
19
- add column "last_worker_id" uuid null,
20
- add constraint "step_tasks_last_worker_id_fkey" foreign key ("last_worker_id") references "pgflow"."workers" (
21
- "worker_id"
22
- ) on update no action on delete set null;
23
- -- Create index "idx_step_tasks_last_worker" to table: "step_tasks"
24
- create index "idx_step_tasks_last_worker" on "pgflow"."step_tasks" ("last_worker_id") where (status = 'started'::text);
25
- -- Create index "idx_step_tasks_queued_msg" to table: "step_tasks"
26
- create index "idx_step_tasks_queued_msg" on "pgflow"."step_tasks" ("message_id") where (status = 'queued'::text);
27
- -- Create index "idx_step_tasks_started" to table: "step_tasks"
28
- create index "idx_step_tasks_started" on "pgflow"."step_tasks" ("started_at") where (status = 'started'::text);
29
- -- Modify "complete_task" function
30
- create or replace function "pgflow"."complete_task"(
31
- "run_id" uuid, "step_slug" text, "task_index" integer, "output" jsonb
32
- ) returns setof "pgflow"."step_tasks" language plpgsql set "search_path"
33
- = '' as $$
34
- begin
35
-
36
- WITH run_lock AS (
37
- SELECT * FROM pgflow.runs
38
- WHERE pgflow.runs.run_id = complete_task.run_id
39
- FOR UPDATE
40
- ),
41
- step_lock AS (
42
- SELECT * FROM pgflow.step_states
43
- WHERE pgflow.step_states.run_id = complete_task.run_id
44
- AND pgflow.step_states.step_slug = complete_task.step_slug
45
- FOR UPDATE
46
- ),
47
- task AS (
48
- UPDATE pgflow.step_tasks
49
- SET
50
- status = 'completed',
51
- completed_at = now(),
52
- output = complete_task.output
53
- WHERE pgflow.step_tasks.run_id = complete_task.run_id
54
- AND pgflow.step_tasks.step_slug = complete_task.step_slug
55
- AND pgflow.step_tasks.task_index = complete_task.task_index
56
- AND pgflow.step_tasks.status = 'started'
57
- RETURNING *
58
- ),
59
- step_state AS (
60
- UPDATE pgflow.step_states
61
- SET
62
- status = CASE
63
- WHEN pgflow.step_states.remaining_tasks = 1 THEN 'completed' -- Will be 0 after decrement
64
- ELSE 'started'
65
- END,
66
- completed_at = CASE
67
- WHEN pgflow.step_states.remaining_tasks = 1 THEN now() -- Will be 0 after decrement
68
- ELSE NULL
69
- END,
70
- remaining_tasks = pgflow.step_states.remaining_tasks - 1
71
- FROM task
72
- WHERE pgflow.step_states.run_id = complete_task.run_id
73
- AND pgflow.step_states.step_slug = complete_task.step_slug
74
- RETURNING pgflow.step_states.*
75
- ),
76
- -- Find all dependent steps if the current step was completed
77
- dependent_steps AS (
78
- SELECT d.step_slug AS dependent_step_slug
79
- FROM pgflow.deps d
80
- JOIN step_state s ON s.status = 'completed' AND d.flow_slug = s.flow_slug
81
- WHERE d.dep_slug = complete_task.step_slug
82
- ORDER BY d.step_slug -- Ensure consistent ordering
83
- ),
84
- -- Lock dependent steps before updating
85
- dependent_steps_lock AS (
86
- SELECT * FROM pgflow.step_states
87
- WHERE pgflow.step_states.run_id = complete_task.run_id
88
- AND pgflow.step_states.step_slug IN (SELECT dependent_step_slug FROM dependent_steps)
89
- FOR UPDATE
90
- ),
91
- -- Update all dependent steps
92
- dependent_steps_update AS (
93
- UPDATE pgflow.step_states
94
- SET remaining_deps = pgflow.step_states.remaining_deps - 1
95
- FROM dependent_steps
96
- WHERE pgflow.step_states.run_id = complete_task.run_id
97
- AND pgflow.step_states.step_slug = dependent_steps.dependent_step_slug
98
- )
99
- -- Only decrement remaining_steps, don't update status
100
- UPDATE pgflow.runs
101
- SET remaining_steps = pgflow.runs.remaining_steps - 1
102
- FROM step_state
103
- WHERE pgflow.runs.run_id = complete_task.run_id
104
- AND step_state.status = 'completed';
105
-
106
- -- For completed tasks: archive the message
107
- PERFORM (
108
- WITH completed_tasks AS (
109
- SELECT r.flow_slug, st.message_id
110
- FROM pgflow.step_tasks st
111
- JOIN pgflow.runs r ON st.run_id = r.run_id
112
- WHERE st.run_id = complete_task.run_id
113
- AND st.step_slug = complete_task.step_slug
114
- AND st.task_index = complete_task.task_index
115
- AND st.status = 'completed'
116
- )
117
- SELECT pgmq.archive(ct.flow_slug, ct.message_id)
118
- FROM completed_tasks ct
119
- WHERE EXISTS (SELECT 1 FROM completed_tasks)
120
- );
121
-
122
- PERFORM pgflow.start_ready_steps(complete_task.run_id);
123
-
124
- PERFORM pgflow.maybe_complete_run(complete_task.run_id);
125
-
126
- RETURN QUERY SELECT *
127
- FROM pgflow.step_tasks AS step_task
128
- WHERE step_task.run_id = complete_task.run_id
129
- AND step_task.step_slug = complete_task.step_slug
130
- AND step_task.task_index = complete_task.task_index;
131
-
132
- end;
133
- $$;
134
- -- Modify "fail_task" function
135
- create or replace function "pgflow"."fail_task"(
136
- "run_id" uuid, "step_slug" text, "task_index" integer, "error_message" text
137
- ) returns setof "pgflow"."step_tasks" language plpgsql set "search_path"
138
- = '' as $$
139
- begin
140
-
141
- WITH run_lock AS (
142
- SELECT * FROM pgflow.runs
143
- WHERE pgflow.runs.run_id = fail_task.run_id
144
- FOR UPDATE
145
- ),
146
- step_lock AS (
147
- SELECT * FROM pgflow.step_states
148
- WHERE pgflow.step_states.run_id = fail_task.run_id
149
- AND pgflow.step_states.step_slug = fail_task.step_slug
150
- FOR UPDATE
151
- ),
152
- flow_info AS (
153
- SELECT r.flow_slug
154
- FROM pgflow.runs r
155
- WHERE r.run_id = fail_task.run_id
156
- ),
157
- config AS (
158
- SELECT
159
- COALESCE(s.opt_max_attempts, f.opt_max_attempts) AS opt_max_attempts,
160
- COALESCE(s.opt_base_delay, f.opt_base_delay) AS opt_base_delay
161
- FROM pgflow.steps s
162
- JOIN pgflow.flows f ON f.flow_slug = s.flow_slug
163
- JOIN flow_info fi ON fi.flow_slug = s.flow_slug
164
- WHERE s.flow_slug = fi.flow_slug AND s.step_slug = fail_task.step_slug
165
- ),
166
-
167
- fail_or_retry_task as (
168
- UPDATE pgflow.step_tasks as task
169
- SET
170
- status = CASE
171
- WHEN task.attempts_count < (SELECT opt_max_attempts FROM config) THEN 'queued'
172
- ELSE 'failed'
173
- END,
174
- failed_at = CASE
175
- WHEN task.attempts_count >= (SELECT opt_max_attempts FROM config) THEN now()
176
- ELSE NULL
177
- END,
178
- started_at = CASE
179
- WHEN task.attempts_count < (SELECT opt_max_attempts FROM config) THEN NULL
180
- ELSE task.started_at
181
- END,
182
- error_message = fail_task.error_message
183
- WHERE task.run_id = fail_task.run_id
184
- AND task.step_slug = fail_task.step_slug
185
- AND task.task_index = fail_task.task_index
186
- AND task.status = 'started'
187
- RETURNING *
188
- ),
189
- maybe_fail_step AS (
190
- UPDATE pgflow.step_states
191
- SET
192
- status = CASE
193
- WHEN (select fail_or_retry_task.status from fail_or_retry_task) = 'failed' THEN 'failed'
194
- ELSE pgflow.step_states.status
195
- END,
196
- failed_at = CASE
197
- WHEN (select fail_or_retry_task.status from fail_or_retry_task) = 'failed' THEN now()
198
- ELSE NULL
199
- END
200
- FROM fail_or_retry_task
201
- WHERE pgflow.step_states.run_id = fail_task.run_id
202
- AND pgflow.step_states.step_slug = fail_task.step_slug
203
- RETURNING pgflow.step_states.*
204
- )
205
- UPDATE pgflow.runs
206
- SET status = CASE
207
- WHEN (select status from maybe_fail_step) = 'failed' THEN 'failed'
208
- ELSE status
209
- END,
210
- failed_at = CASE
211
- WHEN (select status from maybe_fail_step) = 'failed' THEN now()
212
- ELSE NULL
213
- END
214
- WHERE pgflow.runs.run_id = fail_task.run_id;
215
-
216
- -- For queued tasks: delay the message for retry with exponential backoff
217
- PERFORM (
218
- WITH retry_config AS (
219
- SELECT
220
- COALESCE(s.opt_base_delay, f.opt_base_delay) AS base_delay
221
- FROM pgflow.steps s
222
- JOIN pgflow.flows f ON f.flow_slug = s.flow_slug
223
- JOIN pgflow.runs r ON r.flow_slug = f.flow_slug
224
- WHERE r.run_id = fail_task.run_id
225
- AND s.step_slug = fail_task.step_slug
226
- ),
227
- queued_tasks AS (
228
- SELECT
229
- r.flow_slug,
230
- st.message_id,
231
- pgflow.calculate_retry_delay((SELECT base_delay FROM retry_config), st.attempts_count) AS calculated_delay
232
- FROM pgflow.step_tasks st
233
- JOIN pgflow.runs r ON st.run_id = r.run_id
234
- WHERE st.run_id = fail_task.run_id
235
- AND st.step_slug = fail_task.step_slug
236
- AND st.task_index = fail_task.task_index
237
- AND st.status = 'queued'
238
- )
239
- SELECT pgmq.set_vt(qt.flow_slug, qt.message_id, qt.calculated_delay)
240
- FROM queued_tasks qt
241
- WHERE EXISTS (SELECT 1 FROM queued_tasks)
242
- );
243
-
244
- -- For failed tasks: archive the message
245
- PERFORM (
246
- WITH failed_tasks AS (
247
- SELECT r.flow_slug, st.message_id
248
- FROM pgflow.step_tasks st
249
- JOIN pgflow.runs r ON st.run_id = r.run_id
250
- WHERE st.run_id = fail_task.run_id
251
- AND st.step_slug = fail_task.step_slug
252
- AND st.task_index = fail_task.task_index
253
- AND st.status = 'failed'
254
- )
255
- SELECT pgmq.archive(ft.flow_slug, ft.message_id)
256
- FROM failed_tasks ft
257
- WHERE EXISTS (SELECT 1 FROM failed_tasks)
258
- );
259
-
260
- return query select *
261
- from pgflow.step_tasks st
262
- where st.run_id = fail_task.run_id
263
- and st.step_slug = fail_task.step_slug
264
- and st.task_index = fail_task.task_index;
265
-
266
- end;
267
- $$;
268
- -- Modify "poll_for_tasks" function
269
- create or replace function "pgflow"."poll_for_tasks"(
270
- "queue_name" text,
271
- "vt" integer,
272
- "qty" integer,
273
- "max_poll_seconds" integer default 5,
274
- "poll_interval_ms" integer default 100
275
- ) returns setof "pgflow"."step_task_record" language plpgsql set "search_path"
276
- = '' as $$
277
- begin
278
- -- DEPRECATED: This function is deprecated and will be removed in a future version.
279
- -- Please update pgflow to use the new two-phase polling approach.
280
- -- Run 'npx pgflow install' to update your installation.
281
- raise notice 'DEPRECATED: poll_for_tasks is deprecated and will be removed. Please update pgflow via "npx pgflow install".';
282
-
283
- -- Return empty set - no tasks will be processed
284
- return;
285
- end;
286
- $$;
287
- -- Create "start_tasks" function
288
- create function "pgflow"."start_tasks"(
289
- "flow_slug" text, "msg_ids" bigint [], "worker_id" uuid
290
- ) returns setof "pgflow"."step_task_record" language sql set "search_path"
291
- = '' as $$
292
- with tasks as (
293
- select
294
- task.flow_slug,
295
- task.run_id,
296
- task.step_slug,
297
- task.task_index,
298
- task.message_id
299
- from pgflow.step_tasks as task
300
- where task.flow_slug = start_tasks.flow_slug
301
- and task.message_id = any(msg_ids)
302
- and task.status = 'queued'
303
- ),
304
- start_tasks_update as (
305
- update pgflow.step_tasks
306
- set
307
- attempts_count = attempts_count + 1,
308
- status = 'started',
309
- started_at = now(),
310
- last_worker_id = worker_id
311
- from tasks
312
- where step_tasks.message_id = tasks.message_id
313
- and step_tasks.flow_slug = tasks.flow_slug
314
- and step_tasks.status = 'queued'
315
- ),
316
- runs as (
317
- select
318
- r.run_id,
319
- r.input
320
- from pgflow.runs r
321
- where r.run_id in (select run_id from tasks)
322
- ),
323
- deps as (
324
- select
325
- st.run_id,
326
- st.step_slug,
327
- dep.dep_slug,
328
- dep_task.output as dep_output
329
- from tasks st
330
- join pgflow.deps dep on dep.flow_slug = st.flow_slug and dep.step_slug = st.step_slug
331
- join pgflow.step_tasks dep_task on
332
- dep_task.run_id = st.run_id and
333
- dep_task.step_slug = dep.dep_slug and
334
- dep_task.status = 'completed'
335
- ),
336
- deps_outputs as (
337
- select
338
- d.run_id,
339
- d.step_slug,
340
- jsonb_object_agg(d.dep_slug, d.dep_output) as deps_output
341
- from deps d
342
- group by d.run_id, d.step_slug
343
- ),
344
- timeouts as (
345
- select
346
- task.message_id,
347
- task.flow_slug,
348
- coalesce(step.opt_timeout, flow.opt_timeout) + 2 as vt_delay
349
- from tasks task
350
- join pgflow.flows flow on flow.flow_slug = task.flow_slug
351
- join pgflow.steps step on step.flow_slug = task.flow_slug and step.step_slug = task.step_slug
352
- )
353
- select
354
- st.flow_slug,
355
- st.run_id,
356
- st.step_slug,
357
- jsonb_build_object('run', r.input) ||
358
- coalesce(dep_out.deps_output, '{}'::jsonb) as input,
359
- st.message_id as msg_id
360
- from tasks st
361
- join runs r on st.run_id = r.run_id
362
- left join deps_outputs dep_out on
363
- dep_out.run_id = st.run_id and
364
- dep_out.step_slug = st.step_slug
365
- cross join lateral (
366
- -- TODO: this is slow because it calls set_vt for each row, and set_vt
367
- -- builds dynamic query from string every time it is called
368
- -- implement set_vt_batch(msgs_ids bigint[], vt_delays int[])
369
- select pgmq.set_vt(t.flow_slug, st.message_id, t.vt_delay)
370
- from timeouts t
371
- where t.message_id = st.message_id
372
- and t.flow_slug = st.flow_slug
373
- ) set_vt
374
- $$;
375
- `,
376
- };
@@ -1,3 +0,0 @@
1
- import type { Migration } from '../types.js';
2
- export declare const migration: Migration;
3
- //# sourceMappingURL=m_20250610180554.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"m_20250610180554.d.ts","sourceRoot":"","sources":["../../../src/migrations/_generated/m_20250610180554.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,eAAO,MAAM,SAAS,EAAE,SAmIvB,CAAC"}
@@ -1,132 +0,0 @@
1
- export const migration = {
2
- timestamp: '20250610180554',
3
- filename: '20250610180554_pgflow_add_set_vt_batch_and_use_it_in_start_tasks.sql',
4
- content: `-- Create "set_vt_batch" function
5
- CREATE FUNCTION "pgflow"."set_vt_batch" ("queue_name" text, "msg_ids" bigint[], "vt_offsets" integer[]) RETURNS SETOF pgmq.message_record LANGUAGE plpgsql AS $$
6
- DECLARE
7
- qtable TEXT := pgmq.format_table_name(queue_name, 'q');
8
- sql TEXT;
9
- BEGIN
10
- /* ---------- safety checks ---------------------------------------------------- */
11
- IF msg_ids IS NULL OR vt_offsets IS NULL OR array_length(msg_ids, 1) = 0 THEN
12
- RETURN; -- nothing to do, return empty set
13
- END IF;
14
-
15
- IF array_length(msg_ids, 1) IS DISTINCT FROM array_length(vt_offsets, 1) THEN
16
- RAISE EXCEPTION
17
- 'msg_ids length (%) must equal vt_offsets length (%)',
18
- array_length(msg_ids, 1), array_length(vt_offsets, 1);
19
- END IF;
20
-
21
- /* ---------- dynamic statement ------------------------------------------------ */
22
- /* One UPDATE joins with the unnested arrays */
23
- sql := format(
24
- $FMT$
25
- WITH input (msg_id, vt_offset) AS (
26
- SELECT unnest($1)::bigint
27
- , unnest($2)::int
28
- )
29
- UPDATE pgmq.%I q
30
- SET vt = clock_timestamp() + make_interval(secs => input.vt_offset),
31
- read_ct = read_ct -- no change, but keeps RETURNING list aligned
32
- FROM input
33
- WHERE q.msg_id = input.msg_id
34
- RETURNING q.msg_id,
35
- q.read_ct,
36
- q.enqueued_at,
37
- q.vt,
38
- q.message
39
- $FMT$,
40
- qtable
41
- );
42
-
43
- RETURN QUERY EXECUTE sql USING msg_ids, vt_offsets;
44
- END;
45
- $$;
46
- -- Modify "start_tasks" function
47
- CREATE OR REPLACE FUNCTION "pgflow"."start_tasks" ("flow_slug" text, "msg_ids" bigint[], "worker_id" uuid) RETURNS SETOF "pgflow"."step_task_record" LANGUAGE sql SET "search_path" = '' AS $$
48
- with tasks as (
49
- select
50
- task.flow_slug,
51
- task.run_id,
52
- task.step_slug,
53
- task.task_index,
54
- task.message_id
55
- from pgflow.step_tasks as task
56
- where task.flow_slug = start_tasks.flow_slug
57
- and task.message_id = any(msg_ids)
58
- and task.status = 'queued'
59
- ),
60
- start_tasks_update as (
61
- update pgflow.step_tasks
62
- set
63
- attempts_count = attempts_count + 1,
64
- status = 'started',
65
- started_at = now(),
66
- last_worker_id = worker_id
67
- from tasks
68
- where step_tasks.message_id = tasks.message_id
69
- and step_tasks.flow_slug = tasks.flow_slug
70
- and step_tasks.status = 'queued'
71
- ),
72
- runs as (
73
- select
74
- r.run_id,
75
- r.input
76
- from pgflow.runs r
77
- where r.run_id in (select run_id from tasks)
78
- ),
79
- deps as (
80
- select
81
- st.run_id,
82
- st.step_slug,
83
- dep.dep_slug,
84
- dep_task.output as dep_output
85
- from tasks st
86
- join pgflow.deps dep on dep.flow_slug = st.flow_slug and dep.step_slug = st.step_slug
87
- join pgflow.step_tasks dep_task on
88
- dep_task.run_id = st.run_id and
89
- dep_task.step_slug = dep.dep_slug and
90
- dep_task.status = 'completed'
91
- ),
92
- deps_outputs as (
93
- select
94
- d.run_id,
95
- d.step_slug,
96
- jsonb_object_agg(d.dep_slug, d.dep_output) as deps_output
97
- from deps d
98
- group by d.run_id, d.step_slug
99
- ),
100
- timeouts as (
101
- select
102
- task.message_id,
103
- task.flow_slug,
104
- coalesce(step.opt_timeout, flow.opt_timeout) + 2 as vt_delay
105
- from tasks task
106
- join pgflow.flows flow on flow.flow_slug = task.flow_slug
107
- join pgflow.steps step on step.flow_slug = task.flow_slug and step.step_slug = task.step_slug
108
- ),
109
- -- Batch update visibility timeouts for all messages
110
- set_vt_batch as (
111
- select pgflow.set_vt_batch(
112
- start_tasks.flow_slug,
113
- array_agg(t.message_id order by t.message_id),
114
- array_agg(t.vt_delay order by t.message_id)
115
- )
116
- from timeouts t
117
- )
118
- select
119
- st.flow_slug,
120
- st.run_id,
121
- st.step_slug,
122
- jsonb_build_object('run', r.input) ||
123
- coalesce(dep_out.deps_output, '{}'::jsonb) as input,
124
- st.message_id as msg_id
125
- from tasks st
126
- join runs r on st.run_id = r.run_id
127
- left join deps_outputs dep_out on
128
- dep_out.run_id = st.run_id and
129
- dep_out.step_slug = st.step_slug
130
- $$;
131
- `,
132
- };
@@ -1,3 +0,0 @@
1
- import type { Migration } from '../types.js';
2
- export declare const migration: Migration;
3
- //# sourceMappingURL=m_20250614124241.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"m_20250614124241.d.ts","sourceRoot":"","sources":["../../../src/migrations/_generated/m_20250614124241.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,eAAO,MAAM,SAAS,EAAE,SAyfvB,CAAC"}