@pgpm/jobs 0.4.0

package/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Dan Lynch <pyramation@gmail.com>
+Copyright (c) 2025 Interweb, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/Makefile

@@ -0,0 +1,6 @@
+EXTENSION = launchql-jobs
+DATA = sql/launchql-jobs--0.4.6.sql
+
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

package/README.md

@@ -0,0 +1,5 @@
+# @pgpm/jobs
+
+Core job system for background task processing in PostgreSQL.
+
+Provides job queue management, task scheduling, and background processing capabilities for PostgreSQL applications.

package/__tests__/__snapshots__/jobs.test.ts.snap

@@ -0,0 +1,16 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`scheduled jobs schedule jobs 1`] = `
+{
+  "attempts": null,
+  "id": null,
+  "key": null,
+  "last_error": null,
+  "locked_at": null,
+  "locked_by": null,
+  "max_attempts": null,
+  "payload": null,
+  "priority": null,
+  "task_identifier": null,
+}
+`;

package/__tests__/jobs.test.ts

@@ -0,0 +1,139 @@
+import { getConnections, PgTestClient } from 'pgsql-test';
+
+let pg: PgTestClient;
+let teardown:  () => Promise<void>;
+
+const objs: Record<string, any> = {};
+
+describe('scheduled jobs', () => {
+  beforeAll(async () => {
+    ({ pg, teardown } = await getConnections());
+  });
+
+  beforeEach(async () => {
+    await pg.beforeEach();
+  });
+
+  afterEach(async () => {
+    await pg.afterEach();
+  });
+
+  afterAll(async () => {
+    await teardown();
+  });
+
+  it('schedule jobs by cron', async () => {
+    const result = await pg.one(
+      `INSERT INTO app_jobs.scheduled_jobs (task_identifier, schedule_info)
+       VALUES ($1, $2)
+       RETURNING *`,
+      [
+        'my_job',
+        {
+          hour: Array.from({ length: 23 }, (_: unknown, i: number) => i),
+          minute: [0, 15, 30, 45],
+          dayOfWeek: Array.from({ length: 6 }, (_: unknown, i: number) => i),
+        },
+      ]
+    );
+    objs.scheduled1 = result;
+  });
+
+  it('schedule jobs by rule', async () => {
+    const start = new Date(Date.now() + 10000);
+    const end = new Date(start.getTime() + 180000);
+
+    const result = await pg.one(
+      `INSERT INTO app_jobs.scheduled_jobs (task_identifier, payload, schedule_info)
+       VALUES ($1, $2, $3)
+       RETURNING *`,
+      [
+        'my_job',
+        { just: 'run it' },
+        { start, end, rule: '*/1 * * * *' }
+      ]
+    );
+    objs.scheduled2 = result;
+  });
+
+  it('schedule jobs', async () => {
+    const [result] = await pg.any(
+      `SELECT * FROM app_jobs.run_scheduled_job($1)`,
+      [objs.scheduled2.id]
+    );
+
+    const { queue_name, run_at, created_at, updated_at, ...obj } = result;
+    expect(obj).toMatchSnapshot();
+  });
+
+  it('schedule jobs with keys', async () => {
+    const start = new Date(Date.now() + 10000);
+    const end = new Date(start.getTime() + 180000);
+
+    const [result] = await pg.any(
+      `SELECT * FROM app_jobs.add_scheduled_job(
+        identifier := $1::text,
+        payload := $2::json,
+        schedule_info := $3::json,
+        job_key := $4::text,
+        queue_name := $5::text,
+        max_attempts := $6::integer,
+        priority := $7::integer
+      )`,
+      [
+        'my_job',
+        { just: 'run it' },
+        { start, end, rule: '*/1 * * * *' },
+        'new_key',
+        null,
+        25,
+        0
+      ]
+    );
+
+    const {
+      queue_name,
+      run_at,
+      created_at,
+      updated_at,
+      schedule_info: sch,
+      start: s1,
+      end: d1,
+      ...obj
+    } = result;
+
+    const [result2] = await pg.any(
+      `SELECT * FROM app_jobs.add_scheduled_job(
+        identifier := $1,
+        payload := $2,
+        schedule_info := $3,
+        job_key := $4,
+        queue_name := $5,
+        max_attempts := $6,
+        priority := $7
+      )`,
+      [
+        'my_job',
+        { just: 'run it' },
+        { start, end, rule: '*/1 * * * *' },
+        'new_key',
+        null,
+        25,
+        0
+      ]
+    );
+
+    const {
+      queue_name: qn,
+      created_at: ca,
+      updated_at: ua,
+      schedule_info: sch2,
+      start: s,
+      end: e,
+      ...obj2
+    } = result2;
+
+    console.log('First insert:', obj);
+    console.log('Duplicate insert (job_key conflict):', obj2);
+  });
+});

package/deploy/schemas/app_jobs/helpers/json_build_object_apply.sql

@@ -0,0 +1,28 @@
+-- Deploy schemas/app_jobs/helpers/json_build_object_apply to pg
+-- requires: schemas/app_jobs/schema
+
+BEGIN;
+CREATE FUNCTION app_jobs.json_build_object_apply (arguments text[])
+  RETURNS json
+  AS $$
+DECLARE
+  arg text;
+  _sql text;
+  _res json;
+  args text[];
+BEGIN
+  _sql = 'SELECT json_build_object(';
+  FOR arg IN
+  SELECT
+    unnest(arguments)
+    LOOP
+      args = array_append(args, format('''%s''', arg));
+    END LOOP;
+  _sql = _sql || format('%s);', array_to_string(args, ','));
+  EXECUTE _sql INTO _res;
+  RETURN _res;
+END;
+$$
+LANGUAGE 'plpgsql';
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/add_job.sql

@@ -0,0 +1,65 @@
+-- Deploy schemas/app_jobs/procedures/add_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.add_job (
+  identifier text,
+  payload json DEFAULT '{}' ::json,
+  job_key text DEFAULT NULL,
+  queue_name text DEFAULT NULL,
+  run_at timestamptz DEFAULT now(),
+  max_attempts integer DEFAULT 25,
+  priority integer DEFAULT 0
+)
+  RETURNS app_jobs.jobs
+  AS $$
+DECLARE
+  v_job app_jobs.jobs;
+BEGIN
+  IF job_key IS NOT NULL THEN
+    -- Upsert job	
+    INSERT INTO app_jobs.jobs (task_identifier, payload, queue_name, run_at, max_attempts, KEY, priority)	
+      VALUES (identifier, coalesce(payload, '{}'::json), queue_name, coalesce(run_at, now()), coalesce(max_attempts, 25), job_key, coalesce(priority, 0))	
+    ON CONFLICT (KEY)	
+      DO UPDATE SET	
+        task_identifier = excluded.task_identifier, payload = excluded.payload, queue_name = excluded.queue_name, max_attempts = excluded.max_attempts, run_at = excluded.run_at, priority = excluded.priority,	
+        -- always reset error/retry state	
+        attempts = 0, last_error = NULL	
+      WHERE	
+        jobs.locked_at IS NULL	
+      RETURNING	
+        * INTO v_job;	
+
+    -- If upsert succeeded (insert or update), return early	
+    
+    IF NOT (v_job IS NULL) THEN	
+      RETURN v_job;	
+    END IF;	
+
+    -- Upsert failed -> there must be an existing job that is locked. Remove	
+    -- existing key to allow a new one to be inserted, and prevent any	
+    -- subsequent retries by bumping attempts to the max allowed.
+
+    UPDATE	
+      app_jobs.jobs	
+    SET	
+      KEY = NULL,	
+      attempts = jobs.max_attempts	
+    WHERE	
+      KEY = job_key;	
+  END IF;
+
+  INSERT INTO app_jobs.jobs (task_identifier, payload, queue_name, run_at, max_attempts, priority)
+    VALUES (identifier, payload, queue_name, run_at, max_attempts, priority)
+  RETURNING
+    * INTO v_job;
+  RETURN v_job;
+END;
+$$
+LANGUAGE 'plpgsql'
+VOLATILE
+SECURITY DEFINER;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql

@@ -0,0 +1,66 @@
+-- Deploy schemas/app_jobs/procedures/add_scheduled_job to pg
+
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/scheduled_jobs/table
+
+BEGIN;
+
+CREATE FUNCTION app_jobs.add_scheduled_job(
+  identifier text,
+  payload json DEFAULT '{}'::json,
+  schedule_info json DEFAULT '{}'::json,
+  job_key text DEFAULT NULL,
+  queue_name text DEFAULT NULL,
+  max_attempts integer DEFAULT 25,
+  priority integer DEFAULT 0
+)
+  RETURNS app_jobs.scheduled_jobs
+  AS $$
+DECLARE
+  v_job app_jobs.scheduled_jobs;
+BEGIN
+  IF job_key IS NOT NULL THEN
+
+    -- Upsert job	
+    INSERT INTO app_jobs.scheduled_jobs (task_identifier, payload, queue_name, schedule_info, max_attempts, KEY, priority)	
+      VALUES (identifier, coalesce(payload, '{}'::json), queue_name, schedule_info, coalesce(max_attempts, 25), job_key, coalesce(priority, 0))	
+    ON CONFLICT (KEY)	
+      DO UPDATE SET	
+        task_identifier = excluded.task_identifier,
+        payload = excluded.payload,
+        queue_name = excluded.queue_name,
+        max_attempts = excluded.max_attempts,
+        schedule_info = excluded.schedule_info,
+        priority = excluded.priority
+      WHERE	
+        scheduled_jobs.locked_at IS NULL	
+      RETURNING	
+        * INTO v_job;	
+
+    -- If upsert succeeded (insert or update), return early	
+    
+    IF NOT (v_job IS NULL) THEN	
+      RETURN v_job;	
+    END IF;	
+
+    -- Upsert failed -> there must be an existing scheduled job that is locked. Remove	
+    -- and allow a new one to be inserted
+
+    DELETE FROM	
+      app_jobs.scheduled_jobs	
+    WHERE	
+      KEY = job_key;	
+  END IF;
+
+  INSERT INTO app_jobs.scheduled_jobs (task_identifier, payload, queue_name, schedule_info, max_attempts, priority)
+    VALUES (identifier, payload, queue_name, schedule_info, max_attempts, priority)
+  RETURNING
+    * INTO v_job;
+  RETURN v_job;
+END;
+$$
+LANGUAGE 'plpgsql'
+VOLATILE
+SECURITY DEFINER;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/complete_job.sql

@@ -0,0 +1,32 @@
+-- Deploy schemas/app_jobs/procedures/complete_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.complete_job (worker_id text, job_id bigint)
+  RETURNS app_jobs.jobs
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+  v_row app_jobs.jobs;
+BEGIN
+  DELETE FROM app_jobs.jobs
+  WHERE id = job_id
+  RETURNING
+    * INTO v_row;
+  IF v_row.queue_name IS NOT NULL THEN
+    UPDATE
+      app_jobs.job_queues
+    SET
+      locked_by = NULL,
+      locked_at = NULL
+    WHERE
+      queue_name = v_row.queue_name
+      AND locked_by = worker_id;
+  END IF;
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/complete_jobs.sql

@@ -0,0 +1,19 @@
+-- Deploy schemas/app_jobs/procedures/complete_jobs to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/job_queues/table
+-- requires: schemas/app_jobs/tables/jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.complete_jobs (job_ids bigint[])
+  RETURNS SETOF app_jobs.jobs
+  LANGUAGE sql
+  AS $$
+  DELETE FROM app_jobs.jobs
+  WHERE id = ANY (job_ids)
+    AND (locked_by IS NULL
+      OR locked_at < NOW() - interval '4 hours')
+  RETURNING
+    *;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/do_notify.sql

@@ -0,0 +1,16 @@
+-- Deploy schemas/app_jobs/procedures/do_notify to pg
+-- requires: schemas/app_jobs/schema
+
+BEGIN;
+CREATE FUNCTION app_jobs.do_notify ()
+  RETURNS TRIGGER
+  AS $$
+BEGIN
+  PERFORM
+    pg_notify(TG_ARGV[0], '');
+  RETURN NEW;
+END;
+$$
+LANGUAGE plpgsql;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/fail_job.sql

@@ -0,0 +1,41 @@
+-- Deploy schemas/app_jobs/procedures/fail_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.fail_job (worker_id text, job_id bigint, error_message text)
+  RETURNS app_jobs.jobs
+  LANGUAGE plpgsql
+  STRICT
+  AS $$
+DECLARE
+  v_row app_jobs.jobs;
+BEGIN
+  UPDATE
+    app_jobs.jobs
+  SET
+    last_error = error_message,
+    run_at = greatest (now(), run_at) + (exp(least (attempts, 10))::text || ' seconds')::interval,
+    locked_by = NULL,
+    locked_at = NULL
+  WHERE
+    id = job_id
+    AND locked_by = worker_id
+  RETURNING
+    * INTO v_row;
+  IF v_row.queue_name IS NOT NULL THEN
+    UPDATE
+      app_jobs.job_queues
+    SET
+      locked_by = NULL,
+      locked_at = NULL
+    WHERE
+      queue_name = v_row.queue_name
+      AND locked_by = worker_id;
+  END IF;
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/get_job.sql

@@ -0,0 +1,77 @@
+-- Deploy schemas/app_jobs/procedures/get_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/job_queues/table
+-- requires: schemas/app_jobs/tables/jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.get_job (worker_id text, task_identifiers text[] DEFAULT NULL, job_expiry interval DEFAULT '4 hours')
+  RETURNS app_jobs.jobs
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+  v_job_id bigint;
+  v_queue_name text;
+  v_row app_jobs.jobs;
+  v_now timestamptz = now();
+BEGIN
+  IF worker_id IS NULL THEN
+    RAISE exception 'INVALID_WORKER_ID';
+  END IF;
+  SELECT
+    jobs.queue_name,
+    jobs.id INTO v_queue_name,
+    v_job_id
+  FROM
+    app_jobs.jobs
+  WHERE (jobs.locked_at IS NULL
+    OR jobs.locked_at < (v_now - job_expiry))
+    AND (jobs.queue_name IS NULL
+      OR EXISTS (
+        SELECT
+          1
+        FROM
+          app_jobs.job_queues
+        WHERE
+          job_queues.queue_name = jobs.queue_name
+          AND (job_queues.locked_at IS NULL
+            OR job_queues.locked_at < (v_now - job_expiry))
+        FOR UPDATE
+          SKIP LOCKED))
+    AND run_at <= v_now
+    AND attempts < max_attempts
+    AND (task_identifiers IS NULL
+      OR task_identifier = ANY (task_identifiers))
+  ORDER BY
+    priority ASC,
+    run_at ASC,
+    id ASC
+  LIMIT 1
+  FOR UPDATE
+    SKIP LOCKED;
+  IF v_job_id IS NULL THEN
+    RETURN NULL;
+  END IF;
+  IF v_queue_name IS NOT NULL THEN
+    UPDATE
+      app_jobs.job_queues
+    SET
+      locked_by = worker_id,
+      locked_at = v_now
+    WHERE
+      job_queues.queue_name = v_queue_name;
+  END IF;
+  UPDATE
+    app_jobs.jobs
+  SET
+    attempts = attempts + 1,
+    locked_by = worker_id,
+    locked_at = v_now
+  WHERE
+    id = v_job_id
+  RETURNING
+    * INTO v_row;
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/get_scheduled_job.sql

@@ -0,0 +1,46 @@
+-- Deploy schemas/app_jobs/procedures/get_scheduled_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/scheduled_jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.get_scheduled_job (worker_id text, task_identifiers text[] DEFAULT NULL)
+  RETURNS app_jobs.scheduled_jobs
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+  v_job_id bigint;
+  v_row app_jobs.scheduled_jobs;
+BEGIN
+  IF worker_id IS NULL THEN
+    RAISE exception 'INVALID_WORKER_ID';
+  END IF;
+  SELECT
+    scheduled_jobs.id INTO v_job_id
+  FROM
+    app_jobs.scheduled_jobs
+  WHERE (scheduled_jobs.locked_at IS NULL)
+    AND (task_identifiers IS NULL
+      OR task_identifier = ANY (task_identifiers))
+  ORDER BY
+    priority ASC,
+    id ASC
+  LIMIT 1
+  FOR UPDATE
+    SKIP LOCKED;
+  IF v_job_id IS NULL THEN
+    RETURN NULL;
+  END IF;
+  UPDATE
+    app_jobs.scheduled_jobs
+  SET
+    locked_by = worker_id,
+    locked_at = NOW()
+  WHERE
+    id = v_job_id
+  RETURNING
+    * INTO v_row;
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/permanently_fail_jobs.sql

@@ -0,0 +1,24 @@
+-- Deploy schemas/app_jobs/procedures/permanently_fail_jobs to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/job_queues/table
+-- requires: schemas/app_jobs/tables/jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.permanently_fail_jobs (job_ids bigint[], error_message text DEFAULT NULL)
+  RETURNS SETOF app_jobs.jobs
+  LANGUAGE sql
+  AS $$
+  UPDATE
+    app_jobs.jobs
+  SET
+    last_error = coalesce(error_message, 'Manually marked as failed'),
+    attempts = max_attempts
+  WHERE
+    id = ANY (job_ids)
+    AND (locked_by IS NULL
+      OR locked_at < NOW() - interval '4 hours')
+  RETURNING
+    *;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/release_jobs.sql

@@ -0,0 +1,34 @@
+-- Deploy schemas/app_jobs/procedures/release_jobs to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.release_jobs (worker_id text)
+  RETURNS void
+  AS $$
+DECLARE
+BEGIN
+  -- clear the job
+  UPDATE
+    app_jobs.jobs
+  SET
+    locked_at = NULL,
+    locked_by = NULL,
+    attempts = GREATEST (attempts - 1, 0)
+  WHERE
+    locked_by = worker_id;
+  -- clear the queue
+  UPDATE
+    app_jobs.job_queues
+  SET
+    locked_at = NULL,
+    locked_by = NULL
+  WHERE
+    locked_by = worker_id;
+END;
+$$
+LANGUAGE 'plpgsql'
+VOLATILE;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/release_scheduled_jobs.sql

@@ -0,0 +1,26 @@
+-- Deploy schemas/app_jobs/procedures/release_scheduled_jobs to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/scheduled_jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.release_scheduled_jobs (worker_id text, ids bigint[] DEFAULT NULL)
+  RETURNS void
+  AS $$
+DECLARE
+BEGIN
+  -- clear the scheduled job
+  UPDATE
+    app_jobs.scheduled_jobs s
+  SET
+    locked_at = NULL,
+    locked_by = NULL
+  WHERE
+    locked_by = worker_id
+    AND (ids IS NULL
+      OR s.id = ANY (ids));
+END;
+$$
+LANGUAGE 'plpgsql'
+VOLATILE;
+COMMIT;
+