@pgpm/database-jobs 0.4.0

package/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Dan Lynch <pyramation@gmail.com>
+Copyright (c) 2025 Interweb, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/Makefile

@@ -0,0 +1,6 @@
+EXTENSION = launchql-database-jobs
+DATA = sql/launchql-database-jobs--0.4.6.sql
+
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

package/README.md

@@ -0,0 +1,5 @@
+# @pgpm/database-jobs
+
+Database-specific job handling and queue management.
+
+Extends the core jobs system with database-specific functionality for managing background tasks and job processing workflows.

package/__tests__/__snapshots__/jobs.test.ts.snap

@@ -0,0 +1,19 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`scheduled jobs schedule jobs 1`] = `
+{
+  "attempts": 0,
+  "database_id": "5b720132-17d5-424d-9bcb-ee7b17c13d43",
+  "id": "1",
+  "key": null,
+  "last_error": null,
+  "locked_at": null,
+  "locked_by": null,
+  "max_attempts": 25,
+  "payload": {
+    "just": "run it",
+  },
+  "priority": 0,
+  "task_identifier": "my_job",
+}
+`;

package/__tests__/jobs.test.ts

@@ -0,0 +1,138 @@
+import { getConnections, PgTestClient } from 'pgsql-test';
+
+let pg: PgTestClient;
+let teardown: () => Promise<void>;
+
+const database_id = '5b720132-17d5-424d-9bcb-ee7b17c13d43';
+const objs: Record<string, any> = {};
+
+describe('scheduled jobs', () => {
+  beforeAll(async () => {
+    ({ pg, teardown } = await getConnections());
+  });
+
+  afterAll(async () => {
+    await teardown();
+  });
+
+  it('schedule jobs by cron', async () => {
+    const result = await pg.one(
+      `INSERT INTO app_jobs.scheduled_jobs (database_id, task_identifier, schedule_info)
+       VALUES ($1, $2, $3)
+       RETURNING *`,
+      [
+        database_id,
+        'my_job',
+        {
+          hour: Array.from({ length: 23 }, (_, i) => i),
+          minute: [0, 15, 30, 45],
+          dayOfWeek: Array.from({ length: 6 }, (_, i) => i)
+        }
+      ]
+    );
+    objs.scheduled1 = result;
+  });
+
+  it('schedule jobs by rule', async () => {
+    const start = new Date(Date.now() + 10000); // 10s from now
+    const end = new Date(start.getTime() + 180000); // +3min
+
+    const result = await pg.one(
+      `INSERT INTO app_jobs.scheduled_jobs (database_id, task_identifier, payload, schedule_info)
+       VALUES ($1, $2, $3, $4)
+       RETURNING *`,
+      [
+        database_id,
+        'my_job',
+        { just: 'run it' },
+        { start, end, rule: '*/1 * * * *' }
+      ]
+    );
+    objs.scheduled2 = result;
+  });
+
+  it('schedule jobs', async () => {
+    const [result] = await pg.any(
+      `SELECT * FROM app_jobs.run_scheduled_job($1)`,
+      [objs.scheduled2.id]
+    );
+
+    const { queue_name, run_at, created_at, updated_at, ...obj } = result;
+    expect(obj).toMatchSnapshot();
+  });
+
+  it('schedule jobs with keys', async () => {
+    const start = new Date(Date.now() + 10000); // 10s
+    const end = new Date(start.getTime() + 180000); // +3min
+
+    const [result] = await pg.any(
+      `SELECT * FROM app_jobs.add_scheduled_job(
+        db_id := $1::uuid,
+        identifier := $2::text,
+        payload := $3::json,
+        schedule_info := $4::json,
+        job_key := $5::text,
+        queue_name := $6::text,
+        max_attempts := $7::integer,
+        priority := $8::integer
+      )`,
+      [
+        database_id,
+        'my_job',
+        { just: 'run it' },
+        { start, end, rule: '*/1 * * * *' },
+        'new_key',
+        null,
+        25,
+        0
+      ]
+    );
+
+    const {
+      queue_name,
+      run_at,
+      created_at,
+      updated_at,
+      schedule_info: sch,
+      start: s1,
+      end: d1,
+      ...obj
+    } = result;
+
+    const [result2] = await pg.any(
+      `SELECT * FROM app_jobs.add_scheduled_job(
+        db_id := $1,
+        identifier := $2,
+        payload := $3,
+        schedule_info := $4,
+        job_key := $5,
+        queue_name := $6,
+        max_attempts := $7,
+        priority := $8
+      )`,
+      [
+        database_id,
+        'my_job',
+        { just: 'run it' },
+        { start, end, rule: '*/1 * * * *' },
+        'new_key',
+        null,
+        25,
+        0
+      ]
+    );
+
+    const {
+      queue_name: qn,
+      created_at: ca,
+      updated_at: ua,
+      schedule_info: sch2,
+      start: s,
+      end: e,
+      ...obj2
+    } = result2;
+
+    console.log('First insert:', obj);
+    console.log('Duplicate insert (job_key conflict):', obj2);
+  });
+});

package/deploy/schemas/app_jobs/helpers/json_build_object_apply.sql

@@ -0,0 +1,28 @@
+-- Deploy schemas/app_jobs/helpers/json_build_object_apply to pg
+-- requires: schemas/app_jobs/schema
+
+BEGIN;
+CREATE FUNCTION app_jobs.json_build_object_apply (arguments text[])
+  RETURNS json
+  AS $$
+DECLARE
+  arg text;
+  _sql text;
+  _res json;
+  args text[];
+BEGIN
+  _sql = 'SELECT json_build_object(';
+  FOR arg IN
+  SELECT
+    unnest(arguments)
+    LOOP
+      args = array_append(args, format('''%s''', arg));
+    END LOOP;
+  _sql = _sql || format('%s);', array_to_string(args, ','));
+  EXECUTE _sql INTO _res;
+  RETURN _res;
+END;
+$$
+LANGUAGE 'plpgsql';
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/add_job.sql

@@ -0,0 +1,103 @@
+-- Deploy schemas/app_jobs/procedures/add_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.add_job (
+  db_id uuid,
+  identifier text,
+  payload json DEFAULT '{}' ::json,
+  job_key text DEFAULT NULL,
+  queue_name text DEFAULT NULL,
+  run_at timestamptz DEFAULT now(),
+  max_attempts integer DEFAULT 25,
+  priority integer DEFAULT 0
+)
+  RETURNS app_jobs.jobs
+  AS $$
+DECLARE
+  v_job app_jobs.jobs;
+BEGIN
+  IF job_key IS NOT NULL THEN
+    -- Upsert job	
+    INSERT INTO app_jobs.jobs (
+      database_id,
+      task_identifier,
+      payload,
+      queue_name,
+      run_at,
+      max_attempts,
+      key,
+      priority
+    ) VALUES (
+        db_id,
+        identifier,
+        coalesce(payload,
+        '{}'::json),
+        queue_name,
+        coalesce(run_at, now()),
+        coalesce(max_attempts, 25),
+        job_key,
+        coalesce(priority, 0)
+    )	
+    ON CONFLICT (key)	
+      DO UPDATE SET	
+        task_identifier = EXCLUDED.task_identifier,
+        payload = EXCLUDED.payload,
+        queue_name = EXCLUDED.queue_name,
+        max_attempts = EXCLUDED.max_attempts,
+        run_at = EXCLUDED.run_at,
+        priority = EXCLUDED.priority,	
+        -- always reset error/retry state	
+        attempts = 0, last_error = NULL	
+      WHERE	
+        jobs.locked_at IS NULL	
+      RETURNING	
+        * INTO v_job;	
+
+    -- If upsert succeeded (insert or update), return early	
+    
+    IF NOT (v_job IS NULL) THEN	
+      RETURN v_job;	
+    END IF;	
+
+    -- Upsert failed -> there must be an existing job that is locked. Remove	
+    -- existing key to allow a new one to be inserted, and prevent any	
+    -- subsequent retries by bumping attempts to the max allowed.
+
+    UPDATE	
+      app_jobs.jobs	
+    SET	
+      KEY = NULL,	
+      attempts = jobs.max_attempts	
+    WHERE	
+      KEY = job_key;	
+  END IF;
+
+  INSERT INTO app_jobs.jobs (
+    database_id,
+    task_identifier,
+    payload,
+    queue_name,
+    run_at,
+    max_attempts,
+    priority
+  ) VALUES (
+    db_id,
+    identifier,
+    payload,
+    queue_name,
+    run_at,
+    max_attempts,
+    priority
+  )
+  RETURNING * INTO v_job;
+
+  RETURN v_job;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE SECURITY DEFINER;
+
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql

@@ -0,0 +1,97 @@
+-- Deploy schemas/app_jobs/procedures/add_scheduled_job to pg
+
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/scheduled_jobs/table
+
+BEGIN;
+
+CREATE FUNCTION app_jobs.add_scheduled_job(
+  db_id uuid,
+  identifier text,
+  payload json DEFAULT '{}'::json,
+  schedule_info json DEFAULT '{}'::json,
+  job_key text DEFAULT NULL,
+  queue_name text DEFAULT NULL,
+  max_attempts integer DEFAULT 25,
+  priority integer DEFAULT 0
+)
+  RETURNS app_jobs.scheduled_jobs
+  AS $$
+DECLARE
+  v_job app_jobs.scheduled_jobs;
+BEGIN
+  IF job_key IS NOT NULL THEN
+
+    -- Upsert job	
+    INSERT INTO app_jobs.scheduled_jobs (
+      database_id,
+      task_identifier,
+      payload,
+      queue_name,
+      schedule_info,
+      max_attempts,
+      key,
+      priority
+      ) VALUES (
+        db_id,
+        identifier,
+        coalesce(payload, '{}'::json),
+        queue_name,
+        schedule_info,
+        coalesce(max_attempts, 25),
+        job_key,
+        coalesce(priority, 0)
+    )	
+    ON CONFLICT (key)	
+      DO UPDATE SET	
+        task_identifier = EXCLUDED.task_identifier,
+        payload = EXCLUDED.payload,
+        queue_name = EXCLUDED.queue_name,
+        max_attempts = EXCLUDED.max_attempts,
+        schedule_info = EXCLUDED.schedule_info,
+        priority = EXCLUDED.priority
+      WHERE	
+        scheduled_jobs.locked_at IS NULL	
+      RETURNING	
+        * INTO v_job;	
+
+    -- If upsert succeeded (insert or update), return early	
+    
+    IF NOT (v_job IS NULL) THEN	
+      RETURN v_job;	
+    END IF;	
+
+    -- Upsert failed -> there must be an existing scheduled job that is locked. Remove	
+    -- and allow a new one to be inserted
+
+    DELETE FROM	
+      app_jobs.scheduled_jobs	
+    WHERE	
+      KEY = job_key;	
+  END IF;
+
+  INSERT INTO app_jobs.scheduled_jobs (
+    database_id,
+    task_identifier,
+    payload,
+    queue_name,
+    schedule_info,
+    max_attempts,
+    priority
+    ) VALUES (
+    db_id,
+    identifier,
+    payload,
+    queue_name,
+    schedule_info,
+    max_attempts,
+    priority
+  ) RETURNING * INTO v_job;
+  RETURN v_job;
+END;
+$$
+LANGUAGE 'plpgsql'
+VOLATILE
+SECURITY DEFINER;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/complete_job.sql

@@ -0,0 +1,32 @@
+-- Deploy schemas/app_jobs/procedures/complete_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.complete_job (worker_id text, job_id bigint)
+  RETURNS app_jobs.jobs
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+  v_row app_jobs.jobs;
+BEGIN
+  DELETE FROM app_jobs.jobs
+  WHERE id = job_id
+  RETURNING
+    * INTO v_row;
+  IF v_row.queue_name IS NOT NULL THEN
+    UPDATE
+      app_jobs.job_queues
+    SET
+      locked_by = NULL,
+      locked_at = NULL
+    WHERE
+      queue_name = v_row.queue_name
+      AND locked_by = worker_id;
+  END IF;
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/complete_jobs.sql

@@ -0,0 +1,19 @@
+-- Deploy schemas/app_jobs/procedures/complete_jobs to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/job_queues/table
+-- requires: schemas/app_jobs/tables/jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.complete_jobs (job_ids bigint[])
+  RETURNS SETOF app_jobs.jobs
+  LANGUAGE sql
+  AS $$
+  DELETE FROM app_jobs.jobs
+  WHERE id = ANY (job_ids)
+    AND (locked_by IS NULL
+      OR locked_at < NOW() - interval '4 hours')
+  RETURNING
+    *;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/do_notify.sql

@@ -0,0 +1,16 @@
+-- Deploy schemas/app_jobs/procedures/do_notify to pg
+-- requires: schemas/app_jobs/schema
+
+BEGIN;
+CREATE FUNCTION app_jobs.do_notify ()
+  RETURNS TRIGGER
+  AS $$
+BEGIN
+  PERFORM
+    pg_notify(TG_ARGV[0], '');
+  RETURN NEW;
+END;
+$$
+LANGUAGE plpgsql;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/fail_job.sql

@@ -0,0 +1,41 @@
+-- Deploy schemas/app_jobs/procedures/fail_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/jobs/table
+-- requires: schemas/app_jobs/tables/job_queues/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.fail_job (worker_id text, job_id bigint, error_message text)
+  RETURNS app_jobs.jobs
+  LANGUAGE plpgsql
+  STRICT
+  AS $$
+DECLARE
+  v_row app_jobs.jobs;
+BEGIN
+  UPDATE
+    app_jobs.jobs
+  SET
+    last_error = error_message,
+    run_at = greatest (now(), run_at) + (exp(least (attempts, 10))::text || ' seconds')::interval,
+    locked_by = NULL,
+    locked_at = NULL
+  WHERE
+    id = job_id
+    AND locked_by = worker_id
+  RETURNING
+    * INTO v_row;
+  IF v_row.queue_name IS NOT NULL THEN
+    UPDATE
+      app_jobs.job_queues
+    SET
+      locked_by = NULL,
+      locked_at = NULL
+    WHERE
+      queue_name = v_row.queue_name
+      AND locked_by = worker_id;
+  END IF;
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/get_job.sql

@@ -0,0 +1,92 @@
+-- Deploy schemas/app_jobs/procedures/get_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/job_queues/table
+-- requires: schemas/app_jobs/tables/jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.get_job (worker_id text, task_identifiers text[] DEFAULT NULL, job_expiry interval DEFAULT '4 hours')
+  RETURNS app_jobs.jobs
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+  v_job_id bigint;
+  v_queue_name text;
+  v_row app_jobs.jobs;
+  v_now timestamptz = now();
+BEGIN
+
+  IF worker_id IS NULL THEN
+    RAISE exception 'INVALID_WORKER_ID';
+  END IF;
+
+  --
+
+  SELECT
+    jobs.queue_name,
+    jobs.id INTO v_queue_name,
+    v_job_id
+  FROM
+    app_jobs.jobs
+  WHERE (jobs.locked_at IS NULL
+    OR jobs.locked_at < (v_now - job_expiry))
+    AND (jobs.queue_name IS NULL
+      OR EXISTS (
+        SELECT
+          1
+        FROM
+          app_jobs.job_queues
+        WHERE
+          job_queues.queue_name = jobs.queue_name
+          AND (job_queues.locked_at IS NULL
+            OR job_queues.locked_at < (v_now - job_expiry))
+        FOR UPDATE
+          SKIP LOCKED))
+    AND run_at <= v_now
+    AND attempts < max_attempts
+    AND (task_identifiers IS NULL
+      OR task_identifier = ANY (task_identifiers))
+  ORDER BY
+    priority ASC,
+    run_at ASC,
+    id ASC
+  LIMIT 1
+  FOR UPDATE
+    SKIP LOCKED;
+
+  --
+
+  IF v_job_id IS NULL THEN
+    RETURN NULL;
+  END IF;
+
+  --
+
+  IF v_queue_name IS NOT NULL THEN
+    UPDATE
+      app_jobs.job_queues
+    SET
+      locked_by = worker_id,
+      locked_at = v_now
+    WHERE
+      job_queues.queue_name = v_queue_name;
+  END IF;
+
+  --
+
+  UPDATE
+    app_jobs.jobs
+  SET
+    attempts = attempts + 1,
+    locked_by = worker_id,
+    locked_at = v_now
+  WHERE
+    id = v_job_id
+  RETURNING
+    * INTO v_row;
+
+  --
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+

package/deploy/schemas/app_jobs/procedures/get_scheduled_job.sql

@@ -0,0 +1,61 @@
+-- Deploy schemas/app_jobs/procedures/get_scheduled_job to pg
+-- requires: schemas/app_jobs/schema
+-- requires: schemas/app_jobs/tables/scheduled_jobs/table
+
+BEGIN;
+CREATE FUNCTION app_jobs.get_scheduled_job (worker_id text, task_identifiers text[] DEFAULT NULL)
+  RETURNS app_jobs.scheduled_jobs
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+  v_job_id bigint;
+  v_row app_jobs.scheduled_jobs;
+BEGIN
+
+  --
+
+  IF worker_id IS NULL THEN
+    RAISE exception 'INVALID_WORKER_ID';
+  END IF;
+
+  --
+
+  SELECT
+    scheduled_jobs.id INTO v_job_id
+  FROM
+    app_jobs.scheduled_jobs
+  WHERE (scheduled_jobs.locked_at IS NULL)
+    AND (task_identifiers IS NULL
+      OR task_identifier = ANY (task_identifiers))
+  ORDER BY
+    priority ASC,
+    id ASC
+  LIMIT 1
+  FOR UPDATE
+    SKIP LOCKED;
+
+  --
+
+  IF v_job_id IS NULL THEN
+    RETURN NULL;
+  END IF;
+
+  --
+
+  UPDATE
+    app_jobs.scheduled_jobs
+  SET
+    locked_by = worker_id,
+    locked_at = NOW()
+  WHERE
+    id = v_job_id
+  RETURNING
+    * INTO v_row;
+
+  --
+
+  RETURN v_row;
+END;
+$$;
+COMMIT;
+