@constructive-io/knative-job-worker 0.7.15 → 0.8.1

package/README.md

@@ -1,3 +1,61 @@
 # knative-job-worker
 
 Knative-compatible job worker that uses the existing Constructive PostgreSQL job queue and job utilities, invoking HTTP functions via `KNATIVE_SERVICE_URL` (or `INTERNAL_GATEWAY_URL` as a fallback) while preserving the same headers and payload shape as the OpenFaaS worker.
+
+---
+
+## Education and Tutorials
+
+ 1. 🚀 [Quickstart: Getting Up and Running](https://constructive.io/learn/quickstart)
+Get started with modular databases in minutes. Install prerequisites and deploy your first module.
+
+ 2. 📦 [Modular PostgreSQL Development with Database Packages](https://constructive.io/learn/modular-postgres)
+Learn to organize PostgreSQL projects with pgpm workspaces and reusable database modules.
+
+ 3. ✏️ [Authoring Database Changes](https://constructive.io/learn/authoring-database-changes)
+Master the workflow for adding, organizing, and managing database changes with pgpm.
+
+ 4. 🧪 [End-to-End PostgreSQL Testing with TypeScript](https://constructive.io/learn/e2e-postgres-testing)
+Master end-to-end PostgreSQL testing with ephemeral databases, RLS testing, and CI/CD automation.
+
+ 5. ⚡ [Supabase Testing](https://constructive.io/learn/supabase)
+Use TypeScript-first tools to test Supabase projects with realistic RLS, policies, and auth contexts.
+
+ 6. 💧 [Drizzle ORM Testing](https://constructive.io/learn/drizzle-testing)
+Run full-stack tests with Drizzle ORM, including database setup, teardown, and RLS enforcement.
+
+ 7. 🔧 [Troubleshooting](https://constructive.io/learn/troubleshooting)
+Common issues and solutions for pgpm, PostgreSQL, and testing.
+
+## Related Constructive Tooling
+
+### 📦 Package Management
+
+* [pgpm](https://github.com/constructive-io/constructive/tree/main/pgpm/pgpm): **🖥️ PostgreSQL Package Manager** for modular Postgres development. Works with database workspaces, scaffolding, migrations, seeding, and installing database packages.
+
+### 🧪 Testing
+
+* [pgsql-test](https://github.com/constructive-io/constructive/tree/main/postgres/pgsql-test): **📊 Isolated testing environments** with per-test transaction rollbacks—ideal for integration tests, complex migrations, and RLS simulation.
+* [pgsql-seed](https://github.com/constructive-io/constructive/tree/main/postgres/pgsql-seed): **🌱 PostgreSQL seeding utilities** for CSV, JSON, SQL data loading, and pgpm deployment.
+* [supabase-test](https://github.com/constructive-io/constructive/tree/main/postgres/supabase-test): **🧪 Supabase-native test harness** preconfigured for the local Supabase stack—per-test rollbacks, JWT/role context helpers, and CI/GitHub Actions ready.
+* [graphile-test](https://github.com/constructive-io/constructive/tree/main/graphile/graphile-test): **🔐 Authentication mocking** for Graphile-focused test helpers and emulating row-level security contexts.
+* [pg-query-context](https://github.com/constructive-io/constructive/tree/main/postgres/pg-query-context): **🔒 Session context injection** to add session-local context (e.g., `SET LOCAL`) into queries—ideal for setting `role`, `jwt.claims`, and other session settings.
+
+### 🧠 Parsing & AST
+
+* [pgsql-parser](https://www.npmjs.com/package/pgsql-parser): **🔄 SQL conversion engine** that interprets and converts PostgreSQL syntax.
+* [libpg-query-node](https://www.npmjs.com/package/libpg-query): **🌉 Node.js bindings** for `libpg_query`, converting SQL into parse trees.
+* [pg-proto-parser](https://www.npmjs.com/package/pg-proto-parser): **📦 Protobuf parser** for parsing PostgreSQL Protocol Buffers definitions to generate TypeScript interfaces, utility functions, and JSON mappings for enums.
+* [@pgsql/enums](https://www.npmjs.com/package/@pgsql/enums): **🏷️ TypeScript enums** for PostgreSQL AST for safe and ergonomic parsing logic.
+* [@pgsql/types](https://www.npmjs.com/package/@pgsql/types): **📝 Type definitions** for PostgreSQL AST nodes in TypeScript.
+* [@pgsql/utils](https://www.npmjs.com/package/@pgsql/utils): **🛠️ AST utilities** for constructing and transforming PostgreSQL syntax trees.
+
+## Credits
+
+**🛠 Built by the [Constructive](https://constructive.io) team — creators of modular Postgres tooling for secure, composable backends. If you like our work, contribute on [GitHub](https://github.com/constructive-io).**
+
+## Disclaimer
+
+AS DESCRIBED IN THE LICENSES, THE SOFTWARE IS PROVIDED "AS IS", AT YOUR OWN RISK, AND WITHOUT WARRANTIES OF ANY KIND.
+
+No developer or entity involved in creating this software will be liable for any claims or damages whatsoever associated with your use, inability to use, or your interaction with other users of the code, including any direct, indirect, incidental, special, exemplary, punitive or consequential damages, or loss of profits, cryptocurrencies, tokens, or anything else of value.

package/esm/index.js

@@ -0,0 +1,208 @@
+import poolManager from '@constructive-io/job-pg';
+import * as jobs from '@constructive-io/job-utils';
+import { Logger } from '@pgpmjs/logger';
+import { request as req } from './req';
+const log = new Logger('jobs:worker');
+export default class Worker {
+    idleDelay;
+    supportedTaskNames;
+    workerId;
+    doNextTimer;
+    pgPool;
+    _initialized;
+    listenClient;
+    listenRelease;
+    stopped;
+    constructor({ tasks, idleDelay = 15000, pgPool = poolManager.getPool(), workerId = 'worker-0' }) {
+        /*
+         * idleDelay: This is how long to wait between polling for jobs.
+         *
+         * Note: this does NOT need to be short, because we use LISTEN/NOTIFY to be
+         * notified when new jobs are added - this is just used in the case where
+         * LISTEN/NOTIFY fails for whatever reason.
+         */
+        this.idleDelay = idleDelay;
+        this.supportedTaskNames = tasks;
+        this.workerId = workerId;
+        this.doNextTimer = undefined;
+        this.pgPool = pgPool;
+        poolManager.onClose(async () => {
+            await jobs.releaseJobs(pgPool, { workerId: this.workerId });
+        });
+    }
+    async initialize(client) {
+        if (this._initialized === true)
+            return;
+        // release any jobs not finished from before if fatal error prevented cleanup
+        await jobs.releaseJobs(client, { workerId: this.workerId });
+        this._initialized = true;
+        await this.doNext(client);
+    }
+    async handleFatalError(client, { err, fatalError, jobId }) {
+        const when = err ? `after failure '${err.message}'` : 'after success';
+        log.error(`Failed to release job '${jobId}' ${when}; committing seppuku`);
+        await poolManager.close();
+        log.error(String(fatalError));
+        process.exit(1);
+    }
+    async handleError(client, { err, job, duration }) {
+        log.error(`Failed task ${job.id} (${job.task_identifier}) with error ${err.message} (${duration}ms)`);
+        if (err.stack) {
+            log.debug(err.stack);
+        }
+        await jobs.failJob(client, {
+            workerId: this.workerId,
+            jobId: job.id,
+            message: err.message
+        });
+    }
+    async handleSuccess(client, { job, duration }) {
+        log.info(`Async task ${job.id} (${job.task_identifier}) to be processed`);
+    }
+    async doWork(job) {
+        const { payload, task_identifier } = job;
+        log.debug('starting work on job', {
+            id: job.id,
+            task: task_identifier,
+            databaseId: job.database_id
+        });
+        if (!jobs.getJobSupportAny() &&
+            !this.supportedTaskNames.includes(task_identifier)) {
+            throw new Error('Unsupported task');
+        }
+        await req(task_identifier, {
+            body: payload,
+            databaseId: job.database_id,
+            workerId: this.workerId,
+            jobId: job.id
+        });
+    }
+    async doNext(client) {
+        if (this.stopped)
+            return;
+        if (!this._initialized) {
+            return await this.initialize(client);
+        }
+        log.debug('checking for jobs...');
+        if (this.doNextTimer) {
+            clearTimeout(this.doNextTimer);
+            this.doNextTimer = undefined;
+        }
+        try {
+            const job = (await jobs.getJob(client, {
+                workerId: this.workerId,
+                supportedTaskNames: jobs.getJobSupportAny()
+                    ? null
+                    : this.supportedTaskNames
+            }));
+            if (!job || !job.id) {
+                if (!this.stopped) {
+                    this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+                }
+                return;
+            }
+            const start = process.hrtime();
+            let err = null;
+            try {
+                await this.doWork(job);
+            }
+            catch (error) {
+                err = error;
+            }
+            const durationRaw = process.hrtime(start);
+            const duration = ((durationRaw[0] * 1e9 + durationRaw[1]) / 1e6).toFixed(2);
+            const jobId = job.id;
+            try {
+                if (err) {
+                    await this.handleError(client, { err, job, duration });
+                }
+                else {
+                    await this.handleSuccess(client, { job, duration });
+                }
+            }
+            catch (fatalError) {
+                await this.handleFatalError(client, { err, fatalError, jobId });
+            }
+            if (!this.stopped) {
+                return this.doNext(client);
+            }
+            return;
+        }
+        catch (err) {
+            if (!this.stopped) {
+                this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+            }
+        }
+    }
+    listen() {
+        if (this.stopped)
+            return;
+        const listenForChanges = (err, client, release) => {
+            if (err) {
+                log.error('Error connecting with notify listener', err);
+                if (err instanceof Error && err.stack) {
+                    log.debug(err.stack);
+                }
+                // Try again in 5 seconds
+                // should this really be done in the node process?
+                if (!this.stopped) {
+                    setTimeout(this.listen, 5000);
+                }
+                return;
+            }
+            if (this.stopped) {
+                release();
+                return;
+            }
+            this.listenClient = client;
+            this.listenRelease = release;
+            client.on('notification', () => {
+                if (this.doNextTimer) {
+                    // Must be idle, do something!
+                    this.doNext(client);
+                }
+            });
+            client.query('LISTEN "jobs:insert"');
+            client.on('error', (e) => {
+                if (this.stopped) {
+                    release();
+                    return;
+                }
+                log.error('Error with database notify listener', e);
+                if (e instanceof Error && e.stack) {
+                    log.debug(e.stack);
+                }
+                release();
+                if (!this.stopped) {
+                    this.listen();
+                }
+            });
+            log.info(`${this.workerId} connected and looking for jobs...`);
+            this.doNext(client);
+        };
+        this.pgPool.connect(listenForChanges);
+    }
+    async stop() {
+        this.stopped = true;
+        if (this.doNextTimer) {
+            clearTimeout(this.doNextTimer);
+            this.doNextTimer = undefined;
+        }
+        const client = this.listenClient;
+        const release = this.listenRelease;
+        this.listenClient = undefined;
+        this.listenRelease = undefined;
+        if (client && release) {
+            client.removeAllListeners('notification');
+            client.removeAllListeners('error');
+            try {
+                await client.query('UNLISTEN "jobs:insert"');
+            }
+            catch {
+                // Ignore listener cleanup errors during shutdown.
+            }
+            release();
+        }
+    }
+}
+export { Worker };

package/esm/req.js

@@ -0,0 +1,55 @@
+import requestLib from 'request';
+import { getCallbackBaseUrl, getJobGatewayConfig, getJobGatewayDevMap, getNodeEnvironment } from '@constructive-io/job-utils';
+import { Logger } from '@pgpmjs/logger';
+const log = new Logger('jobs:req');
+// callback URL for job completion
+const completeUrl = getCallbackBaseUrl();
+// Development override map (e.g. point a function name at localhost)
+const nodeEnv = getNodeEnvironment();
+const DEV_MAP = nodeEnv !== 'production' ? getJobGatewayDevMap() : null;
+const getFunctionUrl = (fn) => {
+    if (DEV_MAP && DEV_MAP[fn]) {
+        return DEV_MAP[fn] || completeUrl;
+    }
+    const { gatewayUrl } = getJobGatewayConfig();
+    const base = gatewayUrl.replace(/\/$/, '');
+    return `${base}/${fn}`;
+};
+const request = (fn, { body, databaseId, workerId, jobId }) => {
+    const url = getFunctionUrl(fn);
+    log.info(`dispatching job`, {
+        fn,
+        url,
+        callbackUrl: completeUrl,
+        workerId,
+        jobId,
+        databaseId
+    });
+    return new Promise((resolve, reject) => {
+        requestLib.post({
+            headers: {
+                'Content-Type': 'application/json',
+                // these are used by job-worker/job-fn
+                'X-Worker-Id': workerId,
+                'X-Job-Id': jobId,
+                'X-Database-Id': databaseId,
+                // async HTTP completion callback
+                'X-Callback-Url': completeUrl
+            },
+            url,
+            json: true,
+            body
+        }, function (error) {
+            if (error) {
+                log.error(`request error for job[${jobId}] fn[${fn}]`, error);
+                if (error instanceof Error && error.stack) {
+                    log.debug(error.stack);
+                }
+                return reject(error);
+            }
+            log.debug(`request success for job[${jobId}] fn[${fn}]`);
+            return resolve(true);
+        });
+    });
+};
+export { request };

package/{src/run.ts → esm/run.js}

@@ -1,18 +1,11 @@
 #!/usr/bin/env node
-
 import Worker from './index';
 import poolManager from '@constructive-io/job-pg';
-import {
-  getWorkerHostname,
-  getJobSupported
-} from '@constructive-io/job-utils';
-
+import { getWorkerHostname, getJobSupported } from '@constructive-io/job-utils';
 const pgPool = poolManager.getPool();
-
 const worker = new Worker({
-  pgPool,
-  workerId: getWorkerHostname(),
-  tasks: getJobSupported()
+    pgPool,
+    workerId: getWorkerHostname(),
+    tasks: getJobSupported()
 });
-
 worker.listen();

package/{dist/index.d.ts → index.d.ts}

@@ -1,5 +1,5 @@
 import type { PgClientLike } from '@constructive-io/job-utils';
-import type { Pool } from 'pg';
+import type { Pool, PoolClient } from 'pg';
 export interface JobRow {
     id: number | string;
     task_identifier: string;
@@ -13,6 +13,9 @@ export default class Worker {
     doNextTimer?: NodeJS.Timeout;
     pgPool: Pool;
     _initialized?: boolean;
+    listenClient?: PoolClient;
+    listenRelease?: () => void;
+    stopped?: boolean;
     constructor({ tasks, idleDelay, pgPool, workerId }: {
         tasks: string[];
         idleDelay?: number;
@@ -37,5 +40,6 @@ export default class Worker {
     doWork(job: JobRow): Promise<void>;
     doNext(client: PgClientLike): Promise<void>;
     listen(): void;
+    stop(): Promise<void>;
 }
 export { Worker };

package/{dist/index.js → index.js}

@@ -49,6 +49,9 @@ class Worker {
     doNextTimer;
     pgPool;
     _initialized;
+    listenClient;
+    listenRelease;
+    stopped;
     constructor({ tasks, idleDelay = 15000, pgPool = job_pg_1.default.getPool(), workerId = 'worker-0' }) {
         /*
          * idleDelay: This is how long to wait between polling for jobs.
@@ -114,6 +117,8 @@ class Worker {
         });
     }
     async doNext(client) {
+        if (this.stopped)
+            return;
         if (!this._initialized) {
             return await this.initialize(client);
         }
@@ -130,7 +135,9 @@ class Worker {
                     : this.supportedTaskNames
             }));
             if (!job || !job.id) {
-                this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+                if (!this.stopped) {
+                    this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+                }
                 return;
             }
             const start = process.hrtime();
@@ -155,13 +162,20 @@ class Worker {
             catch (fatalError) {
                 await this.handleFatalError(client, { err, fatalError, jobId });
             }
-            return this.doNext(client);
+            if (!this.stopped) {
+                return this.doNext(client);
+            }
+            return;
         }
         catch (err) {
-            this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+            if (!this.stopped) {
+                this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+            }
         }
     }
     listen() {
+        if (this.stopped)
+            return;
         const listenForChanges = (err, client, release) => {
             if (err) {
                 log.error('Error connecting with notify listener', err);
@@ -170,9 +184,17 @@ class Worker {
                 }
                 // Try again in 5 seconds
                 // should this really be done in the node process?
-                setTimeout(this.listen, 5000);
+                if (!this.stopped) {
+                    setTimeout(this.listen, 5000);
+                }
+                return;
+            }
+            if (this.stopped) {
+                release();
                 return;
             }
+            this.listenClient = client;
+            this.listenRelease = release;
             client.on('notification', () => {
                 if (this.doNextTimer) {
                     // Must be idle, do something!
@@ -181,18 +203,46 @@ class Worker {
             });
             client.query('LISTEN "jobs:insert"');
             client.on('error', (e) => {
+                if (this.stopped) {
+                    release();
+                    return;
+                }
                 log.error('Error with database notify listener', e);
                 if (e instanceof Error && e.stack) {
                     log.debug(e.stack);
                 }
                 release();
-                this.listen();
+                if (!this.stopped) {
+                    this.listen();
+                }
             });
             log.info(`${this.workerId} connected and looking for jobs...`);
             this.doNext(client);
         };
         this.pgPool.connect(listenForChanges);
     }
+    async stop() {
+        this.stopped = true;
+        if (this.doNextTimer) {
+            clearTimeout(this.doNextTimer);
+            this.doNextTimer = undefined;
+        }
+        const client = this.listenClient;
+        const release = this.listenRelease;
+        this.listenClient = undefined;
+        this.listenRelease = undefined;
+        if (client && release) {
+            client.removeAllListeners('notification');
+            client.removeAllListeners('error');
+            try {
+                await client.query('UNLISTEN "jobs:insert"');
+            }
+            catch {
+                // Ignore listener cleanup errors during shutdown.
+            }
+            release();
+        }
+    }
 }
 exports.default = Worker;
 exports.Worker = Worker;

package/package.json

@@ -1,21 +1,24 @@
 {
   "name": "@constructive-io/knative-job-worker",
-  "version": "0.7.15",
+  "version": "0.8.1",
   "description": "knative job worker",
   "author": "Constructive <developers@constructive.io>",
   "homepage": "https://github.com/constructive-io/jobs/tree/master/packages/knative-job-worker#readme",
   "license": "SEE LICENSE IN LICENSE",
-  "main": "dist/index.js",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
   "directories": {
     "lib": "src",
     "test": "__tests__"
   },
   "bin": {
-    "faas-jobs": "src/run.ts",
-    "knative-jobs": "src/run.ts"
+    "faas-jobs": "run.js",
+    "knative-jobs": "run.js"
   },
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "directory": "dist"
   },
   "repository": {
     "type": "git",
@@ -25,21 +28,27 @@
     "test": "jest --passWithNoTests",
     "test:watch": "jest --watch",
     "test:debug": "node --inspect node_modules/.bin/jest --runInBand",
-    "build": "tsc -p tsconfig.json",
-    "build:watch": "tsc -p tsconfig.json -w"
+    "clean": "makage clean",
+    "prepack": "npm run build",
+    "build": "makage build",
+    "build:dev": "makage build --dev"
   },
   "bugs": {
     "url": "https://github.com/constructive-io/jobs/issues"
   },
   "dependencies": {
-    "@constructive-io/job-pg": "^0.3.20",
-    "@constructive-io/job-utils": "^0.5.15",
-    "@pgpmjs/logger": "^1.3.7",
-    "pg": "8.16.3",
+    "@constructive-io/job-pg": "^0.4.1",
+    "@constructive-io/job-utils": "^0.6.1",
+    "@pgpmjs/logger": "^1.4.0",
+    "pg": "8.17.1",
     "request": "2.88.2"
   },
   "devDependencies": {
-    "pgsql-test": "^2.24.15"
+    "@pgpm/database-jobs": "^0.16.0",
+    "@pgpm/verify": "^0.16.0",
+    "@pgpmjs/core": "^4.16.1",
+    "makage": "^0.1.10",
+    "pgsql-test": "^2.25.1"
   },
-  "gitHead": "b9719cd48487af70e6e065510197c9d2d6b331b6"
+  "gitHead": "3ffd5718e86ea5fa9ca6e0930aeb510cf392f343"
 }