@constructive-io/knative-job-worker 0.5.9

package/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+## 0.5.9 (2025-12-18)
+
+**Note:** Version bump only for package @constructive-io/knative-job-worker
+
+## [0.5.8](https://github.com/constructive-io/jobs/compare/@launchql/knative-job-worker@0.5.7...@launchql/knative-job-worker@0.5.8) (2025-12-17)
+
+**Note:** Version bump only for package @launchql/knative-job-worker

package/LICENSE

@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Dan Lynch <pyramation@gmail.com>
+Copyright (c) 2025 Constructive <developers@constructive.io>
+Copyright (c) 2020-present, Interweb, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,3 @@
+# knative-job-worker
+
+Knative-compatible job worker that uses the existing Constructive PostgreSQL job queue and job utilities, invoking HTTP functions via `KNATIVE_SERVICE_URL` (or `INTERNAL_GATEWAY_URL` as a fallback) while preserving the same headers and payload shape as the OpenFaaS worker.

package/__tests__/req.test.ts

@@ -0,0 +1,130 @@
+const postMock = jest.fn();
+
+jest.mock('request', () => ({
+  __esModule: true,
+  default: { post: postMock },
+  post: postMock
+}));
+
+describe('knative request wrapper', () => {
+  beforeEach(() => {
+    jest.resetModules();
+    postMock.mockReset();
+
+    process.env.PGUSER = 'postgres';
+    process.env.PGHOST = 'localhost';
+    process.env.PGPASSWORD = 'password';
+    process.env.PGPORT = '5432';
+    process.env.PGDATABASE = 'jobs';
+    process.env.JOBS_SCHEMA = 'app_jobs';
+    process.env.INTERNAL_JOBS_CALLBACK_URL =
+      'http://callback.internal/jobs-complete';
+    process.env.NODE_ENV = 'test';
+    delete process.env.INTERNAL_GATEWAY_URL;
+    delete process.env.KNATIVE_SERVICE_URL;
+    delete process.env.INTERNAL_GATEWAY_DEVELOPMENT_MAP;
+  });
+
+  it('uses KNATIVE_SERVICE_URL as base and preserves headers and body', async () => {
+    process.env.KNATIVE_SERVICE_URL = 'http://knative.internal';
+
+    postMock.mockImplementation(
+      (options: any, callback: (err: any) => void) => callback(null)
+    );
+
+    const { request } = await import('../src/req');
+
+    await request('example-fn', {
+      body: { value: 1 },
+      databaseId: 'db-123',
+      workerId: 'worker-1',
+      jobId: 42
+    });
+
+    expect(postMock).toHaveBeenCalledTimes(1);
+    const [options] = postMock.mock.calls[0];
+
+    expect(options.url).toBe('http://knative.internal/example-fn');
+    expect(options.headers['Content-Type']).toBe('application/json');
+    expect(options.headers['X-Worker-Id']).toBe('worker-1');
+    expect(options.headers['X-Job-Id']).toBe(42);
+    expect(options.headers['X-Database-Id']).toBe('db-123');
+    expect(options.headers['X-Callback-Url']).toBe(
+      'http://callback.internal/jobs-complete'
+    );
+    expect(options.body).toEqual({ value: 1 });
+  });
+
+  it('falls back to INTERNAL_GATEWAY_URL when KNATIVE_SERVICE_URL is not set', async () => {
+    process.env.INTERNAL_GATEWAY_URL =
+      'http://gateway.internal/async-function';
+
+    postMock.mockImplementation(
+      (options: any, callback: (err: any) => void) => callback(null)
+    );
+
+    const { request } = await import('../src/req');
+
+    await request('example-fn', {
+      body: { value: 2 },
+      databaseId: 'db-456',
+      workerId: 'worker-2',
+      jobId: 43
+    });
+
+    const [options] = postMock.mock.calls[0];
+    expect(options.url).toBe(
+      'http://gateway.internal/async-function/example-fn'
+    );
+  });
+
+  it('uses development map override when provided', async () => {
+    process.env.KNATIVE_SERVICE_URL = 'http://knative.internal';
+    process.env.INTERNAL_GATEWAY_DEVELOPMENT_MAP = JSON.stringify({
+      'example-fn': 'http://localhost:3000/dev-fn'
+    });
+    process.env.NODE_ENV = 'development';
+
+    postMock.mockImplementation(
+      (options: any, callback: (err: any) => void) => callback(null)
+    );
+
+    const { request } = await import('../src/req');
+
+    await request('example-fn', {
+      body: {},
+      databaseId: 'db-789',
+      workerId: 'worker-3',
+      jobId: 44
+    });
+
+    const [options] = postMock.mock.calls[0];
+    expect(options.url).toBe('http://localhost:3000/dev-fn');
+  });
+
+  it('rejects when HTTP request errors', async () => {
+    process.env.KNATIVE_SERVICE_URL = 'http://knative.internal';
+
+    postMock.mockImplementation(
+      (options: any, callback: (err: any) => void) =>
+        callback(new Error('network failure'))
+    );
+
+    const { request } = await import('../src/req');
+
+    await expect(
+      request('example-fn', {
+        body: {},
+        databaseId: 'db-000',
+        workerId: 'worker-4',
+        jobId: 45
+      })
+    ).rejects.toThrow('network failure');
+  });
+
+  it('throws on startup when no base URL env vars are set', async () => {
+    await expect(import('../src/env')).rejects.toThrow(
+      /KNATIVE_SERVICE_URL \(or INTERNAL_GATEWAY_URL as fallback\) is required/
+    );
+  });
+});

package/__tests__/worker.integration.test.ts

@@ -0,0 +1,117 @@
+process.env.KNATIVE_SERVICE_URL =
+  process.env.KNATIVE_SERVICE_URL || 'knative.internal';
+process.env.INTERNAL_JOBS_CALLBACK_URL =
+  process.env.INTERNAL_JOBS_CALLBACK_URL ||
+  'http://callback.internal/jobs-complete';
+
+const postMock = jest.fn();
+
+jest.mock('request', () => ({
+  __esModule: true,
+  default: { post: postMock },
+  post: postMock
+}));
+
+jest.mock('@constructive-io/job-pg', () => ({
+  __esModule: true,
+  default: {
+    getPool: jest.fn(),
+    onClose: jest.fn(),
+    close: jest.fn()
+  }
+}));
+
+import path from 'path';
+import { getConnections, seed } from 'pgsql-test';
+import type { PgTestClient } from 'pgsql-test/test-client';
+import * as jobUtils from '@constructive-io/job-utils';
+import Worker from '../src';
+
+let db: PgTestClient;
+let teardown: () => Promise<void>;
+
+beforeAll(async () => {
+  const modulePath = path.resolve(
+    __dirname,
+    '../../../extensions/@pgpm/database-jobs'
+  );
+  ({ db, teardown } = await getConnections({}, [seed.loadPgpm(modulePath)]));
+  db.setContext({ role: 'administrator' });
+});
+
+afterAll(async () => {
+  await teardown();
+});
+
+beforeEach(async () => {
+  await db.beforeEach();
+  postMock.mockReset();
+});
+
+afterEach(async () => {
+  await db.afterEach();
+});
+
+describe('knative worker integration with job queue', () => {
+  const databaseId = '5b720132-17d5-424d-9bcb-ee7b17c13d43';
+
+  it('pulls a job from the queue and posts to Knative service', async () => {
+    const insertedJob = await db.one(
+      'SELECT * FROM app_jobs.add_job($1::uuid, $2::text, $3::json);',
+      [databaseId, 'example-fn', { hello: 'world' }]
+    );
+
+    const worker = new Worker({
+      tasks: ['example-fn'],
+      pgPool: {} as any,
+      workerId: 'worker-integration-1'
+    });
+
+    const job = await jobUtils.getJob(db as any, {
+      workerId: 'worker-integration-1',
+      supportedTaskNames: null
+    });
+
+    expect(job).toBeTruthy();
+    expect(job.id).toBe(insertedJob.id);
+
+    postMock.mockImplementation(
+      (options: any, callback: (err: any) => void) => callback(null)
+    );
+
+    await (worker as any).doWork(job);
+
+    expect(postMock).toHaveBeenCalledTimes(1);
+    const [options] = postMock.mock.calls[0];
+    expect(options.url).toBe('http://example-fn.knative.internal');
+    expect(options.headers['X-Job-Id']).toBe(job.id);
+    expect(options.headers['X-Database-Id']).toBe(databaseId);
+  });
+
+  it('propagates errors from failed Knative calls while keeping job queue interactions intact', async () => {
+    await db.one(
+      'SELECT * FROM app_jobs.add_job($1::uuid, $2::text, $3::json);',
+      [databaseId, 'example-fn', { hello: 'world' }]
+    );
+
+    const worker = new Worker({
+      tasks: ['example-fn'],
+      pgPool: {} as any,
+      workerId: 'worker-integration-2'
+    });
+
+    const job = await jobUtils.getJob(db as any, {
+      workerId: 'worker-integration-2',
+      supportedTaskNames: null
+    });
+
+    postMock.mockImplementation(
+      (options: any, callback: (err: any) => void) =>
+        callback(new Error('knative failure'))
+    );
+
+    await expect((worker as any).doWork(job)).rejects.toThrow(
+      'knative failure'
+    );
+  });
+});

package/dist/env.d.ts

@@ -0,0 +1,20 @@
+declare const _default: {
+    KNATIVE_SERVICE_URL: string;
+    INTERNAL_GATEWAY_DEVELOPMENT_MAP: string;
+    PGUSER: string;
+    PGHOST: string;
+    PGPASSWORD: string;
+    PGPORT: number;
+    PGDATABASE: string;
+    JOBS_SCHEMA: string;
+    JOBS_SUPPORT_ANY: boolean;
+    JOBS_SUPPORTED: string[];
+    HOSTNAME: string;
+    INTERNAL_JOBS_CALLBACK_URL: string;
+    isDevelopment: boolean;
+    isDev: boolean;
+    isTest: boolean;
+    isProduction: boolean;
+    isProd: boolean;
+};
+export default _default;

package/dist/env.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const envalid_1 = require("envalid");
+const array = (0, envalid_1.makeValidator)((x) => x.split(',').filter((i) => i), '');
+const baseEnv = (0, envalid_1.cleanEnv)(process.env, {
+    PGUSER: (0, envalid_1.str)({ default: 'postgres' }),
+    PGHOST: (0, envalid_1.str)({ default: 'localhost' }),
+    PGPASSWORD: (0, envalid_1.str)({ default: 'password' }),
+    PGPORT: (0, envalid_1.port)({ default: 5432 }),
+    PGDATABASE: (0, envalid_1.str)({ default: 'jobs' }),
+    JOBS_SCHEMA: (0, envalid_1.str)({ default: 'app_jobs' }),
+    JOBS_SUPPORT_ANY: (0, envalid_1.bool)({ default: true }),
+    JOBS_SUPPORTED: array({ default: '' }),
+    HOSTNAME: (0, envalid_1.str)({
+        default: 'worker-0'
+    }),
+    INTERNAL_JOBS_CALLBACK_URL: (0, envalid_1.url)()
+}, { dotEnvPath: null });
+const KNATIVE_SERVICE_URL = process.env.KNATIVE_SERVICE_URL;
+if (!KNATIVE_SERVICE_URL) {
+    throw new Error('KNATIVE_SERVICE_URL (or INTERNAL_GATEWAY_URL as fallback) is required for @launchql/knative-job-worker');
+}
+const INTERNAL_GATEWAY_DEVELOPMENT_MAP = process.env.INTERNAL_GATEWAY_DEVELOPMENT_MAP;
+exports.default = {
+    ...baseEnv,
+    KNATIVE_SERVICE_URL,
+    INTERNAL_GATEWAY_DEVELOPMENT_MAP
+};

package/dist/index.d.ts

@@ -0,0 +1,41 @@
+import type { PgClientLike } from '@constructive-io/job-utils';
+import type { Pool } from 'pg';
+export interface JobRow {
+    id: number | string;
+    task_identifier: string;
+    payload?: unknown;
+    database_id?: string;
+}
+export default class Worker {
+    idleDelay: number;
+    supportedTaskNames: string[];
+    workerId: string;
+    doNextTimer?: NodeJS.Timeout;
+    pgPool: Pool;
+    _initialized?: boolean;
+    constructor({ tasks, idleDelay, pgPool, workerId }: {
+        tasks: string[];
+        idleDelay?: number;
+        pgPool?: Pool;
+        workerId?: string;
+    });
+    initialize(client: PgClientLike): Promise<void>;
+    handleFatalError(client: PgClientLike, { err, fatalError, jobId }: {
+        err?: Error;
+        fatalError: unknown;
+        jobId: JobRow['id'];
+    }): Promise<void>;
+    handleError(client: PgClientLike, { err, job, duration }: {
+        err: Error;
+        job: JobRow;
+        duration: string;
+    }): Promise<void>;
+    handleSuccess(client: PgClientLike, { job, duration }: {
+        job: JobRow;
+        duration: string;
+    }): Promise<void>;
+    doWork(job: JobRow): Promise<void>;
+    doNext(client: PgClientLike): Promise<void>;
+    listen(): void;
+}
+export { Worker };

package/dist/index.js

@@ -0,0 +1,198 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const job_pg_1 = __importDefault(require("@constructive-io/job-pg"));
+const jobs = __importStar(require("@constructive-io/job-utils"));
+const logger_1 = require("@pgpmjs/logger");
+const req_1 = require("./req");
+const log = new logger_1.Logger('jobs:worker');
+class Worker {
+    idleDelay;
+    supportedTaskNames;
+    workerId;
+    doNextTimer;
+    pgPool;
+    _initialized;
+    constructor({ tasks, idleDelay = 15000, pgPool = job_pg_1.default.getPool(), workerId = 'worker-0' }) {
+        /*
+         * idleDelay: This is how long to wait between polling for jobs.
+         *
+         * Note: this does NOT need to be short, because we use LISTEN/NOTIFY to be
+         * notified when new jobs are added - this is just used in the case where
+         * LISTEN/NOTIFY fails for whatever reason.
+         */
+        this.idleDelay = idleDelay;
+        this.supportedTaskNames = tasks;
+        this.workerId = workerId;
+        this.doNextTimer = undefined;
+        this.pgPool = pgPool;
+        job_pg_1.default.onClose(async () => {
+            await jobs.releaseJobs(pgPool, { workerId: this.workerId });
+        });
+    }
+    async initialize(client) {
+        if (this._initialized === true)
+            return;
+        // release any jobs not finished from before if fatal error prevented cleanup
+        await jobs.releaseJobs(client, { workerId: this.workerId });
+        this._initialized = true;
+        await this.doNext(client);
+    }
+    async handleFatalError(client, { err, fatalError, jobId }) {
+        const when = err ? `after failure '${err.message}'` : 'after success';
+        log.error(`Failed to release job '${jobId}' ${when}; committing seppuku`);
+        await job_pg_1.default.close();
+        log.error(String(fatalError));
+        process.exit(1);
+    }
+    async handleError(client, { err, job, duration }) {
+        log.error(`Failed task ${job.id} (${job.task_identifier}) with error ${err.message} (${duration}ms)`);
+        if (err.stack) {
+            log.debug(err.stack);
+        }
+        await jobs.failJob(client, {
+            workerId: this.workerId,
+            jobId: job.id,
+            message: err.message
+        });
+    }
+    async handleSuccess(client, { job, duration }) {
+        log.info(`Async task ${job.id} (${job.task_identifier}) to be processed`);
+    }
+    async doWork(job) {
+        const { payload, task_identifier } = job;
+        log.debug('starting work on job', {
+            id: job.id,
+            task: task_identifier,
+            databaseId: job.database_id
+        });
+        if (!jobs.getJobSupportAny() &&
+            !this.supportedTaskNames.includes(task_identifier)) {
+            throw new Error('Unsupported task');
+        }
+        await (0, req_1.request)(task_identifier, {
+            body: payload,
+            databaseId: job.database_id,
+            workerId: this.workerId,
+            jobId: job.id
+        });
+    }
+    async doNext(client) {
+        if (!this._initialized) {
+            return await this.initialize(client);
+        }
+        log.debug('checking for jobs...');
+        if (this.doNextTimer) {
+            clearTimeout(this.doNextTimer);
+            this.doNextTimer = undefined;
+        }
+        try {
+            const job = (await jobs.getJob(client, {
+                workerId: this.workerId,
+                supportedTaskNames: jobs.getJobSupportAny()
+                    ? null
+                    : this.supportedTaskNames
+            }));
+            if (!job || !job.id) {
+                this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+                return;
+            }
+            const start = process.hrtime();
+            let err = null;
+            try {
+                await this.doWork(job);
+            }
+            catch (error) {
+                err = error;
+            }
+            const durationRaw = process.hrtime(start);
+            const duration = ((durationRaw[0] * 1e9 + durationRaw[1]) / 1e6).toFixed(2);
+            const jobId = job.id;
+            try {
+                if (err) {
+                    await this.handleError(client, { err, job, duration });
+                }
+                else {
+                    await this.handleSuccess(client, { job, duration });
+                }
+            }
+            catch (fatalError) {
+                await this.handleFatalError(client, { err, fatalError, jobId });
+            }
+            return this.doNext(client);
+        }
+        catch (err) {
+            this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+        }
+    }
+    listen() {
+        const listenForChanges = (err, client, release) => {
+            if (err) {
+                log.error('Error connecting with notify listener', err);
+                if (err instanceof Error && err.stack) {
+                    log.debug(err.stack);
+                }
+                // Try again in 5 seconds
+                // should this really be done in the node process?
+                setTimeout(this.listen, 5000);
+                return;
+            }
+            client.on('notification', () => {
+                if (this.doNextTimer) {
+                    // Must be idle, do something!
+                    this.doNext(client);
+                }
+            });
+            client.query('LISTEN "jobs:insert"');
+            client.on('error', (e) => {
+                log.error('Error with database notify listener', e);
+                if (e instanceof Error && e.stack) {
+                    log.debug(e.stack);
+                }
+                release();
+                this.listen();
+            });
+            log.info(`${this.workerId} connected and looking for jobs...`);
+            this.doNext(client);
+        };
+        this.pgPool.connect(listenForChanges);
+    }
+}
+exports.default = Worker;
+exports.Worker = Worker;

package/dist/req.d.ts

@@ -0,0 +1,8 @@
+interface RequestOptions {
+    body: unknown;
+    databaseId: string;
+    workerId: string;
+    jobId: string | number;
+}
+declare const request: (fn: string, { body, databaseId, workerId, jobId }: RequestOptions) => Promise<boolean>;
+export { request };

package/dist/req.js

@@ -0,0 +1,61 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.request = void 0;
+const request_1 = __importDefault(require("request"));
+const job_utils_1 = require("@constructive-io/job-utils");
+const logger_1 = require("@pgpmjs/logger");
+const log = new logger_1.Logger('jobs:req');
+// callback URL for job completion
+const completeUrl = (0, job_utils_1.getCallbackBaseUrl)();
+// Development override map (e.g. point a function name at localhost)
+const nodeEnv = (0, job_utils_1.getNodeEnvironment)();
+const DEV_MAP = nodeEnv !== 'production' ? (0, job_utils_1.getJobGatewayDevMap)() : null;
+const getFunctionUrl = (fn) => {
+    if (DEV_MAP && DEV_MAP[fn]) {
+        return DEV_MAP[fn] || completeUrl;
+    }
+    const { gatewayUrl } = (0, job_utils_1.getJobGatewayConfig)();
+    const base = gatewayUrl.replace(/\/$/, '');
+    return `${base}/${fn}`;
+};
+const request = (fn, { body, databaseId, workerId, jobId }) => {
+    const url = getFunctionUrl(fn);
+    log.info(`dispatching job`, {
+        fn,
+        url,
+        callbackUrl: completeUrl,
+        workerId,
+        jobId,
+        databaseId
+    });
+    return new Promise((resolve, reject) => {
+        request_1.default.post({
+            headers: {
+                'Content-Type': 'application/json',
+                // these are used by job-worker/job-fn
+                'X-Worker-Id': workerId,
+                'X-Job-Id': jobId,
+                'X-Database-Id': databaseId,
+                // async HTTP completion callback
+                'X-Callback-Url': completeUrl
+            },
+            url,
+            json: true,
+            body
+        }, function (error) {
+            if (error) {
+                log.error(`request error for job[${jobId}] fn[${fn}]`, error);
+                if (error instanceof Error && error.stack) {
+                    log.debug(error.stack);
+                }
+                return reject(error);
+            }
+            log.debug(`request success for job[${jobId}] fn[${fn}]`);
+            return resolve(true);
+        });
+    });
+};
+exports.request = request;

package/dist/run.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/run.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = __importDefault(require("./index"));
+const job_pg_1 = __importDefault(require("@constructive-io/job-pg"));
+const job_utils_1 = require("@constructive-io/job-utils");
+const pgPool = job_pg_1.default.getPool();
+const worker = new index_1.default({
+    pgPool,
+    workerId: (0, job_utils_1.getWorkerHostname)(),
+    tasks: (0, job_utils_1.getJobSupported)()
+});
+worker.listen();

package/jest.config.js

@@ -0,0 +1,18 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        babelConfig: false,
+        tsconfig: 'tsconfig.json',
+      },
+    ],
+  },
+  transformIgnorePatterns: [`/node_modules/*`],
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePathIgnorePatterns: ['dist/*'],
+};

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@constructive-io/knative-job-worker",
+  "version": "0.5.9",
+  "description": "knative job worker",
+  "author": "Constructive <developers@constructive.io>",
+  "homepage": "https://github.com/constructive-io/jobs/tree/master/packages/knative-job-worker#readme",
+  "license": "SEE LICENSE IN LICENSE",
+  "main": "dist/index.js",
+  "directories": {
+    "lib": "src",
+    "test": "__tests__"
+  },
+  "bin": {
+    "faas-jobs": "src/run.ts",
+    "knative-jobs": "src/run.ts"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/constructive-io/jobs"
+  },
+  "scripts": {
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:debug": "node --inspect node_modules/.bin/jest --runInBand",
+    "build": "tsc -p tsconfig.json",
+    "build:watch": "tsc -p tsconfig.json -w"
+  },
+  "bugs": {
+    "url": "https://github.com/constructive-io/jobs/issues"
+  },
+  "dependencies": {
+    "@constructive-io/job-pg": "^0.3.5",
+    "@constructive-io/job-utils": "^0.5.0",
+    "@pgpmjs/logger": "^1.3.3",
+    "pg": "8.16.3",
+    "request": "2.88.2"
+  },
+  "devDependencies": {
+    "pgsql-test": "^2.17.1"
+  },
+  "gitHead": "86d74dc4fce9051df0d2b5bcc163607aba42f009"
+}

package/src/index.ts

@@ -0,0 +1,210 @@
+import poolManager from '@constructive-io/job-pg';
+import * as jobs from '@constructive-io/job-utils';
+import type { PgClientLike } from '@constructive-io/job-utils';
+import type { Pool, PoolClient } from 'pg';
+import { Logger } from '@pgpmjs/logger';
+import { request as req } from './req';
+
+export interface JobRow {
+  id: number | string;
+  task_identifier: string;
+  payload?: unknown;
+  database_id?: string;
+}
+
+const log = new Logger('jobs:worker');
+
+export default class Worker {
+  idleDelay: number;
+  supportedTaskNames: string[];
+  workerId: string;
+  doNextTimer?: NodeJS.Timeout;
+  pgPool: Pool;
+  _initialized?: boolean;
+
+  constructor({
+    tasks,
+    idleDelay = 15000,
+    pgPool = poolManager.getPool(),
+    workerId = 'worker-0'
+  }: {
+    tasks: string[];
+    idleDelay?: number;
+    pgPool?: Pool;
+    workerId?: string;
+  }) {
+    /*
+     * idleDelay: This is how long to wait between polling for jobs.
+     *
+     * Note: this does NOT need to be short, because we use LISTEN/NOTIFY to be
+     * notified when new jobs are added - this is just used in the case where
+     * LISTEN/NOTIFY fails for whatever reason.
+     */
+
+    this.idleDelay = idleDelay;
+    this.supportedTaskNames = tasks;
+    this.workerId = workerId;
+    this.doNextTimer = undefined;
+    this.pgPool = pgPool;
+    poolManager.onClose(async () => {
+      await jobs.releaseJobs(pgPool, { workerId: this.workerId });
+    });
+  }
+  async initialize(client: PgClientLike) {
+    if (this._initialized === true) return;
+
+    // release any jobs not finished from before if fatal error prevented cleanup
+    await jobs.releaseJobs(client, { workerId: this.workerId });
+
+    this._initialized = true;
+    await this.doNext(client);
+  }
+  async handleFatalError(
+    client: PgClientLike,
+    {
+      err,
+      fatalError,
+      jobId
+    }: { err?: Error; fatalError: unknown; jobId: JobRow['id'] }
+  ) {
+    const when = err ? `after failure '${err.message}'` : 'after success';
+    log.error(`Failed to release job '${jobId}' ${when}; committing seppuku`);
+    await poolManager.close();
+    log.error(String(fatalError));
+    process.exit(1);
+  }
+  async handleError(
+    client: PgClientLike,
+    { err, job, duration }: { err: Error; job: JobRow; duration: string }
+  ) {
+    log.error(
+      `Failed task ${job.id} (${job.task_identifier}) with error ${err.message} (${duration}ms)`
+    );
+    if (err.stack) {
+      log.debug(err.stack);
+    }
+    await jobs.failJob(client, {
+      workerId: this.workerId,
+      jobId: job.id,
+      message: err.message
+    });
+  }
+  async handleSuccess(
+    client: PgClientLike,
+    { job, duration }: { job: JobRow; duration: string }
+  ) {
+    log.info(
+      `Async task ${job.id} (${job.task_identifier}) to be processed`
+    );
+  }
+  async doWork(job: JobRow) {
+    const { payload, task_identifier } = job;
+    log.debug('starting work on job', {
+      id: job.id,
+      task: task_identifier,
+      databaseId: job.database_id
+    });
+    if (
+      !jobs.getJobSupportAny() &&
+      !this.supportedTaskNames.includes(task_identifier)
+    ) {
+      throw new Error('Unsupported task');
+    }
+    await req(task_identifier, {
+      body: payload,
+      databaseId: job.database_id,
+      workerId: this.workerId,
+      jobId: job.id
+    });
+  }
+  async doNext(client: PgClientLike): Promise<void> {
+    if (!this._initialized) {
+      return await this.initialize(client);
+    }
+
+    log.debug('checking for jobs...');
+    if (this.doNextTimer) {
+      clearTimeout(this.doNextTimer);
+      this.doNextTimer = undefined;
+    }
+    try {
+      const job = (await jobs.getJob<JobRow>(client, {
+        workerId: this.workerId,
+        supportedTaskNames: jobs.getJobSupportAny()
+          ? null
+          : this.supportedTaskNames
+      })) as JobRow | undefined;
+
+      if (!job || !job.id) {
+        this.doNextTimer = setTimeout(
+          () => this.doNext(client),
+          this.idleDelay
+        );
+        return;
+      }
+      const start = process.hrtime();
+
+      let err: Error | null = null;
+      try {
+        await this.doWork(job);
+      } catch (error) {
+        err = error as Error;
+      }
+      const durationRaw = process.hrtime(start);
+      const duration = ((durationRaw[0] * 1e9 + durationRaw[1]) / 1e6).toFixed(
+        2
+      );
+      const jobId = job.id;
+      try {
+        if (err) {
+          await this.handleError(client, { err, job, duration });
+        } else {
+          await this.handleSuccess(client, { job, duration });
+        }
+      } catch (fatalError: unknown) {
+        await this.handleFatalError(client, { err, fatalError, jobId });
+      }
+      return this.doNext(client);
+    } catch (err: unknown) {
+      this.doNextTimer = setTimeout(() => this.doNext(client), this.idleDelay);
+    }
+  }
+  listen() {
+    const listenForChanges = (
+      err: Error | null,
+      client: PoolClient,
+      release: () => void
+    ) => {
+      if (err) {
+        log.error('Error connecting with notify listener', err);
+        if (err instanceof Error && err.stack) {
+          log.debug(err.stack);
+        }
+        // Try again in 5 seconds
+        // should this really be done in the node process?
+        setTimeout(this.listen, 5000);
+        return;
+      }
+      client.on('notification', () => {
+        if (this.doNextTimer) {
+          // Must be idle, do something!
+          this.doNext(client);
+        }
+      });
+      client.query('LISTEN "jobs:insert"');
+      client.on('error', (e: unknown) => {
+        log.error('Error with database notify listener', e);
+        if (e instanceof Error && e.stack) {
+          log.debug(e.stack);
+        }
+        release();
+        this.listen();
+      });
+      log.info(`${this.workerId} connected and looking for jobs...`);
+      this.doNext(client);
+    };
+    this.pgPool.connect(listenForChanges);
+  }
+}
+
+export { Worker };

package/src/req.ts

@@ -0,0 +1,82 @@
+import requestLib from 'request';
+import {
+  getCallbackBaseUrl,
+  getJobGatewayConfig,
+  getJobGatewayDevMap,
+  getNodeEnvironment
+} from '@constructive-io/job-utils';
+import { Logger } from '@pgpmjs/logger';
+
+const log = new Logger('jobs:req');
+
+// callback URL for job completion
+const completeUrl = getCallbackBaseUrl();
+
+// Development override map (e.g. point a function name at localhost)
+const nodeEnv = getNodeEnvironment();
+const DEV_MAP = nodeEnv !== 'production' ? getJobGatewayDevMap() : null;
+
+const getFunctionUrl = (fn: string): string => {
+  if (DEV_MAP && DEV_MAP[fn]) {
+    return DEV_MAP[fn] || completeUrl;
+  }
+
+  const { gatewayUrl } = getJobGatewayConfig();
+  const base = gatewayUrl.replace(/\/$/, '');
+  return `${base}/${fn}`;
+};
+
+interface RequestOptions {
+  body: unknown;
+  databaseId: string;
+  workerId: string;
+  jobId: string | number;
+}
+
+const request = (
+  fn: string,
+  { body, databaseId, workerId, jobId }: RequestOptions
+) => {
+  const url = getFunctionUrl(fn);
+  log.info(`dispatching job`, {
+    fn,
+    url,
+    callbackUrl: completeUrl,
+    workerId,
+    jobId,
+    databaseId
+  });
+  return new Promise<boolean>((resolve, reject) => {
+    requestLib.post(
+      {
+        headers: {
+          'Content-Type': 'application/json',
+
+          // these are used by job-worker/job-fn
+          'X-Worker-Id': workerId,
+          'X-Job-Id': jobId,
+          'X-Database-Id': databaseId,
+
+          // async HTTP completion callback
+          'X-Callback-Url': completeUrl
+        },
+        url,
+        json: true,
+        body
+      },
+      function (error: unknown) {
+        if (error) {
+          log.error(`request error for job[${jobId}] fn[${fn}]`, error);
+          if (error instanceof Error && error.stack) {
+            log.debug(error.stack);
+          }
+          return reject(error);
+        }
+        log.debug(`request success for job[${jobId}] fn[${fn}]`);
+        return resolve(true);
+      }
+    );
+  });
+};
+
+export { request };

package/src/run.ts

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+
+import Worker from './index';
+import poolManager from '@constructive-io/job-pg';
+import {
+  getWorkerHostname,
+  getJobSupported
+} from '@constructive-io/job-utils';
+
+const pgPool = poolManager.getPool();
+
+const worker = new Worker({
+  pgPool,
+  workerId: getWorkerHostname(),
+  tasks: getJobSupported()
+});
+
+worker.listen();

package/tsconfig.esm.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist/esm",
+    "module": "es2022",
+    "rootDir": "src/",
+    "declaration": false
+  }
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src/"
+  },
+  "include": ["src/**/*.ts", "../../types/**/*.d.ts"],
+  "exclude": ["dist", "node_modules", "**/*.spec.*", "**/*.test.*"]
+}