duron 0.1.0

package/dist/step-manager.js

@@ -0,0 +1,216 @@
+import fastq from 'fastq';
+import { StepOptionsSchema, } from './action.js';
+import { STEP_STATUS_CANCELLED, STEP_STATUS_COMPLETED, STEP_STATUS_FAILED } from './constants.js';
+import { ActionCancelError, isCancelError, isNonRetriableError, NonRetriableError, StepAlreadyExecutedError, StepTimeoutError, serializeError, } from './errors.js';
+import pRetry from './utils/p-retry.js';
+import waitForAbort from './utils/wait-for-abort.js';
+export class StepStore {
+    #adapter;
+    constructor(adapter) {
+        this.#adapter = adapter;
+    }
+    async getOrCreate(jobId, name, timeoutMs, retriesLimit) {
+        try {
+            return await this.#adapter.createOrRecoverJobStep({
+                jobId,
+                name,
+                timeoutMs,
+                retriesLimit,
+            });
+        }
+        catch (error) {
+            throw new NonRetriableError(`Failed to get or create step "${name}" for job "${jobId}"`, { cause: error });
+        }
+    }
+    async updateStatus(stepId, status, output, error) {
+        if (status === STEP_STATUS_COMPLETED) {
+            return this.#adapter.completeJobStep({ stepId, output });
+        }
+        else if (status === STEP_STATUS_FAILED) {
+            return this.#adapter.failJobStep({ stepId, error });
+        }
+        else if (status === STEP_STATUS_CANCELLED) {
+            return this.#adapter.cancelJobStep({ stepId });
+        }
+        return false;
+    }
+    async delay(stepId, delayMs, error) {
+        return this.#adapter.delayJobStep({ stepId, delayMs, error });
+    }
+}
+export class StepManager {
+    #jobId;
+    #actionName;
+    #stepStore;
+    #queue;
+    #logger;
+    #historySteps = new Set();
+    constructor(options) {
+        this.#jobId = options.jobId;
+        this.#actionName = options.actionName;
+        this.#logger = options.logger;
+        this.#stepStore = new StepStore(options.adapter);
+        this.#queue = fastq.promise(async (task) => {
+            if (this.#historySteps.has(task.name)) {
+                throw new StepAlreadyExecutedError(task.name, this.#jobId, this.#actionName);
+            }
+            this.#historySteps.add(task.name);
+            return this.#executeStep(task.name, task.cb, task.options, task.abortSignal);
+        }, options.concurrencyLimit);
+    }
+    createActionContext(job, action, variables, abortSignal, logger) {
+        return new ActionContext(this, job, action, variables, abortSignal, logger);
+    }
+    async push(task) {
+        return this.#queue.push(task);
+    }
+    async drain() {
+        await this.#queue.drain();
+    }
+    async #executeStep(name, cb, options, abortSignal) {
+        const expire = options.expire;
+        const retryOptions = options.retry;
+        let step = null;
+        const executeStep = async () => {
+            if (!step) {
+                if (abortSignal.aborted) {
+                    throw new ActionCancelError(this.#actionName, this.#jobId, { cause: 'step cancelled before create step' });
+                }
+                const newStep = await this.#stepStore.getOrCreate(this.#jobId, name, expire, retryOptions.limit);
+                if (!newStep) {
+                    throw new NonRetriableError(`Failed to create step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`, { cause: 'step not created' });
+                }
+                step = newStep;
+                if (abortSignal.aborted) {
+                    throw new ActionCancelError(this.#actionName, this.#jobId, { cause: 'step cancelled after create step' });
+                }
+                if (step.status === STEP_STATUS_COMPLETED) {
+                    this.#logger.debug({ jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id }, '[StepManager] Step recovered (already completed)');
+                    return step.output;
+                }
+                else if (step.status === STEP_STATUS_FAILED) {
+                    throw new NonRetriableError(`Cannot recover a failed step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`, {
+                        cause: step.error,
+                    });
+                }
+                else if (step.status === STEP_STATUS_CANCELLED) {
+                    throw new NonRetriableError(`Cannot recover a cancelled step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`, { cause: step.error });
+                }
+                this.#logger.debug({ jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id }, '[StepManager] Step started executing');
+            }
+            const stepAbortController = new AbortController();
+            const timeoutId = setTimeout(() => {
+                const timeoutError = new StepTimeoutError(name, this.#jobId, expire);
+                stepAbortController.abort(timeoutError);
+            }, expire);
+            timeoutId?.unref?.();
+            const signal = AbortSignal.any([abortSignal, stepAbortController.signal]);
+            try {
+                const abortPromise = waitForAbort(signal);
+                const callbackPromise = cb({ signal });
+                let result = null;
+                await Promise.race([
+                    abortPromise.promise,
+                    callbackPromise
+                        .then((res) => {
+                        if (res !== undefined && res !== null) {
+                            result = res;
+                        }
+                    })
+                        .finally(() => {
+                        abortPromise.release();
+                    }),
+                ]);
+                const completed = await this.#stepStore.updateStatus(step.id, 'completed', result);
+                if (!completed) {
+                    throw new Error(`Failed to complete step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`);
+                }
+                this.#logger.debug({ jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id }, '[StepManager] Step finished executing');
+                return result;
+            }
+            finally {
+                clearTimeout(timeoutId);
+            }
+        };
+        return pRetry(executeStep, {
+            retries: retryOptions.limit,
+            factor: retryOptions.factor,
+            randomize: false,
+            signal: abortSignal,
+            minTimeout: retryOptions.minTimeout,
+            maxTimeout: retryOptions.maxTimeout,
+            onFailedAttempt: async (ctx) => {
+                const error = ctx.error;
+                if (isNonRetriableError(error) ||
+                    (error.cause && isNonRetriableError(error.cause)) ||
+                    (error instanceof Error && error.name === 'AbortError' && isNonRetriableError(error.cause))) {
+                    throw error;
+                }
+                if (ctx.retriesLeft > 0 && step) {
+                    const delayed = await this.#stepStore.delay(step.id, ctx.finalDelay, serializeError(error));
+                    if (!delayed) {
+                        throw new Error(`Failed to delay step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`);
+                    }
+                }
+            },
+        }).catch(async (error) => {
+            if (step) {
+                if (isCancelError(error)) {
+                    await this.#stepStore.updateStatus(step.id, 'cancelled');
+                }
+                else {
+                    await this.#stepStore.updateStatus(step.id, STEP_STATUS_FAILED, null, serializeError(error));
+                }
+            }
+            throw error;
+        });
+    }
+}
+class ActionContext {
+    #stepManager;
+    #variables;
+    #abortSignal;
+    #logger;
+    #input;
+    #jobId;
+    #groupKey = '@default';
+    #action;
+    constructor(stepManager, job, action, variables, abortSignal, logger) {
+        this.#stepManager = stepManager;
+        this.#variables = variables;
+        this.#abortSignal = abortSignal;
+        this.#logger = logger;
+        this.#action = action;
+        this.#jobId = job.id;
+        this.#groupKey = job.groupKey ?? '@default';
+        if (action.input) {
+            this.#input = action.input.parse(job.input, {
+                error: () => 'Error parsing action input',
+                reportInput: true,
+            });
+        }
+        this.#input = job.input ?? {};
+    }
+    get input() {
+        return this.#input;
+    }
+    get jobId() {
+        return this.#jobId;
+    }
+    get groupKey() {
+        return this.#groupKey;
+    }
+    get var() {
+        return this.#variables;
+    }
+    get logger() {
+        return this.#logger;
+    }
+    async step(name, cb, options = {}) {
+        const parsedOptions = StepOptionsSchema.parse({
+            ...this.#action.steps,
+            ...options,
+        });
+        return this.#stepManager.push({ name, cb, options: parsedOptions, abortSignal: this.#abortSignal });
+    }
+}

package/dist/utils/checksum.d.ts

@@ -0,0 +1,2 @@
+export default function generateChecksum(code: string): string;
+//# sourceMappingURL=checksum.d.ts.map

package/dist/utils/checksum.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checksum.d.ts","sourceRoot":"","sources":["../../src/utils/checksum.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,gBAAgB,CAAC,IAAI,EAAE,MAAM,UAIpD"}

package/dist/utils/checksum.js

@@ -0,0 +1,6 @@
+import { createHash } from 'node:crypto';
+export default function generateChecksum(code) {
+    const hash = createHash('md5');
+    hash.update(code);
+    return hash.digest('hex');
+}

package/dist/utils/p-retry.d.ts

@@ -0,0 +1,19 @@
+interface RetryOptions {
+    retries: number;
+    factor: number;
+    minTimeout: number;
+    maxTimeout: number;
+    maxRetryTime?: number;
+    randomize: boolean;
+    signal: AbortSignal;
+    onFailedAttempt: (context: {
+        error: Error;
+        attemptNumber: number;
+        retriesLeft: number;
+        retriesConsumed: number;
+        finalDelay: number;
+    }) => Promise<void>;
+}
+export default function pRetry<TResult>(input: (attemptNumber: number) => Promise<TResult>, options: RetryOptions): Promise<TResult>;
+export {};
+//# sourceMappingURL=p-retry.d.ts.map

package/dist/utils/p-retry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"p-retry.d.ts","sourceRoot":"","sources":["../../src/utils/p-retry.ts"],"names":[],"mappings":"AAMA,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,MAAM,EAAE,WAAW,CAAA;IACnB,eAAe,EAAE,CAAC,OAAO,EAAE;QACzB,KAAK,EAAE,KAAK,CAAA;QACZ,aAAa,EAAE,MAAM,CAAA;QACrB,WAAW,EAAE,MAAM,CAAA;QACnB,eAAe,EAAE,MAAM,CAAA;QACvB,UAAU,EAAE,MAAM,CAAA;KACnB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACpB;AA6HD,wBAA8B,MAAM,CAAC,OAAO,EAC1C,KAAK,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,EAClD,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,OAAO,CAAC,CA+DlB"}

package/dist/utils/p-retry.js

@@ -0,0 +1,130 @@
+function validateRetries(retries) {
+    if (typeof retries === 'number') {
+        if (retries < 0) {
+            throw new TypeError('Expected `retries` to be a non-negative number.');
+        }
+        if (Number.isNaN(retries)) {
+            throw new TypeError('Expected `retries` to be a valid number or Infinity, got NaN.');
+        }
+    }
+    else if (retries !== undefined) {
+        throw new TypeError('Expected `retries` to be a number or Infinity.');
+    }
+}
+function validateNumberOption(name, value, { min = 0, allowInfinity = false } = {}) {
+    if (value === undefined) {
+        return;
+    }
+    if (typeof value !== 'number' || Number.isNaN(value)) {
+        throw new TypeError(`Expected \`${name}\` to be a number${allowInfinity ? ' or Infinity' : ''}.`);
+    }
+    if (!allowInfinity && !Number.isFinite(value)) {
+        throw new TypeError(`Expected \`${name}\` to be a finite number.`);
+    }
+    if (value < min) {
+        throw new TypeError(`Expected \`${name}\` to be \u2265 ${min}.`);
+    }
+}
+function calculateDelay(retriesConsumed, options) {
+    const attempt = Math.max(1, retriesConsumed + 1);
+    const random = options.randomize ? Math.random() + 1 : 1;
+    let timeout = Math.round(random * options.minTimeout * options.factor ** (attempt - 1));
+    timeout = Math.min(timeout, options.maxTimeout);
+    return timeout;
+}
+function calculateRemainingTime(start, max) {
+    if (!Number.isFinite(max)) {
+        return max;
+    }
+    return max - (performance.now() - start);
+}
+async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTime, options, }) {
+    const normalizedError = error instanceof Error ? error : new TypeError(`Non-error was thrown: "${error}". You should only throw errors.`);
+    const retriesLeft = Number.isFinite(options.retries)
+        ? Math.max(0, options.retries - retriesConsumed)
+        : options.retries;
+    const maxRetryTime = options.maxRetryTime ?? Number.POSITIVE_INFINITY;
+    const delayTime = calculateDelay(retriesConsumed, options);
+    const remainingTime = calculateRemainingTime(startTime, maxRetryTime);
+    const finalDelay = Math.min(delayTime, remainingTime);
+    if (remainingTime <= 0) {
+        throw normalizedError;
+    }
+    const context = Object.freeze({
+        error: normalizedError,
+        attemptNumber,
+        retriesLeft,
+        retriesConsumed,
+        finalDelay,
+    });
+    await options.onFailedAttempt(context);
+    if (remainingTime <= 0 || retriesLeft <= 0) {
+        throw normalizedError;
+    }
+    if (normalizedError instanceof TypeError) {
+        options.signal?.throwIfAborted();
+        return false;
+    }
+    if (finalDelay > 0) {
+        await new Promise((resolve, reject) => {
+            const onAbort = () => {
+                clearTimeout(timeoutToken);
+                options.signal?.removeEventListener('abort', onAbort);
+                reject(options.signal.reason);
+            };
+            const timeoutToken = setTimeout(() => {
+                options.signal?.removeEventListener('abort', onAbort);
+                resolve(undefined);
+            }, finalDelay);
+            timeoutToken.unref?.();
+            options.signal?.addEventListener('abort', onAbort, { once: true });
+        });
+    }
+    options.signal?.throwIfAborted();
+    return true;
+}
+export default async function pRetry(input, options) {
+    options = { ...options };
+    validateRetries(options.retries);
+    if (Object.hasOwn(options, 'forever')) {
+        throw new Error('The `forever` option is no longer supported. For many use-cases, you can set `retries: Infinity` instead.');
+    }
+    options.retries ??= 10;
+    options.factor ??= 2;
+    options.minTimeout ??= 1000;
+    options.maxTimeout ??= Number.POSITIVE_INFINITY;
+    options.maxRetryTime ??= Number.POSITIVE_INFINITY;
+    options.randomize ??= false;
+    validateNumberOption('factor', options.factor, { min: 0, allowInfinity: false });
+    validateNumberOption('minTimeout', options.minTimeout, { min: 0, allowInfinity: false });
+    validateNumberOption('maxTimeout', options.maxTimeout, { min: 0, allowInfinity: true });
+    validateNumberOption('maxRetryTime', options.maxRetryTime, { min: 0, allowInfinity: true });
+    if (!(options.factor > 0)) {
+        options.factor = 1;
+    }
+    options.signal?.throwIfAborted();
+    let attemptNumber = 0;
+    let retriesConsumed = 0;
+    const startTime = performance.now();
+    while (Number.isFinite(options.retries) ? retriesConsumed <= options.retries : true) {
+        attemptNumber++;
+        try {
+            options.signal?.throwIfAborted();
+            const result = await input(attemptNumber);
+            options.signal?.throwIfAborted();
+            return result;
+        }
+        catch (error) {
+            if (await onAttemptFailure({
+                error: error,
+                attemptNumber,
+                retriesConsumed,
+                startTime,
+                options,
+            })) {
+                retriesConsumed++;
+            }
+        }
+    }
+    throw new Error('Retry attempts exhausted without throwing an error.');
+}

package/dist/utils/wait-for-abort.d.ts

@@ -0,0 +1,5 @@
+export default function waitForAbort(signal: AbortSignal): {
+    promise: Promise<unknown>;
+    release: () => void;
+};
+//# sourceMappingURL=wait-for-abort.d.ts.map

package/dist/utils/wait-for-abort.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wait-for-abort.d.ts","sourceRoot":"","sources":["../../src/utils/wait-for-abort.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,YAAY,CAAC,MAAM,EAAE,WAAW;;;EAqCvD"}

package/dist/utils/wait-for-abort.js

@@ -0,0 +1,32 @@
+export default function waitForAbort(signal) {
+    let done = false;
+    let globalResolve = null;
+    const promise = new Promise((resolve, reject) => {
+        if (done) {
+            resolve(undefined);
+            return;
+        }
+        if (signal.aborted) {
+            done = true;
+            reject(signal.reason);
+            return;
+        }
+        globalResolve = resolve;
+        signal.addEventListener('abort', () => {
+            done = true;
+            reject(signal.reason);
+        });
+    });
+    return {
+        promise,
+        release: () => {
+            if (done) {
+                return;
+            }
+            setTimeout(() => {
+                globalResolve?.();
+            }, 0);
+            done = true;
+        },
+    };
+}

package/migrations/postgres/0000_lethal_speed_demon.sql

@@ -0,0 +1,64 @@
+CREATE SCHEMA IF NOT EXISTS "duron";
+--> statement-breakpoint
+CREATE TABLE "duron"."job_steps" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"job_id" uuid NOT NULL,
+	"name" text NOT NULL,
+	"status" text DEFAULT 'active' NOT NULL,
+	"output" jsonb,
+	"error" jsonb,
+	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"finished_at" timestamp with time zone,
+	"timeout_ms" integer NOT NULL,
+	"expires_at" timestamp with time zone,
+	"retries_limit" integer DEFAULT 0 NOT NULL,
+	"retries_count" integer DEFAULT 0 NOT NULL,
+	"delayed_ms" integer,
+	"history_failed_attempts" jsonb DEFAULT '{}'::jsonb NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "unique_job_step_name" UNIQUE("job_id","name"),
+	CONSTRAINT "job_steps_status_check" CHECK ("duron"."job_steps"."status" IN ('active','completed','failed','cancelled'))
+);
+--> statement-breakpoint
+CREATE TABLE "duron"."jobs" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"action_name" text NOT NULL,
+	"group_key" text NOT NULL,
+	"status" text DEFAULT 'created' NOT NULL,
+	"checksum" text NOT NULL,
+	"input" jsonb DEFAULT '{}'::jsonb NOT NULL,
+	"output" jsonb,
+	"error" jsonb,
+	"timeout_ms" integer NOT NULL,
+	"expires_at" timestamp with time zone,
+	"started_at" timestamp with time zone,
+	"finished_at" timestamp with time zone,
+	"owner_id" text,
+	"concurrency_limit" integer DEFAULT 10 NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "jobs_status_check" CHECK ("duron"."jobs"."status" IN ('created','active','completed','failed','cancelled'))
+);
+--> statement-breakpoint
+ALTER TABLE "duron"."job_steps" ADD CONSTRAINT "job_steps_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "duron"."jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "idx_job_steps_job_id" ON "duron"."job_steps" USING btree ("job_id");--> statement-breakpoint
+CREATE INDEX "idx_job_steps_status" ON "duron"."job_steps" USING btree ("status");--> statement-breakpoint
+CREATE INDEX "idx_job_steps_name" ON "duron"."job_steps" USING btree ("name");--> statement-breakpoint
+CREATE INDEX "idx_job_steps_expires_at" ON "duron"."job_steps" USING btree ("expires_at");--> statement-breakpoint
+CREATE INDEX "idx_job_steps_job_status" ON "duron"."job_steps" USING btree ("job_id","status");--> statement-breakpoint
+CREATE INDEX "idx_job_steps_job_name" ON "duron"."job_steps" USING btree ("job_id","name");--> statement-breakpoint
+CREATE INDEX "idx_job_steps_output_fts" ON "duron"."job_steps" USING gin (to_tsvector('english', "output"::text));--> statement-breakpoint
+CREATE INDEX "idx_jobs_action_name" ON "duron"."jobs" USING btree ("action_name");--> statement-breakpoint
+CREATE INDEX "idx_jobs_status" ON "duron"."jobs" USING btree ("status");--> statement-breakpoint
+CREATE INDEX "idx_jobs_group_key" ON "duron"."jobs" USING btree ("group_key");--> statement-breakpoint
+CREATE INDEX "idx_jobs_started_at" ON "duron"."jobs" USING btree ("started_at");--> statement-breakpoint
+CREATE INDEX "idx_jobs_finished_at" ON "duron"."jobs" USING btree ("finished_at");--> statement-breakpoint
+CREATE INDEX "idx_jobs_expires_at" ON "duron"."jobs" USING btree ("expires_at");--> statement-breakpoint
+CREATE INDEX "idx_jobs_owner_id" ON "duron"."jobs" USING btree ("owner_id");--> statement-breakpoint
+CREATE INDEX "idx_jobs_checksum" ON "duron"."jobs" USING btree ("checksum");--> statement-breakpoint
+CREATE INDEX "idx_jobs_concurrency_limit" ON "duron"."jobs" USING btree ("concurrency_limit");--> statement-breakpoint
+CREATE INDEX "idx_jobs_action_status" ON "duron"."jobs" USING btree ("action_name","status");--> statement-breakpoint
+CREATE INDEX "idx_jobs_action_group" ON "duron"."jobs" USING btree ("action_name","group_key");--> statement-breakpoint
+CREATE INDEX "idx_jobs_input_fts" ON "duron"."jobs" USING gin (to_tsvector('english', "input"::text));--> statement-breakpoint
+CREATE INDEX "idx_jobs_output_fts" ON "duron"."jobs" USING gin (to_tsvector('english', "output"::text));