@unchainedshop/core-worker 3.0.0-alpha6 → 3.0.0-rc.11

package/src/schedulers/FailedRescheduler.ts

@@ -1,83 +0,0 @@
-import { log } from '@unchainedshop/logger';
-import { subscribe } from '@unchainedshop/events';
-import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
-import { UnchainedCore } from '@unchainedshop/core';
-import { WorkData } from '../worker-index.js';
-import { Work } from '../types.js';
-
-export interface FailedReschedulerParams {
-  retryInput?: (
-    workData: WorkData,
-    priorInput: Record<string, any>,
-  ) => Promise<Record<string, any> | null>;
-}
-
-export type IScheduler<P> = {
-  key: string;
-  label: string;
-  version: string;
-
-  actions: (
-    params: P,
-    unchainedAPI: UnchainedCore,
-  ) => {
-    start: () => void;
-    stop: () => void;
-  };
-};
-
-export const FailedRescheduler: IScheduler<FailedReschedulerParams> = {
-  key: 'shop.unchained.scheduler.failed',
-  label: 'Reschedule failed works',
-  version: '1.0.0',
-
-  actions: ({ retryInput }, unchainedAPI) => {
-    const handleFinishedWork = async ({ payload: work }: { payload: Work }) => {
-      if (!work.success && work.retries > 0) {
-        const now = new Date();
-        const workDelayMs = work.scheduled.getTime() - work.created.getTime();
-
-        // In short: Double the delay of the old work or delay for 5 seconds
-        const scheduled =
-          workDelayMs > 1000
-            ? new Date(now.getTime() + workDelayMs * 2)
-            : new Date(now.setSeconds(now.getSeconds() + 5));
-
-        log(
-          `${FailedRescheduler.key} -> Reschedule failed work ${work._id} ${
-            work.type
-          } for ${scheduled.toISOString()} (in ${Math.round(workDelayMs / 1000)}). Remaining retries: ${
-            work.retries
-          }`,
-        );
-
-        const workData: WorkData = {
-          type: work.type,
-          priority: work.priority,
-          originalWorkId: work.originalWorkId || work._id,
-          retries: work.retries - 1,
-          timeout: work.timeout,
-          scheduled,
-        };
-
-        if (retryInput) {
-          workData.input = await retryInput(workData, work.input);
-        } else {
-          workData.input = work.input;
-        }
-
-        unchainedAPI.modules.worker.addWork(workData);
-      }
-    };
-
-    return {
-      start() {
-        subscribe<Work>(WorkerEventTypes.FINISHED, handleFinishedWork);
-      },
-
-      stop() {
-        /* */
-      },
-    };
-  },
-};

package/src/types.ts

@@ -1,20 +0,0 @@
-import type { TimestampFields } from '@unchainedshop/mongodb';
-
-export type Work = {
-  _id?: string;
-  priority: number;
-  retries: number;
-  scheduled: Date;
-  type: string;
-  input: Record<string, any>;
-  error?: any;
-  finished?: Date;
-  originalWorkId?: string;
-  result?: any;
-  started?: Date;
-  success?: boolean;
-  timeout?: number;
-  worker?: string;
-  autoscheduled?: boolean;
-  scheduleId?: string;
-} & TimestampFields;

package/src/workers/BaseWorker.ts

@@ -1,125 +0,0 @@
-import later from '@breejs/later';
-import { log } from '@unchainedshop/logger';
-import { WorkerDirector } from '../director/WorkerDirector.js';
-import { UnchainedCore } from '@unchainedshop/core';
-import { Work } from '../types.js';
-
-export type WorkData = Pick<
-  Partial<Work>,
-  'input' | 'originalWorkId' | 'priority' | 'retries' | 'timeout' | 'scheduled' | 'worker' | 'scheduleId'
-> & { type: string };
-
-export type IWorker<P extends { workerId?: string }> = {
-  key: string;
-  label: string;
-  version: string;
-  type: string;
-  external: boolean;
-
-  getFloorDate: (date?: Date) => Date;
-
-  actions: (
-    params: P,
-    unchainedAPI: UnchainedCore,
-  ) => {
-    autorescheduleTypes: (options: { referenceDate: Date }) => Promise<Array<Work | null>>;
-    process: (options: { maxWorkItemCount?: number; referenceDate?: Date }) => Promise<void>;
-    reset: () => Promise<void>;
-    start: () => void;
-    stop: () => void;
-  };
-};
-
-interface WorkerParams {
-  workerId?: string;
-  worker: IWorker<any>;
-}
-
-export const BaseWorker: IWorker<WorkerParams> = {
-  key: 'shop.unchained.worker.base',
-  label: 'Base worker. Do not use this directly.',
-  version: '1.0.0',
-  type: 'BASE',
-  external: false,
-
-  getFloorDate: (date = new Date()) => {
-    const floored = new Date(Math.floor(date.getTime() / 1000) * 1000);
-    return floored;
-  },
-
-  actions: ({ workerId, worker }: WorkerParams, unchainedAPI) => {
-    log(`${worker.key} -> Initialized ${worker.type}`);
-
-    const workerActions = {
-      start() {
-        throw new Error(`Not implemented on ${this.constructor.key}`);
-      },
-
-      stop() {
-        throw new Error(`Not implemented on ${this.constructor.key}`);
-      },
-
-      reset: async (referenceDate = new Date()) => {
-        await unchainedAPI.modules.worker.markOldWorkAsFailed({
-          types: WorkerDirector.getActivePluginTypes({ external: false }),
-          worker: workerId,
-          referenceDate,
-        });
-      },
-
-      autorescheduleTypes: async ({ referenceDate }) => {
-        return Promise.all(
-          WorkerDirector.getAutoSchedules().map(async ([scheduleId, workConfig]) => {
-            if (!workConfig.schedule) {
-              await unchainedAPI.modules.worker.ensureNoWork({
-                type: workConfig.type,
-                priority: workConfig.priority || 0,
-                scheduleId,
-              });
-              return null;
-            }
-
-            const fixedSchedule = { ...workConfig.schedule };
-            fixedSchedule.schedules[0].s = [0]; // ignore seconds, always run on second 0
-            const nextDate = later.schedule(fixedSchedule).next(1, referenceDate);
-            nextDate.setMilliseconds(0);
-            const workData: WorkData = {
-              type: workConfig.type,
-              scheduleId,
-              scheduled: nextDate,
-              timeout: workConfig.timeout,
-              priority: workConfig.priority || 0,
-              retries: workConfig.retries || 0,
-            };
-            if (workConfig.input) {
-              workData.input = await workConfig.input(workData);
-              // A work input fn can skip auto scheduling a new record
-              // when it explicitly returns a falsish input instead of a dictionary
-              if (!workData.input) return null;
-            }
-            return unchainedAPI.modules.worker.ensureOneWork(workData);
-          }),
-        );
-      },
-
-      process: async ({ maxWorkItemCount, referenceDate }) => {
-        await workerActions.autorescheduleTypes({
-          referenceDate,
-        });
-
-        const processRecursively = async (recursionCounter = 0) => {
-          if (maxWorkItemCount && maxWorkItemCount < recursionCounter) return null;
-          const doneWork = await unchainedAPI.modules.worker.processNextWork(unchainedAPI, workerId);
-          if (doneWork) return processRecursively(recursionCounter + 1);
-          return null;
-        };
-
-        await processRecursively();
-      },
-    };
-
-    workerActions.reset();
-
-    return workerActions;
-  },
-};

package/src/workers/EventListenerWorker.ts

@@ -1,61 +0,0 @@
-import { subscribe } from '@unchainedshop/events';
-import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
-import { BaseWorker, IWorker } from './BaseWorker.js';
-
-function debounce<T extends (...args: any) => any>(func: T, wait) {
-  let timeout;
-  return (...args) => {
-    clearTimeout(timeout);
-    timeout = setTimeout(() => {
-      timeout = null;
-      func(...args);
-    }, wait);
-  };
-}
-
-export interface EventListenerWorkerParams {
-  workerId?: string;
-}
-
-export const EventListenerWorker: IWorker<EventListenerWorkerParams> = {
-  ...BaseWorker,
-
-  key: 'shop.unchained.worker.event-listener',
-  label: 'Allocates work on events. This worker does not make sense on multiple containers.',
-  version: '1.0.0',
-  type: 'EVENT_LISTENER',
-
-  actions: ({ workerId }, unchainedAPI) => {
-    const baseWorkerActions = BaseWorker.actions(
-      { workerId, worker: EventListenerWorker },
-      unchainedAPI,
-    );
-
-    // Debounce in the event of many work queue events conflicting with each other
-    const processWorkQueue = debounce<() => Promise<void>>(async () => {
-      await baseWorkerActions.process({
-        maxWorkItemCount: 1, // only one work item at a time, else we could end up in a loop
-        referenceDate: EventListenerWorker.getFloorDate(),
-      });
-    }, 300);
-
-    return {
-      ...baseWorkerActions,
-
-      start() {
-        subscribe(WorkerEventTypes.ADDED, processWorkQueue);
-        subscribe(WorkerEventTypes.FINISHED, processWorkQueue);
-
-        setTimeout(async () => {
-          await baseWorkerActions.autorescheduleTypes({
-            referenceDate: EventListenerWorker.getFloorDate(),
-          });
-        }, 300);
-      },
-
-      stop() {
-        /* */
-      },
-    };
-  },
-};

package/src/workers/IntervalWorker.ts

@@ -1,56 +0,0 @@
-import later from '@breejs/later';
-import { BaseWorker, IWorker } from './BaseWorker.js';
-import { WorkerSchedule } from '../worker-index.js';
-
-const { NODE_ENV } = process.env;
-
-export interface IntervalWorkerParams {
-  workerId?: string;
-  batchCount?: number;
-  schedule?: WorkerSchedule | string;
-}
-
-const defaultSchedule = later.parse.text(
-  NODE_ENV !== 'production' ? 'every 2 seconds' : 'every 30 seconds',
-) as WorkerSchedule;
-
-export const scheduleToInterval = (scheduleRaw: WorkerSchedule | string) => {
-  const referenceDate = new Date(1000);
-  const schedule = typeof scheduleRaw === 'string' ? later.parse.text(scheduleRaw) : scheduleRaw;
-  const [one, two] = later.schedule(schedule).next(2, referenceDate);
-  const diff = new Date(two).getTime() - new Date(one).getTime();
-  return Math.min(1000 * 60 * 60, diff); // at least once every hour!
-};
-
-export const IntervalWorker: IWorker<IntervalWorkerParams> = {
-  ...BaseWorker,
-
-  key: 'shop.unchained.worker.cron',
-  label: 'Allocates work on fixed intervals with native node setInterval',
-  version: '1.0.0',
-  type: 'CRON',
-
-  actions: ({ workerId, batchCount = 0, schedule = defaultSchedule }, unchainedAPI) => {
-    const baseWorkerActions = BaseWorker.actions({ workerId, worker: IntervalWorker }, unchainedAPI);
-
-    const intervalDelay = scheduleToInterval(schedule);
-    let intervalHandle: NodeJS.Timeout;
-
-    return {
-      ...baseWorkerActions,
-
-      start() {
-        intervalHandle = setInterval(() => {
-          baseWorkerActions.process({
-            maxWorkItemCount: batchCount,
-            referenceDate: IntervalWorker.getFloorDate(),
-          });
-        }, intervalDelay);
-      },
-
-      stop() {
-        clearInterval(intervalHandle);
-      },
-    };
-  },
-};