@unchainedshop/core-worker 3.0.0-alpha7 → 3.0.0-rc2

package/src/director/WorkerAdapter.ts

@@ -1,28 +0,0 @@
-import { log, LogLevel } from '@unchainedshop/logger';
-import type { IBaseAdapter } from '@unchainedshop/utils';
-import type { UnchainedCore } from '@unchainedshop/core';
-export interface WorkResult<Result> {
-  success: boolean;
-  result?: Result;
-  error?: any;
-}
-
-export type IWorkerAdapter<Input, Output> = IBaseAdapter & {
-  type: string;
-  external: boolean;
-  maxParallelAllocations?: number;
-
-  doWork: (input: Input, unchainedAPI: UnchainedCore, workId: string) => Promise<WorkResult<Output>>;
-};
-
-export const WorkerAdapter: Omit<IWorkerAdapter<any, void>, 'key' | 'label' | 'type' | 'version'> = {
-  external: false,
-
-  async doWork() {
-    return { success: false, result: null };
-  },
-
-  log(message: string, { level = LogLevel.Debug, ...options } = {}) {
-    return log(message, { level, ...options });
-  },
-};

package/src/director/WorkerDirector.ts

@@ -1,120 +0,0 @@
-import type { WorkData, IWorkerAdapter, WorkResult } from '../worker-index.js';
-import type { UnchainedCore } from '@unchainedshop/core';
-
-import { log, LogLevel } from '@unchainedshop/logger';
-import { BaseDirector, IBaseDirector } from '@unchainedshop/utils';
-import { Work } from '../types.js';
-
-export const DIRECTOR_MARKED_FAILED_ERROR = 'DIRECTOR_MARKED_FAILED';
-
-export interface WorkerSchedule {
-  schedules: Array<Record<string, any>>;
-  exceptions: Array<Record<string, any>>;
-}
-
-export type WorkScheduleConfiguration = Pick<
-  WorkData,
-  'timeout' | 'retries' | 'priority' | 'worker' | 'scheduleId'
-> & {
-  type: string;
-  input?: (workData: Omit<WorkData, 'input'>) => Promise<Record<string, any> | null>;
-  schedule: WorkerSchedule;
-  scheduleId?: string;
-};
-
-export type IWorkerDirector = IBaseDirector<IWorkerAdapter<any, any>> & {
-  getActivePluginTypes: (options?: { external?: boolean }) => Array<string>;
-  getAdapterByType: (type: string) => IWorkerAdapter<any, any>;
-  disableAutoscheduling: (scheduleId: string) => void;
-  configureAutoscheduling: (workScheduleConfiguration: WorkScheduleConfiguration) => void;
-  getAutoSchedules: () => Array<[string, WorkScheduleConfiguration]>;
-  doWork: (work: Work, unchainedAPI: UnchainedCore) => Promise<WorkResult<any>>;
-};
-
-const AutoScheduleMap = new Map<string, WorkScheduleConfiguration>();
-
-const baseDirector = BaseDirector<IWorkerAdapter<any, any>>('WorkerDirector', {
-  adapterKeyField: 'key',
-});
-
-export const WorkerDirector: IWorkerDirector = {
-  ...baseDirector,
-
-  getActivePluginTypes: ({ external } = { external: null }) => {
-    return WorkerDirector.getAdapters()
-      .filter((adapter) => {
-        if (external === null || external === undefined) return true;
-        return Boolean(adapter.external) === external;
-      })
-      .map((adapter) => adapter.type);
-  },
-
-  registerAdapter: (Adapter) => {
-    if (WorkerDirector.getAdapterByType(Adapter.type))
-      throw new Error(
-        `WorkerDirector: There is already a adapter registered with type: ${Adapter.type}`,
-      );
-
-    baseDirector.registerAdapter(Adapter);
-  },
-
-  disableAutoscheduling: (type) => {
-    const config = AutoScheduleMap.get(type);
-    AutoScheduleMap.set(type, { ...config, schedule: null });
-  },
-
-  unregisterAdapter: (key) => {
-    const adapter = WorkerDirector.getAdapter(key);
-    const result = baseDirector.unregisterAdapter(key);
-    if (adapter) {
-      AutoScheduleMap.forEach((workItemConfiguration, scheduleId) => {
-        if (workItemConfiguration.type === adapter.type) {
-          AutoScheduleMap.delete(scheduleId);
-        }
-      });
-    }
-    return result;
-  },
-
-  configureAutoscheduling: (workItemConfiguration) => {
-    const adapter = WorkerDirector.getAdapterByType(workItemConfiguration.type);
-    AutoScheduleMap.set(
-      workItemConfiguration.scheduleId || workItemConfiguration.type,
-      workItemConfiguration,
-    );
-    log(
-      `WorkerDirector -> Configured ${adapter.type} ${adapter.key}@${adapter.version} (${adapter.label}) for Autorun at ${JSON.stringify(workItemConfiguration.schedule?.schedules)}`,
-    );
-  },
-
-  getAutoSchedules: () => Array.from(AutoScheduleMap),
-
-  getAdapterByType: (type: string) => {
-    const adapters = WorkerDirector.getAdapters({
-      adapterFilter: (Adapter) => {
-        return type === Adapter.type;
-      },
-    });
-    return adapters?.[0];
-  },
-
-  doWork: async ({ type, input, _id: workId }, unchainedAPI) => {
-    const adapter = WorkerDirector.getAdapterByType(type);
-
-    if (!adapter) {
-      log(`WorkerDirector: No registered adapter for type: ${type}`);
-    }
-
-    try {
-      const output = await adapter.doWork(input, unchainedAPI, workId);
-      return output;
-    } catch (error) {
-      // DO not use this as flow control. The adapter should catch expected errors and return status: FAILED
-      log('DO not use this as flow control.', { level: LogLevel.Verbose });
-      log(`WorkerDirector -> Error doing work ${type}: ${error.message}`);
-
-      const errorOutput = { error, success: false };
-      return errorOutput;
-    }
-  },
-};

package/src/director/WorkerEventTypes.ts

@@ -1,7 +0,0 @@
-export enum WorkerEventTypes {
-  ADDED = 'WORK_ADDED',
-  ALLOCATED = 'WORK_ALLOCATED',
-  FINISHED = 'WORK_FINISHED',
-  DELETED = 'WORK_DELETED',
-  RESCHEDULED = 'WORK_RESCHEDULED',
-}

package/src/schedulers/FailedRescheduler.ts

@@ -1,83 +0,0 @@
-import { log } from '@unchainedshop/logger';
-import { subscribe } from '@unchainedshop/events';
-import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
-import { UnchainedCore } from '@unchainedshop/core';
-import { WorkData } from '../worker-index.js';
-import { Work } from '../types.js';
-
-export interface FailedReschedulerParams {
-  retryInput?: (
-    workData: WorkData,
-    priorInput: Record<string, any>,
-  ) => Promise<Record<string, any> | null>;
-}
-
-export type IScheduler<P> = {
-  key: string;
-  label: string;
-  version: string;
-
-  actions: (
-    params: P,
-    unchainedAPI: UnchainedCore,
-  ) => {
-    start: () => void;
-    stop: () => void;
-  };
-};
-
-export const FailedRescheduler: IScheduler<FailedReschedulerParams> = {
-  key: 'shop.unchained.scheduler.failed',
-  label: 'Reschedule failed works',
-  version: '1.0.0',
-
-  actions: ({ retryInput }, unchainedAPI) => {
-    const handleFinishedWork = async ({ payload: work }: { payload: Work }) => {
-      if (!work.success && work.retries > 0) {
-        const now = new Date();
-        const workDelayMs = work.scheduled.getTime() - work.created.getTime();
-
-        // In short: Double the delay of the old work or delay for 5 seconds
-        const scheduled =
-          workDelayMs > 1000
-            ? new Date(now.getTime() + workDelayMs * 2)
-            : new Date(now.setSeconds(now.getSeconds() + 5));
-
-        log(
-          `${FailedRescheduler.key} -> Reschedule failed work ${work._id} ${
-            work.type
-          } for ${scheduled.toISOString()} (in ${Math.round(workDelayMs / 1000)}). Remaining retries: ${
-            work.retries
-          }`,
-        );
-
-        const workData: WorkData = {
-          type: work.type,
-          priority: work.priority,
-          originalWorkId: work.originalWorkId || work._id,
-          retries: work.retries - 1,
-          timeout: work.timeout,
-          scheduled,
-        };
-
-        if (retryInput) {
-          workData.input = await retryInput(workData, work.input);
-        } else {
-          workData.input = work.input;
-        }
-
-        unchainedAPI.modules.worker.addWork(workData);
-      }
-    };
-
-    return {
-      start() {
-        subscribe<Work>(WorkerEventTypes.FINISHED, handleFinishedWork);
-      },
-
-      stop() {
-        /* */
-      },
-    };
-  },
-};

package/src/types.ts

@@ -1,20 +0,0 @@
-import type { TimestampFields } from '@unchainedshop/mongodb';
-
-export type Work = {
-  _id?: string;
-  priority: number;
-  retries: number;
-  scheduled: Date;
-  type: string;
-  input: Record<string, any>;
-  error?: any;
-  finished?: Date;
-  originalWorkId?: string;
-  result?: any;
-  started?: Date;
-  success?: boolean;
-  timeout?: number;
-  worker?: string;
-  autoscheduled?: boolean;
-  scheduleId?: string;
-} & TimestampFields;

package/src/workers/BaseWorker.ts

@@ -1,125 +0,0 @@
-import later from '@breejs/later';
-import { log } from '@unchainedshop/logger';
-import { WorkerDirector } from '../director/WorkerDirector.js';
-import { UnchainedCore } from '@unchainedshop/core';
-import { Work } from '../types.js';
-
-export type WorkData = Pick<
-  Partial<Work>,
-  'input' | 'originalWorkId' | 'priority' | 'retries' | 'timeout' | 'scheduled' | 'worker' | 'scheduleId'
-> & { type: string };
-
-export type IWorker<P extends { workerId?: string }> = {
-  key: string;
-  label: string;
-  version: string;
-  type: string;
-  external: boolean;
-
-  getFloorDate: (date?: Date) => Date;
-
-  actions: (
-    params: P,
-    unchainedAPI: UnchainedCore,
-  ) => {
-    autorescheduleTypes: (options: { referenceDate: Date }) => Promise<Array<Work | null>>;
-    process: (options: { maxWorkItemCount?: number; referenceDate?: Date }) => Promise<void>;
-    reset: () => Promise<void>;
-    start: () => void;
-    stop: () => void;
-  };
-};
-
-interface WorkerParams {
-  workerId?: string;
-  worker: IWorker<any>;
-}
-
-export const BaseWorker: IWorker<WorkerParams> = {
-  key: 'shop.unchained.worker.base',
-  label: 'Base worker. Do not use this directly.',
-  version: '1.0.0',
-  type: 'BASE',
-  external: false,
-
-  getFloorDate: (date = new Date()) => {
-    const floored = new Date(Math.floor(date.getTime() / 1000) * 1000);
-    return floored;
-  },
-
-  actions: ({ workerId, worker }: WorkerParams, unchainedAPI) => {
-    log(`${worker.key} -> Initialized ${worker.type}`);
-
-    const workerActions = {
-      start() {
-        throw new Error(`Not implemented on ${this.constructor.key}`);
-      },
-
-      stop() {
-        throw new Error(`Not implemented on ${this.constructor.key}`);
-      },
-
-      reset: async (referenceDate = new Date()) => {
-        await unchainedAPI.modules.worker.markOldWorkAsFailed({
-          types: WorkerDirector.getActivePluginTypes({ external: false }),
-          worker: workerId,
-          referenceDate,
-        });
-      },
-
-      autorescheduleTypes: async ({ referenceDate }) => {
-        return Promise.all(
-          WorkerDirector.getAutoSchedules().map(async ([scheduleId, workConfig]) => {
-            if (!workConfig.schedule) {
-              await unchainedAPI.modules.worker.ensureNoWork({
-                type: workConfig.type,
-                priority: workConfig.priority || 0,
-                scheduleId,
-              });
-              return null;
-            }
-
-            const fixedSchedule = { ...workConfig.schedule };
-            fixedSchedule.schedules[0].s = [0]; // ignore seconds, always run on second 0
-            const nextDate = later.schedule(fixedSchedule).next(1, referenceDate);
-            nextDate.setMilliseconds(0);
-            const workData: WorkData = {
-              type: workConfig.type,
-              scheduleId,
-              scheduled: nextDate,
-              timeout: workConfig.timeout,
-              priority: workConfig.priority || 0,
-              retries: workConfig.retries || 0,
-            };
-            if (workConfig.input) {
-              workData.input = await workConfig.input(workData);
-              // A work input fn can skip auto scheduling a new record
-              // when it explicitly returns a falsish input instead of a dictionary
-              if (!workData.input) return null;
-            }
-            return unchainedAPI.modules.worker.ensureOneWork(workData);
-          }),
-        );
-      },
-
-      process: async ({ maxWorkItemCount, referenceDate }) => {
-        await workerActions.autorescheduleTypes({
-          referenceDate,
-        });
-
-        const processRecursively = async (recursionCounter = 0) => {
-          if (maxWorkItemCount && maxWorkItemCount < recursionCounter) return null;
-          const doneWork = await unchainedAPI.modules.worker.processNextWork(unchainedAPI, workerId);
-          if (doneWork) return processRecursively(recursionCounter + 1);
-          return null;
-        };
-
-        await processRecursively();
-      },
-    };
-
-    workerActions.reset();
-
-    return workerActions;
-  },
-};

package/src/workers/EventListenerWorker.ts

@@ -1,61 +0,0 @@
-import { subscribe } from '@unchainedshop/events';
-import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
-import { BaseWorker, IWorker } from './BaseWorker.js';
-
-function debounce<T extends (...args: any) => any>(func: T, wait) {
-  let timeout;
-  return (...args) => {
-    clearTimeout(timeout);
-    timeout = setTimeout(() => {
-      timeout = null;
-      func(...args);
-    }, wait);
-  };
-}
-
-export interface EventListenerWorkerParams {
-  workerId?: string;
-}
-
-export const EventListenerWorker: IWorker<EventListenerWorkerParams> = {
-  ...BaseWorker,
-
-  key: 'shop.unchained.worker.event-listener',
-  label: 'Allocates work on events. This worker does not make sense on multiple containers.',
-  version: '1.0.0',
-  type: 'EVENT_LISTENER',
-
-  actions: ({ workerId }, unchainedAPI) => {
-    const baseWorkerActions = BaseWorker.actions(
-      { workerId, worker: EventListenerWorker },
-      unchainedAPI,
-    );
-
-    // Debounce in the event of many work queue events conflicting with each other
-    const processWorkQueue = debounce<() => Promise<void>>(async () => {
-      await baseWorkerActions.process({
-        maxWorkItemCount: 1, // only one work item at a time, else we could end up in a loop
-        referenceDate: EventListenerWorker.getFloorDate(),
-      });
-    }, 300);
-
-    return {
-      ...baseWorkerActions,
-
-      start() {
-        subscribe(WorkerEventTypes.ADDED, processWorkQueue);
-        subscribe(WorkerEventTypes.FINISHED, processWorkQueue);
-
-        setTimeout(async () => {
-          await baseWorkerActions.autorescheduleTypes({
-            referenceDate: EventListenerWorker.getFloorDate(),
-          });
-        }, 300);
-      },
-
-      stop() {
-        /* */
-      },
-    };
-  },
-};

package/src/workers/IntervalWorker.ts

@@ -1,56 +0,0 @@
-import later from '@breejs/later';
-import { BaseWorker, IWorker } from './BaseWorker.js';
-import { WorkerSchedule } from '../worker-index.js';
-
-const { NODE_ENV } = process.env;
-
-export interface IntervalWorkerParams {
-  workerId?: string;
-  batchCount?: number;
-  schedule?: WorkerSchedule | string;
-}
-
-const defaultSchedule = later.parse.text(
-  NODE_ENV !== 'production' ? 'every 2 seconds' : 'every 30 seconds',
-) as WorkerSchedule;
-
-export const scheduleToInterval = (scheduleRaw: WorkerSchedule | string) => {
-  const referenceDate = new Date(1000);
-  const schedule = typeof scheduleRaw === 'string' ? later.parse.text(scheduleRaw) : scheduleRaw;
-  const [one, two] = later.schedule(schedule).next(2, referenceDate);
-  const diff = new Date(two).getTime() - new Date(one).getTime();
-  return Math.min(1000 * 60 * 60, diff); // at least once every hour!
-};
-
-export const IntervalWorker: IWorker<IntervalWorkerParams> = {
-  ...BaseWorker,
-
-  key: 'shop.unchained.worker.cron',
-  label: 'Allocates work on fixed intervals with native node setInterval',
-  version: '1.0.0',
-  type: 'CRON',
-
-  actions: ({ workerId, batchCount = 0, schedule = defaultSchedule }, unchainedAPI) => {
-    const baseWorkerActions = BaseWorker.actions({ workerId, worker: IntervalWorker }, unchainedAPI);
-
-    const intervalDelay = scheduleToInterval(schedule);
-    let intervalHandle: NodeJS.Timeout;
-
-    return {
-      ...baseWorkerActions,
-
-      start() {
-        intervalHandle = setInterval(() => {
-          baseWorkerActions.process({
-            maxWorkItemCount: batchCount,
-            referenceDate: IntervalWorker.getFloorDate(),
-          });
-        }, intervalDelay);
-      },
-
-      stop() {
-        clearInterval(intervalHandle);
-      },
-    };
-  },
-};