@unchainedshop/core-worker 1.1.3

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@unchainedshop/core-worker",
+  "version": "1.1.3",
+  "main": "lib/worker-index.js",
+  "types": "lib/worker-index.d.ts",
+  "type": "module",
+  "scripts": {
+    "clean": "rm -rf lib",
+    "build": "npm run clean && tsc -p tsconfig.build.json",
+    "watch": "tsc -p tsconfig.build.json --watch",
+    "link:core": "npm link  @unchainedshop/types",
+    "test": "cross-env METEOR_PACKAGE_DIRS=../ TEST_CLIENT=0 TEST_WATCH=1 meteor test-packages ./ --driver-package meteortesting:mocha --port 4200"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/unchainedshop/unchained.git"
+  },
+  "keywords": [
+    "unchained",
+    "ecommerce",
+    "core"
+  ],
+  "author": "Joël Meiller",
+  "license": "EUPL-1.2",
+  "bugs": {
+    "url": "https://github.com/unchainedshop/unchained/issues"
+  },
+  "homepage": "https://github.com/unchainedshop/unchained#readme",
+  "dependencies": {
+    "later": "1.2.0",
+    "simpl-schema": "1.12.0",
+    "@unchainedshop/logger": "latest",
+    "@unchainedshop/utils": "latest"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.1",
+    "@types/mocha": "^9.1.1",
+    "@unchainedshop/types": "latest",
+    "chai": "^4.3.6",
+    "cross-env": "^7.0.3",
+    "typescript": "^4.7.4"
+  }
+}

package/src/db/WorkQueueCollection.ts

@@ -0,0 +1,37 @@
+import { Db } from '@unchainedshop/types/common';
+import { Work } from '@unchainedshop/types/worker';
+import { buildDbIndexes } from '@unchainedshop/utils';
+
+const ONE_DAY_IN_SECONDS = 86400;
+
+export const WorkQueueCollection = async (db: Db) => {
+  const WorkQueue = db.collection<Work>('work_queue');
+
+  await buildDbIndexes<Work>(WorkQueue, [
+    {
+      index: { created: -1 },
+      options: { expireAfterSeconds: 30 * ONE_DAY_IN_SECONDS },
+    },
+    { index: { started: -1 } },
+    { index: { finished: 1 } },
+    { index: { scheduled: 1 } },
+    { index: { priority: -1 } },
+    { index: { type: 1 } },
+    { index: { originalWorkId: 1 } },
+    {
+      index: { originalWorkId: 'text', _id: 'text', worker: 'text', input: 'text', type: 'text' },
+      options: {
+        weights: {
+          _id: 8,
+          originalWorkId: 6,
+          type: 5,
+          worker: 4,
+          input: 2,
+        },
+        name: 'workqueue_fulltext_search',
+      },
+    },
+  ]);
+
+  return WorkQueue;
+};

package/src/db/WorkQueueSchema.js

@@ -0,0 +1,23 @@
+import { Schemas } from '@unchainedshop/utils';
+import SimpleSchema from 'simpl-schema';
+
+export const WorkQueueSchema = new SimpleSchema(
+  {
+    finished: Date,
+    originalWorkId: { type: String },
+    priority: { type: Number, required: true },
+    retries: { type: Number, required: true },
+    scheduled: { type: Date, required: true },
+    started: Date,
+    success: Boolean,
+    timeout: Number,
+    type: { type: String, required: true },
+    worker: String,
+    // Worker data
+    input: { type: Object, blackbox: true },
+    result: { type: Object, blackbox: true },
+    error: { type: Object, blackbox: true },
+    ...Schemas.timestampFields,
+  },
+  { requiredByDefault: false },
+);

package/src/director/WorkStatus.ts

@@ -0,0 +1,7 @@
+export enum WorkStatus {
+  NEW = 'NEW',
+  ALLOCATED = 'ALLOCATED',
+  SUCCESS = 'SUCCESS',
+  FAILED = 'FAILED',
+  DELETED = 'DELETED',
+}

package/src/director/WorkerAdapter.ts

@@ -0,0 +1,12 @@
+import { IWorkerAdapter } from '@unchainedshop/types/worker';
+import { log, LogLevel } from '@unchainedshop/logger';
+
+export const WorkerAdapter: Omit<IWorkerAdapter<any, void>, 'key' | 'label' | 'type' | 'version'> = {
+  async doWork() {
+    return { success: false, result: null };
+  },
+
+  log(message: string, { level = LogLevel.Debug, ...options } = {}) {
+    return log(message, { level, ...options });
+  },
+};

package/src/director/WorkerDirector.ts

@@ -0,0 +1,73 @@
+import { Context } from '@unchainedshop/types/api';
+import {
+  IWorkerAdapter,
+  IWorkerDirector,
+  WorkScheduleConfigureation,
+} from '@unchainedshop/types/worker';
+import { EventEmitter } from 'events';
+import { log, LogLevel } from '@unchainedshop/logger';
+import { BaseDirector } from '@unchainedshop/utils';
+import { WorkerEventTypes } from './WorkerEventTypes';
+
+export const DIRECTOR_MARKED_FAILED_ERROR = 'DIRECTOR_MARKED_FAILED';
+
+const AutoScheduleMap = new Map<string, WorkScheduleConfigureation>();
+
+const baseDirector = BaseDirector<IWorkerAdapter<any, any>>('WorkerDirector', {
+  adapterKeyField: 'type',
+});
+
+export const WorkerDirector: IWorkerDirector = {
+  ...baseDirector,
+
+  events: new EventEmitter(),
+
+  getActivePluginTypes: () => {
+    return baseDirector.getAdapters().map((adapter) => adapter.type);
+  },
+
+  registerAdapter: (Adapter) => {
+    if (baseDirector.getAdapter(Adapter.type))
+      throw new Error(
+        `WorkderDirector: There is already a adapter registered with type: ${Adapter.type}`,
+      );
+
+    baseDirector.registerAdapter(Adapter);
+  },
+
+  configureAutoscheduling: (adapter, workQueue) => {
+    const { schedule } = workQueue;
+    AutoScheduleMap.set(adapter.type, workQueue);
+    log(
+      `WorkderDirector -> Configured ${adapter.type} ${adapter.key}@${adapter.version} (${adapter.label}) for Autorun at ${schedule}`,
+    );
+  },
+  getAutoSchedules: () => Array.from(AutoScheduleMap),
+
+  doWork: async ({ type, input, _id: workId }, requestContext: Context) => {
+    const adapter = baseDirector.getAdapter(type);
+
+    if (!adapter) {
+      log(`WorkderDirector: No registered adapter for type: ${type}`);
+    }
+
+    const output = await adapter.doWork(input, requestContext, workId).catch((error) => {
+      // DO not use this as flow control. The adapter should catch expected errors and return status: FAILED
+      log('DO not use this as flow control.', { level: LogLevel.Verbose });
+
+      log(`WorkderDirector -> Error doing work ${type}: ${error.message}`);
+
+      const errorOutput = { error, success: false };
+
+      WorkerDirector.events.emit(WorkerEventTypes.DONE, {
+        output: errorOutput,
+      });
+
+      return errorOutput;
+    });
+
+    WorkerDirector.events.emit(WorkerEventTypes.DONE, { output });
+
+    return output;
+  },
+};

package/src/director/WorkerEventTypes.ts

@@ -0,0 +1,11 @@
+export enum WorkerEventTypes {
+  ADDED = 'added',
+  ALLOCATED = 'allocated',
+  DONE = 'done',
+  FINISHED = 'finished',
+  DELETED = 'deleted',
+  RESCHEDULED = 'rescheduled',
+}
+// The difference between `done` and `finished` is, that work is `done` after
+// it was computed (no DB write, could be external) and `finished` after
+// the changes are written to the DB

package/src/module/configureWorkerModule.ts

@@ -0,0 +1,397 @@
+import {
+  ModifyResult,
+  ModuleInput,
+  ModuleMutations,
+  Projection,
+  Query,
+} from '@unchainedshop/types/common';
+import { Work, WorkerModule } from '@unchainedshop/types/worker';
+import { log, LogLevel } from '@unchainedshop/logger';
+import { generateDbFilterById, generateDbMutations, buildSortOptions } from '@unchainedshop/utils';
+import os from 'os';
+import { WorkQueueCollection } from '../db/WorkQueueCollection';
+import { WorkQueueSchema } from '../db/WorkQueueSchema';
+import { DIRECTOR_MARKED_FAILED_ERROR, WorkerDirector } from '../director/WorkerDirector';
+import { WorkerEventTypes } from '../director/WorkerEventTypes';
+import { WorkStatus } from '../director/WorkStatus';
+
+const { UNCHAINED_WORKER_ID = os.hostname() } = process.env;
+
+const buildQuerySelector = ({
+  created,
+  scheduled,
+  types,
+  status,
+  workId,
+  queryString,
+  ...rest
+}: Query & {
+  created?: { end?: Date; start?: Date };
+  scheduled?: { end?: Date; start?: Date };
+  status?: Array<WorkStatus>;
+  workId?: string;
+  queryString?: string;
+  types?: Array<string>;
+}) => {
+  const filterMap = {
+    [WorkStatus.DELETED]: { deleted: { $exists: true } },
+    [WorkStatus.NEW]: {
+      started: { $exists: false },
+      deleted: { $exists: false },
+    },
+    [WorkStatus.ALLOCATED]: {
+      started: { $exists: true },
+      finished: { $exists: false },
+      deleted: { $exists: false },
+    },
+    [WorkStatus.SUCCESS]: {
+      finished: { $exists: true },
+      success: true,
+      deleted: { $exists: false },
+    },
+    [WorkStatus.FAILED]: {
+      finished: { $exists: true },
+      success: false,
+      deleted: { $exists: false },
+    },
+  };
+  const statusQuery = {
+    $or: Object.entries(filterMap).reduce(
+      (acc, [key, filter]) => (status?.includes(key as WorkStatus) ? [...acc, filter] : acc),
+      [],
+    ),
+  };
+
+  let query: Query = statusQuery.$or.length > 0 ? statusQuery : { deleted: { $exists: false } };
+
+  if (created) {
+    query.created = created?.end
+      ? { $gte: created.start || new Date(0), $lte: created.end }
+      : { $gte: created.start || new Date(0) };
+  }
+  if (scheduled) {
+    query.scheduled = scheduled?.end
+      ? { $gte: scheduled.start || new Date(0), $lte: scheduled.end }
+      : { $gte: scheduled.start || new Date(0) };
+  }
+  if (types) {
+    query.type = { $in: types };
+  }
+
+  if (workId) {
+    query = generateDbFilterById(workId, query);
+  }
+  if (queryString) query.$text = { $search: queryString };
+
+  return { ...query, ...rest };
+};
+
+const defaultSort: Array<{ key: string; value: 'DESC' | 'ASC' }> = [
+  { key: 'started', value: 'DESC' },
+  { key: 'priority', value: 'DESC' },
+  { key: 'originalWorkId', value: 'ASC' },
+  { key: 'created', value: 'ASC' },
+];
+
+export const configureWorkerModule = async ({
+  db,
+}: ModuleInput<Record<string, never>>): Promise<WorkerModule> => {
+  const WorkQueue = await WorkQueueCollection(db);
+
+  const mutations = generateDbMutations<Work>(WorkQueue, WorkQueueSchema) as ModuleMutations<Work>;
+
+  const finishWork: WorkerModule['finishWork'] = async (
+    workId,
+    {
+      error,
+      finished = new Date(),
+      result,
+      started = new Date(),
+      success,
+      worker = UNCHAINED_WORKER_ID,
+    },
+    userId,
+  ) => {
+    const workBeforeUpdate = await WorkQueue.findOne(
+      buildQuerySelector({ workId, status: [WorkStatus.ALLOCATED] }),
+    );
+
+    if (!workBeforeUpdate) return null;
+
+    await mutations.update(
+      workId,
+      {
+        $set: {
+          finished,
+          success,
+          error,
+          result,
+          ...(!workBeforeUpdate.started ? { started } : {}),
+          worker,
+        },
+      },
+      userId,
+    );
+
+    const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
+
+    log(`Finished work ${workId}`, {
+      level: LogLevel.Verbose,
+      work,
+    });
+
+    WorkerDirector.events.emit(WorkerEventTypes.FINISHED, { work, userId });
+
+    return work;
+  };
+
+  return {
+    // Queries
+    activeWorkTypes: async () => {
+      const typeList = await WorkQueue.aggregate([{ $group: { _id: '$type' } }]).toArray();
+      return typeList
+        .map((t) => t._id as string)
+        .filter((type) => {
+          return WorkerDirector.getActivePluginTypes().includes(type);
+        });
+    },
+
+    findWork: async ({ workId, originalWorkId }) =>
+      WorkQueue.findOne(workId ? generateDbFilterById(workId) : { originalWorkId }, {}),
+
+    findWorkQueue: async ({ limit, skip, sort, ...selectorOptions }) => {
+      const selector = buildQuerySelector(selectorOptions);
+      const workQueues = WorkQueue.find(selector, {
+        skip,
+        limit,
+        sort: buildSortOptions(sort),
+      });
+
+      return workQueues.toArray();
+    },
+
+    count: async (query) => {
+      return WorkQueue.countDocuments(buildQuerySelector(query));
+    },
+
+    workExists: async ({ workId, originalWorkId }) => {
+      const queueCount = await WorkQueue.countDocuments(
+        workId ? generateDbFilterById(workId) : { originalWorkId },
+        { limit: 1 },
+      );
+      return !!queueCount;
+    },
+
+    // Transformations
+
+    type: (work) => {
+      if (WorkerDirector.getActivePluginTypes().includes(work.type)) {
+        return work.type;
+      }
+      return 'UNKNOWN';
+    },
+
+    status: (work) => {
+      if (work.deleted) {
+        return WorkStatus.DELETED;
+      }
+      if (!work.started && !work.finished) {
+        return WorkStatus.NEW;
+      }
+      if (work.started && !work.finished) {
+        return WorkStatus.ALLOCATED;
+      }
+      if (work.started && work.finished && work.success) {
+        return WorkStatus.SUCCESS;
+      }
+      if (work.started && work.finished && !work.success) {
+        return WorkStatus.FAILED;
+      }
+
+      log('Unexpected work status', { level: LogLevel.Warning });
+
+      throw new Error('Unexpected work status');
+    },
+
+    // Mutations
+    addWork: async ({ type, input, priority = 0, scheduled, originalWorkId, retries = 20 }, userId) => {
+      if (!WorkerDirector.getAdapter(type)) {
+        throw new Error(`No plugin registered for type ${type}`);
+      }
+
+      const created = new Date();
+      const workId = await mutations.create(
+        {
+          type,
+          input,
+          priority,
+          scheduled: scheduled || created,
+          originalWorkId,
+          retries,
+          created,
+        },
+        userId,
+      );
+
+      log(`WorkerDirector -> Work added ${workId} (${type} / ${scheduled || created} / ${retries})`, {
+        userId,
+      });
+
+      const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
+
+      WorkerDirector.events.emit(WorkerEventTypes.ADDED, { work, userId });
+
+      return work;
+    },
+
+    rescheduleWork: async (currentWork, scheduled, { userId }) => {
+      await mutations.update(
+        currentWork._id,
+        {
+          $set: {
+            scheduled,
+          },
+        },
+        userId,
+      );
+
+      const work = await WorkQueue.findOne(generateDbFilterById(currentWork._id), {});
+
+      WorkerDirector.events.emit(WorkerEventTypes.RESCHEDULED, {
+        work,
+        oldScheduled: currentWork.scheduled,
+        userId,
+      });
+
+      return work;
+    },
+
+    allocateWork: async ({ types, worker = UNCHAINED_WORKER_ID }) => {
+      // Find a work item that is scheduled for now and is not started.
+      // Also:
+      // - Restrict by types and worker if provided
+      // - Sort by default queue order
+      const query = buildQuerySelector({
+        status: [WorkStatus.NEW],
+        scheduled: { end: new Date() },
+        worker: { $in: [null, worker] },
+        ...(types ? { type: { $in: types } } : {}),
+      });
+      const result = await WorkQueue.findOneAndUpdate(
+        query,
+        {
+          $set: { started: new Date(), worker },
+        },
+        { sort: buildSortOptions(defaultSort), returnDocument: 'after' },
+      );
+
+      WorkerDirector.events.emit(WorkerEventTypes.ALLOCATED, {
+        work: result.value,
+      });
+
+      return result.value;
+    },
+
+    ensureOneWork: async ({ type, input, priority = 0, scheduled, originalWorkId, retries = 20 }) => {
+      const created = new Date();
+      const query = buildQuerySelector({
+        type,
+        status: [WorkStatus.NEW],
+      });
+      try {
+        const workId = `${type}:${scheduled.getTime()}`;
+        const result = await (WorkQueue.findOneAndUpdate(
+          query,
+          {
+            $set: {
+              input,
+              priority,
+              worker: null,
+              updated: created,
+            },
+            $setOnInsert: {
+              _id: workId,
+              type,
+              originalWorkId,
+              scheduled,
+              retries,
+              created,
+            },
+          },
+          {
+            sort: buildSortOptions(defaultSort),
+            returnDocument: 'after',
+            upsert: true,
+          },
+        ) as Promise<ModifyResult<Work>>);
+
+        if (!result.lastErrorObject.updatedExisting) {
+          log(`WorkerDirector -> Work added again (ensure) ${type} ${scheduled} ${retries}`);
+
+          WorkerDirector.events.emit(WorkerEventTypes.ADDED, {
+            work: result.value,
+          });
+        }
+        return result.value;
+      } catch (e) {
+        return null;
+      }
+    },
+
+    doWork: (work, requestContext) => {
+      return WorkerDirector.doWork(work, requestContext);
+    },
+
+    finishWork,
+
+    deleteWork: async (workId, userId) => {
+      const workBeforeRemoval = await WorkQueue.findOne(
+        buildQuerySelector({
+          workId,
+          status: [WorkStatus.NEW, WorkStatus.ALLOCATED],
+        }),
+      );
+      if (!workBeforeRemoval) return null;
+
+      await mutations.delete(workId, userId);
+
+      const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
+
+      WorkerDirector.events.emit(WorkerEventTypes.DELETED, { work, userId });
+
+      return work;
+    },
+
+    markOldWorkAsFailed: async ({ types, worker, referenceDate }, userId) => {
+      const workQueue = await WorkQueue.find(
+        buildQuerySelector({
+          status: [WorkStatus.ALLOCATED],
+          started: { $lte: referenceDate },
+          worker,
+          type: { $in: types },
+        }),
+        { type: 1, sort: { test: 1 } } as Projection<Work>,
+      ).toArray();
+
+      return Promise.all(
+        workQueue.map(({ _id }) =>
+          finishWork(
+            _id as string,
+            {
+              finished: new Date(),
+              result: null,
+              success: false,
+              error: {
+                name: DIRECTOR_MARKED_FAILED_ERROR,
+                message:
+                  'Director marked old work as failed after restart. This work was eventually running at the moment when node.js exited.',
+              },
+              worker,
+            },
+            userId,
+          ),
+        ),
+      );
+    },
+  };
+};

package/src/schedulers/FailedRescheduler.ts

@@ -0,0 +1,53 @@
+import { IScheduler, Work } from '@unchainedshop/types/worker';
+import { log } from '@unchainedshop/logger';
+import { WorkerDirector } from '../director/WorkerDirector';
+import { WorkerEventTypes } from '../director/WorkerEventTypes';
+
+export const FailedRescheduler: IScheduler = {
+  key: 'shop.unchained.scheduler.failed',
+  label: 'Reschedule failed works',
+  version: '1.0',
+
+  actions: (requestContext) => {
+    const handleFinishedWork = ({ work, userId }: { work: Work; userId: string }) => {
+      if (!work.success && work.retries > 0) {
+        const now = new Date();
+        const workDelayMs = work.scheduled.getTime() - work.created.getTime();
+
+        // In short: Double the delay of the old work or delay for 5 seconds
+        const scheduled =
+          workDelayMs > 1000
+            ? new Date(now.getTime() + workDelayMs * 2)
+            : new Date(now.setSeconds(now.getSeconds() + 5));
+
+        log(
+          `${FailedRescheduler.key} -> Reschedule failed work ${work._id} ${
+            work.type
+          } for ${scheduled} (in ${Math.round(workDelayMs / 1000)}). Remaining retries: ${work.retries}`,
+        );
+
+        requestContext.modules.worker.addWork(
+          {
+            type: work.type,
+            input: work.input,
+            priority: work.priority,
+            originalWorkId: work.originalWorkId || work._id,
+            retries: work.retries - 1,
+            scheduled,
+          },
+          userId,
+        );
+      }
+    };
+
+    return {
+      start() {
+        WorkerDirector.events.on(WorkerEventTypes.FINISHED, handleFinishedWork);
+      },
+
+      stop() {
+        WorkerDirector.events.off(WorkerEventTypes.FINISHED, handleFinishedWork);
+      },
+    };
+  },
+};

package/src/worker-index.ts

@@ -0,0 +1,12 @@
+export { configureWorkerModule } from './module/configureWorkerModule';
+
+export { WorkerDirector } from './director/WorkerDirector';
+export { WorkerAdapter } from './director/WorkerAdapter';
+export { WorkStatus } from './director/WorkStatus';
+export { WorkerEventTypes } from './director/WorkerEventTypes';
+
+export { FailedRescheduler } from './schedulers/FailedRescheduler';
+
+export { BaseWorker } from './workers/BaseWorker';
+export { EventListenerWorker } from './workers/EventListenerWorker';
+export { IntervalWorker } from './workers/IntervalWorker';