0z2i6v3u5t 1.0.0

package/src/initializers/tasks.ts

@@ -0,0 +1,188 @@
+import * as path from "path";
+import { Plugin } from "node-resque";
+import * as TaskModule from "./../modules/task";
+import { api, config, log, utils, task, Initializer } from "../index";
+import { Task } from "../classes/task";
+import { safeGlobSync } from "../modules/utils/safeGlob";
+
+const taskModule = task;
+
+export interface TaskApi {
+  tasks: { [key: string]: Task };
+  jobs: { [key: string]: any };
+  middleware: { [key: string]: TaskModule.task.TaskMiddleware };
+  globalMiddleware: Array<string>;
+  loadFile: TasksInitializer["loadFile"];
+  jobWrapper: TasksInitializer["jobWrapper"];
+  loadTasks: TasksInitializer["loadTasks"];
+}
+
+/**
+ * Tools for enqueuing and inspecting the task system (delayed jobs).
+ */
+export class TasksInitializer extends Initializer {
+  constructor() {
+    super();
+    this.name = "tasks";
+    this.loadPriority = 699;
+    this.startPriority = 975;
+  }
+
+  loadFile = async (fullFilePath: string, reload = false) => {
+    let task;
+    let collection = await import(fullFilePath);
+    for (const i in collection) {
+      const TaskClass = collection[i];
+      task = new TaskClass();
+      task.validate();
+
+      if (api.tasks.tasks[task.name] && !reload) {
+        log(
+          `an existing task with the same name \`${task.name}\` will be overridden by the file ${fullFilePath}`,
+          "crit",
+        );
+      }
+
+      api.tasks.tasks[task.name] = task;
+      api.tasks.jobs[task.name] = api.tasks.jobWrapper(task.name);
+      log(
+        `task ${reload ? "(re)" : ""} loaded: ${task.name}, ${fullFilePath}`,
+        "debug",
+      );
+    }
+  };
+
+  jobWrapper = (taskName: string) => {
+    const task = api.tasks.tasks[taskName];
+
+    const middleware = task.middleware || [];
+    const plugins = task.plugins || [];
+    const pluginOptions = task.pluginOptions || {};
+
+    if (task.frequency > 0) {
+      if (plugins.indexOf("JobLock") < 0) {
+        plugins.push("JobLock");
+        pluginOptions.JobLock = { reEnqueue: false };
+      }
+      if (plugins.indexOf("QueueLock") < 0) {
+        plugins.push("QueueLock");
+      }
+      if (plugins.indexOf("DelayQueueLock") < 0) {
+        plugins.push("DelayQueueLock");
+      }
+    }
+
+    // load middleware into plugins
+    const processMiddleware = (m: string) => {
+      if (api.tasks.middleware[m]) {
+        class NodeResquePlugin extends Plugin {
+          constructor(...args: ConstructorParameters<typeof Plugin>) {
+            super(...args);
+          }
+
+          async beforeEnqueue() {
+            return api.tasks.middleware[m].preEnqueue
+              ? api.tasks.middleware[m].preEnqueue.call(this)
+              : true;
+          }
+
+          async afterEnqueue() {
+            return api.tasks.middleware[m].postEnqueue
+              ? api.tasks.middleware[m].postEnqueue.call(this)
+              : true;
+          }
+
+          async beforePerform() {
+            return api.tasks.middleware[m].preProcessor
+              ? api.tasks.middleware[m].preProcessor.call(this)
+              : true;
+          }
+
+          async afterPerform() {
+            return api.tasks.middleware[m].postProcessor
+              ? api.tasks.middleware[m].postProcessor.call(this)
+              : true;
+          }
+        }
+
+        //@ts-ignore
+        plugins.push(NodeResquePlugin);
+      }
+    };
+
+    api.tasks.globalMiddleware.forEach(processMiddleware);
+    middleware.forEach(processMiddleware);
+
+    return {
+      plugins,
+      pluginOptions,
+      perform: async function () {
+        const combinedArgs = [].concat(Array.prototype.slice.call(arguments));
+        combinedArgs.push(this);
+        let response = null;
+        try {
+          response = await task.run.apply(task, combinedArgs);
+          await taskModule.enqueueRecurrentTask(taskName);
+        } catch (error) {
+          if (task.frequency > 0 && task.reEnqueuePeriodicTaskIfException) {
+            await taskModule.enqueueRecurrentTask(taskName);
+          }
+          throw error;
+        }
+        return response;
+      },
+    };
+  };
+
+  loadTasks = async (reload: boolean) => {
+    for (const p of config.general.paths.task) {
+      await Promise.all(
+        utils
+          .ensureNoTsHeaderOrSpecFiles(
+            safeGlobSync(path.join(p, "**", "**/*(*.js|*.ts)")),
+          )
+          .map((f) => api.tasks.loadFile(f, reload)),
+      );
+    }
+
+    for (const plugin of Object.values(config.plugins)) {
+      if (plugin.tasks !== false) {
+        const pluginPath = path.normalize(plugin.path);
+
+        // old style at the root of the project
+        let files = safeGlobSync(path.join(pluginPath, "tasks", "**", "*.js"));
+
+        files = files.concat(
+          safeGlobSync(path.join(pluginPath, "dist", "tasks", "**", "*.js")),
+        );
+
+        utils.ensureNoTsHeaderOrSpecFiles(files).forEach((f) => {
+          api.tasks.loadFile(f, reload);
+        });
+      }
+    }
+  };
+
+  async initialize() {
+    api.tasks = {
+      tasks: {},
+      jobs: {},
+      middleware: {},
+      globalMiddleware: [],
+      loadFile: this.loadFile,
+      jobWrapper: this.jobWrapper,
+      loadTasks: this.loadTasks,
+    };
+
+    await api.tasks.loadTasks(false);
+
+    // we want to start the queue now, so that it's available for other initializers and CLI commands
+    await api.resque.startQueue();
+  }
+
+  async start() {
+    if (config.tasks.scheduler === true) {
+      await taskModule.enqueueAllRecurrentTasks();
+    }
+  }
+}

package/src/modules/action.ts

@@ -0,0 +1,89 @@
+import { Connection } from "../classes/connection";
+import { ActionProcessor } from "../classes/actionProcessor";
+import { api, utils, config } from "../index";
+
+export namespace action {
+  /**
+   * var middleware = {
+   *  name: 'userId checker',
+   *  global: false,
+   *  priority: 1000,
+   *  preProcessor: async (data) => {
+   *    if(!data.params.userId){
+   *      throw new Error('All actions require a userId')
+   *    }
+   *  },
+   *  postProcessor: async (data) => {
+   *    if(data.thing.stuff == false){ data.toRender = false }
+   *  }
+   *}
+   */
+  export interface ActionMiddleware {
+    /**Unique name for the middleware. */
+    name: string;
+    /**Is this middleware applied to all actions? */
+    global: boolean;
+    /**Module load order. Defaults to `api.config.general.defaultMiddlewarePriority`. */
+    priority?: number;
+    /**Called before the action runs.  Has access to all params, before sanitization.  Can modify the data object for use in actions. */
+    preProcessor?: Function;
+    /**Called after the action runs. */
+    postProcessor?: Function;
+  }
+
+  /**
+   * Add a middleware component available to pre or post-process actions.
+   */
+  export function addMiddleware(data: ActionMiddleware) {
+    if (!data.name) {
+      throw new Error("middleware.name is required");
+    }
+    if (!data.priority) {
+      data.priority = config.general.defaultMiddlewarePriority;
+    }
+    data.priority = Number(data.priority);
+    api.actions.middleware[data.name] = data;
+    if (data.global === true) {
+      api.actions.globalMiddleware.push(data.name);
+      utils.sortGlobalMiddleware(
+        api.actions.globalMiddleware,
+        api.actions.middleware,
+      );
+    }
+  }
+
+  /**
+   * Run an Action in-line, perhaps from within another Action or Task.
+   */
+  export async function run<ActionClass>(
+    actionName: string,
+    actionVersion?: string | number,
+    params: { [key: string]: any } = {},
+    connectionProperties = {},
+  ) {
+    const connection = new Connection({
+      type: "in-line-action",
+      remotePort: "0",
+      remoteIP: "0",
+      rawConnection: {},
+      canChat: false,
+    });
+
+    connection.params = params;
+    Object.assign(connection, connectionProperties);
+
+    try {
+      const actionProcessor = new ActionProcessor(connection);
+      const data = await actionProcessor.processAction(
+        actionName,
+        actionVersion,
+      );
+
+      if (data.response.error) throw new Error(data.response.error);
+
+      return data.response;
+    } finally {
+      await connection.destroy();
+    }
+  }
+}

package/src/modules/cache.ts

@@ -0,0 +1,326 @@
+import * as fs from "fs";
+import { api, id, utils, config } from "../index";
+
+export namespace cache {
+  export enum CacheErrorMessages {
+    locked = "Object locked",
+    notFound = "Object not found",
+  }
+
+  export interface CacheObject {
+    key: string;
+    value: any;
+    createdAt: number;
+  }
+
+  export interface CacheOptions {
+    expireTimeMS?: number;
+    retry?: boolean | number;
+  }
+
+  export const redisPrefix: string = config.general.cachePrefix;
+  export const lockPrefix: string = config.general.lockPrefix;
+  export const lockDuration: number = config.general.lockDuration;
+  export const scanCount: number = config.redis.scanCount || 1000;
+  export const lockRetry: number = 100;
+
+  export function client() {
+    if (api.redis.clients && api.redis.clients.client) {
+      return api.redis.clients.client;
+    } else {
+      throw new Error("redis not connected, cache cannot be used");
+    }
+  }
+
+  let lockNameOverride: string;
+  export function lockName() {
+    if (lockNameOverride) return lockNameOverride;
+    return id;
+  }
+
+  export function overrideLockName(name: string) {
+    lockNameOverride = name;
+  }
+
+  /**
+   * A generic method to find all keys which match a pattern.
+   * @pattern likely has a * at the end of the string.  Other arguments are for recursion and not required.
+   */
+  export async function getKeys(
+    pattern: string,
+    count: number = scanCount,
+    keysAry: string[] = [],
+    cursor = 0,
+  ): Promise<Array<string>> {
+    // return client().keys(redisPrefix + "*");
+
+    const [newCursor, matches] = await client().scan(
+      cursor,
+      "MATCH",
+      pattern,
+      "COUNT",
+      count,
+    );
+
+    if (matches && matches.length > 0) keysAry = keysAry.concat(matches);
+    if (newCursor === "0") return keysAry;
+    return cache.getKeys(pattern, count, keysAry, parseInt(newCursor));
+  }
+
+  /**
+   * Returns all the keys in redis which are under this Actionhero namespace.  Potentially very slow.
+   */
+  export async function keys(optionalScopePrefix = ""): Promise<Array<string>> {
+    // return client().keys(redisPrefix + "*");
+    return getKeys(redisPrefix + optionalScopePrefix + "*");
+  }
+
+  /**
+   * Returns all the locks in redis which are under this Actionhero namespace.  Potentially slow.
+   */
+  export async function locks(
+    optionalScopePrefix = "",
+  ): Promise<Array<string>> {
+    // return client().keys(lockPrefix + "*");
+    return getKeys(lockPrefix + optionalScopePrefix + "*");
+  }
+
+  /**
+   * Returns the number of keys in redis which are under this Actionhero namespace.  Potentially very slow.
+   */
+  export async function size(pattern = redisPrefix + "*") {
+    let length = 0;
+    const keys = await cache.getKeys(pattern);
+    if (keys) length = keys.length;
+    return length;
+  }
+
+  /**
+   * Removes all keys in redis which are under this Actionhero namespace.  Potentially very slow.
+   * Returns the deleted keys.
+   */
+  export async function clear(pattern = redisPrefix + "*") {
+    const keys = await cache.getKeys(pattern);
+
+    const pipelineArgs: Array<[string, string]> = [];
+    keys.forEach((key: string) => {
+      pipelineArgs.push(["del", key]);
+    });
+
+    await client().pipeline(pipelineArgs).exec();
+
+    return keys;
+  }
+
+  /**
+   * Write the current concents of redis (only the keys in Actionhero's namespace) to a file.
+   */
+  export async function dumpWrite(file: string) {
+    const data: Record<string, any> = {};
+    const jobs: Array<Promise<void>> = [];
+    const keys = await cache.keys();
+
+    keys.forEach((key: string) => {
+      jobs.push(
+        client()
+          .get(key)
+          .then((content) => {
+            data[key] = content;
+          }),
+      );
+    });
+
+    await Promise.all(jobs);
+
+    fs.writeFileSync(file, JSON.stringify(data));
+    return keys.length;
+  }
+
+  /**
+   * Load in contents for redis (and api.cache) saved to a file
+   * Warning! Any existing keys in redis (under this Actionhero namespace) will be removed.
+   */
+  export async function dumpRead(file: string) {
+    const jobs: Array<Promise<void>> = [];
+    await cache.clear();
+    const fileData = fs.readFileSync(file).toString();
+    const data = JSON.parse(fileData);
+    const count = Object.keys(data).length;
+
+    const saveDumpedElement = async (key: string, content: any) => {
+      await client().set(key, content, "KEEPTTL");
+    };
+
+    Object.keys(data).forEach((key) => {
+      const content = data[key];
+      jobs.push(saveDumpedElement(key, content));
+    });
+
+    await Promise.all(jobs);
+    return count;
+  }
+
+  /**
+   * Load an item from the cache.  Will throw an error if the item named by `key` cannot be found.
+   * Automatically handles `api.cache.redisPrefix`
+   */
+  export async function load(
+    key: string,
+    options: CacheOptions = {},
+  ): Promise<CacheObject> {
+    let cacheObj: CacheObject;
+
+    let lockOk = await cache.checkLock(key, options.retry);
+    if (lockOk !== true) throw new Error(CacheErrorMessages.locked);
+
+    let cachedStringifiedObjet = await client().get(`${redisPrefix}${key}`);
+    try {
+      cacheObj = JSON.parse(cachedStringifiedObjet);
+    } catch (e) {}
+
+    if (!cacheObj) throw new Error(CacheErrorMessages.notFound);
+
+    lockOk = await cache.checkLock(key, options.retry);
+    if (lockOk !== true) throw new Error(CacheErrorMessages.locked);
+
+    if (options.expireTimeMS) {
+      await client().pexpire(redisPrefix + key, options.expireTimeMS);
+      return {
+        key,
+        value: cacheObj.value,
+        createdAt: cacheObj.createdAt,
+      };
+    } else {
+      return {
+        key,
+        value: cacheObj.value,
+        createdAt: cacheObj.createdAt,
+      };
+    }
+  }
+
+  /**
+   * Delete an item in the cache.  Will throw an error if the item named by `key` is locked.
+   * Automatically handles `api.cache.redisPrefix`
+   */
+  export async function destroy(key: string): Promise<boolean> {
+    const lockOk = await cache.checkLock(key, null);
+    if (!lockOk) throw new Error(CacheErrorMessages.locked);
+
+    const count = await client().del(redisPrefix + key);
+    let response = true;
+    if (count !== 1) {
+      response = false;
+    }
+    return response;
+  }
+
+  /**
+   * Save an item in the cache.  If an item is already in the cache with the same key, it will be overwritten.  Throws an error if the object is already in the cache and is locked.
+   * Automatically handles `api.cache.redisPrefix`
+   */
+  export async function save(
+    key: string,
+    value: any,
+    expireTimeMS?: number,
+  ): Promise<boolean> {
+    const cacheObj = {
+      value: value,
+      createdAt: new Date().getTime(),
+    };
+
+    const lockOk = await cache.checkLock(key, null);
+    if (!lockOk) throw new Error(CacheErrorMessages.locked);
+
+    await client().set(redisPrefix + key, JSON.stringify(cacheObj));
+    if (expireTimeMS) {
+      await client().pexpire(redisPrefix + key, expireTimeMS);
+    }
+    return true;
+  }
+
+  /**
+   * Push an item to a shared queue/list in redis.
+   * Automatically handles `api.cache.redisPrefix`
+   */
+  export async function push(key: string, item: any): Promise<boolean> {
+    const object = JSON.stringify({ data: item });
+    await client().rpush(redisPrefix + key, object);
+    return true;
+  }
+
+  /**
+   * Pop (get) an item to a shared queue/list in redis.
+   * Automatically handles `api.cache.redisPrefix`
+   */
+  export async function pop(key: string): Promise<boolean> {
+    const object = await client().lpop(redisPrefix + key);
+    if (!object) {
+      return null;
+    }
+    const item = JSON.parse(object);
+    return item.data;
+  }
+
+  /**
+   * Check how many items are stored in a shared queue/list in redis.
+   */
+  export async function listLength(key: string): Promise<number> {
+    return client().llen(redisPrefix + key);
+  }
+
+  /**
+   * Lock an item in redis (can be a list or a saved item) to this Actionhero process.
+   */
+  export async function lock(
+    key: string,
+    expireTimeMS: number = lockDuration,
+  ): Promise<boolean> {
+    const lockOk = await cache.checkLock(key, null);
+    if (!lockOk) {
+      return false;
+    }
+
+    const result = await client().setnx(lockPrefix + key, lockName());
+    if (!result) {
+      return false;
+    } // value was already set, so we cannot obtain the lock
+
+    await client().expire(lockPrefix + key, Math.ceil(expireTimeMS / 1000));
+
+    return true;
+  }
+
+  /**
+   * Unlock an item in redis (can be a list or a saved item) which was previously locked by this Actionhero process.
+   */
+  export async function unlock(key: string): Promise<boolean> {
+    const lockOk = await cache.checkLock(key, null);
+
+    if (!lockOk) {
+      return false;
+    }
+
+    await client().del(lockPrefix + key);
+    return true;
+  }
+
+  export async function checkLock(
+    key: string,
+    retry: boolean | number = false,
+    startTime: number = new Date().getTime(),
+  ): Promise<boolean> {
+    const lockedBy = await client().get(lockPrefix + key);
+    if (lockedBy === lockName() || lockedBy === null) {
+      return true;
+    } else {
+      const delta = new Date().getTime() - startTime;
+      if (!retry) return false;
+      else if (typeof retry === "boolean") return retry;
+      else if (delta > retry) return false;
+
+      await utils.sleep(lockRetry);
+      return cache.checkLock(key, retry, startTime);
+    }
+  }
+}