clever-queue 0.1.0

package/exemples/index01.ts

@@ -0,0 +1,98 @@
+import * as CleverQueue from "../dist/index.js";
+
+/* openTelemetry */
+/*
+import * as OTL from "@opentelemetry/api";
+import * as OTLTrace from "@opentelemetry/sdk-trace-node";
+import * as OTLTraceBase from "@opentelemetry/sdk-trace-base";
+
+const { trace, context } = require("@opentelemetry/api");
+
+const activeSpan = OTL.trace.getSpan(OTL.context.active());
+
+const traceId = activeSpan.spanContext().traceId;
+const spanId = activeSpan.spanContext().spanId;
+const traceFlag = activeSpan.spanContext().traceFlags;
+
+console.log(traceId, spanId, traceFlag);
+
+/*
+import { NodeSDK as OTLNodeSDK } from "@opentelemetry/sdk-node";
+
+const otl = new OTLNodeSDK({
+  traceExporter: new OTLConsoleSpanExporter(),
+});
+
+otl.start();
+*/
+/*
+OTL.diag.setLogger(new OTL.DiagConsoleLogger(), OTL.DiagLogLevel.DEBUG);
+const tracerProvider = new OTLTrace.NodeTracerProvider();
+const exporter = new OTLTraceBase.ConsoleSpanExporter();
+const processor = new OTLTraceBase.SimpleSpanProcessor(exporter);
+tracerProvider.addSpanProcessor(processor);
+tracerProvider.register();
+
+const tracer = OTL.trace.getTracer("esm-tracer");
+tracer.startActiveSpan("manual", (span) => {
+  span.addEvent("myEvent");
+  span.addEvent("myEvent2");
+  span.end();
+});
+*/
+
+const myFunction: CleverQueue.Task.FunctionToExecute = function (message: string) {
+  // eslint-disable-next-line no-console
+  console.log("Executing", message);
+};
+
+/*
+const myPromise: CleverQueue.Task.FunctionToExecute = function (message: string): Promise<string> {
+  console.log(message);
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+  console.log(message);
+  return message;
+};
+*/
+
+const myAsync: CleverQueue.Task.FunctionToExecute = async function (message: string) {
+  logFunction({ body: "Executing " + message });
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+  return message;
+};
+
+const logFunction: CleverQueue.Engine.LogFunction = function (logRecord) {
+  // eslint-disable-next-line no-console
+  console.log(JSON.stringify(logRecord));
+};
+
+async function run(): Promise<void> {
+  logFunction({ body: "01 - Creating Engine" });
+  const engine = CleverQueue.createEngine({ logFunction: logFunction, bestEffortRunners: 2 });
+  logFunction({ body: "02 - Creating Queue" });
+  const queue = engine.createQueue({ priority: CleverQueue.Queue.Priorities.Standard, weight: 50 });
+  logFunction({ body: "03 - Creating Tasks" });
+  const task1 = engine.createTask(() => myAsync("myAsync01"), {});
+  const task2 = engine.createTask(() => myAsync("myAsync02"), {});
+  const task3 = engine.createTask(() => myAsync("myAsync03"), {});
+  logFunction({ body: "04 - Enqueue Task" });
+  const result = await queue.enqueue(task1);
+  queue.enqueue(task2);
+  queue.enqueue(task3);
+  logFunction({ body: JSON.stringify(engine.statistics()) });
+  logFunction({ body: result as string });
+  logFunction({ body: "05 - Stopping Engine" });
+  logFunction({ body: JSON.stringify(engine.statistics()) });
+  engine.stop();
+  logFunction({ body: JSON.stringify(engine.statistics()) });
+  while (engine.statistics().engine.tasks > 0) {
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  logFunction({ body: JSON.stringify(engine.statistics()) });
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+  logFunction({ body: JSON.stringify(engine.statistics()) });
+  logFunction({ body: { task1: task1.result as string, task2: task2.result as string, task3: task3.result as string } });
+}
+
+// eslint-disable-next-line unicorn/prefer-top-level-await
+run();

package/exemples/index02.mjs

@@ -0,0 +1,13 @@
+import * as CleverQueue from "../dist/index.js";
+
+async function run() {
+  console.log("01");
+  const engine = CleverQueue.createEngine({ bestEffortRunners: 2 });
+  const queue = engine.createQueue({ priority: CleverQueue.Queue.Priorities.Standard, weight: 50 });
+  console.log("02");
+  // engine.run();
+  console.log("03");
+  queue.createTask("aaaa");
+}
+
+run();

package/exemples/tsconfig.json

@@ -0,0 +1,24 @@
+{
+    "compilerOptions": {
+      "module": "CommonJS",
+      "esModuleInterop": true,
+      "target": "es2022",
+      "moduleResolution": "node",
+      "lib": [
+        "es2022",
+      ],
+      "strict": true,
+      "useUnknownInCatchVariables": true,
+      "noImplicitAny": true,
+      "strictNullChecks": true,
+      "strictFunctionTypes": true,
+      "strictBindCallApply": true,
+      "strictPropertyInitialization": true,
+      "noImplicitThis": true,
+      "alwaysStrict": true,
+      "forceConsistentCasingInFileNames": true
+    },
+    "$schema": "https://json.schemastore.org/tsconfig",
+    "display": "Recommended"
+  }
+  

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "clever-queue",
+  "version": "0.1.0",
+  "description": "Queuing system for promises that handle concurring, throttling, weighting and prioritizing in a clever fashion.",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "npx tsc -p src/ --declaration",
+    "build:watch": "npx tsc -p src/ -w",
+    "prestart": "npx npm run build",
+    "start": "node dist/index.js",
+    "debug": "node --nolazy --inspect-brk=9229 dist/index.js",
+    "test": "node --test ./test/**/*.mjs",
+    "test:unit:watch": "node --test --watch ./test/units/*.mjs"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/job-so/clever-queue.git"
+  },
+  "keywords": [
+    "queue",
+    "clever",
+    "concurrency",
+    "throttling",
+    "priority",
+    "weight",
+    "promise"
+  ],
+  "author": "Olivier GROSJEANNE",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://gitlab.com/job-so/clever-queue/issues"
+  },
+  "homepage": "https://gitlab.com/job-so/clever-queue#readme",
+  "devDependencies": {
+    "@eslint/eslintrc": "^3.1",
+    "@eslint/js": "^9.8",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/sdk-metrics": "^1.26",
+    "@opentelemetry/sdk-node": "^0.53",
+    "@opentelemetry/sdk-trace-node": "^1.26",
+    "@types/eslint__js": "^8.42",
+    "@types/node": "^22.1",
+    "eslint": "^9.8",
+    "eslint-config-prettier": "^9.1",
+    "eslint-plugin-node": "^11.1",
+    "eslint-plugin-prettier": "^5.1",
+    "eslint-plugin-promise": "^6.4",
+    "eslint-plugin-security": "^3.0",
+    "eslint-plugin-unicorn": "^54.0",
+    "globals": "^15.9",
+    "node": "^22.4",
+    "typescript": "^5.5",
+    "typescript-eslint": "^8.0"
+  }
+}

package/src/engine/index.ts

@@ -0,0 +1,166 @@
+// clever-queue
+
+import * as _ from "../helpers";
+import * as Queue from "../queue";
+import * as Runner from "../runner";
+import * as Task from "../task";
+
+type Status = "unknown" | "initialiazing" | "running" | "stopping" | "destroying";
+
+const TICK_START = 42;
+
+import { Options, Statistics, ErrorsList } from "./interfaces";
+
+const defaultOptions: Options = {
+  bestEffortRunners: 1,
+  autostart: true,
+  logFunction: undefined,
+};
+
+class Engine {
+  status: Status = "unknown";
+  readonly runners: Runner.Runner[] = [];
+  readonly queues: Queue.Queue[] = [];
+
+  constructor(opt: Options) {
+    this.status = "initialiazing";
+    const options = { ...defaultOptions, ...opt };
+    this.#checkOptionsConsistancy(options);
+
+    // create runners
+    for (let index = 0; index < (options.bestEffortRunners ?? 0); index++) this.createRunner({ priority: Runner.Priorities.BestEffort });
+    if (options.logFunction) options.logFunction({ body: "engine has been initialized" });
+    if (options.autostart) this.start();
+  }
+
+  start(): void {
+    this.status = "running";
+  }
+
+  stop(): void {
+    this.status = "stopping";
+    for (const queue of this.queues) queue.stop();
+    for (const runner of this.runners) runner.stop();
+  }
+
+  createQueue(options: Queue.Options): Queue.Queue {
+    const queue = new Queue.Queue({ logFunction: options.logFunction, ...options });
+    this.queues.push(queue);
+    this.#resetLastTick(options.priority);
+    return queue;
+  }
+
+  createRunner(options: Runner.Options): Runner.Runner {
+    const runner = new Runner.Runner({ logFunction: options.logFunction, ...options }, this);
+    this.runners.push(runner);
+    return runner;
+  }
+
+  createTask(function_: Task.FunctionToExecute, options: Task.Options): Task.Task {
+    return new Task.Task(function_, { logFunction: options.logFunction, ...options });
+  }
+
+  #checkOptionsConsistancy(options: Options): void {
+    if (!options) throw new _.Errors.CQError(ErrorsList.NoOptionsOnEngineInitialization);
+    if (!options.bestEffortRunners || options.bestEffortRunners < 1) throw new _.Errors.CQError(ErrorsList.bestEffortRunnersOptionMustBeGreaterThanZero);
+  }
+
+  #getSumWeightFunction: Queue.GetSumWeightFunction = (priority: number): number => {
+    let result = 0;
+    for (const q of this.queues) if (q.options.priority === priority) result += q.options.weight;
+    return result;
+  };
+
+  #resetLastTick(priority: number): void {
+    for (const q of this.queues) if (q.options.priority === priority) q.tick = TICK_START;
+  }
+
+  #getNextWeightedQueue(priority: number): Queue.Queue | undefined {
+    let minTick = Infinity;
+    let result: Queue.Queue | undefined = undefined;
+    for (const q of this.queues)
+      if (q.options.priority === priority && q.tick <= minTick) {
+        minTick = q.tick;
+        result = q;
+      }
+    return result;
+  }
+
+  #howMuchTasksWaitingInQueues(priority: number): number {
+    let result: number = 0;
+    for (const q of this.queues) if (q.options.priority === priority) result += q.tasks.length;
+    return result;
+  }
+  /*
+   async getNextTask(priorityGreaterOrEqualTo: number): Promise<Task.Task> {
+    for (let priority = Queue.Priorities.Absolute; priority <= priorityGreaterOrEqualTo; priority++) {
+      // Iterate on Each PriorityLevel From Absolute to param function
+      while (this.#howMuchTasksWaitingInQueues(priority) > 0) {
+        let task: Task.Task | undefined;
+        while (!task) {
+          const nextPriorityQueue = this.#getNextQueue(priority); // Find Next Priority Queue to process
+          if (nextPriorityQueue) {
+            nextPriorityQueue.tick += this.#getSumWeightFunction(priority) / nextPriorityQueue.options.weight; // Update tick with related weight
+            if (nextPriorityQueue.tasks.length > 0) {
+              task = nextPriorityQueue.tasks.shift(); // If queue is not empty, this is our next Task to process
+              if (task) task.runner();
+              console.log(nextPriorityQueue.options.name, nextPriorityQueue.tick);
+            }
+          }
+        }
+      }
+    }
+  }
+  */
+
+  async getNextTask(priorityGreaterOrEqualTo: number): Promise<Task.Task | undefined> {
+    let task: Task.Task | undefined = undefined;
+    // Iterate on Each PriorityLevel From Absolute to parameter
+    for (let priority = Queue.Priorities.Absolute; priority <= priorityGreaterOrEqualTo; priority++) {
+      // Check if there is task waiting of this priority level
+      while (this.#howMuchTasksWaitingInQueues(priority) > 0) {
+        // Find Next Weighted Queue of this priority level to check
+        const nextPriorityQueue = this.#getNextWeightedQueue(priority);
+        if (nextPriorityQueue) {
+          // update Tick (next check) according to queue related weigth
+          nextPriorityQueue.tick += this.#getSumWeightFunction(priority) / nextPriorityQueue.options.weight;
+          // If queue is not empty, we've got our next Task to process
+          if (nextPriorityQueue.tasks.length > 0) {
+            // dequeue task from queue
+            task = nextPriorityQueue.tasks.shift();
+            if (task) return task;
+          }
+        }
+      }
+    }
+    return undefined;
+  }
+
+  statistics(): Statistics {
+    let taskCount = 0;
+    const runners: Runner.Statistics[] = [];
+    const queues: Queue.Statistics[] = [];
+    for (const runner of this.runners) {
+      if (runner.status == "running") taskCount += 1;
+      runners.push(runner.statistics());
+    }
+    for (const queue of this.queues) {
+      taskCount += queue.tasks.length;
+      queues.push(queue.statistics());
+    }
+    const stat: Statistics = {
+      engine: {
+        status: this.status,
+        runners: this.runners.length,
+        queues: this.queues.length,
+        tasks: taskCount,
+      },
+      runners,
+      queues,
+    };
+    return stat;
+  }
+}
+
+export { Options, Statistics, ErrorsList } from "./interfaces";
+export { Engine };

package/src/engine/interfaces.ts

@@ -0,0 +1,33 @@
+import * as _ from "../helpers";
+
+import * as Runner from "../runner/interfaces";
+import * as Queue from "../runner/interfaces";
+
+interface Options {
+  autostart?: boolean;
+  bestEffortRunners?: number;
+  logFunction?: _.Logs.LogFunction;
+}
+
+interface Statistics {
+  engine: {
+    status: string;
+    runners: number;
+    queues: number;
+    tasks: number;
+  };
+  runners: Runner.Statistics[];
+  queues: Queue.Statistics[];
+}
+type ErrorListKeys = "NoOptionsOnEngineInitialization" | "bestEffortRunnersOptionMustBeGreaterThanZero";
+
+const ErrorsList: {
+  [index in ErrorListKeys]: { name: string; message: string };
+} = {
+  NoOptionsOnEngineInitialization: { name: "NoOptionsOnEngineInitialization", message: "You must provide an options parameter to Engine" },
+  bestEffortRunnersOptionMustBeGreaterThanZero: { name: "bestEffortRunnersOptionMustBeGreaterThanZero", message: "bestEffortRunners Option must be equal or superior to 1" },
+};
+
+export { LogFunction } from "../helpers/logs";
+export { TraceFunction } from "../helpers/traces";
+export { Options, Statistics, ErrorsList };

package/src/helpers/errors.ts

@@ -0,0 +1,16 @@
+class CQError extends Error {
+  constructor(error: { name: string; message: string }, context?: object) {
+    if (!error) {
+      error = {
+        name: "undocumentedError",
+        message: "Undocumented Error, please raise an issue to developpers",
+      };
+    }
+    super(error.message);
+    this.name = error.name;
+    this.message = error.message;
+    if (context) this.cause = JSON.stringify(context);
+  }
+}
+
+export { CQError };

package/src/helpers/index.ts

@@ -0,0 +1,3 @@
+export * as Errors from "./errors";
+export * as Logs from "./logs";
+export * as Traces from "./traces";

package/src/helpers/logs.ts

@@ -0,0 +1,91 @@
+export declare type HrTime = [number, number];
+export declare type TimeInput = HrTime | number | Date;
+
+export type AnyValueScalar = string | number | boolean;
+export type AnyValueArray = Array<AnyValue>;
+/**
+ * AnyValueMap is a map from string to AnyValue (attribute value or a nested map)
+ */
+export interface AnyValueMap {
+  [attributeKey: string]: AnyValue;
+}
+
+/**
+ * AnyValue can be one of the following:
+ * - a scalar value
+ * - a byte array
+ * - array of any value
+ * - map from string to any value
+ * - empty value
+ */
+export type AnyValue = AnyValueScalar | Uint8Array | AnyValueArray | AnyValueMap | null | undefined;
+
+export type LogBody = AnyValue;
+export type LogAttributes = AnyValueMap;
+
+export enum SeverityNumber {
+  UNSPECIFIED = 0,
+  TRACE = 1,
+  TRACE2 = 2,
+  TRACE3 = 3,
+  TRACE4 = 4,
+  DEBUG = 5,
+  DEBUG2 = 6,
+  DEBUG3 = 7,
+  DEBUG4 = 8,
+  INFO = 9,
+  INFO2 = 10,
+  INFO3 = 11,
+  INFO4 = 12,
+  WARN = 13,
+  WARN2 = 14,
+  WARN3 = 15,
+  WARN4 = 16,
+  ERROR = 17,
+  ERROR2 = 18,
+  ERROR3 = 19,
+  ERROR4 = 20,
+  FATAL = 21,
+  FATAL2 = 22,
+  FATAL3 = 23,
+  FATAL4 = 24,
+}
+
+export interface LogRecord {
+  /**
+   * The time when the log record occurred as UNIX Epoch time in nanoseconds.
+   */
+  timestamp?: TimeInput;
+
+  /**
+   * Time when the event was observed by the collection system.
+   */
+  observedTimestamp?: TimeInput;
+
+  /**
+   * Numerical value of the severity.
+   */
+  severityNumber?: SeverityNumber;
+
+  /**
+   * The severity text.
+   */
+  severityText?: string;
+
+  /**
+   * A value containing the body of the log record.
+   */
+  body?: LogBody;
+
+  /**
+   * Attributes that define the log record.
+   */
+  attributes?: LogAttributes;
+
+  /**
+   * The Context associated with the LogRecord.
+   */
+  // context?: Context;
+}
+
+export type LogFunction = (logRecord: LogRecord) => void;

package/src/helpers/traces.ts

@@ -0,0 +1,24 @@
+/*
+interface EventRecord {
+  // https://opentelemetry.io/docs/specs/otel/logs/data-model/
+  timestamp?: number; // Time when the event occurred.
+  observedTimestamp: number; // Time when the event was observed.
+  traceId?: string; // Request trace id.
+  spanId?: string; // Request span id.
+  traceFlags?: number; // W3C trace flag.
+  SeverityText; // The severity text (also known as log level).
+  SeverityNumber; // Numerical value of the severity.
+  Body; // The body of the log record.
+  Resource; // Describes the source of the log.
+  InstrumentationScope; // Describes the scope that emitted the log.
+  Attributes; // Additional information about the event.
+}
+
+declare type AttributeValue = string | number | boolean | Array<null | undefined | string> | Array<null | undefined | number> | Array<null | undefined | boolean>;
+interface Attributes {
+  [attributeKey: string]: AttributeValue | undefined;
+}
+*/
+
+// export type TraceFunction = (message: string, attributes: Attributes) => void;
+export type TraceFunction = () => void;

package/src/index.ts

@@ -0,0 +1,13 @@
+// clever-queue
+import * as Engine from "./engine";
+
+function createEngine(options: Engine.Options): Engine.Engine {
+  return new Engine.Engine(options);
+}
+
+export * as Engine from "./engine/interfaces";
+export * as Runner from "./runner/interfaces";
+export * as Queue from "./queue/interfaces";
+export * as Task from "./task/interfaces";
+
+export { createEngine };

package/src/queue/index.ts

@@ -0,0 +1,93 @@
+// clever-queue
+
+import * as _ from "../helpers";
+import * as Task from "../task";
+import { Options, Statistics, ErrorsList, Priorities } from "./interfaces";
+
+type GetSumWeightFunction = (priority: number) => number;
+
+type Status = "unknown" | "initialiazing" | "running" | "stopping" | "destroying";
+
+const defaultOptions: Options = {
+  autostart: true,
+  priority: Priorities.Standard,
+  weight: 128,
+  logFunction: undefined,
+};
+
+class Queue {
+  status: Status = "unknown";
+  readonly options: Options;
+  tick: number = 0;
+  tasks: Task.Task[] = [];
+
+  constructor(opt: Options) {
+    this.status = "initialiazing";
+    this.options = this.#checkOptionsConsistancy({ ...defaultOptions, ...opt });
+    if (this.options.autostart) this.start();
+  }
+
+  #checkOptionsConsistancy(options: Options): Options {
+    if (!options) throw new _.Errors.CQError(ErrorsList.NoOptionsOnQueueInitialization);
+    if (!(typeof options.priority === "number") || options.priority < 0 || options.priority > 255)
+      throw new _.Errors.CQError(ErrorsList.BadPriorityOptionsOnQueueInitialization, options);
+    if (!options.weight) throw new _.Errors.CQError(ErrorsList.NoWeightOptionsOnQueueInitialization);
+    return options;
+  }
+
+  start(): void {
+    this.status = "running";
+  }
+
+  stop(): void {
+    this.status = "stopping";
+  }
+
+  /*
+  async createTask(function_: Task._function): Promise<unknown> {
+    if (typeof function_ !== "function") {
+      console.error("raisong Task.ErrorsList.FunctionIsNotAFunction", Task.ErrorsList.FunctionIsNotAFunction);
+      throw new _.Errors.CQError(Task.ErrorsList.FunctionIsNotAFunction);
+    }
+    return new Promise((resolve, reject) => {
+      try {
+        const taskRunner = async (): Promise<void> => {
+          console.log("running Task");
+          const result = await function_();
+          console.log("task runned", result);
+          resolve(result);
+        };
+        const task = new Task.Task(taskRunner, {});
+        this.tasks.push(task);
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+*/
+  async createTaskAndEnqueue(function_: Task.FunctionToExecute, options: Task.Options): Promise<unknown> {
+    const task = new Task.Task(function_, options);
+    return this.enqueue(task);
+  }
+
+  async enqueue(task: Task.Task): Promise<unknown> {
+    return new Promise((resolve, reject) => {
+      task.resolver = resolve;
+      task.rejecter = reject;
+      this.tasks.push(task);
+    });
+  }
+
+  statistics(): Statistics {
+    const stat: Statistics = {
+      status: this.status,
+      priority: this.options.priority,
+      weight: this.options.weight,
+      tasks: this.tasks.length,
+    };
+    return stat;
+  }
+}
+
+export { Options, Statistics, ErrorsList, Priorities } from "./interfaces";
+export { Queue, GetSumWeightFunction };

package/src/queue/interfaces.ts

@@ -0,0 +1,38 @@
+import * as _ from "../helpers";
+
+const Priorities = { Absolute: 0, Standard: 128, BestEffort: 255 };
+
+interface Options {
+  name?: string;
+  priority: number;
+  weight: number;
+  autostart?: boolean;
+  logFunction?: _.Logs.LogFunction;
+}
+
+interface Statistics {
+  status: string;
+  priority: number;
+  weight: number;
+  tasks: number;
+}
+
+type ErrorListKeys = "BadPriorityOptionsOnQueueInitialization" | "NoWeightOptionsOnQueueInitialization" | "NoOptionsOnQueueInitialization";
+
+const ErrorsList: {
+  [index in ErrorListKeys]: { name: string; message: string };
+} = {
+  BadPriorityOptionsOnQueueInitialization: {
+    name: "BadPriorityOptionsOnQueueInitialization",
+    message:
+      "You must provide a 'priority' parameter for Queue initialization from 0 (absolute priority) to 255 (best effort priority), you may use CleverQueue.Priorities constants",
+  },
+  NoWeightOptionsOnQueueInitialization: {
+    name: "NoWeightOptionsOnQueueInitialization",
+    message: "You must provide a 'weight' parameter for Queue initialization",
+  },
+
+  NoOptionsOnQueueInitialization: { name: "NoOptionsOnQueueInitialization", message: "You must provide an options parameter to Queue" },
+};
+
+export { Options, Statistics, ErrorsList, Priorities };