@deephaven-enterprise/query-utils 1.20240723.113-beta → 1.20240723.115-beta

package/dist/DraftQuery.d.ts

@@ -13,8 +13,10 @@ export type ScriptDraftQuery = IDraftQuery & {
 export type IDraftQuery = EditableQueryInfo & {
     isClientSide: boolean;
     draftOwner: string;
+    initializationThreads?: string;
+    updateThreads?: string;
 };
-export type DraftQueryConstructorObject = Partial<IDraftQuery> & Pick<IDraftQuery, 'name' | 'owner' | 'draftOwner' | 'dbServerName'>;
+export type DraftQueryConstructorObject = Partial<IDraftQuery> & Pick<IDraftQuery, 'name' | 'owner' | 'draftOwner' | 'dbServerName' | 'initializationThreads' | 'updateThreads'>;
 /**
  * A Draft Query is a client side object that contains mutable data for a Peristent Query along with some metadata.
  */
@@ -58,6 +60,8 @@ export declare class DraftQuery implements IDraftQuery {
         readonly SPARE_COUNT: "spareCount";
         readonly ASSIGNMENT_POLICY: "assignmentPolicy";
         readonly ASSIGNMENT_POLICY_PARAMS: "assignmentPolicyParams";
+        readonly INITIALIZATION_THREADS: "initializationThreads";
+        readonly UPDATE_THREADS: "updateThreads";
     }>;
     /**
      * Checks if a serial id from the server is actually a draft id
@@ -71,7 +75,7 @@ export declare class DraftQuery implements IDraftQuery {
      * @param serial a serial that contains a draft id
      */
     static serialToDraftId(serial: string): string;
-    constructor({ isClientSide, draftOwner, serial, name, owner, type, enabled, enableGcLogs, envVars, heapSize, additionalMemory, dataMemoryRatio, jvmArgs, extraClasspaths, jvmProfile, dbServerName, scriptLanguage, scriptPath, scriptCode, scheduling, adminGroups, viewerGroups, restartUsers, timeout, workerKind, kubernetesControl, pythonControl, genericWorkerControl, typeSpecificFields, replicaCount, spareCount, assignmentPolicy, assignmentPolicyParams, }: DraftQueryConstructorObject);
+    constructor({ isClientSide, draftOwner, serial, name, owner, type, enabled, enableGcLogs, envVars, heapSize, additionalMemory, dataMemoryRatio, jvmArgs, extraClasspaths, jvmProfile, dbServerName, scriptLanguage, scriptPath, scriptCode, scheduling, adminGroups, viewerGroups, restartUsers, timeout, workerKind, kubernetesControl, pythonControl, genericWorkerControl, typeSpecificFields, replicaCount, spareCount, assignmentPolicy, assignmentPolicyParams, initializationThreads, updateThreads, }: DraftQueryConstructorObject);
     id: string;
     isModified: boolean;
     isClientSide: boolean;
@@ -114,6 +118,14 @@ export declare class DraftQuery implements IDraftQuery {
     spareCount: number;
     assignmentPolicy: string | null;
     assignmentPolicyParams: string | null;
+    /**
+     * initialiZationThreads and updateThreads are thread counts only used for Core workers.
+     * Additionally, they are not part of a Query object. These values are in Extra JVM Args.
+     * When a draft is created, they will be parsed and removed from JVM args.
+     * When the query is saved, they will be written to JVM args.
+     */
+    initializationThreads: string;
+    updateThreads: string;
     updateSchedule(): void;
     /**
      * Creates a deep copy of this draft query.
@@ -128,6 +140,14 @@ export declare class DraftQuery implements IDraftQuery {
      * @param editableQuery the Editable Query from the Server
      */
     saveDraftToQuery(editableQuery: Partial<EditableQueryInfo>): void;
+    /**
+     * Updates the JVM args to include the thread count for initialization and update threads.
+     * It is safe to call from saveDraftToQuery because the the thead counts will be set to
+     * system defaults for Legacy workers, which will not affect the JVM args.
+     * However, saving a new draft does not call saveDraftToQuery, so this method should be called
+     * manually in that case.
+     */
+    addThreadCountToJVMArgs(): void;
     toEditableQueryInfo(): {
         serial: string | null;
     };

package/dist/DraftQuery.js

@@ -3,6 +3,7 @@ import shortid from 'shortid';
 import QueryColumns from './QueryColumns';
 import QueryScheduler from './QueryScheduler';
 import { QueryType } from './QueryType';
+import WorkerThreadUtils from './WorkerThreadUtils';
 const DRAFT_SERIAL_PREFIX = 'Draft ID: ';
 /**
  * A Draft Query is a client side object that contains mutable data for a Peristent Query along with some metadata.
@@ -25,7 +26,7 @@ export class DraftQuery {
         // Remove the 'Draft ID: ' marking
         return serial === null || serial === void 0 ? void 0 : serial.substring(DRAFT_SERIAL_PREFIX.length);
     }
-    constructor({ isClientSide = false, draftOwner, serial = DraftQuery.NO_SERIAL, name, owner, type = QueryType.SCRIPT, enabled = true, enableGcLogs = true, envVars = '', heapSize = 4.0, additionalMemory = 0.0, dataMemoryRatio = 0.25, jvmArgs = '', extraClasspaths = '', jvmProfile = 'Default', dbServerName, scriptLanguage = 'Groovy', scriptPath = null, scriptCode = '', scheduling, adminGroups = [], viewerGroups = [], restartUsers = DraftQuery.RESTART_MODE.ADMIN, timeout = 0, workerKind, kubernetesControl, pythonControl, genericWorkerControl, typeSpecificFields = null, replicaCount = 1, spareCount = 0, assignmentPolicy = null, assignmentPolicyParams = null, }) {
+    constructor({ isClientSide = false, draftOwner, serial = DraftQuery.NO_SERIAL, name, owner, type = QueryType.SCRIPT, enabled = true, enableGcLogs = true, envVars = '', heapSize = 4.0, additionalMemory = 0.0, dataMemoryRatio = 0.25, jvmArgs = '', extraClasspaths = '', jvmProfile = 'Default', dbServerName, scriptLanguage = 'Groovy', scriptPath = null, scriptCode = '', scheduling, adminGroups = [], viewerGroups = [], restartUsers = DraftQuery.RESTART_MODE.ADMIN, timeout = 0, workerKind, kubernetesControl, pythonControl, genericWorkerControl, typeSpecificFields = null, replicaCount = 1, spareCount = 0, assignmentPolicy = null, assignmentPolicyParams = null, initializationThreads = undefined, updateThreads = undefined, }) {
         this.scheduling = [];
         // Meta Data
         this.id = shortid.generate();
@@ -45,7 +46,6 @@ export class DraftQuery {
         this.spareCount = spareCount;
         this.additionalMemory = additionalMemory;
         this.dataMemoryRatio = dataMemoryRatio;
-        this.jvmArgs = jvmArgs;
         this.extraClasspaths = extraClasspaths;
         this.jvmProfile = jvmProfile;
         this.dbServerName = dbServerName;
@@ -64,6 +64,14 @@ export class DraftQuery {
         this.spareCount = spareCount;
         this.assignmentPolicy = assignmentPolicy;
         this.assignmentPolicyParams = assignmentPolicyParams;
+        // If the thread counts are not provided, they are read from the JVM args
+        // And they need to be removed from the JVM args
+        this.initializationThreads =
+            initializationThreads !== null && initializationThreads !== void 0 ? initializationThreads : WorkerThreadUtils.getDisplayStringFromJvmArgs(jvmArgs, WorkerThreadUtils.INITIALIZATION_THREADS_ARG);
+        this.updateThreads =
+            updateThreads !== null && updateThreads !== void 0 ? updateThreads : WorkerThreadUtils.getDisplayStringFromJvmArgs(jvmArgs, WorkerThreadUtils.UPDATE_THREADS_ARG);
+        // The thread count args should always be removed from the JVM args as they are stored in the DraftQuery
+        this.jvmArgs = WorkerThreadUtils.removeArgs(jvmArgs);
         // The scheduling array is parsed into a scheduler object
         this.scheduling = [];
         this.scheduler = new QueryScheduler(scheduling);
@@ -98,6 +106,7 @@ export class DraftQuery {
      */
     saveDraftToQuery(editableQuery) {
         var _a, _b;
+        this.addThreadCountToJVMArgs();
         editableQuery.name = this.name;
         editableQuery.owner = this.owner;
         editableQuery.type = this.type;
@@ -129,6 +138,17 @@ export class DraftQuery {
         editableQuery.assignmentPolicy = this.assignmentPolicy;
         editableQuery.assignmentPolicyParams = this.assignmentPolicyParams;
     }
+    /**
+     * Updates the JVM args to include the thread count for initialization and update threads.
+     * It is safe to call from saveDraftToQuery because the the thead counts will be set to
+     * system defaults for Legacy workers, which will not affect the JVM args.
+     * However, saving a new draft does not call saveDraftToQuery, so this method should be called
+     * manually in that case.
+     */
+    addThreadCountToJVMArgs() {
+        const { initializationThreads, updateThreads } = this;
+        this.jvmArgs = WorkerThreadUtils.addArgs(this.jvmArgs, initializationThreads, updateThreads);
+    }
     toEditableQueryInfo() {
         const editableQuery = { serial: this.serial };
         this.saveDraftToQuery(editableQuery);
@@ -204,5 +224,7 @@ DraftQuery.FIELDS = Object.freeze({
     SPARE_COUNT: 'spareCount',
     ASSIGNMENT_POLICY: `assignmentPolicy`,
     ASSIGNMENT_POLICY_PARAMS: `assignmentPolicyParams`,
+    INITIALIZATION_THREADS: 'initializationThreads',
+    UPDATE_THREADS: 'updateThreads',
 });
 export default DraftQuery;

package/dist/WorkerThreadUtils.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Utility class for managing worker thread configuration arguments.
+ */
+export declare abstract class WorkerThreadUtils {
+    static readonly INITIALIZATION_THREADS_ARG = "OperationInitializationThreadPool.threads";
+    static readonly UPDATE_THREADS_ARG = "PeriodicUpdateGraph.updateThreads";
+    static readonly ALL_ARGS: string[];
+    static readonly SYSTEM_DEFAULT_STRING = "Default";
+    static readonly ALL_AVAILABLE_PROCESSORS_STRING = "All available processors";
+    static readonly ALL_AVAILABLE_PROCESSORS_VALUE = "-1";
+    static readonly THREAD_ITEMS: string[];
+    static readonly INITIALIZATION_THREADS_HELP_TEXT = "Determines the number of threads available for parallel processing of initialization operations. Select a value or type a positive integer.";
+    static readonly UPDATE_THREADS_HELP_TEXT = "Determines the number of threads available for parallel processing of the Periodic Update Graph refresh cycle. Select a value or type a positive integer.";
+    /**
+     * Converts the display string to the value string.
+     *
+     * @param display the display string
+     * @returns the value string
+     */
+    static getValue(display: string): string;
+    /**
+     * Gets the display string for the provided JVM arguments and argument name.
+     *
+     * @param jvmArgs the JVM arguments
+     * @param argName the name of the argument to search for
+     * @returns the display string or the system default string if not found
+     */
+    static getDisplayStringFromJvmArgs(jvmArgs: string, argName: string): string;
+    /**
+     * Removes the thread count arguments from the provided jvmArgs string.
+     *
+     * @param jvmArgs the string containing JVM arguments
+     * @returns a new string with the thread count arguments removed
+     */
+    static removeArgs(jvmArgs: string): string;
+    /**
+     * Adds the thread count arguments to the provided jvmArgs string.
+     *
+     * @param jvmArgs the string containing JVM arguments
+     * @param initializationThreads the display string for the initialization threads
+     * @param updateThreads the display string for the update threads
+     * @returns the new string with the thread count arguments added
+     */
+    static addArgs(jvmArgs: string, initializationThreads: string, updateThreads: string): string;
+    /**
+     * Gets the display string for the provided value.
+     *
+     * @param value the value to convert
+     * @returns the display string
+     */
+    static getDisplayString(value?: string): string;
+    /**
+     * Checks if the provided value is a valid thread display string. This includes blank and partial strings, so it is not strictly validated.
+     *
+     * @param value the string to validate
+     * @returns true if the string is valid for display, false otherwise
+     */
+    static isValidThreadDisplayString(value: string): boolean;
+    /**
+     * Checks if the provided value is a valid thread count. This only allows positive integers or specific strings.
+     *
+     * @param value the string to validate
+     * @returns true if the string is a valid thread count, false otherwise
+     */
+    static isValidThreadCount(value: string): boolean;
+}
+export default WorkerThreadUtils;

package/dist/WorkerThreadUtils.js

@@ -0,0 +1,134 @@
+/**
+ * Utility class for managing worker thread configuration arguments.
+ */
+export class WorkerThreadUtils {
+    /**
+     * Converts the display string to the value string.
+     *
+     * @param display the display string
+     * @returns the value string
+     */
+    static getValue(display) {
+        if (display === WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_STRING) {
+            return WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_VALUE;
+        }
+        return display;
+    }
+    /**
+     * Gets the display string for the provided JVM arguments and argument name.
+     *
+     * @param jvmArgs the JVM arguments
+     * @param argName the name of the argument to search for
+     * @returns the display string or the system default string if not found
+     */
+    static getDisplayStringFromJvmArgs(jvmArgs, argName) {
+        const args = jvmArgs.split(/[\n\r\s]+/);
+        for (let i = 0; i < args.length; i += 1) {
+            if (args[i].includes(argName)) {
+                return WorkerThreadUtils.getDisplayString(args[i].split('=')[1]);
+            }
+        }
+        // If the value is not found, return the system default string
+        return WorkerThreadUtils.SYSTEM_DEFAULT_STRING;
+    }
+    /**
+     * Removes the thread count arguments from the provided jvmArgs string.
+     *
+     * @param jvmArgs the string containing JVM arguments
+     * @returns a new string with the thread count arguments removed
+     */
+    static removeArgs(jvmArgs) {
+        const args = jvmArgs.split(/[\n\r\s]+/);
+        const filteredArgs = args.filter(arg => !WorkerThreadUtils.ALL_ARGS.some(argName => arg.includes(argName)));
+        return filteredArgs.join(' ');
+    }
+    /**
+     * Adds the thread count arguments to the provided jvmArgs string.
+     *
+     * @param jvmArgs the string containing JVM arguments
+     * @param initializationThreads the display string for the initialization threads
+     * @param updateThreads the display string for the update threads
+     * @returns the new string with the thread count arguments added
+     */
+    static addArgs(jvmArgs, initializationThreads, updateThreads) {
+        // The default value means "do not set an extra JVM arg"
+        if (WorkerThreadUtils.isValidThreadCount(initializationThreads) &&
+            initializationThreads !== WorkerThreadUtils.SYSTEM_DEFAULT_STRING) {
+            jvmArgs += ` -D${WorkerThreadUtils.INITIALIZATION_THREADS_ARG}=${WorkerThreadUtils.getValue(initializationThreads)}`;
+        }
+        if (WorkerThreadUtils.isValidThreadCount(updateThreads) &&
+            updateThreads !== WorkerThreadUtils.SYSTEM_DEFAULT_STRING) {
+            jvmArgs += ` -D${WorkerThreadUtils.UPDATE_THREADS_ARG}=${WorkerThreadUtils.getValue(updateThreads)}`;
+        }
+        return jvmArgs.trim();
+    }
+    /**
+     * Gets the display string for the provided value.
+     *
+     * @param value the value to convert
+     * @returns the display string
+     */
+    static getDisplayString(value) {
+        if (value == null) {
+            return WorkerThreadUtils.SYSTEM_DEFAULT_STRING;
+        }
+        if (value === WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_VALUE) {
+            return WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_STRING;
+        }
+        if (WorkerThreadUtils.isValidThreadDisplayString(value)) {
+            return value;
+        }
+        return WorkerThreadUtils.SYSTEM_DEFAULT_STRING;
+    }
+    /**
+     * Checks if the provided value is a valid thread display string. This includes blank and partial strings, so it is not strictly validated.
+     *
+     * @param value the string to validate
+     * @returns true if the string is valid for display, false otherwise
+     */
+    static isValidThreadDisplayString(value) {
+        // Allow empty and partial strings so user can clear the input
+        if (value.length === 0 ||
+            WorkerThreadUtils.SYSTEM_DEFAULT_STRING.startsWith(value) ||
+            WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_STRING.startsWith(value)) {
+            return true;
+        }
+        return WorkerThreadUtils.isValidThreadCount(value);
+    }
+    /**
+     * Checks if the provided value is a valid thread count. This only allows positive integers or specific strings.
+     *
+     * @param value the string to validate
+     * @returns true if the string is a valid thread count, false otherwise
+     */
+    static isValidThreadCount(value) {
+        if (value === WorkerThreadUtils.SYSTEM_DEFAULT_STRING ||
+            value === WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_STRING) {
+            return true;
+        }
+        return Number.isInteger(Number(value)) && Number(value) >= 1;
+    }
+}
+WorkerThreadUtils.INITIALIZATION_THREADS_ARG = 'OperationInitializationThreadPool.threads';
+WorkerThreadUtils.UPDATE_THREADS_ARG = 'PeriodicUpdateGraph.updateThreads';
+WorkerThreadUtils.ALL_ARGS = [
+    WorkerThreadUtils.INITIALIZATION_THREADS_ARG,
+    WorkerThreadUtils.UPDATE_THREADS_ARG,
+];
+WorkerThreadUtils.SYSTEM_DEFAULT_STRING = 'Default';
+WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_STRING = 'All available processors';
+WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_VALUE = '-1';
+WorkerThreadUtils.THREAD_ITEMS = [
+    WorkerThreadUtils.SYSTEM_DEFAULT_STRING,
+    WorkerThreadUtils.ALL_AVAILABLE_PROCESSORS_STRING,
+    '1',
+    '2',
+    '4',
+    '8',
+    '16',
+    '32',
+    '64',
+];
+WorkerThreadUtils.INITIALIZATION_THREADS_HELP_TEXT = 'Determines the number of threads available for parallel processing of initialization operations. Select a value or type a positive integer.';
+WorkerThreadUtils.UPDATE_THREADS_HELP_TEXT = `Determines the number of threads available for parallel processing of the Periodic Update Graph refresh cycle. Select a value or type a positive integer.`;
+export default WorkerThreadUtils;

package/dist/index.d.ts

@@ -3,3 +3,4 @@ export * from './QueryColumns';
 export * from './QueryScheduler';
 export * from './QuerySchedulerValidation';
 export * from './QueryType';
+export * from './WorkerThreadUtils';

package/dist/index.js

@@ -3,3 +3,4 @@ export * from './QueryColumns';
 export * from './QueryScheduler';
 export * from './QuerySchedulerValidation';
 export * from './QueryType';
+export * from './WorkerThreadUtils';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deephaven-enterprise/query-utils",
-  "version": "1.20240723.113-beta",
+  "version": "1.20240723.115-beta",
   "description": "Deephaven Enterprise Query Utils",
   "author": "Deephaven Data Labs LLC",
   "license": "SEE LICENSE IN LICENSE.md",
@@ -29,5 +29,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "04c09a4d3c5ebdf01ad261a85388f9455726f5c6"
+  "gitHead": "1091fd27057aa3f9b60d89a82ffe5070490774e4"
 }