@hpcc-js/comms 2.82.1 → 2.82.2

package/src/ecl/dfuWorkunit.ts

@@ -0,0 +1,306 @@
+import { Cache, IEvent, scopedLogger, StateCallback, StateEvents, StateObject, StatePropCallback } from "@hpcc-js/util";
+import { IConnection, IOptions } from "../connection";
+import { ESPExceptions } from "../espConnection";
+import { WsSMC } from "../services/wsSMC";
+import { FileSpray, SprayFixedEx, FileSprayService, SprayVariableEx } from "../services/fileSpray";
+import * as WsTopology from "../services/wsTopology";
+
+const logger = scopedLogger("@hpcc-js/comms/dfuWorkunit.ts");
+
+enum States {
+    Unknown = 0,
+    Scheduled,
+    Queued,
+    Started,
+    Aborted,
+    Failed,
+    Finished,
+    Monitoring,
+    Aborting,
+    NotFound = 999
+}
+
+export class DFUWorkunitCache extends Cache<{ BaseUrl: string, ID: string }, DFUWorkunit> {
+    constructor() {
+        super((obj) => {
+            return `${obj.BaseUrl}-${obj.ID}`;
+        });
+    }
+}
+const _workunits = new DFUWorkunitCache();
+
+export type DFUWorkunitEvents = "finished" | StateEvents;
+export type UDFUWorkunitState = FileSpray.DFUWorkunit;
+export type IDFUWorkunitState = FileSpray.DFUWorkunit | WsSMC.ActiveWorkunit;
+export class DFUWorkunit extends StateObject<UDFUWorkunitState, IDFUWorkunitState> implements FileSpray.DFUWorkunit {
+    connection: FileSprayService;
+    topologyConnection: WsTopology.TopologyService;
+    get BaseUrl() { return this.connection.baseUrl; }
+
+    //  Accessors  ---
+    get properties(): FileSpray.DFUWorkunit { return this.get(); }
+    get ID(): string { return this.get("ID"); }
+    get DFUServerName(): string { return this.get("DFUServerName"); }
+    get ClusterName(): string { return this.get("ClusterName"); }
+    get JobName(): string { return this.get("JobName"); }
+    get Queue(): string { return this.get("Queue"); }
+    get User(): string { return this.get("User"); }
+    get isProtected(): boolean { return this.get("isProtected"); }
+    get Command(): number { return this.get("Command"); }
+    get CommandMessage(): string { return this.get("CommandMessage"); }
+    get PercentDone(): number { return this.get("PercentDone"); }
+    get SecsLeft(): number { return this.get("SecsLeft"); }
+    get ProgressMessage(): string { return this.get("ProgressMessage"); }
+    get SummaryMessage(): string { return this.get("SummaryMessage"); }
+    get State(): number { return this.get("State", States.Unknown); }
+    get SourceLogicalName(): string { return this.get("SourceLogicalName"); }
+    get SourceIP(): string { return this.get("SourceIP"); }
+    get SourceFilePath(): string { return this.get("SourceFilePath"); }
+    get SourceDali(): string { return this.get("SourceDali"); }
+    get SourceRecordSize(): number { return this.get("SourceRecordSize"); }
+    get SourceFormat(): number { return this.get("SourceFormat"); }
+    get RowTag(): string { return this.get("RowTag"); }
+    get SourceNumParts(): number { return this.get("SourceNumParts"); }
+    get SourceDirectory(): string { return this.get("SourceDirectory"); }
+    get DestLogicalName(): string { return this.get("DestLogicalName"); }
+    get DestGroupName(): string { return this.get("DestGroupName"); }
+    get DestDirectory(): string { return this.get("DestDirectory"); }
+    get DestIP(): string { return this.get("DestIP"); }
+    get DestFilePath(): string { return this.get("DestFilePath"); }
+    get DestFormat(): number { return this.get("DestFormat"); }
+    get DestNumParts(): number { return this.get("DestNumParts"); }
+    get DestRecordSize(): number { return this.get("DestRecordSize"); }
+    get Replicate(): boolean { return this.get("Replicate"); }
+    get Overwrite(): boolean { return this.get("Overwrite"); }
+    get Compress(): boolean { return this.get("Compress"); }
+    get SourceCsvSeparate(): string { return this.get("SourceCsvSeparate"); }
+    get SourceCsvQuote(): string { return this.get("SourceCsvQuote"); }
+    get SourceCsvTerminate(): string { return this.get("SourceCsvTerminate"); }
+    get SourceCsvEscape(): string { return this.get("SourceCsvEscape"); }
+    get TimeStarted(): string { return this.get("TimeStarted"); }
+    get TimeStopped(): string { return this.get("TimeStopped"); }
+    get StateMessage(): string { return this.get("StateMessage"); }
+    get MonitorEventName(): string { return this.get("MonitorEventName"); }
+    get MonitorSub(): boolean { return this.get("MonitorSub"); }
+    get MonitorShotLimit(): number { return this.get("MonitorShotLimit"); }
+    get SourceDiffKeyName(): string { return this.get("SourceDiffKeyName"); }
+    get DestDiffKeyName(): string { return this.get("DestDiffKeyName"); }
+    get Archived(): boolean { return this.get("Archived"); }
+    get encrypt(): string { return this.get("encrypt"); }
+    get decrypt(): string { return this.get("decrypt"); }
+    get failIfNoSourceFile(): boolean { return this.get("failIfNoSourceFile"); }
+    get recordStructurePresent(): boolean { return this.get("recordStructurePresent"); }
+    get quotedTerminator(): boolean { return this.get("quotedTerminator"); }
+    get preserveCompression(): boolean { return this.get("preserveCompression"); }
+    get expireDays(): number { return this.get("expireDays"); }
+    get PreserveFileParts(): boolean { return this.get("PreserveFileParts"); }
+    get FileAccessCost(): number { return this.get("FileAccessCost"); }
+
+    //  Factories  ---
+    static create(optsConnection: IOptions | IConnection, dfuServerQueue: string): Promise<DFUWorkunit> {
+        const retVal: DFUWorkunit = new DFUWorkunit(optsConnection);
+        return retVal.connection.CreateDFUWorkunit({ DFUServerQueue: dfuServerQueue }).then((response) => {
+            _workunits.set(retVal);
+            retVal.set(response.result);
+            return retVal;
+        });
+    }
+
+    static attach(optsConnection: IOptions | IConnection, wuid: string, state?: IDFUWorkunitState): DFUWorkunit {
+        const retVal: DFUWorkunit = _workunits.get({ BaseUrl: optsConnection.baseUrl, ID: wuid }, () => {
+            return new DFUWorkunit(optsConnection, wuid);
+        });
+        if (state) {
+            retVal.set(state);
+        }
+        return retVal;
+    }
+
+    static sprayFixed(server: IOptions | IConnection, request: Partial<SprayFixedEx>): Promise<DFUWorkunit> {
+        const service = new FileSprayService(server);
+        return service.SprayFixedEx({
+            ...request
+        }).then(response => {
+            const wuid = response.wuid;
+            return service.GetDFUWorkunit({ wuid }).then(response => {
+                return DFUWorkunit.attach(server, wuid, response.result);
+            });
+        });
+    }
+
+    static sprayVariable(server: IOptions | IConnection, request: Partial<SprayVariableEx>): Promise<DFUWorkunit> {
+        const service = new FileSprayService(server);
+        return service.SprayVariableEx({
+            ...request
+        }).then(response => {
+            const wuid = response.wuid;
+            return service.GetDFUWorkunit({ wuid }).then(response => {
+                return DFUWorkunit.attach(server, wuid, response.result);
+            });
+        });
+    }
+
+    static despray(server: IOptions | IConnection, request: Partial<FileSpray.Despray>): Promise<DFUWorkunit> {
+        const service = new FileSprayService(server);
+        return service.DesprayEx({
+            ...request
+        }).then(response => {
+            const wuid = response.wuid;
+            return service.GetDFUWorkunit({ wuid }).then(response => {
+                return DFUWorkunit.attach(server, wuid, response.result);
+            });
+        });
+    }
+
+    isComplete(): boolean {
+        switch (this.State) {
+            case States.Finished:
+            case States.Failed:
+            case States.Aborted:
+            case States.NotFound:
+                return true;
+            default:
+        }
+        return false;
+    }
+
+    isFailed() {
+        if (this.isComplete() && this.State !== States.Finished) {
+            return true;
+        }
+        return false;
+    }
+
+    isDeleted() {
+        switch (this.State) {
+            case States.NotFound:
+                return true;
+            default:
+        }
+        return false;
+    }
+
+    isRunning(): boolean {
+        return !this.isComplete();
+    }
+
+    async refresh(full: boolean = false): Promise<this> {
+        await this.GetDFUWorkunit();
+        return this;
+    }
+
+    //  Monitoring  ---
+    protected _monitor(): void {
+        if (this.isComplete()) {
+            this._monitorTickCount = 0;
+            return;
+        }
+        super._monitor();
+    }
+
+    protected _monitorTimeoutDuration(): number {
+        const retVal = super._monitorTimeoutDuration();
+        if (this._monitorTickCount <= 1) {          //  Once
+            return 3000;
+        } else if (this._monitorTickCount <= 5) {   //  Twice
+            return 6000;
+        } else if (this._monitorTickCount <= 7) {   //  Twice
+            return 12000;
+        }
+        return retVal;
+    }
+
+    //  Events  ---
+    on(eventID: DFUWorkunitEvents, propIDorCallback: StateCallback | keyof UDFUWorkunitState, callback?: StatePropCallback): this {
+        if (this.isCallback(propIDorCallback)) {
+            switch (eventID) {
+                case "finished":
+                    super.on("propChanged", "State", (changeInfo: IEvent) => {
+                        if (this.isComplete()) {
+                            propIDorCallback([changeInfo]);
+                        }
+                    });
+                    break;
+                case "changed":
+                    super.on(eventID, propIDorCallback);
+                    break;
+                default:
+            }
+        } else {
+            switch (eventID) {
+                case "changed":
+                    super.on(eventID, propIDorCallback, callback!);
+                    break;
+                default:
+            }
+        }
+        this._monitor();
+        return this;
+    }
+
+    watchUntilComplete(callback?: StateCallback): Promise<this> {
+        return new Promise((resolve, _) => {
+            const watchHandle = this.watch((changes) => {
+                if (callback) {
+                    callback(changes);
+                }
+                if (this.isComplete()) {
+                    watchHandle.release();
+                    resolve(this);
+                }
+            });
+        });
+    }
+
+    watchUntilRunning(callback?: StateCallback): Promise<this> {
+        return new Promise((resolve, _) => {
+            const watchHandle = this.watch((changes) => {
+                if (callback) {
+                    callback(changes);
+                }
+                if (this.isComplete() || this.isRunning()) {
+                    watchHandle.release();
+                    resolve(this);
+                }
+            });
+        });
+    }
+
+    //  ---  ---  ---
+    protected constructor(optsConnection: IOptions | IConnection, wuid?: string) {
+        super();
+        this.connection = new FileSprayService(optsConnection);
+        this.topologyConnection = new WsTopology.TopologyService(optsConnection);
+        this.clearState(wuid);
+    }
+
+    clearState(wuid?: string) {
+        this.clear({
+            ID: wuid,
+            State: States.Unknown
+        });
+    }
+
+    //  FileSpray passthroughs  ---
+    protected GetDFUWorkunit(_request: Partial<FileSpray.GetDFUWorkunit> = {}): Promise<FileSpray.GetDFUWorkunitResponse> {
+        return this.connection.GetDFUWorkunit({ ..._request, wuid: this.ID }).then((response) => {
+            this.set(response.result);
+            return response;
+        }).catch((e: ESPExceptions) => {
+            //  deleted  ---
+            const wuMissing = e.Exception.some((exception) => {
+                if (exception.Code === 20081) {
+                    this.clearState(this.ID);
+                    this.set("State", States.NotFound);
+                    return true;
+                }
+                return false;
+            });
+            if (!wuMissing) {
+                logger.warning("Unexpected exception:  ");
+                throw e;
+            }
+            return {} as FileSpray.GetDFUWorkunitResponse;
+        });
+    }
+}

package/src/ecl/workunit.ts

@@ -724,8 +724,8 @@ export class Workunit extends StateObject<UWorkunitState, IWorkunitState> implem
         super._monitor();
     }
 
-    protected _monitorTimeoutDuraction(): number {
-        const retVal = super._monitorTimeoutDuraction();
+    protected _monitorTimeoutDuration(): number {
+        const retVal = super._monitorTimeoutDuration();
         if (this._monitorTickCount <= 1) {          //  Once
             return 1000;
         } else if (this._monitorTickCount <= 3) {   //  Twice

package/src/index-common.ts

@@ -32,6 +32,7 @@ export * from "./ecl/store";
 export * from "./ecl/timer";
 export * from "./ecl/topology";
 export * from "./ecl/targetCluster";
+export * from "./ecl/dfuWorkunit";
 export * from "./ecl/workunit";
 export * from "./ecl/xsdParser";
 

package/src/services/fileSpray.ts

@@ -4,5 +4,113 @@ export {
     FileSpray
 };
 
+export interface SprayFixedEx {
+    sourceIP?: string;
+    sourcePlane?: string;
+    sourcePath?: string;
+    srcxml?: string;
+    sourceFormat?: string;
+    sourceRecordSize?: number;
+    destGroup?: string;
+    destLogicalName?: string;
+    overwrite?: boolean;
+    replicate?: boolean;
+    ReplicateOffset?: number;
+    maxConnections?: number;
+    throttle?: number;
+    transferBufferSize?: number;
+    prefix?: string;
+    nosplit?: boolean;
+    norecover?: boolean;
+    compress?: boolean;
+    push?: boolean;
+    pull?: boolean;
+    noCommon?: boolean;
+    encrypt?: string;
+    decrypt?: string;
+    wrap?: boolean;
+    failIfNoSourceFile?: boolean;
+    recordStructurePresent?: boolean;
+    quotedTerminator?: boolean;
+    expireDays?: number;
+    DFUServerQueue?: string;
+}
+
+export interface SprayVariableEx {
+    sourceIP?: string;
+    sourcePlane?: string;
+    sourcePath?: string;
+    srcxml?: string;
+    sourceMaxRecordSize?: number;
+    sourceFormat?: number;
+    NoSourceCsvSeparator?: boolean;
+    sourceCsvSeparate?: string;
+    sourceCsvTerminate?: string;
+    sourceCsvQuote?: string;
+    sourceCsvEscape?: string;
+    sourceRowTag?: string;
+    destGroup?: string;
+    destLogicalName?: string;
+    overwrite?: boolean;
+    replicate?: boolean;
+    ReplicateOffset?: number;
+    maxConnections?: number;
+    throttle?: number;
+    transferBufferSize?: number;
+    prefix?: string;
+    nosplit?: boolean;
+    norecover?: boolean;
+    compress?: boolean;
+    push?: boolean;
+    pull?: boolean;
+    noCommon?: boolean;
+    encrypt?: string;
+    decrypt?: string;
+    failIfNoSourceFile?: boolean;
+    recordStructurePresent?: boolean;
+    quotedTerminator?: boolean;
+    sourceRowPath?: string;
+    isJSON?: boolean;
+    expireDays?: number;
+    DFUServerQueue?: string;
+    srcUsername?: string;
+    srcPassword?: string;
+}
+
+export interface DesprayEx {
+    destGroup?: string;
+    sourceLogicalName?: string;
+    destIP?: string;
+    destPath?: string;
+    destPlane?: string;
+    dstxml?: string;
+    overwrite?: boolean;
+    maxConnections?: number;
+    throttle?: number;
+    transferBufferSize?: number;
+    splitprefix?: string;
+    norecover?: boolean;
+    wrap?: boolean;
+    multiCopy?: boolean;
+    SingleConnection?: boolean;
+    DFUServerQueue?: string;
+    compress?: boolean;
+    encrypt?: string;
+    decrypt?: string;
+}
+
 export class FileSprayService extends FileSprayServiceBase {
+
+    SprayFixedEx(request: SprayFixedEx): Promise<FileSpray.SprayFixedResponse> {
+        return this._connection.send("SprayFixed", request);
+    }
+
+    SprayVariableEx(request: SprayVariableEx): Promise<FileSpray.SprayResponse> {
+        return this._connection.send("SprayVariable", request, "json", false, null, "SprayResponse");
+    }
+
+    DesprayEx(request: DesprayEx): Promise<FileSpray.DesprayResponse> {
+        return this._connection.send("Despray", request);
+    }
+
 }

package/src/services/wsLogaccess.ts

@@ -1,5 +1,5 @@
 import { scopedLogger } from "@hpcc-js/util";
-import { LogaccessServiceBase, WsLogaccess } from "./wsdl/ws_logaccess/v1.03/ws_logaccess";
+import { LogaccessServiceBase, WsLogaccess } from "./wsdl/ws_logaccess/v1.04/ws_logaccess";
 
 const logger = scopedLogger("@hpcc-js/comms/services/wsLogaccess.ts");
 
@@ -9,7 +9,7 @@ export {
 
 export interface GetLogsExRequest {
     audience?: string;
-    class?: string;
+    class?: string[];
     jobId?: string;
     message?: string;
     procId?: string;
@@ -35,7 +35,7 @@ export const enum LogType {
 export const enum TargetAudience {
     Operator = "OPR",
     User = "USR",
-    Programmer = "PRO",
+    Programmer = "PRG",
     Audit = "ADT"
 }
 
@@ -133,7 +133,7 @@ export class LogaccessService extends LogaccessServiceBase {
                     BinaryLogFilter: [{
                         leftFilter: {
                             LogCategory: WsLogaccess.LogAccessType.All,
-                        } as WsLogaccess.leftFilter,
+                        },
                     } as WsLogaccess.BinaryLogFilter]
                 }
             },
@@ -149,34 +149,78 @@ export class LogaccessService extends LogaccessServiceBase {
         const filters: WsLogaccess.leftFilter[] = [];
         for (const key in request) {
             if (key in ElasticKnownColumns) {
-                filters.push({
-                    LogCategory: WsLogaccess.LogAccessType.ByFieldName,
-                    SearchField: ElasticKnownColumns[key],
-                    SearchByValue: request[key]
-                });
+                if (Array.isArray(request[key])) {
+                    request[key].forEach(value => {
+                        filters.push({
+                            LogCategory: WsLogaccess.LogAccessType.ByFieldName,
+                            SearchField: ElasticKnownColumns[key],
+                            SearchByValue: value
+                        });
+                    });
+                } else {
+                    filters.push({
+                        LogCategory: WsLogaccess.LogAccessType.ByFieldName,
+                        SearchField: ElasticKnownColumns[key],
+                        SearchByValue: request[key]
+                    });
+                }
+
             }
         }
 
-        let binaryLogFilter = getLogsRequest.Filter.leftBinaryFilter.BinaryLogFilter[0];
-        filters.forEach((filter, i) => {
-            if (i === 0) {
-                binaryLogFilter.leftFilter = filter;
-            } else if (i === filters.length - 1) {
-                binaryLogFilter.Operator = WsLogaccess.LogAccessFilterOperator.AND;
-                binaryLogFilter.rightFilter = filter;
-            } else {
-                binaryLogFilter.Operator = WsLogaccess.LogAccessFilterOperator.AND;
-                binaryLogFilter.rightBinaryFilter = {
-                    BinaryLogFilter: [{
-                        leftFilter: filter
-                    } as WsLogaccess.BinaryLogFilter]
+        if (filters.length > 2) {
+            let binaryLogFilter = getLogsRequest.Filter.leftBinaryFilter.BinaryLogFilter[0];
+            filters.forEach((filter, i) => {
+                let operator = WsLogaccess.LogAccessFilterOperator.AND;
+                if (i > 0) {
+                    if (filters[i - 1].SearchField === filter.SearchField) {
+                        operator = WsLogaccess.LogAccessFilterOperator.OR;
+                    }
+                    if (i === filters.length - 1) {
+                        binaryLogFilter.Operator = operator;
+                        binaryLogFilter.rightFilter = filter as WsLogaccess.rightFilter;
+                    } else {
+                        binaryLogFilter.Operator = operator;
+                        binaryLogFilter.rightBinaryFilter = {
+                            BinaryLogFilter: [{
+                                leftFilter: filter
+                            } as WsLogaccess.BinaryLogFilter]
+                        };
+                        binaryLogFilter = binaryLogFilter.rightBinaryFilter.BinaryLogFilter[0];
+                    }
+                } else {
+                    binaryLogFilter.leftFilter = filter as WsLogaccess.leftFilter;
+                }
+            });
+        } else {
+            delete getLogsRequest.Filter.leftBinaryFilter;
+            getLogsRequest.Filter.leftFilter = {
+                LogCategory: WsLogaccess.LogAccessType.All
+            } as WsLogaccess.leftFilter;
+            if (filters[0]?.SearchField) {
+                getLogsRequest.Filter.leftFilter = {
+                    LogCategory: filters[0]?.LogCategory,
+                    SearchField: filters[0]?.SearchField,
+                    SearchByValue: filters[0]?.SearchByValue
                 };
-                binaryLogFilter = binaryLogFilter.rightBinaryFilter.BinaryLogFilter[0];
             }
-        });
+            if (filters[1]?.SearchField) {
+                getLogsRequest.Filter.Operator = WsLogaccess.LogAccessFilterOperator.AND;
+                if (filters[0].SearchField === filters[1].SearchField) {
+                    getLogsRequest.Filter.Operator = WsLogaccess.LogAccessFilterOperator.OR;
+                }
+                getLogsRequest.Filter.rightFilter = {
+                    LogCategory: filters[0]?.LogCategory,
+                    SearchField: filters[1]?.SearchField,
+                    SearchByValue: filters[1]?.SearchByValue
+                };
+            }
+        }
 
-        if (request.StartDate && request.EndDate) {
+        if (request.StartDate) {
             getLogsRequest.Range.StartDate = request.StartDate.toISOString();
+        }
+        if (request.EndDate) {
             getLogsRequest.Range.EndDate = request.EndDate.toISOString();
         }
 
@@ -197,7 +241,7 @@ export class LogaccessService extends LogaccessServiceBase {
                 }
                 return {
                     lines: lines,
-                    total: response.TotalLogLinesAvailable || 10000
+                    total: response.TotalLogLinesAvailable ?? 10000
                 };
             } catch (e) {
                 logger.error(e);