@mindline/sync 1.0.37 → 1.0.39

package/.vs/VSWorkspaceState.json

@@ -2,5 +2,6 @@
   "ExpandedNodes": [
     ""
   ],
+  "SelectedNode": "\\index.ts",
   "PreviewInSolutionExplorer": false
 }

package/hybridspa.ts

@@ -3,6 +3,7 @@ import {
     User,
     Tenant,
     Config,
+    TenantConfigInfo,
     APIResult
 } from "./index";
 import {
@@ -26,6 +27,7 @@ export const graphConfig = {
         "https://dev-configurationapi-westus.azurewebsites.net/api/v1/configurations",
     initEndpoint:
         "https://dev-configurationapi-westus.azurewebsites.net/api/v1/configuration/init",
+    readerStartSyncEndpoint: "https://dev-configurationapi-westus.azurewebsites.net/api/v1/startSync",
     tenantEndpoint:
         "https://dev-configurationapi-westus.azurewebsites.net/api/v1/tenant",
     tenantsEndpoint:
@@ -47,9 +49,6 @@ export const graphConfig = {
     authorityUSRegex: /^(https:\/\/login\.microsoftonline\.(?:us|com)\/)([\dA-Fa-f]{8}-[\dA-Fa-f]{4}-[\dA-Fa-f]{4}-[\dA-Fa-f]{4}-[\dA-Fa-f]{12})\/oauth2\/authorize$/,
     authorityCN: "https://login.partner.microsoftonline.cn/",
     authorityCNRegex: /^(https:\/\/login\.partner\.microsoftonline\.cn\/)([\dA-Fa-f]{8}-[\dA-Fa-f]{4}-[\dA-Fa-f]{4}-[\dA-Fa-f]{4}-[\dA-Fa-f]{12})\/oauth2\/authorize$/,
-    // reader endpoint to trigger sync start
-    readerStartSyncEndpoint: "https://dev-fn-reader-westus.azurewebsites.net/api/startSync/",
-    readerApiEndpoint: "https://dev-fn-reader-westus.azurewebsites.net/api/lookup/"
 };
 // helper functions
 async function defineHeaders(
@@ -134,7 +133,7 @@ export async function adminDelete(
         console.log("Attempting DELETE from /admin: " + url!.href);
         let response = await fetch(url!.href, options);
         if (response.status === 200 && response.statusText === "OK") {
-            console.log(`Successful DELETE from //admin: ${url!.href}`);
+            console.log(`Successful DELETE from /admin: ${url!.href}`);
             return result;
         } else {
             result.error = await processErrors(response);
@@ -335,13 +334,20 @@ export async function configPost(
       "isEnabled": ${config.isEnabled},
       "tenants": [`;
     config.tenants.map((tci) => {
+        // be sure we send null and not "null" in body
+        let sourceGroupId: string = tci.sourceGroupId != "" ? `"${tci.sourceGroupId}"` : "null";
+        let sourceGroupName: string = tci.sourceGroupName != "" ? `"${tci.sourceGroupName}"` : "null";
+        let targetGroupId: string = tci.targetGroupId != "" ? `"${tci.targetGroupId}"` : "null";
+        let targetGroupName: string = tci.targetGroupName != "" ? `"${tci.targetGroupName}"` : "null";
         // if last character is } we need a comma first
         let needComma: boolean = configBody.slice(-1) === "}";
         if (needComma) configBody += ",";
         configBody += `{
         "tenantId": "${tci.tid}",
-        "sourceGroupId": "${tci.sourceGroupId}",
-        "sourceGroupName": "${tci.sourceGroupName}",
+        "sourceGroupId": ${sourceGroupId},
+        "sourceGroupName": ${sourceGroupName},
+        "targetGroupId": ${targetGroupId},
+        "targetGroupName": ${targetGroupName},
         "configurationTenantType": "${tci.configurationTenantType}"
       }`;
     });
@@ -404,15 +410,21 @@ export async function configPut(
       "description": "${config.description}",
       "isEnabled": ${config.isEnabled},
       "tenants": [`;
-    config.tenants.map((tci) => {
+    config.tenants.map((tci: TenantConfigInfo) => {
         // if last character is } we need a comma first
         let needComma: boolean = configBody.slice(-1) === "}";
         if (needComma) configBody += ",";
-        // TODO: more sophisticated source tenant user filtering
+        // be sure we send null and not "null" in body
+        let sourceGroupId: string = tci.sourceGroupId != "" ? `"${tci.sourceGroupId}"` : "null";
+        let sourceGroupName: string = tci.sourceGroupName != "" ? `"${tci.sourceGroupName}"` : "null";
+        let targetGroupId: string = tci.targetGroupId != "" ? `"${tci.targetGroupId}"` : "null";
+        let targetGroupName: string = tci.targetGroupName != "" ? `"${tci.targetGroupName}"` : "null";
         configBody += `{
           "tenantId": "${tci.tid}",
-          "sourceGroupId": "${tci.sourceGroupId}",
-          "sourceGroupName": "${tci.sourceGroupName}",
+          "sourceGroupId": ${sourceGroupId},
+          "sourceGroupName": ${sourceGroupName},
+          "targetGroupId": ${targetGroupId},
+          "targetGroupName": ${targetGroupName},
           "configurationTenantType": "${tci.configurationTenantType}",
           "deltaToken": "${tci.deltaToken}"
         }`;
@@ -857,21 +869,21 @@ export async function readerPost(
         result.status = 500;
         return result;
     }
-    // create reader endpoint with config ID
-    let readerEndpoint: string = graphConfig.readerStartSyncEndpoint + config.id;
+    // create reader endpoint
+    let readerEndpoint: string = graphConfig.readerStartSyncEndpoint;
+    let url: URL = new URL(readerEndpoint);
+    url.searchParams.append("configurationId", config.id);
     // create headers
     const headers = await defineHeaders(instance, authorizedUser);
     // make reader endpoint call
     let options = { method: "POST", headers: headers };
     try {
-        console.log("Attempting POST to /startSync: " + readerEndpoint);
-        let response = await fetch(readerEndpoint, options);
+        console.log("Attempting POST to /startSync: " + url.href);
+        let response = await fetch(url.href, options);
         if (response.status === 200 && response.statusText === "OK") {
             console.log(`Successful POST to /startSync: ${readerEndpoint}`);
-
-            let textResponse = await response.text();
-            textResponse = textResponse;
-
+            let jsonResponse = await response.json();
+            result.array = JSON.parse(jsonResponse.PayloadStr);
             return result;
         } else {
             result.error = await processErrors(response);

package/index.d.ts

@@ -65,11 +65,14 @@ declare module "@mindline/sync" {
     export class TenantConfigInfo {
         tid: string;              // tenant identifier
         sourceGroupId: string;    // source group - we can configure source group for reading
-        sourceGroupName: string;  // source group - we can configure source group for reading
+        sourceGroupName: string;  // 
+        targetGroupId: string;    // target group - we can configure target group for writing
+        targetGroupName: string;  // 
         configurationTenantType: TenantConfigTypeStrings;
         deltaToken: string;
-        filesWritten: number;
+        usersWritten: number;
         configId: string;
+        batchId: string;
     }
     export class Config {
         id: string;
@@ -134,12 +137,63 @@ declare module "@mindline/sync" {
         setEnd(endDate: Date): void;
         setStart(startDate: Date): void;
     }
+    export class Milestone {
+        Run: number;
+        Start: Date;
+        startDisplay: string;
+        POST: Date;
+        postDisplay: string;
+        Read: Date;
+        readDisplay: string;
+        Write: Date;
+        writeDisplay: string;
+        Duration: Date;
+        durationDisplay: string;
+        constructor(run: number);
+        start(start: string): void;
+        post(post: string): void;
+        read(read: string): void;
+        write(write: string): void;
+    }
+    export class MilestoneArray {
+        milestones: Milestone[];
+        constructor(bClearLocalStorage: boolean);
+        init(bClearLocalStorage: boolean): void;
+        save(): void;
+        start(setMilestones: (milestones: Milestone[]) => void): void;
+        unstart(setMilestones: (milestones: Milestone[]) => void): void;
+        post(setMilestones: (milestones: Milestone[]) => void): void;
+        read(setMilestones: (milestones: Milestone[]) => void): void;
+        write(setMilestones: (milestones: Milestone[]) => void): void;
+        #initFromObjects(milestones: Object[]): void;
+    }
     export class BatchArray {
         tenantNodes: TenantNode[];
+        pb_startTS: number;
+        pb_progress: number;
+        pb_increment: number;
+        pb_idle: number;
+        pb_idleMax: number;
+        pb_timer: NodeJS.Timer;
+        milestoneArray: MilestoneArray;
         constructor(config: Config|null, syncPortalGlobalState: InitInfo|null, bClearLocalStorage: boolean);
         // populate tenantNodes based on config tenants
-        init(config: Config|null, syncPortalGlobalState: InitInfo|null, bClearLocalStorage: boolean): void;
-        startSync(instance: IPublicClientApplication, authorizedUser: User | null | undefined, config: Config | null | undefined, setConfigSyncResult: (syncUpdate: string) => void): APIResult;
+        init(config: Config|null|undefined, syncPortalGlobalState: InitInfo|null, bClearLocalStorage: boolean): void;
+        initializeProgressBar(setSyncProgress: (progress: number) => void, setConfigSyncResult: (result: string) => void, setIdleText: (idleText: string) => void, setMilestones: (milestones: Milestone[]) => void): void;
+        uninitializeProgressBar(setSyncProgress: (progress: number) => void, setConfigSyncResult: (result: string) => void, setIdleText: (idleText: string) => void, setMilestones: (milestones: Milestone[]) => void): void;
+        initializeSignalR(
+            config: Config | null | undefined,
+            syncPortalGlobalState: InitInfo | null,
+            batchIdArray: Array<Object>,
+            setRefreshDeltaTrigger: (trigger: boolean) => void,
+            setReadersTotal: (readersTotal: number) => void,
+            setReadersCurrent: (readersCurrent: number) => void,
+            setWritersTotal: (writersTotal: number) => void,
+            setWritersCurrent: (writersCurrent: number) => void,
+            setMilestones: (milestones: Milestone[]) => void,
+            setConfigSyncResult: (result: string) => void,
+            bClearLocalStorage: boolean): void;
+        startSync(instance: IPublicClientApplication, authorizedUser: User | null | undefined, config: Config | null | undefined): APIResult;
     }
     export class TenantNode {
         expanded: boolean;
@@ -159,6 +213,7 @@ declare module "@mindline/sync" {
         result: boolean;
         status: number;
         error: string;
+        array: Array<Object> | null;
         constructor();
     }  
     //

package/index.ts

@@ -1,7 +1,7 @@
 //index.ts - published interface - AAD implementations, facade to Mindline Config API
 import * as signalR from "@microsoft/signalr"
 import { IPublicClientApplication, AuthenticationResult } from "@azure/msal-browser"
-import { deserializeArray } from 'class-transformer';
+import { deserializeArray, instanceToPlain, ClassTransformOptions } from 'class-transformer';
 import { adminDelete, adminPost, adminsGet, configDelete, configsGet, configPost, configPut, graphConfig, initPost, readerPost, tenantPut, tenantPost, tenantDelete, tenantsGet, workspacesGet } from './hybridspa';
 import { version } from './package.json';
 import users from "./users.json";
@@ -9,6 +9,7 @@ import tenants from "./tenants.json";
 import configs from "./configs.json";
 import workspaces from "./workspaces.json";
 import tasksData from "./tasks";
+import syncmilestones from './syncmilestones';
 import { log } from "console";
 const FILTER_FIELD = "workspaceIDs";
 // called by unit tests
@@ -112,18 +113,24 @@ export class TenantConfigInfo {
     tid: string;
     sourceGroupId: string;
     sourceGroupName: string;
+    targetGroupId: string;
+    targetGroupName: string;
     configurationTenantType: TenantConfigTypeStrings;
     deltaToken: string;
-    filesWritten: number;
+    usersWritten: number;
     configId: string;
+    batchId: string;
     constructor() {
         this.tid = "";
         this.sourceGroupId = "";
         this.sourceGroupName = "";
+        this.targetGroupId = "";
+        this.targetGroupName = "";
         this.configurationTenantType = "source";
         this.deltaToken = "";
-        this.filesWritten = 0;
+        this.usersWritten = 0;
         this.configId = "";
+        this.batchId = "";
     }
 }
 export class Config {
@@ -435,8 +442,196 @@ export class Task {
     };
 }
 // class corresponding to an execution of a Config - a *TenantNode* for each source tenant, each with a *TenantNode* array of target tenants
+export class Milestone {
+    Run: number;
+    Start: Date;
+    startDisplay: string;
+    POST: Date;
+    postDisplay: string;
+    Read: Date;
+    readDisplay: string;
+    Write: Date;
+    writeDisplay: string;
+    Duration: Date;
+    durationDisplay: string;
+    constructor(run: number) {
+        this.Run = run;
+        this.start("");
+        this.POST = null;
+        this.postDisplay = "";
+        this.Read = null;
+        this.readDisplay = "";
+        this.Write = null;
+        this.writeDisplay = "";
+        this.Duration = null;
+        this.durationDisplay = "";
+    }
+    start(start: string): void {
+        start == "" ? this.Start = new Date() : this.Start = new Date(start);
+        this.startDisplay = `${this.Start.getMinutes().toString().padStart(2, "0")}:${this.Start.getSeconds().toString().padStart(2, "0")}`;
+    }
+    post(post: string): void {
+        post == "" ? this.POST = new Date() : this.POST = new Date(post);
+        this.postDisplay = `${this.POST.getMinutes().toString().padStart(2, "0")}:${this.POST.getSeconds().toString().padStart(2, "0")}`;
+    }
+    read(read: string): void {
+        read == "" ? this.Read = new Date() : this.Read = new Date(read);
+        this.readDisplay = `${this.Read.getMinutes().toString().padStart(2, "0")}:${this.Read.getSeconds().toString().padStart(2, "0")}`;
+    }
+    write(write: string): void {
+        write == "" ? this.Write = new Date() : this.Write = new Date(write);
+        this.writeDisplay = `${this.Write.getMinutes().toString().padStart(2, "0")}:${this.Write.getSeconds().toString().padStart(2, "0")}`;
+        this.Duration = new Date(this.Write.getTime() - this.Start.getTime());
+        this.durationDisplay = `${this.Duration.getMinutes().toString().padStart(2, "0")}:${this.Duration.getSeconds().toString().padStart(2, "0")}`;
+    }
+}
+export class MilestoneArray {
+    milestones: Milestone[];
+    constructor(bClearLocalStorage: boolean) {
+        this.init(bClearLocalStorage);
+    }
+    init(bClearLocalStorage: boolean): void {
+        // read from localstorage by default
+        if (storageAvailable("localStorage")) {
+            let result = localStorage.getItem("syncmilestones");
+            if (result != null && typeof result === "string" && result !== "") {
+                let milestonesString: string = result;
+                let milestones: Object [] = JSON.parse(milestonesString);
+                if (milestones.length !== 0) {
+                    if (bClearLocalStorage) {
+                        localStorage.removeItem("syncmilestones");
+                    }
+                    else {
+                        this.#initFromObjects(milestones);
+                        return;
+                    }
+                }
+            }
+        }
+        // if storage unavailable or we were just asked to clear, read from default syncmilestone file
+        this.#initFromObjects(syncmilestones);
+    }
+    save(): void {
+        let milestonesString: string = JSON.stringify(this.milestones);
+        if (storageAvailable("localStorage")) {
+            localStorage.setItem("syncmilestones", milestonesString);
+        }
+    }
+    // milestone tracking during a sync
+    start(setMilestones: (milestones: Milestone[]) => void): void {
+        // we should always have a milestone array and a first milestone
+        if (this.milestones == null || this.milestones.length < 1) { debugger; return; }
+        let currentRun: number = Number(this.milestones[0].Run);
+        // create a new milestone and prepend to front of array
+        let newMilestone: Milestone = new Milestone(currentRun+1);
+        this.milestones.unshift(newMilestone);
+        // re-define milestone array to trigger render
+        this.milestones = this.milestones.map((ms: Milestone) => {
+            let newms = new Milestone(ms.Run);
+            newms.Start = ms.Start;
+            newms.startDisplay = ms.startDisplay;
+            newms.POST = ms.POST;
+            newms.postDisplay = ms.postDisplay;
+            newms.Read = ms.Read;
+            newms.readDisplay = ms.readDisplay;
+            newms.Write = ms.Write;
+            newms.writeDisplay = ms.writeDisplay;
+            newms.Duration = ms.Duration;
+            newms.durationDisplay = ms.durationDisplay;
+            return newms;
+        });
+        setMilestones(this.milestones);
+        console.log(`Start milestone: ${this.milestones[0].Run}:${this.milestones[0].Start}`);
+    }
+    unstart(setMilestones: (milestones: Milestone[]) => void): void {
+        // we should always have a milestone array and a first milestone
+        if (this.milestones == null || this.milestones.length < 1) { debugger; return; }
+        let currentRun: number = Number(this.milestones[0].Run);
+        // remove first milestone from front of array
+        let removedMilestone: Milestone = this.milestones.shift();
+        // re-define milestone array to trigger render
+        this.milestones = this.milestones.map((ms: Milestone) => {
+            let newms = new Milestone(ms.Run);
+            newms.Start = ms.Start;
+            newms.startDisplay = ms.startDisplay;
+            newms.POST = ms.POST;
+            newms.postDisplay = ms.postDisplay;
+            newms.Read = ms.Read;
+            newms.readDisplay = ms.readDisplay;
+            newms.Write = ms.Write;
+            newms.writeDisplay = ms.writeDisplay;
+            newms.Duration = ms.Duration;
+            newms.durationDisplay = ms.durationDisplay;
+            return newms;
+        });
+        setMilestones(this.milestones);
+        console.log(`Unstart removed first milestone: ${removedMilestone.Run}:${removedMilestone.Start}`);
+    }
+    post(setMilestones: (milestones: Milestone[]) => void): void {
+        // update the post value of the first milestone
+        if (this.milestones == null || this.milestones.length < 1) { debugger; return; }
+        this.milestones[0].post("");
+        setMilestones(this.milestones);
+        console.log(`POST milestone: ${this.milestones[0].Run}:${this.milestones[0].POST}`);
+    }
+    read(setMilestones: (milestones: Milestone[]) => void): void {
+        if (this.milestones == null || this.milestones.length < 1) { debugger; return; }
+        this.milestones[0].read("");
+        setMilestones(this.milestones);
+        console.log(`Read milestone: ${this.milestones[0].Run}:${this.milestones[0].Read}`);
+    }
+    write(setMilestones: (milestones: Milestone[]) => void): void {
+        if (this.milestones == null || this.milestones.length < 1) { debugger; return; }
+        this.milestones[0].write("");
+        // while we have >10 complete milestones, remove the last
+        while (this.milestones.length > 10) {
+            let removed: Milestone = this.milestones.pop();
+            console.log(`Removed milestone #${removed.Run}: ${removed.Start}`);
+        }
+        // save to localstorage
+        this.save();
+        // re-define milestone array to trigger render
+        this.milestones = this.milestones.map((ms: Milestone) => {
+            let newms = new Milestone(ms.Run);
+            newms.Start = ms.Start;
+            newms.startDisplay = ms.startDisplay;
+            newms.POST = ms.POST;
+            newms.postDisplay = ms.postDisplay;
+            newms.Read = ms.Read;
+            newms.readDisplay = ms.readDisplay;
+            newms.Write = ms.Write;
+            newms.writeDisplay = ms.writeDisplay;
+            newms.Duration = ms.Duration;
+            newms.durationDisplay = ms.durationDisplay;
+            return newms;
+        });
+        setMilestones(this.milestones);
+    }
+    #initFromObjects(milestones: Object[]): void {
+        if (milestones == null) {
+            this.milestones = new Array();
+        }
+        else {
+            this.milestones = milestones.map((milestone: Object) => {
+                let ms: Milestone = new Milestone(Number(milestone.Run));
+                ms.start(milestone.Start);
+                ms.post(milestone.POST);
+                ms.read(milestone.Read);
+                ms.write(milestone.Write);
+                return ms;
+            });
+        }
+    }
+}
 export class BatchArray {
     tenantNodes: TenantNode[];
+    pb_startTS: number;
+    pb_progress: number;
+    pb_increment: number;
+    pb_idle: number;
+    pb_idleMax: number;
+    pb_timer: NodeJS.Timer;
+    milestoneArray: MilestoneArray;
     constructor(
         config: Config | null,
         syncPortalGlobalState: InitInfo | null,
@@ -444,30 +639,36 @@ export class BatchArray {
     ) {
         this.tenantNodes = new Array<TenantNode>();
         this.init(config, syncPortalGlobalState, bClearLocalStorage);
+        this.pb_startTS = 0;
+        this.pb_progress = 0;
+        this.pb_increment = 0;
+        this.pb_timer = null;
+        this.pb_idle = 0;
+        this.pb_idleMax = 0;
+        this.milestoneArray = new MilestoneArray(false);
     }
     // populate tenantNodes based on config tenants
     init(
-        config: Config | null,
+        config: Config | null | undefined,
         syncPortalGlobalState: InitInfo | null,
         bClearLocalStorage: boolean
     ): void {
         console.log(
-            `Calling BatchArray::init(config: "${config ? config.name : "null"
-            }", bClearLocalStorage: ${bClearLocalStorage ? "true" : "false"})`
+            `Calling BatchArray::init(config: "${config ? config.name : "null"}", bClearLocalStorage: ${bClearLocalStorage ? "true" : "false"})`
         );
-        // first clear batch  array
-        this.tenantNodes.length = 0;
-        // then clear localStorage if we have been asked to
+        // clear localStorage if we have been asked to
         if (bClearLocalStorage) {
-            if (storageAvailable("localStorage"))
+            if (storageAvailable("localStorage")) {
                 localStorage.removeItem(config.name);
+                this.milestoneArray.init(bClearLocalStorage);
+            }
         }
         // create BatchArray if passed Config and InitInfo
-        if (
-            config != null &&
+        if (config != null &&
             config.tenants != null &&
-            syncPortalGlobalState != null
-        ) {
+            syncPortalGlobalState != null) {
+            // clear batch array only if we have been passed something with which to replace it
+            this.tenantNodes.length = 0;
             // create a sourceTenantNode for each Source and SourceTarget
             config.tenants.map((tciPotentialSource: TenantConfigInfo) => {
                 if (
@@ -480,7 +681,8 @@ export class BatchArray {
                     if (sourceTenant != null) {
                         let sourceTenantNode: TenantNode = new TenantNode(
                             tciPotentialSource.tid,
-                            sourceTenant.name
+                            sourceTenant.name,
+                            tciPotentialSource.batchId
                         );
                         this.tenantNodes.push(sourceTenantNode);
                     } else {
@@ -508,7 +710,8 @@ export class BatchArray {
                             if (targetTenant != null) {
                                 let targetTenantNode: TenantNode = new TenantNode(
                                     tciPotentialTarget.tid,
-                                    targetTenant.name
+                                    targetTenant.name,
+                                    tciPotentialTarget.batchId
                                 );
                                 sourceTenantNode.targets.push(targetTenantNode);
                                 sourceTenantNode.expanded = true;
@@ -523,195 +726,255 @@ export class BatchArray {
                     }
                 });
             });
-            // then try localStorage to find any matching source tenant metrics
-            if (storageAvailable("localStorage")) {
-                let result = localStorage.getItem(config.name);
-                if (result != null && typeof result === "string" && result !== "") {
-                    // TODO: retrieve any relevant stored statistics from localStorage
-                    // let batchArrayString: string = result;
-                    // let batchArray: BatchArray = JSON.parse(batchArrayString);
-                    // batchArray.batches.map((batch: Batch) => {
-                    //   config.tenants.map((tciTarget: TenantConfigInfo) => {
-                    //     if(tciTarget.tid !== batch.tid) {
-                    //       let target: Target = new Target(tciTarget.tid);
-                    //       batch.targets.push(target);
-                    //     }
-                    //   });
-                    // });
+        }
+    }
+    initializeProgressBar(setSyncProgress: (progress: number) => void, setConfigSyncResult: (result: string) => void, setIdleText: (idleText: string) => void, setMilestones: (milestones: Milestone[]) => void): void {
+        this.pb_startTS = Date.now();
+        this.pb_progress = 0;
+        this.pb_increment = 1;
+        this.pb_idle = 0;
+        this.pb_idleMax = 0;
+        setIdleText(`No updates seen for ${this.pb_idle} seconds. [max idle: ${this.pb_idleMax}]`);
+        this.pb_timer = setInterval(() => {
+            // if we go 20 seconds without a signalR message, finish the sync
+            this.pb_idle = this.pb_idle + 1;
+            this.pb_idleMax = Math.max(this.pb_idle, this.pb_idleMax);
+            setIdleText(`No updates seen for ${this.pb_idle} seconds. [max idle: ${this.pb_idleMax}]`);
+            if (this.pb_idle >= 20) {
+                clearInterval(this.pb_timer);
+                this.pb_timer = null;
+                if (this.milestoneArray.milestones[0].Write == null) {
+                    this.milestoneArray.write(setMilestones);
                 }
+                setConfigSyncResult(`finished sync, no updates for ${this.pb_idle} seconds`);
             }
-        }
+            // if we get to 100, stop the timer, let SignalR or countdown timer finish sync
+            if (this.pb_progress < 100) {
+                this.pb_progress = Math.min(100, this.pb_progress + this.pb_increment);
+                setSyncProgress(this.pb_progress);
+            }
+        }, 1000);
+        this.milestoneArray.start(setMilestones);
     }
-    // start a sync cycle
-    async startSync(instance: IPublicClientApplication, authorizedUser: User | null | undefined, config: Config | null | undefined, setConfigSyncResult: (syncUpdate: string) => void): Promise<APIResult> {
-        let result: APIResult = new APIResult();
-        if (this.tenantNodes == null || this.tenantNodes.length == 0) {
-            // we should not have an empty batch array for a test
-            debugger;
-            result.result = false;
-            result.error = "startSync: invalid parameters";
-            result.status = 500;
-            return result;
-        }
+    uninitializeProgressBar(setSyncProgress: (progress: number) => void, setConfigSyncResult: (result: string) => void, setIdleText: (idleText: string) => void, setMilestones: (milestones: Milestone[]) => void): void {
+        this.pb_startTS = 0;
+        this.pb_progress = 0;
+        setSyncProgress(this.pb_progress);
+        setConfigSyncResult("sync failed to execute");
+        this.pb_increment = 0;
+        clearInterval(this.pb_timer);
+        this.pb_timer = null;
+        this.pb_idle = 0;
+        this.pb_idleMax = 0;
+        setIdleText(`No updates seen for ${this.pb_idle} seconds. [max idle: ${this.pb_idleMax}]`);
+        this.milestoneArray.unstart(setMilestones);
+    }
+    initializeSignalR(
+        config: Config | null | undefined,
+        syncPortalGlobalState: InitInfo | null,
+        batchIdArray: Array<Object>,
+        setRefreshDeltaTrigger: (trigger: boolean) => void,
+        setReadersTotal: (readersTotal: number) => void,
+        setReadersCurrent: (readersCurrent: number) => void,
+        setWritersTotal: (writersTotal: number) => void,
+        setWritersCurrent: (writersCurrent: number) => void ,
+        setMilestones: (milestones: Milestone[]) => void,
+        setConfigSyncResult: (result: string) => void,
+        bClearLocalStorage: boolean
+    ): void {
+        // we have just completed a successful POST to startSync
+        this.milestoneArray.post(setMilestones);
+        setConfigSyncResult("started sync, waiting for updates...");
+        // re-initialize batch array with Configuration updated by the succcessful POST to startSync
+        this.init(config, syncPortalGlobalState, false);
         // define newMessage handler that can access *this*
         let handler = (message) => {
             console.log(message);
             let item = JSON.parse(message);
+            // reset the countdown timer every time we get a message
+            this.pb_idle = 0;
             // find the associated tenant for this SignalR message
-            let tenantNode: TenantNode = this.tenantNodes.find((t: TenantNode) => t.tid === item.TargetID);
+            let matchingPair: Object = batchIdArray.find((o: Object) => o.BatchId == item.TargetID);
+            if (matchingPair == null) {
+                console.log(`Batch ${item.TargetID} not found in batchIdArray.`);
+                debugger;
+                return;
+            }
+            let tenantNode: TenantNode = this.tenantNodes.find((t: TenantNode) => t.tid === matchingPair.SourceId);
             if (tenantNode == null) { // null OR undefined
-                console.log(`${item.TargetID} not found in BatchArray.`);
+                console.log(`Tenant ${matchingPair.SourceId} not found in BatchArray.`);
                 debugger;
                 return;
             }
-            let writerNode: TenantNode|null = null;
-            // process stats for this SignalR message
+            tenantNode.batchId = matchingPair.BatchId;
+            // process stats for this SignalR message batch
             let statsarray = item.Stats; // get the array of statistics
             let statskeys = Object.keys(statsarray); // get the keys of the array
             let statsvalues = Object.values(statsarray); // get the values of the array
             for (let j = 0; j < statskeys.length; j++) {
+                let bTotalCount = statskeys[j].endsWith("TotalCount");
+                let bCurrentCount = statskeys[j].endsWith("CurrentCount");
+                let bDeferredCount = statskeys[j].endsWith("DeferredCount");
                 if (statskeys[j].startsWith("Reader")) {
-                    if (statskeys[j].endsWith("TotalCount")) {
-                        // parse batchId from key and store in TenantNode for this batch
-                        let batchidRegexp = /Reader\/BID:(.+)\/TotalCount/;
-                        let matchBID = statskeys[j].match(batchidRegexp);
-                        if (matchBID == null) {
-                            console.log(`batchId not found in ${statskeys[j]}.`);
-                            debugger;
-                            return;
-                        }
-                        // integrate any queued Writers that have been waiting for this batchId to be assigned to a Reader
-                        let idx: number = this.tenantNodes.findIndex((t: TenantNode) => (t.name === "QUEUED WRITER" && t.batchId === matchBID[1]));
-                        while (idx !== -1) {
-                            let queuedWriter: TenantNode = this.tenantNodes.splice(idx, 1)[0];
-                            let actualWriter: TenantNode = tenantNode.targets.find((t: TenantNode) => t.tid == queuedWriter.tid);
-                            if (actualWriter == null) {
-                                console.log(`could not find queued writer ${queuedWriter.tid}.`);
-                                debugger;
-                            }
-                            else {
-                                actualWriter.update(queuedWriter.total, queuedWriter.read, queuedWriter.written, queuedWriter.deferred);
-                            }
-                            idx = this.tenantNodes.findIndex((t: TenantNode) => (t.name === "QUEUED WRITER" && t.batchId === matchBID[1]));
-                        }
-                        // update Read node
-                        tenantNode.batchId = matchBID[1];
-                        tenantNode.total = Number(statsvalues[j]);
-                        console.log(`----- ${tenantNode.name} batchId: ${tenantNode.batchId}`);
-                        console.log(`----- ${tenantNode.name} Total Read: ${tenantNode.total}`);
+                    // parse tid from Reader key
+                    let tidRegexp = /Reader\/TID:(.+)\/TotalCount/;
+                    if (bCurrentCount) tidRegexp = /Reader\/TID:(.+)\/CurrentCount/;
+                    if (bDeferredCount) tidRegexp = /Reader\/TID:(.+)\/DeferredCount/;
+                    let matchTID = statskeys[j].match(tidRegexp);
+                    if (matchTID == null) {
+                        console.log(`tid not found in ${statskeys[j]}.`);
+                        debugger;
+                        return;
+                    }
+                    if (bTotalCount) {
+                        tenantNode.total = Math.max(Number(statsvalues[j]), tenantNode.total);
+                        console.log(`----- ${tenantNode.name} TID: ${tenantNode.tid} batchId: ${tenantNode.batchId}`);
+                        console.log(`----- ${tenantNode.name} Total To Read: ${tenantNode.total}`);
                     }
-                    if (statskeys[j].endsWith("CurrentCount")) {
-                        tenantNode.read = Number(statsvalues[j]);
+                    else {
+                        tenantNode.read = Math.max(Number(statsvalues[j]), tenantNode.read);
                         console.log(`----- ${tenantNode.name} Currently Read: ${tenantNode.read}`);
                     }
                 }
                 if (statskeys[j].startsWith("Writer")) {
-                    if (statskeys[j].endsWith("TotalCount")) {
-                        // parse batchId from Writer key
-                        let batchidRegexp = /Writer\/BID:(.+)\/TotalCount/;
-                        let matchBID = statskeys[j].match(batchidRegexp);
-                        if (matchBID == null) {
-                            console.log(`batchId not found in ${statskeys[j]}.`);
-                            debugger;
-                            return;
-                        }
-                        // find the Writer node for this tenant under the Reader node of a different tenant
-                        let readerNode: TenantNode = this.tenantNodes.find((t: TenantNode) => t.batchId === matchBID[1]);
-                        if (readerNode == null) {
-                            // reader for this batch has not yet been encountered, queue this writer for processing when reader arrives
-                            writerNode = new TenantNode(tenantNode.tid, "QUEUED WRITER");
-                            writerNode.batchId = matchBID[1];
-                            writerNode.total = Number(statsvalues[j]);
-                            this.tenantNodes.push(writerNode);
-                            console.log(`----- QUEUED batchId: ${writerNode.batchId}`);
-                            console.log(`----- QUEUED Total To Write: ${writerNode.total}`);
-                        }
-                        else {
-                            if (readerNode.name == "QUEUED WRITER") {
-                                // update queued node
-                                writerNode = readerNode;
-                            }
-                            else {
-                                // reader is legit, find writer node under this reader node
-                                writerNode = readerNode.targets.find((t: TenantNode) => t.tid === tenantNode.tid);
-                                if (writerNode == null) {
-                                    console.log(`Writer ${tenantNode.name} not found under Reader ${readerNode.name}.`);
-                                    debugger;
-                                    return;
-                                }
-                            }
-                            writerNode.batchId = matchBID[1];
-                            writerNode.total = Number(statsvalues[j]);
-                            console.log(`----- ${writerNode.name} batchId: ${writerNode.batchId}`);
-                            console.log(`----- ${writerNode.name} Total To Write: ${writerNode.total}`);
-                        }
+                    // parse tid from Writer key
+                    let tidRegexp = /Writer\/TID:(.+)\/TotalCount/;
+                    if (bCurrentCount) tidRegexp = /Writer\/TID:(.+)\/CurrentCount/;
+                    if (bDeferredCount) tidRegexp = /Writer\/TID:(.+)\/DeferredCount/;
+                    let matchTID = statskeys[j].match(tidRegexp);
+                    if (matchTID == null) {
+                        console.log(`tid not found in ${statskeys[j]}.`);
+                        debugger;
+                        return;
                     }
-                    if (statskeys[j].endsWith("CurrentCount")) {
-                        // parse batchId from Writer key
-                        let batchidRegexp = /Writer\/BID:(.+)\/CurrentCount/;
-                        let matchBID = statskeys[j].match(batchidRegexp);
-                        if (matchBID == null) {
-                            console.log(`batchId not found in ${statskeys[j]}.`);
-                            debugger;
-                            return;
-                        }
-                        // find the Writer node for this tenant under the Reader node of a different tenant
-                        let readerNode: TenantNode = this.tenantNodes.find((t: TenantNode) => t.batchId === matchBID[1]);
-                        if (readerNode == null) {
-                            // no reader or queued writer for this batch
-                            console.log(`No reader or queued writer for batch ${matchBID[1]}.`);
-                            debugger;
-                            return;
-                        }
-                        if (readerNode.name == "QUEUED WRITER") {
-                            // update queued node
-                            writerNode = readerNode;
-                        }
-                        else {
-                            // reader is legit, find writer node under this reader node
-                            writerNode = readerNode.targets.find((t: TenantNode) => t.tid === tenantNode.tid);
-                            if (writerNode == null) {
-                                console.log(`Writer ${tenantNode.name} not found under Reader ${readerNode.name}.`);
-                                debugger;
-                                return;
-                            }
-                        }
-                        writerNode.batchId = matchBID[1];
-                        writerNode.written = Number(statsvalues[j]);
-                        console.log(`${writerNode.name} batchId: ${writerNode.batchId}`);
-                        console.log(`${writerNode.name} Total Written: ${writerNode.written}`);
+                    // this Writer node should exist precisely under the Reader for this SignalR message
+                    let writerNode: TenantNode = tenantNode.targets.find((t: TenantNode) => t.tid === matchTID[1]);
+                    if (writerNode == null) {
+                        console.log(`Writer ${tenantNode.name} not found under Reader ${tenantNode.name}.`);
+                        debugger;
+                        return;
                     }
+                    writerNode.batchId = matchingPair.BatchId;
+                    if (bTotalCount) {
+                        writerNode.total = Math.max(Number(statsvalues[j]), writerNode.total);
+                        console.log(`----- ${writerNode.name} TID: ${writerNode.tid} batchId: ${writerNode.batchId}`);
+                        console.log(`----- ${writerNode.name} Total To Write: ${writerNode.total}`);
+                    }
+                    else if (bCurrentCount) {
+                        writerNode.written = Math.max(Number(statsvalues[j]), writerNode.written);
+                        console.log(`----- ${writerNode.name} Total Written: ${writerNode.written}`);
+                    }
+                    else if (bDeferredCount) {
+                        writerNode.deferred = Math.max(Number(statsvalues[j]), writerNode.deferred);
+                        console.log(`----- ${writerNode.name} Total Deferred: ${writerNode.deferred}`);
+                    }
+                    else {
+                        console.log(`unknown writer type`);
+                        debugger;
+                        return;
+                    }
+                    writerNode.update(writerNode.total, writerNode.read, writerNode.written, writerNode.deferred);
                 }
             }
+            // update status based on all updates in this message
             tenantNode.update(tenantNode.total, tenantNode.read, tenantNode.written, tenantNode.deferred);
-            if (writerNode != null) {
-                writerNode.update(writerNode.total, writerNode.read, writerNode.written, writerNode.deferred);
-                if (tenantNode.total === tenantNode.read && writerNode.total === writerNode.written) {
-                    setConfigSyncResult("complete");
-                    console.log(`Setting config sync result: "complete"`);
-                }
-                else {
-                    setConfigSyncResult("writing in progress");
-                    console.log(`Setting config sync result: "writing in progress"`);
-                }
+            // for each message, enumerate nodes to assess completion state
+            let bReadingComplete: boolean = true;
+            let bWritingComplete: boolean = true;
+            let bWritingStarted: boolean = false;
+            let readerTotal: number = 0;
+            let readerCurrent: number = 0;
+            let writerTotal: number = 0;
+            let writerCurrent: number = 0;
+            this.tenantNodes.map((sourceTenantNode: TenantNode) => {
+                sourceTenantNode.targets.map((writerNode: TenantNode) => {
+                    bWritingComplete &&= (writerNode.status == "complete" || writerNode.status == "failed");
+                    bWritingStarted ||= (writerNode.total > 0 || writerNode.status != "not started");
+                    writerTotal += Math.max(writerNode.total, sourceTenantNode.total);
+                    writerCurrent += writerNode.written;
+                });
+                bReadingComplete &&= (sourceTenantNode.status == "complete" || sourceTenantNode.status == "failed");
+                readerTotal += sourceTenantNode.total;
+                readerCurrent += sourceTenantNode.read;
+            });
+            // set linear gauge max and current values
+            setReadersTotal(readerTotal);
+            setReadersCurrent(readerCurrent);
+            setWritersTotal(Math.max(writerTotal, readerTotal));
+            setWritersCurrent(writerCurrent);
+            // because it is an important milestone, we always check if we have *just* completed reading
+            if (bReadingComplete && this.milestoneArray.milestones[0].Read == null) {
+                this.milestoneArray.read(setMilestones);
+                setConfigSyncResult("reading complete");
+                console.log(`Setting config sync result: "reading complete"`);
+                // trigger refresh delta tokens
+                setRefreshDeltaTrigger(true);
+                // change to % per second to complete in 7x as long as it took to get here
+                let readTS = Date.now();
+                let secsElapsed = (readTS - this.pb_startTS) / 1000;
+                let expectedPercentDone = 7;
+                let expectedPercentPerSecond = secsElapsed / expectedPercentDone;
+                this.pb_increment = expectedPercentPerSecond;
+                console.log(`Setting increment: ${this.pb_increment}% per second`);
             }
-            else {
+            // with that out of the way, is writing complete?
+            if (bWritingComplete) {
+                this.milestoneArray.write(setMilestones);
+                setConfigSyncResult("sync complete");
+                console.log(`Setting config sync result: "complete"`);
+                this.pb_progress = 99;
+            }
+            // if not, has writing even started?
+            else if (bWritingStarted) {
+                setConfigSyncResult("writing in progress");
+                console.log(`Setting config sync result: "writing in progress"`);
+            }
+            // else, we must be reading (unless we already completed reading)
+            else if (this.milestoneArray.milestones[0].Read == null){
                 setConfigSyncResult("reading in progress");
                 console.log(`Setting config sync result: "reading in progress"`);
             }
         }
-        // start SignalR connection for each source tenant in the configuration
-        this.tenantNodes.map((sourceTenantNode: TenantNode) => {
-            const endpointUrl = `https://dev-signalrdispatcher-westus.azurewebsites.net/statsHub?statsId=${sourceTenantNode.tid}`;
+        // start SignalR connection based on each batchId
+        batchIdArray.map((batchPair: Object) => {
+            const endpointUrl = `https://dev-signalrdispatcher-westus.azurewebsites.net/statsHub?statsId=${batchPair.BatchId}`;
+            console.log(`Creating SignalR Hub for TID: ${batchPair.SourceId} ${endpointUrl}`);
             const connection: signalR.HubConnection = new signalR.HubConnectionBuilder()
                 .withUrl(endpointUrl)
-                // .withAutomaticReconnect() TODO: full auto-reconnect implementation
+                .withAutomaticReconnect()
                 .configureLogging(signalR.LogLevel.Information)
                 .build();
             // when you get a message, process the message
             connection.on("newMessage", handler);
+            connection.onreconnecting(error => {
+                console.assert(connection.state === signalR.HubConnectionState.Reconnecting);
+                console.log(`Connection lost due to error "${error}". Reconnecting.`);
+            });
+            connection.onreconnected(connectionId => {
+                console.assert(connection.state === signalR.HubConnectionState.Connected);
+                console.log(`Connection reestablished. Connected with connectionId "${connectionId}".`);
+            });
+            // restart when you get a close event
+            connection.onclose(async () => {
+                console.log(`Connection closing. Attempting restart.`);
+                await connection.start();
+            });
+            // start and display any caught exceptions in the console
             connection.start().catch(console.error);
         });
+    }
+    // start a sync cycle
+    async startSync(instance: IPublicClientApplication, authorizedUser: User | null | undefined, config: Config | null | undefined): Promise<APIResult>
+    {
+        let result: APIResult = new APIResult();
+        if (this.tenantNodes == null || this.tenantNodes.length == 0) {
+            // we should not have an empty batch array for a test
+            debugger;
+            result.result = false;
+            result.error = "startSync: invalid parameters";
+            result.status = 500;
+            return result;
+        }
         // execute post to reader endpoint
         result = await readerPost(instance, authorizedUser, config);
         return result;
@@ -728,11 +991,11 @@ export class TenantNode {
     written: number;
     deferred: number;
     targets: TenantNode[];
-    constructor(tid: string, name: string) {
+    constructor(tid: string, name: string, batchId: string) {
         this.expanded = false;
         this.name = name;
         this.tid = tid;
-        this.batchId = "";
+        this.batchId = batchId;
         this.targets = new Array<TenantNode>();
         this.update(0, 0, 0, 0);
     }
@@ -749,7 +1012,7 @@ export class TenantNode {
         else if (this.written > 0) {
             if (this.written + this.deferred < this.total) this.status = "in progress";
             else if (this.written === this.total) this.status = "complete";
-            else if (this.written + this.deferred === this.total) this.status = "failed";
+            else if (this.written + this.deferred >= this.total) this.status = "failed";
         }
     }
 }
@@ -826,7 +1089,7 @@ export function signIn(user: User, tasks: TaskArray): void {
     tenantURL += "MicrosoftIdentity/Account/Challenge";
     let url: URL = new URL(tenantURL);
     url.searchParams.append("redirectUri", window.location.origin);
-    url.searchParams.append("scope", "openid offline_access profile user.read contacts.read CrossTenantInformation.ReadBasic.All");
+    url.searchParams.append("scope", "openid offline_access Directory.AccessAsUser.All CrossTenantInformation.ReadBasic.All");
     url.searchParams.append("domainHint", "organizations");
     if (user.oid !== "1") {
         url.searchParams.append("loginHint", user.mail);
@@ -1173,7 +1436,7 @@ function processReturnedAdmins(workspace: Workspace, ii: InitInfo, returnedAdmin
     returnedAdmins.map((item) => {
         // are we already tracking this user?
         let user: User | null = null;
-        let usIndex = ii.us.findIndex((u) => u.oid === item.userId);
+        let usIndex = ii.us.findIndex((u) => (u.oid === item.userId || u.oid === item.email));
         if (usIndex === -1) {
             // start tracking
             let dummyIndex = ii.us.findIndex((u) => u.oid === "1");
@@ -1192,13 +1455,13 @@ function processReturnedAdmins(workspace: Workspace, ii: InitInfo, returnedAdmin
             user = ii.us.at(usIndex);
         }
         // refresh all the data available from the server
-        user.oid = item.userId;
+        user.oid = item.userId ? item.userId : item.email;
         user.name = item.firstName;
         user.mail = item.email;
         user.tid = item.tenantId;
         // ensure this workspace tracks this user
-        let idx = workspace.associatedUsers.findIndex((u) => u === item.userId);
-        if (idx == -1) workspace.associatedUsers.push(item.userId);
+        let idx = workspace.associatedUsers.findIndex((u) => u === user.oid);
+        if (idx == -1) workspace.associatedUsers.push(user.oid);
     });
 }
 function processReturnedTenants(workspace: Workspace, ii: InitInfo, returnedTenants: Array<Object>) {
@@ -1227,7 +1490,12 @@ function processReturnedTenants(workspace: Workspace, ii: InitInfo, returnedTena
         tenant.tenantType = item.type.toLowerCase(); // should now be strings
         tenant.permissionType = item.permissionType.toLowerCase(); // should now be strings
         tenant.onboarded = item.isOnboarded ? "true" : "false";
-        tenant.authority = item.authority;
+
+        // canonicalize authority when getting it from config backend
+        const regex = /^(https:\/\/login.microsoftonline.(?:us|com)\/)organizations\/v2.0$/;
+        const regexMatch = item.authority.match(regex);
+        tenant.authority = regexMatch ? regexMatch[1] : item.authority;
+
         tenant.readServicePrincipal = item.readServicePrincipal;
         tenant.writeServicePrincipal = item.writeServicePrincipal;
         // ensure this workspace tracks this tenant
@@ -1265,11 +1533,14 @@ function processReturnedConfigs(workspace: Workspace, ii: InitInfo, returnedConf
         item.tenants.map((tci) => {
             let tenantConfigInfo = new TenantConfigInfo();
             tenantConfigInfo.tid = tci.tenantId;
-            tenantConfigInfo.sourceGroupId = tci.sourceGroupId;
-            tenantConfigInfo.sourceGroupName = tci.sourceGroupName;
+            tenantConfigInfo.sourceGroupId = tci.sourceGroupId ?? "";
+            tenantConfigInfo.sourceGroupName = tci.sourceGroupName ?? "";
+            tenantConfigInfo.targetGroupId = tci.targetGroupId ?? "";
+            tenantConfigInfo.targetGroupName = tci.targetGroupName ?? "";
             tenantConfigInfo.configurationTenantType = tci.configurationTenantType.toLowerCase();
             tenantConfigInfo.deltaToken = tci.deltaToken ?? "";
             tenantConfigInfo.configId = config!.id;
+            tenantConfigInfo.batchId = tci.batchId ?? "";
             config!.tenants.push(tenantConfigInfo);
         });
         // ensure this workspace tracks this config

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mindline/sync",
   "type": "module",
-  "version": "1.0.37",
+  "version": "1.0.39",
   "types": "index.d.ts",
   "exports": "./index.ts",
   "description": "sync is a node.js package encapsulating javscript classes required for configuring Mindline sync service.",

package/syncmilestones.json

@@ -0,0 +1,23 @@
+[
+  {
+    "Run": "3",
+    "Start": "2023-09-17T12:00:00.000-07:00",
+    "POST": "2023-09-17T12:00:09.500-07:00",
+    "Read": "2023-09-17T12:00:15.500-07:00",
+    "Write": "2023-09-17T12:01:30.000-07:00"
+  },
+  {
+    "Run": "2",
+    "Start": "2023-09-17T12:00:00.000-07:00",
+    "POST": "2023-09-17T12:00:09.500-07:00",
+    "Read": "2023-09-17T12:00:15.500-07:00",
+    "Write": "2023-09-17T12:01:30.000-07:00"
+  },
+  {
+    "Run": "1",
+    "Start": "2023-09-17T12:00:00.000-07:00",
+    "POST": "2023-09-17T12:00:09.500-07:00",
+    "Read": "2023-09-17T12:00:15.500-07:00",
+    "Write": "2023-09-17T12:01:30.000-07:00"
+  }
+]