@rancher/shell 0.3.0 → 0.3.1

package/plugins/steve/subscribe.js

@@ -1,6 +1,16 @@
+/**
+ * Handles subscriptions to websockets which receive updates to resources
+ *
+ * Covers three use cases
+ * 1) Handles subscription within this file
+ * 2) Handles `cluster` subscriptions for some basic types in a web worker (SETTING.UI_PERFORMANCE advancedWorker = false)
+ * 2) Handles `cluster` subscriptions and optimisations in an advanced worker (SETTING.UI_PERFORMANCE advancedWorker = true)
+ */
+
 import { addObject, clear, removeObject } from '@shell/utils/array';
 import { get } from '@shell/utils/object';
-import { COUNT, SCHEMA } from '@shell/config/types';
+import { SCHEMA } from '@shell/config/types';
+import { CSRF } from '@shell/config/cookies';
 import { getPerformanceSetting } from '@shell/utils/settings';
 import Socket, {
   EVENT_CONNECTED,
@@ -8,25 +18,47 @@ import Socket, {
   EVENT_MESSAGE,
   //  EVENT_FRAME_TIMEOUT,
   EVENT_CONNECT_ERROR,
-  EVENT_DISCONNECT_ERROR
+  EVENT_DISCONNECT_ERROR,
+  NO_WATCH,
+  NO_SCHEMA,
 } from '@shell/utils/socket';
 import { normalizeType } from '@shell/plugins/dashboard-store/normalize';
 import day from 'dayjs';
 import { DATE_FORMAT, TIME_FORMAT } from '@shell/store/prefs';
 import { escapeHtml } from '@shell/utils/string';
+import { keyForSubscribe } from '@shell/plugins/steve/resourceWatcher';
+import { waitFor } from '@shell/utils/async';
 
 // eslint-disable-next-line
-import webworker from './web-worker.steve-sub-worker.js';
-
-export const NO_WATCH = 'NO_WATCH';
-export const NO_SCHEMA = 'NO_SCHEMA';
+import storeWorker from './worker/index.js';
+import { BLANK_CLUSTER } from '@shell/store/index.js';
 
 // minimum length of time a disconnect notification is shown
 const MINIMUM_TIME_NOTIFIED = 3000;
 
+const waitForManagement = (store) => {
+  const managementReady = () => store.state?.managementReady;
+
+  return waitFor(managementReady, 'Management');
+};
+
+const isAdvancedWorker = (ctx) => {
+  const { rootGetters, getters } = ctx;
+  const storeName = getters.storeName;
+  const clusterId = rootGetters.clusterId;
+
+  if (storeName !== 'cluster' || clusterId === BLANK_CLUSTER) {
+    return false;
+  }
+
+  const perfSetting = getPerformanceSetting(rootGetters);
+
+  return perfSetting?.advancedWorker.enabled;
+};
+
 // We only create a worker for the cluster store
-export function createWorker(store, ctx) {
-  const { getters } = ctx;
+export async function createWorker(store, ctx) {
+  const { getters, dispatch } = ctx;
   const storeName = getters.storeName;
 
   store.$workers = store.$workers || {};
@@ -35,24 +67,45 @@ export function createWorker(store, ctx) {
     return;
   }
 
+  await waitForManagement(store);
+  // getting perf setting in a separate constant here because it'll provide other values we'll want later.
+  const advancedWorker = isAdvancedWorker(ctx);
+
   const workerActions = {
     load: (resource) => {
       queueChange(ctx, resource, true, 'Change');
     },
     destroyWorker: () => {
       if (store.$workers) {
+        store.$workers[storeName].terminate();
         delete store.$workers[storeName];
       }
+    },
+    batchChanges: (batch) => {
+      dispatch('batchChanges', batch);
+    },
+    dispatch: (msg) => {
+      dispatch(`ws.${ msg.name }`, msg);
+    },
+    [EVENT_CONNECT_ERROR]: (e) => {
+      dispatch('error', e );
+    },
+    [EVENT_DISCONNECT_ERROR]: (e) => {
+      dispatch('error', e );
     }
   };
 
   if (!store.$workers[storeName]) {
-    const worker = new webworker();
+    const workerMode = advancedWorker ? 'advanced' : 'basic';
+    const worker = storeWorker(workerMode);
 
     store.$workers[storeName] = worker;
 
     worker.postMessage({ initWorker: { storeName } });
 
+    /**
+     * Covers message from Worker to UI thread
+     */
     store.$workers[storeName].onmessage = (e) => {
       /* on the off chance there's more than key in the message, we handle them in the order that they "keys" method provides which is
       // good enough for now considering that we never send more than one message action at a time right now */
@@ -65,14 +118,11 @@ export function createWorker(store, ctx) {
   }
 }
 
-export function keyForSubscribe({
-  resourceType, type, namespace, id, selector
-} = {}) {
-  return `${ resourceType || type || '' }/${ namespace || '' }/${ id || '' }/${ selector || '' }`;
-}
-
 export function equivalentWatch(a, b) {
-  if ( a.type !== b.type ) {
+  const aresourceType = a.resourceType || a.type;
+  const bresourceType = b.resourceType || b.type;
+
+  if ( aresourceType !== bresourceType ) {
     return false;
   }
 
@@ -136,12 +186,17 @@ function growlsDisabled(rootGetters) {
   return getPerformanceSetting(rootGetters)?.disableWebsocketNotification;
 }
 
-export const actions = {
-  subscribe(ctx, opt) {
+/**
+ * Actions that cover all cases (see file description)
+ */
+const sharedActions = {
+  async subscribe(ctx, opt) {
     const {
       state, commit, dispatch, getters, rootGetters
     } = ctx;
 
+    // ToDo: need to keep the worker up to date on CSRF cookie
+
     if (rootGetters['isSingleProduct']?.disableSteveSockets) {
       return;
     }
@@ -157,13 +212,28 @@ export const actions = {
     state.debugSocket && console.info(`Subscribe [${ getters.storeName }]`); // eslint-disable-line no-console
 
     const url = `${ state.config.baseUrl }/subscribe`;
+    const maxTries = growlsDisabled(rootGetters) ? null : 3;
+    const connectionMetadata = get(opt, 'metadata');
 
-    if ( socket ) {
+    if (isAdvancedWorker(ctx)) {
+      if (!this.$workers[getters.storeName]) {
+        await createWorker(this, ctx);
+      }
+
+      // if the worker is in advanced mode then it'll contain it's own socket which it calls a 'watcher'
+      this.$workers[getters.storeName].postMessage({
+        createWatcher: {
+          connectionMetadata,
+          url:  `${ state.config.baseUrl }/subscribe`,
+          csrf: this.$cookies.get(CSRF, { parseJSON: false }),
+          maxTries
+        }
+      });
+    } else if ( socket ) {
       socket.setAutoReconnect(true);
       socket.setUrl(url);
+      socket.connect(connectionMetadata);
     } else {
-      const maxTries = growlsDisabled(rootGetters) ? null : 3;
-
       socket = new Socket(`${ state.config.baseUrl }/subscribe`, true, null, null, maxTries);
 
       commit('setSocket', socket);
@@ -194,75 +264,28 @@ export const actions = {
           }
         }
       });
+      socket.connect(connectionMetadata);
     }
-
-    socket.connect(get(opt, 'metadata') );
   },
 
   unsubscribe({ commit, getters, state }) {
     const socket = state.socket;
-    const worker = (this.$workers || {})[getters.storeName];
 
     commit('setWantSocket', false);
+    const cleanupTasks = [];
+
+    const worker = (this.$workers || {})[getters.storeName];
 
     if (worker) {
       worker.postMessage({ destroyWorker: true }); // we're only passing the boolean here because the key needs to be something truthy to ensure it's passed on the object.
+      cleanupTasks.push(waitFor(() => !this.$workers[getters.storeName], 'Worker is destroyed'));
     }
 
     if ( socket ) {
-      return socket.disconnect();
-    }
-  },
-
-  async flush({
-    state, commit, dispatch, getters
-  }) {
-    const queue = state.queue;
-    let toLoad = [];
-
-    if ( !queue.length ) {
-      return;
-    }
-
-    const started = new Date().getTime();
-
-    state.queue = [];
-
-    state.debugSocket && console.debug(`Subscribe Flush [${ getters.storeName }]`, queue.length, 'items'); // eslint-disable-line no-console
-
-    for ( const { action, event, body } of queue ) {
-      if ( action === 'dispatch' && event === 'load' ) {
-        // Group loads into one loadMulti when possible
-        toLoad.push(body);
-      } else {
-        // When we hit a different kind of event, process all the previous loads, then the other event.
-        if ( toLoad.length ) {
-          await dispatch('loadMulti', toLoad);
-          toLoad = [];
-        }
-
-        if ( action === 'dispatch' ) {
-          await dispatch(event, body);
-        } else if ( action === 'commit' ) {
-          commit(event, body);
-        } else {
-          throw new Error('Invalid queued action');
-        }
-      }
+      cleanupTasks.push(socket.disconnect());
     }
 
-    // Process any remaining loads
-    if ( toLoad.length ) {
-      await dispatch('loadMulti', toLoad);
-    }
-
-    state.debugSocket && console.debug(`Subscribe Flush [${ getters.storeName }] finished`, (new Date().getTime()) - started, 'ms'); // eslint-disable-line no-console
-  },
-
-  rehydrateSubscribe({ state, dispatch }) {
-    if ( process.client && state.wantSocket && !state.socket ) {
-      dispatch('subscribe');
-    }
+    return Promise.all(cleanupTasks);
   },
 
   watch({
@@ -283,6 +306,7 @@ export const actions = {
       return;
     }
 
+    // If socket is in error don't try to watch.... unless we `force` it
     if ( !stop && !force && !getters.canWatch(params) ) {
       console.error(`Cannot Watch [${ getters.storeName }]`, JSON.stringify(params)); // eslint-disable-line no-console
 
@@ -323,9 +347,122 @@ export const actions = {
       msg.selector = selector;
     }
 
+    const worker = this.$workers[getters.storeName] || {};
+
+    if (worker.mode === 'advanced') {
+      if ( force ) {
+        msg.force = true;
+      }
+
+      worker.postMessage({ watch: msg });
+
+      return;
+    }
+
     return dispatch('send', msg);
   },
 
+  unwatch(ctx, type) {
+    const { commit, getters, dispatch } = ctx;
+
+    if (getters['schemaFor'](type)) {
+      const obj = {
+        type,
+        stop: true, // Stops the watch on a type
+      };
+
+      if (isAdvancedWorker(ctx)) {
+        dispatch('watch', obj); // Ask the backend to stop watching the type
+      } else if (getters['watchStarted'](obj)) {
+        // Set that we don't want to watch this type
+        // Otherwise, the dispatch to unwatch below will just cause a re-watch when we
+        // detect the stop message from the backend over the web socket
+        commit('setWatchStopped', obj);
+        dispatch('watch', obj); // Ask the backend to stop watching the type
+        // Make sure anything in the pending queue for the type is removed, since we've now removed the type
+        commit('clearFromQueue', type);
+      }
+    }
+  },
+
+  'ws.ping'({ getters, dispatch }, msg) {
+    if ( getters.storeName === 'management' ) {
+      const version = msg?.data?.version || null;
+
+      dispatch('updateServerVersion', version, { root: true });
+      console.info(`Ping [${ getters.storeName }] from ${ version || 'unknown version' }`); // eslint-disable-line no-console
+    }
+  },
+};
+
+/**
+ * Mutations that cover all cases (both subscriptions here and in advanced worker)
+ */
+const sharedMutations = {
+  debug(state, on, store) {
+    state.debugSocket = on !== false;
+    if (store && this.$workers[store]) {
+      this.$workers[store].postMessage({ toggleDebug: on !== false });
+    }
+  },
+};
+
+/**
+ * Actions that cover cases 1 & 2 (see file description)
+ */
+const defaultActions = {
+
+  async flush({
+    state, commit, dispatch, getters
+  }) {
+    const queue = state.queue;
+    let toLoad = [];
+
+    if ( !queue.length ) {
+      return;
+    }
+
+    const started = new Date().getTime();
+
+    state.queue = [];
+
+    state.debugSocket && console.debug(`Subscribe Flush [${ getters.storeName }]`, queue.length, 'items'); // eslint-disable-line no-console
+
+    for ( const { action, event, body } of queue ) {
+      if ( action === 'dispatch' && event === 'load' ) {
+        // Group loads into one loadMulti when possible
+        toLoad.push(body);
+      } else {
+        // When we hit a different kind of event, process all the previous loads, then the other event.
+        if ( toLoad.length ) {
+          await dispatch('loadMulti', toLoad);
+          toLoad = [];
+        }
+
+        if ( action === 'dispatch' ) {
+          await dispatch(event, body);
+        } else if ( action === 'commit' ) {
+          commit(event, body);
+        } else {
+          throw new Error('Invalid queued action');
+        }
+      }
+    }
+
+    // Process any remaining loads
+    if ( toLoad.length ) {
+      await dispatch('loadMulti', toLoad);
+    }
+
+    state.debugSocket && console.debug(`Subscribe Flush [${ getters.storeName }] finished`, (new Date().getTime()) - started, 'ms'); // eslint-disable-line no-console
+  },
+
+  rehydrateSubscribe({ state, dispatch }) {
+    if ( process.client && state.wantSocket && !state.socket ) {
+      dispatch('subscribe');
+    }
+  },
+
   reconnectWatches({
     state, getters, commit, dispatch
   }) {
@@ -535,15 +672,9 @@ export const actions = {
     }
   },
 
-  'ws.ping'({ getters, dispatch }, msg) {
-    if ( getters.storeName === 'management' ) {
-      const version = msg?.data?.version || null;
-
-      dispatch('updateServerVersion', version, { root: true });
-      console.info(`Ping [${ getters.storeName }] from ${ version || 'unknown version' }`); // eslint-disable-line no-console
-    }
-  },
-
+  /**
+   * Steve only event
+   */
   'ws.resource.start'({ state, getters, commit }, msg) {
     state.debugSocket && console.info(`Resource start: [${ getters.storeName }]`, msg); // eslint-disable-line no-console
     commit('setWatchStarted', {
@@ -568,6 +699,14 @@ export const actions = {
     }
   },
 
+  /**
+   * Steve only event
+   *
+   * Steve only seems to send out `resource.stop` messages for two reasons
+   * - We have requested that the resource watch should be stopped and we receive this event as confirmation
+   * - Steve tells us that the resource is no longer watched
+   *
+   */
   'ws.resource.stop'({ getters, commit, dispatch }, msg) {
     const type = msg.resourceType;
     const obj = {
@@ -579,15 +718,44 @@ export const actions = {
 
     // console.warn(`Resource stop: [${ getters.storeName }]`, msg); // eslint-disable-line no-console
 
+    // If we're trying to watch this event, attempt to re-watch
     if ( getters['schemaFor'](type) && getters['watchStarted'](obj) ) {
-      // Try reconnecting once
-
       commit('setWatchStopped', obj);
 
+      // In summary, we need to re-watch but with a reliable `revision` (to avoid `too old` message kicking off a full re-fetch of all
+      // resources). To get a reliable `revision` go out and fetch the latest for that resource type, in theory our local cache should be
+      // up to date with that revision.
+
+      const revisionExisting = getters.nextResourceVersion(type, obj.id);
+
+      let revisionLatest;
+
+      if (revisionExisting) {
+        // Attempt to fetch the latest revision at the time the resource watch was stopped, in theory our local cache should be up to
+        // date with this
+        // Ideally we shouldn't need to fetch here and supply `0`, `-1` or `null` to start watching from the latest revision, however steve
+        // will send the current state of each resource via a `resource.created` event.
+        const opt = { limit: 1 };
+
+        opt.url = getters.urlFor(type, null, opt);
+        revisionLatest = dispatch('request', { opt, type } )
+          .then(res => res.revision)
+          .catch((err) => {
+            // For some reason we can't fetch a reasonable revision, so force a re-fetch
+            console.warn(`Resource error retrieving resourceVersion, forcing re-fetch`, type, ':', err); // eslint-disable-line no-console
+            dispatch('resyncWatch', msg);
+            throw err;
+          });
+      } else {
+        // Some v1 resource types don't have revisions (either at the collection or resource level), so we avoided making an API request
+        // for them
+        revisionLatest = Promise.resolve(null); // Null to ensure we don't go through `nextResourceVersion` again
+      }
+
       setTimeout(() => {
         // Delay a bit so that immediate start/error/stop causes
         // only a slow infinite loop instead of a tight one.
-        dispatch('watch', obj);
+        revisionLatest.then(revision => dispatch('watch', { ...obj, revision }));
       }, 5000);
     }
   },
@@ -600,16 +768,10 @@ export const actions = {
     const data = msg.data;
     const type = data.type;
 
-    // Debounce count changes so we send at most 1 every 5 seconds
-    if (type === COUNT) {
-      const worker = (this.$workers || {})[ctx.getters.storeName];
-
-      if (worker) {
-        worker.postMessage({ countsUpdate: msg });
-
-        // No further processing - let the web worker debounce the counts
-        return;
-      }
+    // Work-around for ws.error messages being sent as change events
+    // These have no id (or other metadata) which breaks lots if they are processed as change events
+    if (data.message && !data.id) {
+      return;
     }
 
     // Web worker can process schemas to check that they are actually changing and
@@ -677,7 +839,10 @@ export const actions = {
   },
 };
 
-export const mutations = {
+/**
+ * Mutations that cover cases 1 & 2 (see file description)
+ */
+const defaultMutations = {
   setSocket(state, socket) {
     state.socket = socket;
   },
@@ -726,21 +891,29 @@ export const mutations = {
     delete state.inError[key];
   },
 
-  debug(state, on) {
-    state.debugSocket = on !== false;
-  },
-
   resetSubscriptions(state) {
+    // Clear out socket state. This is only ever called from reset... which is always called after we `disconnect` above.
+    // This could probably be folded in to there
     clear(state.started);
     clear(state.pendingFrames);
     clear(state.queue);
     clearTimeout(state.queueTimer);
     state.deferredRequests = {};
     state.queueTimer = null;
-  }
+  },
+
+  clearFromQueue(state, type) {
+    // Remove anything in the queue that is a resource update for the given type
+    state.queue = state.queue.filter((item) => {
+      return item.body?.type !== type;
+    });
+  },
 };
 
-export const getters = {
+/**
+ * Getters that cover cases 1 & 2 (see file description)
+ */
+const defaultGetters = {
   canWatch: state => (obj) => {
     return !state.inError[keyForSubscribe(obj)];
   },
@@ -766,7 +939,7 @@ export const getters = {
         return null;
       }
 
-      revision = cache.revision;
+      revision = cache.revision; // This is always zero.....
 
       for ( const obj of cache.list ) {
         if ( obj && obj.metadata ) {
@@ -783,16 +956,16 @@ export const getters = {
 
     return null;
   },
+};
 
-  currentGeneration: state => (type) => {
-    type = normalizeType(type);
-
-    const cache = state.types[type];
-
-    if ( !cache ) {
-      return null;
-    }
+export const actions = {
+  ...sharedActions,
+  ...defaultActions,
+};
 
-    return cache.generation;
-  },
+export const mutations = {
+  ...sharedMutations,
+  ...defaultMutations,
 };
+
+export const getters = { ...defaultGetters };

package/plugins/steve/worker/index.js

@@ -0,0 +1,17 @@
+// eslint-disable-next-line no-unused-vars
+import basicWorkerConstructor from '@shell/plugins/steve/worker/web-worker.basic.js';
+// eslint-disable-next-line no-unused-vars
+import advancedWorkerConstructor from '@shell/plugins/steve/worker/web-worker.advanced.js';
+
+export default function storeWorker(mode, options = {}, closures = {}) {
+  let worker;
+
+  if (mode === 'advanced') {
+    worker = new advancedWorkerConstructor();
+  } else {
+    worker = new basicWorkerConstructor();
+  }
+  worker.mode = mode;
+
+  return worker;
+}