dashboard-shell-shell 3.0.2-rc.105 → 3.0.5-test.1

package/plugins/steve/__tests__/subscribe.spec.ts

@@ -1,4 +1,7 @@
-import { actions } from '../subscribe';
+import { actions, getters, mutations } from '../subscribe';
+import { REVISION_TOO_OLD } from '../../../utils/socket';
+import { STEVE_WATCH_EVENT } from '../../../types/store/subscribe.types';
+import backOff from '../../../utils/back-off';
 
 describe('steve: subscribe', () => {
   describe('actions', () => {
@@ -106,4 +109,324 @@ describe('steve: subscribe', () => {
       });
     });
   });
+
+  describe('getters', () => {
+    describe('nextResourceVersion', () => {
+      const myType = 'myType';
+      const myId = `myId`;
+
+      type Revision = string | number | null | undefined
+
+      const createState = (revision?: Revision, cachedList?: any[] ) => {
+        const typeState: any = {};
+
+        if (revision !== undefined) {
+          typeState.revision = revision;
+        }
+        if (cachedList !== undefined) {
+          typeState.list = cachedList;
+        } else {
+          typeState.list = [];
+        }
+
+        return { types: { [myType.toLocaleLowerCase()]: typeState } };
+      };
+
+      const createGetters = (revision?: Revision) => {
+        return {
+          byId: (type: any, id: any) => {
+            expect(type).toBe(myType.toLowerCase());
+            expect(id).toBe(myId);
+
+            return { metadata: { resourceVersion: revision } };
+          }
+        };
+      };
+
+      it.each([
+        ['Is String, Return String', createGetters('abc'), myId, 'abc'],
+        // This catches old parseInt errors
+        ['Is String, Return String (integer prefix)', createGetters('123-abc'), myId, '123-abc'],
+        // This catches old parseInt errors
+        ['Is String, Return String (integer postfix)', createGetters('abc-123'), myId, 'abc-123'],
+        ['Is 0, Return Null', createGetters(0), myId, null],
+        ['Is Number, Return Number', createGetters(123), myId, 123],
+        ['Is Missing (null), Return Null', createGetters(null), myId, null],
+        ['Is Missing (undefined), Return Null', createGetters(undefined), myId, null],
+      ])('test revision from a single resource - %s', (_, storeGetters, id, expected) => {
+        expect(getters.nextResourceVersion(createState(), storeGetters)(myType, id)).toBe(expected);
+      });
+
+      const createResource = (revision: Revision) => ({ metadata: { resourceVersion: revision } });
+
+      it.each([
+        ['Is String, Return String', createState('abc'), 'abc'],
+        ['Is String, Return String (integer prefix)', createState('123-abc'), '123-abc'],
+        ['Is String, Return String (integer postfix)', createState('abc-123'), 'abc-123'],
+        ['Is 0, Return Null (nothing in cache)', createState(0), null],
+        ['Is 0, Return Number (numbers in cache)', createState(0, [createResource(1), createResource(2), createResource(3)]), 3],
+        ['Is 0, Return Number (numbers and strings in cache)', createState(0, [createResource(1), createResource('abc-2'), createResource('3-abc')]), 1],
+        ['Is 0, Return Null (only strings in cache)', createState(0, [createResource('abc'), createResource('abc-2'), createResource('3-abc')]), null],
+        ['Is missing (undefined), Return Null ', createState(undefined), null],
+        ['Is missing (null), Return Null', createState(null), null],
+      ])('test revision from a collection - %s', (_, state, expected) => {
+        expect(getters.nextResourceVersion(state, {})(myType)).toBe(expected);
+      });
+    });
+  });
+
+  describe('backoff', () => {
+    const waitForBackOff = async(advanceTimersByTime = 20000) => {
+      jest.advanceTimersByTime(advanceTimersByTime);
+      // jest.advanceTimersByTime(advanceTimersByTime);
+      await Promise.resolve();
+      await Promise.resolve();
+      await Promise.resolve();
+    };
+
+    describe('stale cache in replicate that handles watch', () => {
+      /**
+        1. ui makes http request.
+          - it's handled by up-to-date replica A
+          - response contains an up-to-date revision X
+        2. ui makes watch request referencing up-to-date revision X from A
+          - it's received by replica B with a stale cache which does not contain revision X.
+          - replicate B rejects watch with unknown revision message (i.e. 'too old')
+        3. ui receives unknown revision and makes a new request
+          - this should backoff until eventually succeeding
+       */
+
+      const startWatch = ({
+        ctx,
+        obj, msg,
+        revision
+      }) => {
+        const {
+          state, dispatch, getters, rootGetters, commit
+        } = ctx;
+
+        // call watch
+        actions.watch({
+          state, dispatch, getters, rootGetters
+        }, {
+          ...obj,
+          revision,
+          mode:  STEVE_WATCH_EVENT.CHANGES,
+          force: true,
+        });
+
+        expect(dispatch).toHaveBeenNthCalledWith(1, 'unwatchIncompatible', {
+          id: undefined, mode: STEVE_WATCH_EVENT.CHANGES, namespace: undefined, selector: undefined, type: obj.type
+        });
+
+        expect(dispatch).toHaveBeenNthCalledWith(2, 'send', {
+          debounceMs:      4000,
+          mode:            'resource.changes',
+          resourceType:    obj.type,
+          resourceVersion: revision.toString(),
+        });
+
+        // Receive start from BE
+        actions['ws.resource.start']({
+          state, dispatch, getters, commit
+        }, { ...msg });
+
+        expect(dispatch).toHaveBeenCalledTimes(2);
+        dispatch.mockClear();
+      };
+
+      const errorWatch = ({
+        ctx,
+        obj, msg,
+      }) => {
+        const {
+          state, dispatch, getters, commit
+        } = ctx;
+
+        // Receive error from BE
+        actions['ws.resource.error']({
+          dispatch, getters, commit
+        }, {
+          ...msg,
+          data: { error: 'too old' }
+        });
+        expect(state.inError).toStrictEqual(
+          {
+            'type=abc,namespace=,id=,selector=': {
+              obj: {
+                type: msg.resourceType,
+                mode: msg.mode,
+              },
+              reason: REVISION_TOO_OLD
+            }
+          }
+        );
+
+        // Receive stop from BE
+        actions['ws.resource.stop']({
+          state, dispatch, getters, commit
+        }, { ...msg });
+        // stop tries to watch again, however we're in error so will be ignored
+        expect(dispatch).toHaveBeenNthCalledWith(1, 'watch', {
+          id: undefined, mode: STEVE_WATCH_EVENT.CHANGES, namespace: undefined, selector: undefined, type: obj.type
+        });
+
+        dispatch.mockClear();
+      };
+
+      const cycleFail = async({
+        ctx,
+        obj, msg,
+        revision,
+        tooManyTries = false,
+      }) => {
+        const { dispatch } = ctx;
+
+        startWatch({
+          ctx, obj, msg, revision
+        });
+        errorWatch({
+          ctx, obj, msg
+        });
+
+        await waitForBackOff(50000);
+        await waitForBackOff(50000);
+
+        if (tooManyTries) {
+          expect(dispatch).toHaveBeenCalledTimes(0);
+        } else {
+          expect(dispatch).toHaveBeenCalledTimes(1);
+          expect(dispatch).toHaveBeenCalledWith('resyncWatch', {
+            ...msg,
+            data: { error: 'too old' }
+          });
+        }
+
+        await waitForBackOff();
+
+        if (tooManyTries) {
+          expect(dispatch).toHaveBeenCalledTimes(0);
+        } else {
+          expect(dispatch).toHaveBeenCalledTimes(1);
+        }
+
+        dispatch.mockClear();
+      };
+
+      const cycleSucceed = async({
+        ctx,
+        obj, msg,
+        revision
+      }) => {
+        const { dispatch } = ctx;
+
+        dispatch.mockImplementation(async(type: string) => {
+          if (type === 'resyncWatch') {
+            return Promise.resolve();
+          }
+        });
+
+        startWatch({
+          ctx, obj, msg, revision
+        });
+
+        await waitForBackOff();
+
+        expect(dispatch).toHaveBeenCalledTimes(0);
+
+        await waitForBackOff();
+
+        expect(dispatch).toHaveBeenCalledTimes(0);
+
+        dispatch.mockClear();
+      };
+
+      const dispatch = jest.fn();
+      const rootGetters = {
+        'type-map/isSpoofed': () => false,
+        'management/byId':    () => ({ value: true })
+      };
+      const obj = { type: 'abc' };
+      const msg = {
+        resourceType: obj.type,
+        mode:         STEVE_WATCH_EVENT.CHANGES,
+      };
+
+      const initStore = () => {
+        const state = { started: [], inError: {} };
+        const _getters = {
+          normalizeType: (type: string) => type,
+          schemaFor:     () => ({}),
+          storeName:     'test',
+          inError:       (...args) => getters.inError(state)(...args),
+          watchStarted:  (...args) => getters.watchStarted(state)(...args),
+          backOffId:     (...args) => getters.backOffId()(...args),
+          canBackoff:    () => true,
+        };
+        const commit = (type, ...args) => mutations[type](state, ...args);
+
+        return {
+          state, dispatch, getters: _getters, rootGetters, commit
+        };
+      };
+
+      beforeAll(() => {
+        jest.useFakeTimers();
+      });
+
+      afterEach(() => {
+        backOff.resetAll();
+        dispatch.mockClear();
+      });
+
+      // eslint-disable-next-line jest/expect-expect
+      it('succeeds', async() => {
+        jest.useFakeTimers();
+
+        const ctx = initStore();
+
+        await cycleSucceed({
+          ctx, msg, obj, revision: 1
+        });
+      });
+
+      // eslint-disable-next-line jest/expect-expect
+      it('succeeds after a few failures', async() => {
+        jest.useFakeTimers();
+
+        const ctx = initStore();
+
+        await cycleFail({
+          ctx, msg, obj, revision: 1
+        });
+        await cycleFail({
+          ctx, msg, obj, revision: 1
+        });
+        await cycleFail({
+          ctx, msg, obj, revision: 1
+        });
+        await cycleFail({
+          ctx, msg, obj, revision: 1
+        });
+        await cycleSucceed({
+          ctx, msg, obj, revision: 1
+        });
+      });
+
+      // eslint-disable-next-line jest/expect-expect
+      it('never succeeds', async() => {
+        const ctx = initStore();
+
+        for (let i = 0; i < 10; i++) {
+          await cycleFail({
+            ctx, msg, obj, revision: 1
+          });
+        }
+
+        await cycleFail({
+          ctx, msg, obj, revision: 1, tooManyTries: true
+        });
+      });
+    });
+  });
 });

package/plugins/steve/actions.js

@@ -9,6 +9,7 @@ import { classify } from '@shell/plugins/dashboard-store/classify';
 import { NAMESPACE } from '@shell/config/types';
 import { handleKubeApiHeaderWarnings } from '@shell/plugins/steve/header-warnings';
 import { steveCleanForDownload } from '@shell/plugins/steve/resource-utils';
+import paginationUtils from '@shell/utils/pagination-utils';
 
 export default {
 
@@ -17,7 +18,9 @@ export default {
     return await loadSchemas(ctx, watch);
   },
 
-  async request({ state, dispatch, rootGetters }, pOpt ) {
+  async request({
+    state, dispatch, rootGetters, getters
+  }, pOpt ) {
     const opt = pOpt.opt || pOpt;
     const spoofedRes = await handleSpoofedRequest(rootGetters, 'cluster', opt);
 
@@ -55,6 +58,9 @@ export default {
       waiting = state.deferredRequests[key];
 
       if ( waiting ) {
+        // A matching request has already been made and is currently waiting to complete
+        // Avoid making another request, just wait for the original one to complete
+        // and return the result of the first call (see `waiting` being processed far below)
         const later = deferred();
 
         waiting.push(later);
@@ -82,6 +88,7 @@ export default {
     }
 
     let paginatedResult;
+    const isSteveCacheUrl = getters.isSteveCacheUrl(opt.url);
 
     while (true) {
       try {
@@ -92,24 +99,42 @@ export default {
         }
 
         if (!paginatedResult) {
-          // First result, so store it
-          paginatedResult = out;
+          const pageByNumber = isSteveCacheUrl && opt.url.includes(`pagesize=${ paginationUtils.defaultPageSize }`) ? {
+            total: out.count,
+            page:  1,
+            url:   opt.url,
+          } : null;
+          const pageByLimit = !pageByNumber ? { } : null;
+
+          paginatedResult = {
+            // initialise some settings
+            pageByLimit,
+            pageByNumber,
+            // First result, so store it
+            out
+          };
         } else {
           // Subsequent request, so add to it
-          paginatedResult.data = paginatedResult.data.concat(out.data);
+          paginatedResult.out.data = paginatedResult.out.data.concat(out.data);
         }
 
-        if (out?.pagination?.next) {
-          // More results to come, update options
-          opt.url = out.pagination.next;
+        const { total, page, url } = paginatedResult.pageByNumber || {};
+
+        if (paginatedResult.pageByLimit && out?.pagination?.next) {
+          opt.url = out?.pagination?.next;
+        } else if (paginatedResult.pageByNumber && (total > paginationUtils.defaultPageSize * page)) {
+          paginatedResult.pageByNumber.page += 1;
+
+          opt.url = addParam(url, 'page', `${ paginatedResult.pageByNumber.page }`);
         } else {
           // No more results, so clear out the pagination section (which will be stale from the first request)
-          delete paginatedResult.pagination?.first;
-          delete paginatedResult.pagination?.last;
-          delete paginatedResult.pagination?.next;
-          delete paginatedResult.pagination?.partial;
+          delete paginatedResult.out.pagination?.first;
+          delete paginatedResult.out.pagination?.last;
+          delete paginatedResult.out.pagination?.next;
+          delete paginatedResult.out.pagination?.partial;
+          delete paginatedResult.out.continue;
 
-          return paginatedResult;
+          return paginatedResult.out;
         }
       } catch (err) {
         return onError(err);
@@ -196,18 +221,10 @@ export default {
     }
   },
 
-  promptMove({ commit, state }, resources) {
-    commit('action-menu/togglePromptMove', resources, { root: true });
-  },
-
   promptRestore({ commit, state }, resources ) {
     commit('action-menu/togglePromptRestore', resources, { root: true });
   },
 
-  assignTo({ commit, state }, resources = []) {
-    commit('action-menu/toggleAssignTo', resources, { root: true });
-  },
-
   async resourceAction({ getters, dispatch }, {
     resource, actionName, body, opt,
   }) {
@@ -234,10 +251,6 @@ export default {
     }
   },
 
-  promptUpdate({ commit, state }, resources = []) {
-    commit('action-menu/togglePromptUpdate', resources, { root: true });
-  },
-
   async collectionAction({ getters, dispatch }, {
     type, actionName, body, opt
   }) {

package/plugins/steve/getters.js

@@ -15,9 +15,10 @@ import { splitObjectPath } from '@shell/utils/string';
 import { parseType } from '@shell/models/schema';
 import {
   STEVE_AGE_COL,
-  STEVE_ID_COL, STEVE_LIST_GROUPS, STEVE_NAMESPACE_COL, STEVE_STATE_COL
+  STEVE_ID_COL, STEVE_LIST_GROUPS, STEVE_NAME_COL, STEVE_NAMESPACE_COL, STEVE_STATE_COL
 } from '@shell/config/pagination-table-headers';
 import { createHeaders } from '@shell/store/type-map.utils';
+import paginationUtils from '@shell/utils/pagination-utils';
 
 export const STEVE_MODEL_TYPES = {
   NORMAN:  'norman',
@@ -36,16 +37,34 @@ const GC_IGNORE_TYPES = {
 const steveRegEx = new RegExp('(/v1)|(\/k8s\/clusters\/[a-z0-9-]+\/v1)');
 
 export default {
-  urlOptions: () => (url, opt, schema) => {
+  /**
+   * Is the url path a rancher steve one?
+   *
+   * Can be used to change behaviour given steve api
+   */
+  isSteveUrl:      () => (urlPath) => steveRegEx.test(urlPath),
+  /**
+   * Is the url path a rancher steve one AND the steve cache is enabled?
+   *
+   * Can be used to change behaviour given steve cache api functionality
+   */
+  isSteveCacheUrl: (state, getters, rootState, rootGetters) => (urlPath) => getters.isSteveUrl(urlPath) && paginationUtils.isSteveCacheEnabled({ rootGetters }),
+
+  /**
+   * opt: ActionFindPageArgs
+   */
+  urlOptions: (state, getters) => (url, opt, schema) => {
     opt = opt || {};
-    const parsedUrl = parse(url);
-    const isSteve = steveRegEx.test(parsedUrl.path);
+    const parsedUrl = parse(url || '');
 
-    const stevePagination = stevePaginationUtils.createParamsForPagination(schema, opt);
+    const isSteveUrl = getters.isSteveUrl(parsedUrl.path);
+    const stevePagination = stevePaginationUtils.createParamsForPagination({ schema, opt });
 
     if (stevePagination) {
       url += `${ (url.includes('?') ? '&' : '?') + stevePagination }`;
     } else {
+      const isSteveCacheUrl = getters.isSteveCacheUrl(parsedUrl.path);
+
       // labelSelector
       if ( opt.labelSelector ) {
         url += `${ url.includes('?') ? '&' : '?' }labelSelector=${ opt.labelSelector }`;
@@ -54,6 +73,7 @@ export default {
 
       // Filter
       if ( opt.filter ) {
+        // When ui-sql-cache is always on we should look to replace the usages of this with findPage (basically using the new filter definitions)
         url += `${ (url.includes('?') ? '&' : '?') }`;
         const keys = Object.keys(opt.filter);
 
@@ -64,13 +84,12 @@ export default {
             vals = [vals];
           }
 
-          // Steve's filter options now support more complex filtering not yet implemented here #9341
-          if (isSteve) {
+          if (isSteveUrl) {
             url += `${ (url.includes('filter=') ? '&' : 'filter=') }`;
           }
 
           const filterStrings = vals.map((val) => {
-            return `${ encodeURI(key) }=${ encodeURI(val) }`;
+            return `${ encodeURI(key) }${ isSteveCacheUrl ? '~' : '=' }${ encodeURI(val) }`;
           });
           const urlEnding = url.charAt(url.length - 1);
           const nextStringConnector = ['&', '?', '='].includes(urlEnding) ? '' : '&';
@@ -97,13 +116,22 @@ export default {
       }
       // End: Limit
 
+      // Page Size
+      if (isSteveCacheUrl && opt.isCollection) {
+        // This is a steve url and the new cache is being used.
+        // Pre-cache there was always a max page size (given kube proxy). With cache there's not.
+        // So ensure we don't go backwards (and fetch crazy high resource counts) by adding a default
+        url += `${ url.includes('?') ? '&' : '?' }pagesize=${ paginationUtils.defaultPageSize }`;
+      }
+      // End: Page Size
+
       // Sort
       // Steve's sort options supports multi-column sorting and column specific sort orders, not implemented yet #9341
       const sortBy = opt.sortBy;
       const orderBy = opt.sortOrder;
 
       if ( sortBy ) {
-        if (isSteve) {
+        if (isSteveUrl) {
           url += `${ url.includes('?') ? '&' : '?' }sort=${ (orderBy === 'desc' ? '-' : '') + encodeURI(sortBy) }`;
         } else {
           url += `${ url.includes('?') ? '&' : '?' }sort=${ encodeURI(sortBy) }`;
@@ -118,7 +146,7 @@ export default {
     // Exclude
     // excludeFields should be an array of strings representing the paths of the fields to exclude
     // only works on Steve but is ignored without error by Norman
-    if (isSteve) {
+    if (isSteveUrl) {
       if (!Array.isArray(opt?.excludeFields)) {
         const excludeFields = ['metadata.managedFields'];
 
@@ -126,9 +154,15 @@ export default {
         opt.excludeFields = Array.isArray(opt?.omitExcludeFields) ? excludeFields.filter((f) => !f.includes(opt.omitExcludeFields)) : excludeFields;
       }
 
-      const excludeParamsString = opt.excludeFields.map((field) => `exclude=${ field }`).join('&');
+      if (opt.excludeFields.length) {
+        const excludeParamsString = opt.excludeFields.map((field) => `exclude=${ field }`).join('&');
 
-      url += `${ url.includes('?') ? '&' : '?' }${ excludeParamsString }`;
+        url += `${ url.includes('?') ? '&' : '?' }${ excludeParamsString }`;
+      }
+
+      if (opt.revision) {
+        url += `${ url.includes('?') ? '&' : '?' }${ `revision=${ opt.revision }` }`;
+      }
     }
     // End: Exclude
 
@@ -312,6 +346,7 @@ export default {
       typeOptions: typeMapGetters['optionsFor'](schema, true),
       schema,
       columns:     {
+        name:      STEVE_NAME_COL,
         state:     STEVE_STATE_COL,
         namespace: STEVE_NAMESPACE_COL,
         age:       STEVE_AGE_COL,

package/plugins/steve/resourceWatcher.js

@@ -36,12 +36,19 @@ export const WATCH_STATUSES = {
   REMOVE_REQUESTED: 'removed_requested'
 };
 
+/**
+ * Create a unique key for a specific resource watch's params
+ */
 export const keyForSubscribe = ({
   resourceType, type, namespace, id, selector
 } = {}) => {
-  return [(resourceType || type), namespace, id, selector] // each watch param in an array
-    .filter((param) => !!param) // filter out all the empty ones // the filter makes these keys neater
-    .join('/'); // join into a string so we can use it as an object key
+  const keyMap = {
+    type: resourceType || type, namespace, id, selector
+  };
+
+  return Object.entries(keyMap)
+    .map(([prop, value]) => `${ prop }=${ value || '' }`)
+    .join(',');
 };
 
 export const watchKeyFromMessage = (msg) => {

package/plugins/steve/steve-class.js

@@ -11,6 +11,7 @@ const STEVE_ADD = [
   'metadata.clusterName',
   'metadata.deletionGracePeriodSeconds',
   'metadata.generateName',
+  'metadata.ownerReferences',
 ];
 const STEVE_NEVER_SAVE = NEVER_ADD.filter((na) => !STEVE_ADD.includes(na));
 
@@ -58,4 +59,8 @@ export default class SteveModel extends HybridModel {
 
     return val;
   }
+
+  paginationEnabled() {
+    return this.$getters['paginationEnabled'](this.type);
+  }
 }