@rancher/shell 0.3.0 → 0.3.1

package/plugins/dashboard-store/mutations.js

@@ -1,7 +1,8 @@
 import Vue from 'vue';
 import { addObject, addObjects, clear, removeObject } from '@shell/utils/array';
-import { SCHEMA } from '@shell/config/types';
-import { normalizeType } from '@shell/plugins/dashboard-store/normalize';
+import { SCHEMA, COUNT } from '@shell/config/types';
+import { normalizeType, keyFieldFor } from '@shell/plugins/dashboard-store/normalize';
+import { addSchemaIndexFields } from '@shell/plugins/steve/schema.utils';
 import { classify } from '@shell/plugins/dashboard-store/classify';
 import garbageCollect from '@shell/utils/gc/gc';
 
@@ -10,12 +11,13 @@ function registerType(state, type) {
 
   if ( !cache ) {
     cache = {
-      list:         [],
-      haveAll:      false,
-      haveSelector: {},
-      revision:     0, // The highest known resourceVersion from the server for this type
-      generation:   0, // Updated every time something is loaded for this type
-      loadCounter:  0, // Used to cancel incremental loads if the page changes during load
+      list:          [],
+      haveAll:       false,
+      haveSelector:  {},
+      haveNamespace: undefined, // If the cached list only contains resources for a namespace, this will contain the ns name
+      revision:      0, // The highest known resourceVersion from the server for this type
+      generation:    0, // Updated every time something is loaded for this type
+      loadCounter:   0, // Used to cancel incremental loads if the page changes during load
     };
 
     // Not enumerable so they don't get sent back to the client for SSR
@@ -31,6 +33,24 @@ function registerType(state, type) {
   return cache;
 }
 
+export function replace(existing, data) {
+  for ( const k of Object.keys(existing) ) {
+    delete existing[k];
+  }
+
+  for ( const k of Object.keys(data) ) {
+    Vue.set(existing, k, data[k]);
+  }
+
+  return existing;
+}
+
+function replaceResource(existing, data, getters) {
+  data = getters.cleanResource(existing, data);
+
+  return replace(existing, data);
+}
+
 export function load(state, { data, ctx, existing }) {
   const { getters } = ctx;
   let type = normalizeType(data.type);
@@ -40,8 +60,7 @@ export function load(state, { data, ctx, existing }) {
 
   // Inject special fields for indexing schemas
   if ( type === SCHEMA ) {
-    data._id = normalizeType(data.id);
-    data._group = normalizeType(data.attributes?.group);
+    addSchemaIndexFields(data);
   }
 
   const id = data[keyField];
@@ -52,24 +71,10 @@ export function load(state, { data, ctx, existing }) {
 
   let entry;
 
-  function replace(existing, data) {
-    data = getters.cleanResource(existing, data);
-
-    for ( const k of Object.keys(existing) ) {
-      delete existing[k];
-    }
-
-    for ( const k of Object.keys(data) ) {
-      Vue.set(existing, k, data[k]);
-    }
-
-    return existing;
-  }
-
   if ( existing && !existing.id ) {
     // A specific proxy instance to used was passed in (for create -> save),
     // use it instead of making a new proxy
-    entry = replace(existing, data);
+    entry = replaceResource(existing, data, getters);
     addObject(cache.list, entry);
     cache.map.set(id, entry);
     // console.log('### Mutation added from existing proxy', type, id);
@@ -78,7 +83,7 @@ export function load(state, { data, ctx, existing }) {
 
     if ( entry ) {
       // There's already an entry in the store, update it
-      replace(entry, data);
+      replaceResource(entry, data, getters);
       // console.log('### Mutation Updated', type, id);
     } else {
       // There's no entry, make a new proxy
@@ -115,6 +120,7 @@ export function forgetType(state, type) {
   if ( cache ) {
     cache.haveAll = false;
     cache.haveSelector = {};
+    cache.haveNamespace = undefined;
     cache.revision = 0;
     cache.generation = 0;
     clear(cache.list);
@@ -163,11 +169,99 @@ export function remove(state, obj, getters) {
   }
 }
 
+export function batchChanges(state, { ctx, batch }) {
+  const batchTypes = Object.keys(batch);
+  const combinedBatch = {};
+
+  batchTypes.forEach((batchType) => {
+    combinedBatch[batchType] = batch[batchType];
+    const typeOption = ctx.rootGetters['type-map/optionsFor'](batchType);
+
+    if (typeOption?.alias?.length > 0) {
+      const alias = typeOption?.alias || [];
+
+      alias.forEach((aliasType) => {
+        combinedBatch[aliasType] = {};
+        for (const [key, value] of Object.entries(batch[batchType])) {
+          combinedBatch[aliasType][key] = {
+            ...value,
+            type: aliasType
+          };
+        }
+      });
+    }
+  });
+
+  const combinedBatchTypes = Object.keys(combinedBatch);
+
+  combinedBatchTypes.forEach((type) => {
+    const normalizedType = normalizeType(type === 'counts' ? COUNT : type);
+    const keyField = keyFieldFor(normalizedType);
+    const typeCache = registerType(state, normalizedType);
+
+    // making a map for every resource's location in the list is gonna ensure we only have to loop through the big list once.
+    const typeCacheIndexMap = {};
+
+    typeCache.list.forEach((resource, index) => {
+      typeCacheIndexMap[resource[keyField]] = index;
+    });
+
+    const removeAtIndexes = [];
+
+    // looping through the batch, executing changes, deferring creates and removes since they change the array length
+    Object.keys(batch[normalizedType]).forEach((id) => {
+      const index = typeCacheIndexMap[id];
+      const resource = batch[normalizedType][id];
+
+      // an empty resource passed into batch changes is how we'll signal which ones to delete
+      if (Object.keys(resource).length === 0 && index !== undefined) {
+        typeCache.map.delete(id);
+        removeAtIndexes.push(index);
+      } else if (Object.keys(resource).length === 0) {
+        // No op. We're removing it... but we don't have it in the cache
+      } else {
+        if (normalizedType === SCHEMA) {
+          addSchemaIndexFields(resource);
+        }
+        const classyResource = classify(ctx, resource);
+
+        if (index === undefined) {
+          typeCache.list.push(classyResource);
+          typeCache.map.set(id, classyResource);
+
+          typeCacheIndexMap[classyResource[keyField]] = typeCache.list.length - 1;
+        } else {
+          replaceResource(typeCache.list[index], resource, ctx.getters);
+        }
+      }
+    });
+
+    // looping through the removeAtIndexes, making sure to offset by iteration so the array changing doesn't mess us up
+    removeAtIndexes.sort().forEach((cacheIndex, loopIndex) => {
+      typeCache.list.splice(cacheIndex - loopIndex, 1);
+    });
+
+    const opts = ctx.rootGetters[`type-map/optionsFor`](type);
+    const limit = opts?.limit;
+
+    // If there is a limit to the number of resources we can store for this type then
+    // remove the first one to keep the list size to that limit
+    if (limit && typeCache.list.length > limit) {
+      const rm = typeCache.list.shift();
+
+      typeCache.map.delete(rm.id);
+    }
+
+    typeCache.generation++;
+  });
+}
+
 export function loadAll(state, {
   type,
   data,
   ctx,
-  skipHaveAll
+  skipHaveAll,
+  namespace
 }) {
   const { getters } = ctx;
 
@@ -199,7 +293,8 @@ export function loadAll(state, {
 
   // Allow requester to skip setting that everything has loaded
   if (!skipHaveAll) {
-    cache.haveAll = true;
+    cache.haveNamespace = namespace;
+    cache.haveAll = !namespace;
   }
 
   return proxies;
@@ -238,6 +333,11 @@ export default {
 
   loadAll,
 
+  /**
+   * Handles changes (add, update, remove) to multiple resources for multiple types
+   */
+  batchChanges,
+
   loadMerge(state, { type, data: allLatest, ctx }) {
     const { commit, getters } = ctx;
     // const allLatest = await dispatch('findAll', { type, opt: { force: true, load, _NONE } });
@@ -288,6 +388,12 @@ export default {
     cache.haveAll = true;
   },
 
+  setHaveNamespace(state, { type, namespace }) {
+    const cache = registerType(state, type);
+
+    cache.haveNamespace = namespace;
+  },
+
   loadedAll(state, { type }) {
     const cache = registerType(state, type);
 

package/plugins/dashboard-store/resource-class.js

@@ -25,6 +25,7 @@ import {
   validateLength,
 } from '@shell/utils/validators';
 import formRulesGenerator from '@shell/utils/validators/formRules/index';
+import { waitFor } from '@shell/utils/async';
 import jsyaml from 'js-yaml';
 import compact from 'lodash/compact';
 import forIn from 'lodash/forIn';
@@ -752,41 +753,7 @@ export default class Resource {
   // ------------------------------------------------------------------
 
   waitForTestFn(fn, msg, timeoutMs, intervalMs) {
-    console.log('Starting wait for', msg); // eslint-disable-line no-console
-
-    if ( !timeoutMs ) {
-      timeoutMs = DEFAULT_WAIT_TMIMEOUT;
-    }
-
-    if ( !intervalMs ) {
-      intervalMs = DEFAULT_WAIT_INTERVAL;
-    }
-
-    return new Promise((resolve, reject) => {
-      // Do a first check immediately
-      if ( fn.apply(this) ) {
-        console.log('Wait for', msg, 'done immediately'); // eslint-disable-line no-console
-        resolve(this);
-      }
-
-      const timeout = setTimeout(() => {
-        console.log('Wait for', msg, 'timed out'); // eslint-disable-line no-console
-        clearInterval(interval);
-        clearTimeout(timeout);
-        reject(new Error(`Failed waiting for: ${ msg }`));
-      }, timeoutMs);
-
-      const interval = setInterval(() => {
-        if ( fn.apply(this) ) {
-          console.log('Wait for', msg, 'done'); // eslint-disable-line no-console
-          clearInterval(interval);
-          clearTimeout(timeout);
-          resolve(this);
-        } else {
-          console.log('Wait for', msg, 'not done yet'); // eslint-disable-line no-console
-        }
-      }, intervalMs);
-    });
+    return waitFor(() => fn.apply(this), msg, timeoutMs || DEFAULT_WAIT_TMIMEOUT, intervalMs || DEFAULT_WAIT_INTERVAL, true);
   }
 
   waitForState(state, timeout, interval) {
@@ -798,19 +765,19 @@ export default class Resource {
   waitForTransition() {
     return this.waitForTestFn(() => {
       return !this.transitioning;
-    }, 'transition completion');
+    }, 'transition completion', undefined, undefined);
   }
 
   waitForAction(name) {
     return this.waitForTestFn(() => {
       return this.hasAction(name);
-    }, `action=${ name }`);
+    }, `action=${ name }`, undefined, undefined);
   }
 
   waitForLink(name) {
     return this.waitForTestFn(() => {
       return this.hasLink(name);
-    }, `link=${ name }`);
+    }, `link=${ name }`, undefined, undefined);
   }
 
   hasCondition(condition) {
@@ -1075,6 +1042,12 @@ export default class Resource {
     return this._save(...arguments);
   }
 
+  /**
+   * Allow to handle the response of the save request
+   * @param {*} res Full request response
+   */
+  processSaveResponse(res) { }
+
   async _save(opt = {}) {
     delete this.__rehydrate;
     delete this.__clone;
@@ -1151,6 +1124,9 @@ export default class Resource {
     try {
       const res = await this.$dispatch('request', { opt, type: this.type } );
 
+      // Allow to process response independently from the related models
+      this.processSaveResponse(res);
+
       // Steve sometimes returns Table responses instead of the resource you just saved.. ignore
       if ( res && res.kind !== 'Table') {
         await this.$dispatch('load', { data: res, existing: (forNew ? this : undefined ) });
@@ -1323,8 +1299,9 @@ export default class Resource {
 
   async download() {
     const value = await this.followLink('view', { headers: { accept: 'application/yaml' } });
+    const data = await this.$dispatch('cleanForDownload', value.data);
 
-    downloadFile(`${ this.nameDisplay }.yaml`, value.data, 'application/yaml');
+    downloadFile(`${ this.nameDisplay }.yaml`, data, 'application/yaml');
   }
 
   async downloadBulk(items) {
@@ -1343,8 +1320,11 @@ export default class Resource {
     }
 
     await eachLimit(items, 10, (item, idx) => {
-      return item.followLink('view', { headers: { accept: 'application/yaml' } } ).then((data) => {
-        files[`resources/${ names[idx] }`] = data.data || data;
+      return item.followLink('view', { headers: { accept: 'application/yaml' } } ).then(async(data) => {
+        const yaml = data.data || data;
+        const cleanedYaml = await this.$dispatch('cleanForDownload', yaml);
+
+        files[`resources/${ names[idx] }`] = cleanedYaml;
       });
     });
 

package/plugins/steve/actions.js

@@ -7,6 +7,7 @@ import { streamJson, streamingSupported } from '@shell/utils/stream';
 import isObject from 'lodash/isObject';
 import { classify } from '@shell/plugins/dashboard-store/classify';
 import { NAMESPACE } from '@shell/config/types';
+import jsyaml from 'js-yaml';
 
 export default {
 
@@ -323,6 +324,35 @@ export default {
 
     return resource;
   },
+
+  // remove fields added by steve before showing/downloading yamls
+  cleanForDownload(ctx, yaml) {
+    if (!yaml) {
+      return;
+    }
+    const rootKeys = [
+      'id',
+      'links',
+      'type',
+      'actions'
+    ];
+    const metadataKeys = [
+      'fields',
+      'relationships',
+      'state',
+    ];
+    const conditionKeys = [
+      'error',
+      'transitioning',
+    ];
+    const obj = jsyaml.load(yaml);
+
+    dropKeys(obj, rootKeys);
+    dropKeys(obj?.metadata, metadataKeys);
+    (obj?.status?.conditions || []).forEach(condition => dropKeys(condition, conditionKeys));
+
+    return jsyaml.dump(obj);
+  }
 };
 
 const diffRootKeys = [

package/plugins/steve/caches/resourceCache.js

@@ -0,0 +1,60 @@
+import { SCHEMA, COUNT } from '@shell/config/types';
+import { keyFieldFor, normalizeType } from '@shell/plugins/dashboard-store/normalize';
+import { addSchemaIndexFields } from '@shell/plugins/steve/schema.utils';
+import { hashObj } from '@shell/utils/crypto/browserHashUtils';
+
+export default class ResourceCache {
+  resources = {};
+  type;
+  keyField;
+  constructor(type) {
+    this.type = normalizeType(type === 'counts' ? COUNT : type);
+    this.keyField = keyFieldFor(this.type);
+  }
+
+  load(collection = []) {
+    // console.time('startSchemaLoads!!!');
+    for (let i = 0; i < collection.length; i++) {
+      const resource = collection[i];
+
+      if ( this.type === SCHEMA ) {
+        addSchemaIndexFields(resource);
+      }
+      const id = resource[this.keyField];
+
+      // Store the hash instead of the whole object. This means longer load time be reduces memory footprint
+      // Perf Note: 3.328125 ms to load ~2500 schemas as objects into cache
+      // Perf Note: 67.450927734375 ms to load ~2500 schemas as hashes into cache
+      this.resources[id] = hashObj(resource);
+    }
+    // console.timeEnd('startSchemaLoads!!!');
+
+    return this;
+  }
+
+  change(resource, callback) {
+    if ( this.type === SCHEMA ) {
+      addSchemaIndexFields(resource);
+    }
+
+    const existingResourceHash = this.resources[resource[this.keyField]] || {};
+    const newResourceHash = hashObj(resource);
+
+    if (existingResourceHash !== newResourceHash) {
+      this.resources[resource[this.keyField]] = resource;
+      callback();
+    }
+  }
+
+  create(resource, callback) {
+    // ToDo: the logic for create is identical to change in these caches but the worker doesn't know that
+    this.change(resource, callback);
+  }
+
+  remove(id, callback) {
+    if (this.resources[id]) {
+      delete this.resources[id];
+      callback();
+    }
+  }
+}

package/plugins/steve/getters.js

@@ -6,6 +6,8 @@ import { NAMESPACE, SCHEMA, COUNT, UI } from '@shell/config/types';
 import SteveModel from './steve-class';
 import HybridModel, { cleanHybridResources } from './hybrid-class';
 import NormanModel from './norman-class';
+import { urlFor } from '@shell/plugins/dashboard-store/getters';
+import { normalizeType } from '@shell/plugins/dashboard-store/normalize';
 
 export const STEVE_MODEL_TYPES = {
   NORMAN:  'norman',
@@ -67,6 +69,20 @@ export default {
     return url;
   },
 
+  urlFor: (state, getters) => (type, id, opt) => {
+    let url = urlFor(state, getters)(type, id, opt);
+
+    if (opt.namespaced) {
+      const parts = url.split('/');
+
+      url = `${ parts.join('/') }/${ opt.namespaced }`;
+
+      return url;
+    }
+
+    return url;
+  },
+
   defaultModel: state => (obj) => {
     const which = state.config.modelBaseClass || STEVE_MODEL_TYPES.BY_TYPE.STEVE;
 
@@ -106,6 +122,21 @@ export default {
   },
 
   cleanResource: () => (existing, data) => {
+    /**
+   * Resource counts are contained within a single 'count' resource with a 'counts' field that is a map of resource types
+   * When counts are updated through the websocket, only the resources that changed are sent so we can't load the new 'count' resource into the store as we would another resource
+   */
+    if (data?.type === COUNT && existing) {
+      data.counts = { ...existing.counts, ...data.counts };
+
+      return data;
+    }
+
+    // If the existing model has a cleanResource method, use it
+    if (existing?.cleanResource && typeof existing.cleanResource === 'function') {
+      return existing.cleanResource(data);
+    }
+
     const typeSuperClass = Object.getPrototypeOf(Object.getPrototypeOf(existing))?.constructor;
 
     return typeSuperClass === HybridModel ? cleanHybridResources(data) : data;
@@ -120,6 +151,18 @@ export default {
 
   gcIgnoreTypes: () => {
     return GC_IGNORE_TYPES;
-  }
+  },
+
+  currentGeneration: state => (type) => {
+    type = normalizeType(type);
+
+    const cache = state.types[type];
+
+    if ( !cache ) {
+      return null;
+    }
+
+    return cache.generation;
+  },
 
 };