@ember-data/store 4.8.0-alpha.4 → 4.8.0-beta.0

package/addon/-private/managers/record-array-manager.ts

@@ -1,28 +1,73 @@
 /**
   @module @ember-data/store
 */
-
-import { A } from '@ember/array';
-import { assert } from '@ember/debug';
-import { _backburner as emberBackburner } from '@ember/runloop';
-import { DEBUG } from '@glimmer/env';
-
-import type { CollectionResourceDocument, Meta } from '@ember-data/types/q/ember-data-json-api';
+import type { CollectionResourceDocument } from '@ember-data/types/q/ember-data-json-api';
 import type { StableRecordIdentifier } from '@ember-data/types/q/identifier';
 import type { Dict } from '@ember-data/types/q/utils';
 
-import AdapterPopulatedRecordArray from '../record-arrays/adapter-populated-record-array';
-import RecordArray from '../record-arrays/record-array';
+import IdentifierArray, {
+  Collection,
+  CollectionCreateOptions,
+  IDENTIFIER_ARRAY_TAG,
+  SOURCE,
+} from '../record-arrays/identifier-array';
 import type Store from '../store-service';
-import WeakCache from '../utils/weak-cache';
 
-const RecordArraysCache = new WeakCache<StableRecordIdentifier, Set<RecordArray>>(DEBUG ? 'record-arrays' : '');
-RecordArraysCache._generator = () => new Set();
-export function recordArraysForIdentifier(identifier: StableRecordIdentifier): Set<RecordArray> {
-  return RecordArraysCache.lookup(identifier);
+const RecordArraysCache = new Map<StableRecordIdentifier, Set<Collection>>();
+const FAKE_ARR = {};
+
+const SLICE_BATCH_SIZE = 1200;
+/**
+ * This is a clever optimization.
+ *
+ * clever optimizations rarely stand the test of time, so if you're
+ * ever curious or think something better is possible please benchmark
+ * and discuss. The benchmark for this at the time of writing is in
+ * `scripts/benchmark-push.js`
+ *
+ * This approach turns out to be 150x faster in Chrome and node than
+ * simply using push or concat. It's highly susceptible to the specifics
+ * of the batch size, and may require tuning.
+ *
+ * Clever optimizations should always come with a `why`. This optimization
+ * exists for two reasons.
+ *
+ * 1) array.push(...objects) and Array.prototype.push.apply(arr, objects)
+ *   are susceptible to stack overflows. The size of objects at which this
+ *   occurs varies by environment, browser, and current stack depth and memory
+ *   pressure; however, it occurs in all browsers in fairly pristine conditions
+ *   somewhere around 125k to 200k elements. Since EmberData regularly encounters
+ *   arrays larger than this in size, we cannot use push.
+ *
+ * 2) `array.concat` or simply setting the array to a new reference is often an
+ *   easier approach; however, native Proxy to an array cannot swap it's target array
+ *   and attempts at juggling multiple array sources have proven to be victim to a number
+ *   of browser implementation bugs. Should these bugs be addressed then we could
+ *   simplify to using `concat`, however, do note this is currently 150x faster
+ *   than concat, and due to the overloaded signature of concat will likely always
+ *   be faster.
+ *
+ * Sincerely,
+ *   - runspired (Chris Thoburn) 08/21/2022
+ *
+ * @function fastPush
+ * @internal
+ * @param target the array to push into
+ * @param source the items to push into target
+ */
+export function fastPush<T>(target: T[], source: T[]) {
+  let startLength = 0;
+  let newLength = source.length;
+  while (newLength - startLength > SLICE_BATCH_SIZE) {
+    // eslint-disable-next-line prefer-spread
+    target.push.apply(target, source.slice(startLength, SLICE_BATCH_SIZE));
+    startLength += SLICE_BATCH_SIZE;
+  }
+  // eslint-disable-next-line prefer-spread
+  target.push.apply(target, source.slice(startLength));
 }
 
-const pendingForIdentifier: Set<StableRecordIdentifier> = new Set([]);
+type ChangeSet = Map<StableRecordIdentifier, 'add' | 'del'>;
 
 /**
   @class RecordArrayManager
@@ -32,390 +77,301 @@ class RecordArrayManager {
   declare store: Store;
   declare isDestroying: boolean;
   declare isDestroyed: boolean;
-  declare _liveRecordArrays: Dict<RecordArray>;
-  declare _pendingIdentifiers: Dict<StableRecordIdentifier[]>;
-  declare _adapterPopulatedRecordArrays: RecordArray[];
+  declare _live: Map<string, IdentifierArray>;
+  declare _managed: Set<IdentifierArray>;
+  declare _pending: Map<IdentifierArray, ChangeSet>;
+  declare _identifiers: Map<StableRecordIdentifier, Set<Collection>>;
+  declare _staged: Map<string, ChangeSet>;
 
   constructor(options: { store: Store }) {
     this.store = options.store;
     this.isDestroying = false;
     this.isDestroyed = false;
-    this._liveRecordArrays = Object.create(null) as Dict<RecordArray>;
-    this._pendingIdentifiers = Object.create(null) as Dict<StableRecordIdentifier[]>;
-    this._adapterPopulatedRecordArrays = [];
+    this._live = new Map();
+    this._managed = new Set();
+    this._pending = new Map();
+    this._staged = new Map();
+    this._identifiers = RecordArraysCache;
   }
 
-  /**
-   * @method getRecordArraysForIdentifier
-   * @internal
-   * @param {StableIdentifier} param
-   * @return {RecordArray} array
-   */
-  getRecordArraysForIdentifier(identifier: StableRecordIdentifier): Set<RecordArray> {
-    return recordArraysForIdentifier(identifier);
-  }
+  _syncArray(array: IdentifierArray) {
+    const pending = this._pending.get(array);
 
-  _flushPendingIdentifiersForModelName(modelName: string, identifiers: StableRecordIdentifier[]): void {
-    if (this.isDestroying || this.isDestroyed) {
+    if (!pending || this.isDestroying || this.isDestroyed) {
       return;
     }
-    let identifiersToRemove: StableRecordIdentifier[] = [];
-    let cache = this.store._instanceCache;
-
-    for (let j = 0; j < identifiers.length; j++) {
-      let i = identifiers[j];
-      // mark identifiers, so they can once again be processed by the
-      // recordArrayManager
-      pendingForIdentifier.delete(i);
-      // build up a set of models to ensure we have purged correctly;
-      if (!cache.recordIsLoaded(i, true)) {
-        identifiersToRemove.push(i);
-      }
-    }
-
-    let array = this._liveRecordArrays[modelName];
-    if (array) {
-      // TODO: skip if it only changed
-      // process liveRecordArrays
-      updateLiveRecordArray(this.store, array, identifiers);
-    }
-
-    // process adapterPopulatedRecordArrays
-    if (identifiersToRemove.length > 0) {
-      removeFromAdapterPopulatedRecordArrays(this.store, identifiersToRemove);
-    }
-  }
 
-  _flush() {
-    let pending = this._pendingIdentifiers;
-    this._pendingIdentifiers = Object.create(null) as Dict<StableRecordIdentifier[]>;
-
-    for (let modelName in pending) {
-      this._flushPendingIdentifiersForModelName(modelName, pending[modelName]!);
-    }
-  }
-
-  _syncLiveRecordArray(array: RecordArray, modelName: string) {
-    assert(
-      `recordArrayManger.syncLiveRecordArray expects modelName not modelClass as the second param`,
-      typeof modelName === 'string'
-    );
-    let pending = this._pendingIdentifiers[modelName];
-
-    if (!Array.isArray(pending)) {
-      return;
-    }
-    let hasNoPotentialDeletions = pending.length === 0;
-    let listSize = this.store._instanceCache.peekList[modelName]?.size;
-    let hasNoInsertionsOrRemovals = listSize === array.length;
-
-    /*
-      Ideally the recordArrayManager has knowledge of the changes to be applied to
-      liveRecordArrays, and is capable of strategically flushing those changes and applying
-      small diffs if desired.  However, until we've refactored recordArrayManager, this dirty
-      check prevents us from unnecessarily wiping out live record arrays returned by peekAll.
-      */
-    if (hasNoPotentialDeletions && hasNoInsertionsOrRemovals) {
-      return;
-    }
-
-    this._flushPendingIdentifiersForModelName(modelName, pending);
-    // TODO make this a map
-    // TODO we probably do way too much work here, probably can just take
-    // current all-known state and filter visible.
-    // We can then provide this directly to the RecordArray instance.
-    // If we deprecate RecordArray being an ArrayProxy this will be stellar.
-    delete this._pendingIdentifiers[modelName];
-
-    let identifiers = this._visibleIdentifiersByType(modelName);
-    let identifiersToAdd: StableRecordIdentifier[] = [];
-    for (let i = 0; i < identifiers.length; i++) {
-      let identifier = identifiers[i];
-      let recordArrays = recordArraysForIdentifier(identifier);
-      if (recordArrays.has(array) === false) {
-        recordArrays.add(array);
-        identifiersToAdd.push(identifier);
-      }
-    }
-
-    if (identifiersToAdd.length) {
-      array._pushIdentifiers(identifiersToAdd);
-    }
-  }
-
-  _didUpdateAll(modelName: string): void {
-    let recordArray = this._liveRecordArrays[modelName];
-    if (recordArray) {
-      recordArray.isUpdating = false;
-      // TODO potentially we should sync here, currently
-      // this occurs as a side-effect of individual records updating
-      // this._syncLiveRecordArray(recordArray, modelName);
-    }
+    sync(array, pending);
+    this._pending.delete(array);
   }
 
   /**
     Get the `RecordArray` for a modelName, which contains all loaded records of
     given modelName.
 
-    @method liveRecordArrayFor
+    @method liveArrayFor
     @internal
     @param {String} modelName
     @return {RecordArray}
   */
-  liveRecordArrayFor(modelName: string): RecordArray {
-    assert(
-      `recordArrayManger.liveRecordArrayFor expects modelName not modelClass as the param`,
-      typeof modelName === 'string'
-    );
-
-    let array = this._liveRecordArrays[modelName];
+  liveArrayFor(type: string): IdentifierArray {
+    let array = this._live.get(type);
+    let identifiers: StableRecordIdentifier[] = [];
+    let staged = this._staged.get(type);
+    if (staged) {
+      staged.forEach((value, key) => {
+        if (value === 'add') {
+          identifiers.push(key);
+        }
+      });
+      this._staged.delete(type);
+    }
 
-    if (array) {
-      // if the array already exists, synchronize
-      this._syncLiveRecordArray(array, modelName);
-    } else {
-      // if the array is being newly created merely create it with its initial
-      // content already set. This prevents unneeded change events.
-      let identifiers = this._visibleIdentifiersByType(modelName);
-      array = this.createRecordArray(modelName, identifiers);
-      this._liveRecordArrays[modelName] = array;
+    if (!array) {
+      array = new IdentifierArray({
+        type,
+        identifiers,
+        store: this.store,
+        allowMutation: false,
+        manager: this,
+      });
+      this._live.set(type, array);
     }
 
     return array;
   }
 
-  _visibleIdentifiersByType(modelName: string) {
-    const cache = this.store._instanceCache;
-    const list = cache.peekList[modelName];
-    let all = list ? [...list.values()] : [];
-    let visible: StableRecordIdentifier[] = [];
-    for (let i = 0; i < all.length; i++) {
-      let identifier = all[i];
-
-      if (cache.recordIsLoaded(identifier, true)) {
-        visible.push(identifier);
-      }
-    }
-    return visible;
-  }
-
-  /**
-    Create a `RecordArray` for a modelName.
-
-    @method createRecordArray
-    @internal
-    @param {String} modelName
-    @param {Array} [identifiers]
-    @return {RecordArray}
-  */
-  createRecordArray(modelName: string, identifiers: StableRecordIdentifier[] = []): RecordArray {
-    assert(
-      `recordArrayManger.createRecordArray expects modelName not modelClass as the param`,
-      typeof modelName === 'string'
-    );
-
-    let array = RecordArray.create({
-      modelName,
-      content: A(identifiers || []),
+  createArray(config: {
+    type: string;
+    query?: Dict<unknown>;
+    identifiers?: StableRecordIdentifier[];
+    doc?: CollectionResourceDocument;
+  }): Collection {
+    let options: CollectionCreateOptions = {
+      type: config.type,
+      links: config.doc?.links || null,
+      meta: config.doc?.meta || null,
+      query: config.query || null,
+      identifiers: config.identifiers || [],
+      isLoaded: !!config.identifiers?.length,
+      allowMutation: false,
       store: this.store,
-      isLoaded: true,
       manager: this,
-    });
-
-    if (Array.isArray(identifiers)) {
-      this._associateWithRecordArray(identifiers, array);
+    };
+    let array = new Collection(options);
+    this._managed.add(array);
+    if (config.identifiers) {
+      associate(array, config.identifiers);
     }
 
     return array;
   }
 
-  /**
-    Create a `AdapterPopulatedRecordArray` for a modelName with given query.
-
-    @method createAdapterPopulatedRecordArray
-    @internal
-    @param {String} modelName
-    @param {Object} query
-    @return {AdapterPopulatedRecordArray}
-  */
-  createAdapterPopulatedRecordArray(
-    modelName: string,
-    query: Dict<unknown> | undefined,
-    identifiers: StableRecordIdentifier[],
-    payload?: CollectionResourceDocument
-  ): AdapterPopulatedRecordArray {
-    assert(
-      `recordArrayManger.createAdapterPopulatedRecordArray expects modelName not modelClass as the first param, received ${modelName}`,
-      typeof modelName === 'string'
-    );
-
-    let array: AdapterPopulatedRecordArray;
-    if (Array.isArray(identifiers)) {
-      array = AdapterPopulatedRecordArray.create({
-        modelName,
-        query: query,
-        content: A(identifiers),
-        store: this.store,
-        manager: this,
-        isLoaded: true,
-        isUpdating: false,
-        // TODO this assign kills the root reference but a deep-copy would be required
-        // for both meta and links to actually not be by-ref. We whould likely change
-        // this to a dev-only deep-freeze.
-        meta: Object.assign({} as Meta, payload?.meta),
-        links: Object.assign({}, payload?.links),
-      });
-
-      this._associateWithRecordArray(identifiers, array);
-    } else {
-      array = AdapterPopulatedRecordArray.create({
-        modelName,
-        query: query,
-        content: A<StableRecordIdentifier>(),
-        isLoaded: false,
-        store: this.store,
-        manager: this,
-      });
+  dirtyArray(array: IdentifierArray): void {
+    if (array === FAKE_ARR) {
+      return;
+    }
+    let tag = array[IDENTIFIER_ARRAY_TAG];
+    if (!tag.shouldReset) {
+      tag.shouldReset = true;
+      tag.ref = null;
     }
+  }
 
-    this._adapterPopulatedRecordArrays.push(array);
+  _getPendingFor(
+    identifier: StableRecordIdentifier,
+    includeManaged: boolean,
+    isRemove?: boolean
+  ): Map<IdentifierArray, ChangeSet> | void {
+    if (this.isDestroying || this.isDestroyed) {
+      return;
+    }
 
-    return array;
-  }
+    let liveArray = this._live.get(identifier.type);
+    const allPending = this._pending;
+    let pending: Map<IdentifierArray, ChangeSet> = new Map();
+
+    if (includeManaged) {
+      let managed = RecordArraysCache.get(identifier);
+      if (managed) {
+        managed.forEach((arr) => {
+          let changes = allPending.get(arr);
+          if (!changes) {
+            changes = new Map();
+            allPending.set(arr, changes);
+          }
+          pending.set(arr, changes);
+        });
+      }
+    }
 
-  /**
-    Unregister a RecordArray.
-    So manager will not update this array.
+    // during unloadAll we can ignore removes since we've already
+    // cleared the array.
+    if (liveArray && liveArray[SOURCE].length === 0 && isRemove) {
+      return pending;
+    }
 
-    @method unregisterRecordArray
-    @internal
-    @param {RecordArray} array
-  */
-  unregisterRecordArray(array: RecordArray): void {
-    let modelName = array.modelName;
-
-    // remove from adapter populated record array
-    let removedFromAdapterPopulated = removeFromArray(this._adapterPopulatedRecordArrays, array);
-
-    if (!removedFromAdapterPopulated) {
-      let liveRecordArrayForType = this._liveRecordArrays[modelName];
-      // unregister live record array
-      if (liveRecordArrayForType) {
-        if (array === liveRecordArrayForType) {
-          delete this._liveRecordArrays[modelName];
-        }
+    if (!liveArray) {
+      // start building a changeset for when we eventually
+      // do have a live array
+      let changes = this._staged.get(identifier.type);
+      if (!changes) {
+        changes = new Map();
+        this._staged.set(identifier.type, changes);
+      }
+      pending.set(FAKE_ARR as IdentifierArray, changes);
+    } else {
+      let changes = allPending.get(liveArray);
+      if (!changes) {
+        changes = new Map();
+        allPending.set(liveArray, changes);
       }
+      pending.set(liveArray, changes);
     }
+
+    return pending;
   }
 
-  /**
-   * @method _associateWithRecordArray
-   * @internal
-   * @param {StableIdentifier} identifiers
-   * @param {RecordArray} array
-   */
-  _associateWithRecordArray(identifiers: StableRecordIdentifier[], array: RecordArray): void {
-    for (let i = 0, l = identifiers.length; i < l; i++) {
-      let identifier = identifiers[i];
-      let recordArrays = this.getRecordArraysForIdentifier(identifier);
-      recordArrays.add(array);
-    }
+  populateManagedArray(array: Collection, identifiers: StableRecordIdentifier[], payload: CollectionResourceDocument) {
+    this._pending.delete(array);
+    const source = array[SOURCE];
+    const old = source.slice();
+    source.length = 0;
+    fastPush(source, identifiers);
+    array[IDENTIFIER_ARRAY_TAG].ref = null;
+    array.meta = payload.meta || null;
+    array.links = payload.links || null;
+    array.isLoaded = true;
+
+    disassociate(array, old);
+    associate(array, identifiers);
   }
 
-  /**
-    @method recordDidChange
-    @internal
-  */
-  recordDidChange(identifier: StableRecordIdentifier): void {
-    if (this.isDestroying || this.isDestroyed) {
-      return;
+  identifierAdded(identifier: StableRecordIdentifier): void {
+    let changeSets = this._getPendingFor(identifier, false);
+    if (changeSets) {
+      changeSets.forEach((changes, array) => {
+        let existing = changes.get(identifier);
+        if (existing === 'del') {
+          changes.delete(identifier);
+        } else {
+          changes.set(identifier, 'add');
+
+          if (changes.size === 1) {
+            this.dirtyArray(array);
+          }
+        }
+      });
     }
-    let modelName = identifier.type;
+  }
 
-    if (pendingForIdentifier.has(identifier)) {
-      return;
+  identifierRemoved(identifier: StableRecordIdentifier): void {
+    let changeSets = this._getPendingFor(identifier, true, true);
+    if (changeSets) {
+      changeSets.forEach((changes, array) => {
+        let existing = changes.get(identifier);
+        if (existing === 'add') {
+          changes.delete(identifier);
+        } else {
+          changes.set(identifier, 'del');
+
+          if (changes.size === 1) {
+            this.dirtyArray(array);
+          }
+        }
+      });
     }
+  }
 
-    pendingForIdentifier.add(identifier);
+  identifierChanged(identifier: StableRecordIdentifier): void {
+    let newState = this.store._instanceCache.recordIsLoaded(identifier, true);
 
-    let pending = this._pendingIdentifiers;
-    let models = (pending[modelName] = pending[modelName] || []);
-    if (models.push(identifier) !== 1) {
-      return;
+    if (newState) {
+      this.identifierAdded(identifier);
+    } else {
+      this.identifierRemoved(identifier);
     }
-
-    // TODO do we still need this schedule?
-    // eslint-disable-next-line @typescript-eslint/unbound-method
-    emberBackburner.schedule('actions', this, this._flush);
   }
 
-  willDestroy() {
-    Object.keys(this._liveRecordArrays).forEach((modelName) => this._liveRecordArrays[modelName]!.destroy());
-    this._adapterPopulatedRecordArrays.forEach((entry) => entry.destroy());
-    this.isDestroyed = true;
+  clear() {
+    this._live.forEach((array) => array.destroy());
+    this._managed.forEach((array) => array.destroy());
+    this._managed.clear();
+    RecordArraysCache.clear();
   }
 
   destroy() {
     this.isDestroying = true;
-    // TODO do we still need this schedule?
-    // eslint-disable-next-line @typescript-eslint/unbound-method
-    emberBackburner.schedule('actions', this, this.willDestroy);
-  }
-}
-
-function removeFromArray(array: RecordArray[], item: RecordArray): boolean {
-  let index = array.indexOf(item);
-
-  if (index !== -1) {
-    array.splice(index, 1);
-    return true;
+    this.clear();
+    this._live.clear();
+    this.isDestroyed = true;
   }
-
-  return false;
 }
 
-function updateLiveRecordArray(store: Store, recordArray: RecordArray, identifiers: StableRecordIdentifier[]): void {
-  let identifiersToAdd: StableRecordIdentifier[] = [];
-  let identifiersToRemove: StableRecordIdentifier[] = [];
-  const cache = store._instanceCache;
-
+function associate(array: Collection, identifiers: StableRecordIdentifier[]) {
   for (let i = 0; i < identifiers.length; i++) {
     let identifier = identifiers[i];
-    let recordArrays = recordArraysForIdentifier(identifier);
-
-    if (cache.recordIsLoaded(identifier, true)) {
-      if (!recordArrays.has(recordArray)) {
-        identifiersToAdd.push(identifier);
-        recordArrays.add(recordArray);
-      }
-    } else {
-      identifiersToRemove.push(identifier);
-      recordArrays.delete(recordArray);
+    let cache = RecordArraysCache.get(identifier);
+    if (!cache) {
+      cache = new Set();
+      RecordArraysCache.set(identifier, cache);
     }
-  }
-
-  if (identifiersToAdd.length > 0) {
-    recordArray._pushIdentifiers(identifiersToAdd);
-  }
-  if (identifiersToRemove.length > 0) {
-    recordArray._removeIdentifiers(identifiersToRemove);
+    cache.add(array);
   }
 }
 
-function removeFromAdapterPopulatedRecordArrays(store: Store, identifiers: StableRecordIdentifier[]): void {
+function disassociate(array: Collection, identifiers: StableRecordIdentifier[]) {
   for (let i = 0; i < identifiers.length; i++) {
-    removeFromAll(store, identifiers[i]);
+    disassociateIdentifier(array, identifiers[i]);
   }
 }
 
-function removeFromAll(store: Store, identifier: StableRecordIdentifier): void {
-  const recordArrays = recordArraysForIdentifier(identifier);
+export function disassociateIdentifier(array: Collection, identifier: StableRecordIdentifier) {
+  let cache = RecordArraysCache.get(identifier);
+  if (cache) {
+    cache.delete(array);
+  }
+}
 
-  recordArrays.forEach(function (recordArray) {
-    recordArray._removeIdentifiers([identifier]);
+function sync(array: IdentifierArray, changes: Map<StableRecordIdentifier, 'add' | 'del'>) {
+  let state = array[SOURCE];
+  const adds: StableRecordIdentifier[] = [];
+  const removes: StableRecordIdentifier[] = [];
+  changes.forEach((value, key) => {
+    if (value === 'add') {
+      // likely we want to keep a Set along-side
+      if (state.includes(key)) {
+        return;
+      }
+      adds.push(key);
+    } else {
+      removes.push(key);
+    }
   });
+  if (removes.length) {
+    if (removes.length === state.length) {
+      state.length = 0;
+      // changing the reference breaks the Proxy
+      // state = array[SOURCE] = [];
+    } else {
+      removes.forEach((i) => {
+        state.splice(state.indexOf(i), 1);
+      });
+    }
+  }
 
-  recordArrays.clear();
+  if (adds.length) {
+    fastPush(state, adds);
+    // changing the reference breaks the Proxy
+    // else we could do this
+    /*
+    if (state.length === 0) {
+      array[SOURCE] = adds;
+    } else {
+      array[SOURCE] = state.concat(adds);
+    }
+    */
+  }
 }
 
 export default RecordArrayManager;